Update dependencies and support

- Drop support for Python < 3.6 (including Python 2), resolves #94
- Some dependencies have less strict requirements, resolves #121
- Add Python 3.8 and 3.9 support
- Rebase Docker images on Python 3.9.4 and Alpine 3.13
- Update CI pipelines and remove Travis
This commit is contained in:
Fred Park 2021-04-12 20:01:48 +00:00
Родитель 7f7ab7ce7f
Коммит ea0af4a5bc
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 3C4D545F457737EB
60 изменённых файлов: 149 добавлений и 551 удалений

Просмотреть файл

@ -1,16 +0,0 @@
language: python
cache: pip
matrix:
include:
- python: 3.5
- python: 3.6
install:
- travis_retry pip install --upgrade pip
- travis_retry pip install --upgrade setuptools wheel
- travis_retry pip install --upgrade tox-travis
script:
- tox

Просмотреть файл

@ -1,5 +1,12 @@
name: $(SourceBranch)$(Rev:.r)
variables:
VENV_VERSION: '20.4.3'
PY36_VER: '3.6.13'
PY37_VER: '3.7.10'
PY38_VER: '3.8.9'
PY39_VER: '3.9.4'
jobs:
- job: ComponentGovernance
pool:
@ -13,15 +20,14 @@ jobs:
displayName: 'CG Component Detection'
- job: Windows
pool:
vmIamge: vs2017-win2016
vmImage: vs2017-win2016
strategy:
maxParallel: 1
maxParallel: 0
matrix:
Python37:
python.version: '3.7'
PYENV_VERSION: '3.7.5'
PYTOX_ENV: 'py37'
VENV_VERSION: '16.2.0'
Python39:
python.version: '3.9'
PYENV_VERSION: $(PY39_VER)
PYTOX_ENV: 'py39'
steps:
- task: InstallPython@1
inputs:
@ -58,7 +64,7 @@ jobs:
displayName: Pre-build Environment (Branch)
condition: >
and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.7'))
in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.9'))
- powershell: |
git tag -l --points-at $env:BUILD_SOURCEVERSION | Tee-Object -Variable gitTag
$gitTag -match "^([\d\.])+"
@ -76,7 +82,7 @@ jobs:
displayName: Pre-build Environment (Tagged Release)
condition: >
and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.7'))
startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.9'))
- powershell: |
$artifactCliPath = "bin\\" + $env:ARTIFACT_CLI
echo "##vso[task.setvariable variable=ARTIFACT_CLI_PATH;]$artifactCliPath"
@ -136,15 +142,26 @@ jobs:
- job: Linux
pool:
vmImage: ubuntu-16.04
vmImage: ubuntu-20.04
strategy:
maxParallel: 1
maxParallel: 0
matrix:
Python36:
python.version: '3.6'
PYENV_VERSION: $(PY36_VER)
PYTOX_ENV: 'py36'
Python37:
python.version: '3.7'
PYENV_VERSION: '3.7.5'
PYENV_VERSION: $(PY37_VER)
PYTOX_ENV: 'py37'
VENV_VERSION: '16.2.0'
Python38:
python.version: '3.8'
PYENV_VERSION: $(PY38_VER)
PYTOX_ENV: 'py38'
Python39:
python.version: '3.9'
PYENV_VERSION: $(PY39_VER)
PYTOX_ENV: 'py39'
steps:
- task: UsePythonVersion@0
inputs:
@ -200,7 +217,7 @@ jobs:
displayName: Pre-build Environment (Branch)
condition: >
and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.7'))
in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.9'))
- script: |
set -e
set -o pipefail
@ -216,7 +233,7 @@ jobs:
displayName: Pre-build Environment (Tagged Release)
condition: >
and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.7'))
startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.9'))
- template: ./pyenv.yml
- script: |
set -e
@ -269,15 +286,14 @@ jobs:
- job: MacOS
pool:
vmImage: macOS-10.13
vmImage: macOS-10.15
strategy:
maxParallel: 1
maxParallel: 0
matrix:
Python37:
python.version: '3.7'
PYENV_VERSION: '3.7.5'
PYTOX_ENV: 'py37'
VENV_VERSION: '16.2.0'
Python39:
python.version: '3.9'
PYENV_VERSION: $(PY39_VER)
PYTOX_ENV: 'py39'
steps:
- task: UsePythonVersion@0
inputs:
@ -326,7 +342,7 @@ jobs:
displayName: Pre-build Environment (Branch)
condition: >
and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.7'))
in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.9'))
- script: |
GIT_TAG=$(git tag -l --points-at $BUILD_SOURCEVERSION)
ARTIFACT_CLI="blobxfer-${GIT_TAG}-mac-x86_64"
@ -337,7 +353,7 @@ jobs:
displayName: Pre-build Environment (Tagged Release)
condition: >
and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.7'))
startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.9'))
- template: ./pyenv.yml
- script: |
set -e

Просмотреть файл

@ -1,5 +1,4 @@
[![Build Status](https://azurebatch.visualstudio.com/blobxfer/_apis/build/status/blobxfer-CI)](https://azurebatch.visualstudio.com/blobxfer/_build/latest?definitionId=12)
[![Build Status](https://travis-ci.org/Azure/blobxfer.svg?branch=master)](https://travis-ci.org/Azure/blobxfer)
[![Build status](https://ci.appveyor.com/api/projects/status/qgth9p7jlessgp5i/branch/master?svg=true)](https://ci.appveyor.com/project/alfpark/blobxfer)
[![codecov](https://codecov.io/gh/Azure/blobxfer/branch/master/graph/badge.svg)](https://codecov.io/gh/Azure/blobxfer)
[![PyPI](https://img.shields.io/pypi/v/blobxfer.svg)](https://pypi.python.org/pypi/blobxfer)

Просмотреть файл

@ -7,16 +7,21 @@ cache:
environment:
matrix:
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
PYTHON: "C:\\Python35-x64"
PYTHON_VERSION: "3.5"
PYTHON_ARCH: "64"
TOX_ENV: "py35"
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
PYTHON: "C:\\Python36-x64"
PYTHON_VERSION: "3.6"
PYTHON_ARCH: "64"
TOX_ENV: "py36"
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
PYTHON: "C:\\Python37-x64"
PYTHON_VERSION: "3.7"
PYTHON_ARCH: "64"
TOX_ENV: "py37"
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
PYTHON: "C:\\Python38-x64"
PYTHON_VERSION: "3.8"
PYTHON_ARCH: "64"
TOX_ENV: "py38"
init:
- echo %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%

Просмотреть файл

@ -22,12 +22,6 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import absolute_import, division, print_function # noqa
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip
)
# stdlib imports
# non-stdlib imports
# local imports

Просмотреть файл

@ -22,19 +22,9 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import fnmatch
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
import pathlib
# non-stdlib imports
# local imports

Просмотреть файл

@ -22,19 +22,9 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import enum
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
import pathlib
# non-stdlib imports
from azure.storage.blob.models import _BlobTypes as BlobTypes
# local imports

Просмотреть файл

@ -22,13 +22,6 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import base64
import collections

Просмотреть файл

@ -22,22 +22,12 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import collections
import logging
import math
import os
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
import pathlib
import tempfile
import threading
import time

Просмотреть файл

@ -22,13 +22,6 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import collections
import json

Просмотреть файл

@ -22,21 +22,11 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import logging
import multiprocessing
import threading
try:
import queue
except ImportError: # noqa
import Queue as queue
import queue
# create logger
logger = logging.getLogger(__name__)

Просмотреть файл

@ -22,21 +22,11 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import collections
import logging
import multiprocessing
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
import pathlib
import sys
# non-stdlib imports
# local imports

Просмотреть файл

@ -22,13 +22,6 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
# non-stdlib imports
# local imports

Просмотреть файл

@ -22,13 +22,6 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import collections
import logging

Просмотреть файл

@ -22,13 +22,6 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import collections
import enum
@ -36,10 +29,7 @@ import json
import logging
import math
import os
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
import pathlib
import threading
# non-stdlib imports
import bitstring

Просмотреть файл

@ -22,13 +22,6 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import logging
import re

Просмотреть файл

@ -22,12 +22,6 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import absolute_import, division, print_function
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip
)
# stdlib imports
import logging
# non-stdlib imports

Просмотреть файл

@ -22,12 +22,6 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import absolute_import, division, print_function
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip
)
# stdlib imports
import logging
# non-stdlib imports

Просмотреть файл

@ -22,12 +22,6 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import absolute_import, division, print_function
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip
)
# stdlib imports
import datetime
import logging

Просмотреть файл

@ -22,12 +22,6 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import absolute_import, division, print_function
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip
)
# stdlib imports
import logging
# non-stdlib imports

Просмотреть файл

@ -22,18 +22,9 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import absolute_import, division, print_function
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip
)
# stdlib imports
import logging
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
import pathlib
# non-stdlib imports
import azure.common
import azure.storage.file

Просмотреть файл

@ -22,22 +22,12 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import base64
import enum
import logging
import os
try:
import queue
except ImportError: # noqa
import Queue as queue
import queue
import tempfile
# non-stdlib imports
import cryptography.hazmat.backends

Просмотреть файл

@ -22,24 +22,11 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import enum
import logging
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
try:
import queue
except ImportError: # noqa
import Queue as queue
import pathlib
import queue
import threading
import time
# non-stdlib imports
@ -466,7 +453,7 @@ class Downloader(object):
if terminate:
self._download_terminate = terminate
for thr in self._disk_threads:
blobxfer.util.join_thread(thr)
thr.join()
def _wait_for_transfer_threads(self, terminate):
# type: (Downloader, bool) -> None
@ -477,7 +464,7 @@ class Downloader(object):
if terminate:
self._download_terminate = terminate
for thr in self._transfer_threads:
blobxfer.util.join_thread(thr)
thr.join()
def _worker_thread_transfer(self):
# type: (Downloader) -> None

Просмотреть файл

@ -22,18 +22,9 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import absolute_import, division, print_function
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip
)
# stdlib imports
import logging
try:
import queue
except ImportError: # noqa
import Queue as queue
import queue
# non-stdlib imports
# local imports
import blobxfer.models.azure

Просмотреть файл

@ -22,13 +22,6 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import logging
import platform

Просмотреть файл

@ -22,20 +22,10 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import contextlib
import logging
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
import pathlib
import pickle
import shelve
import threading
@ -112,11 +102,7 @@ class _BaseResumeManager(object):
:rtype: str
:return: record key
"""
key = '{}:{}'.format(ase._client.primary_endpoint, ase.path)
if blobxfer.util.on_python2():
return key.encode('utf8')
else:
return key
return '{}:{}'.format(ase._client.primary_endpoint, ase.path)
def get_record(self, ase, key=None, lock=True):
# type: (_BaseResumeManager, str, bool) -> object

Просмотреть файл

@ -22,24 +22,11 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import enum
import logging
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
try:
import queue
except ImportError: # noqa
import Queue as queue
import pathlib
import queue
import threading
# non-stdlib imports
# local imports
@ -283,7 +270,7 @@ class SyncCopy(object):
if terminate:
self._synccopy_terminate = terminate
for thr in self._transfer_threads:
blobxfer.util.join_thread(thr)
thr.join()
def _worker_thread_transfer(self):
# type: (SyncCopy) -> None

Просмотреть файл

@ -22,25 +22,12 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import enum
import logging
import math
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
try:
import queue
except ImportError: # noqa
import Queue as queue
import pathlib
import queue
import threading
import time
# non-stdlib imports

Просмотреть файл

@ -22,13 +22,6 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import errno
import ssl

Просмотреть файл

@ -22,12 +22,6 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import absolute_import, division, print_function
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip
)
# stdlib imports
import base64
import copy
@ -36,17 +30,13 @@ import hashlib
import logging
import logging.handlers
import mimetypes
try:
from os import scandir as scandir
except ImportError: # noqa
from scandir import scandir as scandir
import os
import platform
import re
import sys
# non-stdlib imports
import dateutil.parser
import dateutil.tz
import future.utils
# local imports
# global defines
@ -58,15 +48,6 @@ _REGISTERED_LOGGER_HANDLERS = []
_PAGEBLOB_BOUNDARY = 512
def on_python2():
# type: (None) -> bool
"""Execution on python2
:rtype: bool
:return: if on Python2
"""
return future.utils.PY2
def on_linux(): # noqa
# type: (None) -> bool
"""Execution on Linux
@ -144,20 +125,6 @@ def is_not_empty(obj):
return obj is not None and len(obj) > 0
def join_thread(thr):
# type: (threading.Thread) -> None
"""Join a thread
:type threading.Thread thr: thread to join
"""
if on_python2():
while True:
thr.join(timeout=1)
if not thr.isAlive():
break
else:
thr.join()
def merge_dict(dict1, dict2):
# type: (dict, dict) -> dict
"""Recursively merge dictionaries: dict2 on to dict1. This differs
@ -214,7 +181,7 @@ def scantree(path):
:rtype: DirEntry
:return: DirEntry via generator
"""
for entry in scandir(path):
for entry in os.scandir(path):
if entry.is_dir(follow_symlinks=True):
# due to python2 compat, cannot use yield from here
for t in scantree(entry.path):
@ -241,10 +208,7 @@ def base64_encode_as_string(obj): # noqa
:rtype: str
:return: base64 encoded string
"""
if on_python2():
return base64.b64encode(obj)
else:
return str(base64.b64encode(obj), 'ascii')
return str(base64.b64encode(obj), 'ascii')
def base64_decode_string(string):

Просмотреть файл

@ -22,4 +22,4 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
__version__ = '1.9.4'
__version__ = '1.10.0'

Просмотреть файл

@ -22,19 +22,11 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import absolute_import, division, print_function
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import json
import logging
import multiprocessing
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
import pathlib
# non-stdlib imports
import click
import ruamel.yaml

Просмотреть файл

@ -22,13 +22,6 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# compat imports
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from builtins import ( # noqa
bytes, dict, int, list, object, range, ascii, chr, hex, input,
next, oct, open, pow, round, super, filter, map, zip)
# stdlib imports
import enum
# non-stdlib imports

Просмотреть файл

@ -1,6 +1,6 @@
# Dockerfile for Azure/blobxfer (Linux)
FROM alpine:3.10
FROM python:3.9.4-alpine3.13
MAINTAINER Fred Park <https://github.com/Azure/blobxfer>
ARG GIT_BRANCH
@ -8,10 +8,10 @@ ARG GIT_COMMIT
RUN apk update \
&& apk add --update --no-cache musl \
musl build-base python3 python3-dev openssl-dev libffi-dev \
ca-certificates git \
&& python3 -m pip install --upgrade pip \
&& pip3 install --upgrade setuptools \
musl build-base openssl-dev libffi-dev \
rust cargo ca-certificates git \
&& python3 -m ensurepip --upgrade \
&& pip3 install --no-cache-dir --upgrade pip setuptools setuptools-rust wheel \
&& git clone -b $GIT_BRANCH --single-branch --depth 5 https://github.com/Azure/blobxfer.git /blobxfer \
&& cd /blobxfer \
&& git checkout $GIT_COMMIT \
@ -19,9 +19,12 @@ RUN apk update \
&& python3 setup.py install \
&& cp THIRD_PARTY_NOTICES.txt /BLOBXFER_THIRD_PARTY_NOTICES.txt \
&& cp LICENSE /BLOBXFER_LICENSE.txt \
&& pip3 uninstall -y setuptools-rust wheel \
&& apk del --purge build-base patch openssl-dev libffi-dev rust cargo git \
&& rm /var/cache/apk/* \
&& cd /root \
&& rm -rf .cache .cargo \
&& cd / \
&& rm -rf blobxfer \
&& apk del --purge build-base python3-dev openssl-dev libffi-dev git \
&& rm /var/cache/apk/*
&& rm -rf blobxfer
ENTRYPOINT ["blobxfer"]

Просмотреть файл

@ -1,7 +1,7 @@
# Dockerfile for Azure/blobxfer (Windows)
# Adapted from: https://github.com/StefanScherer/dockerfiles-windows/blob/master/python/Dockerfile
FROM python:3.7.5-windowsservercore-ltsc2016
FROM python:3.9.4-windowsservercore-ltsc2016
MAINTAINER Fred Park <https://github.com/Azure/blobxfer>
ENV chocolateyUseWindowsCompression false
@ -29,8 +29,8 @@ COPY --from=0 /blobxfer/LICENSE /BLOBXFER_LICENSE.txt
SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
ENV PYTHON_VERSION 3.7.5
ENV PYTHON_PIP_VERSION 19.3.1
ENV PYTHON_VERSION 3.9.4
ENV PYTHON_PIP_VERSION 21.0.1
RUN $env:PATH = 'C:\Python;C:\Python\Scripts;{0}' -f $env:PATH ; \
Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment\' -Name Path -Value $env:PATH ; \

Просмотреть файл

@ -16,8 +16,8 @@ compiler and supporting libraries if there is no binary wheel for that
dependency and your platform. Please follow the pre-requisites section first
prior to invoking installation via `pip`.
It is strongly recommended to use a 64-bit Python interpreter. Only Python3
is supported.
It is strongly recommended to use a 64-bit Python interpreter. Python 2
is not supported.
### Pre-requisites
`blobxfer` has dependencies which require a C compiler if your platform does
@ -49,12 +49,12 @@ curl -fSsL https://bootstrap.pypa.io/get-pip.py | python3
```
#### Mac OS X
To install `blobxfer` for Python 3.5+, please follow the steps outlined on
To install `blobxfer` for Python 3.6+, please follow the steps outlined on
[this guide](http://docs.python-guide.org/en/latest/starting/install3/osx/#install3-osx)
to ensure that you have the latest version of Python, a compiler and pip.
#### Windows
Please install at least Python 3.5 or later to avoid requiring a
Please install at least Python 3.6 or later to avoid requiring a
compiler. It is strongly recommended to use a 64-bit interpreter.
#### Windows Subsystem for Linux
@ -69,7 +69,7 @@ After the pre-requisite steps have been completed then install the
pip3 install blobxfer
```
`blobxfer` is compatible with Python 3.5+. To install for Python 3, some
`blobxfer` is compatible with Python 3.6+. To install for Python 3, some
distributions may use `pip3` instead of `pip`. Installing into a virtual
environment or your user area via `--user` is recommended to avoid
installation issues and conflicts with system-wide Python packages.

Просмотреть файл

@ -29,17 +29,14 @@ packages = [
]
install_requires = [
'azure-storage-blob~=2.1.0',
'azure-storage-file~=2.1.0',
'bitstring~=3.1.6',
'click~=7.0',
'cryptography~=2.8',
'future~=0.18.2',
'pathlib2>=2.3.5;python_version<"3.5"',
'python-dateutil~=2.8.1',
'requests~=2.22.0',
'ruamel.yaml~=0.16.5',
'scandir>=1.10.0;python_version<"3.5"',
'azure-storage-blob>=2.1.0,<3',
'azure-storage-file>=2.1.0,<3',
'bitstring>=3.1.7,<4',
'click>=7.0,<8',
'cryptography>=3.3.2',
'python-dateutil>=2.8.1,<3',
'requests>=2.25.1,<3',
'ruamel.yaml>=0.16.5',
]
setup(
@ -71,12 +68,7 @@ setup(
'Intended Audience :: System Administrators',
'License :: OSI Approved :: MIT License',
'Operating System :: OS Independent',
'Programming Language :: Python :: 2',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3'
'Topic :: Software Development :: Libraries :: Python Modules',
'Topic :: Utilities',
],

Просмотреть файл

@ -1,5 +1,4 @@
coverage==4.5.4
flake8==3.6.0
mock==3.0.5; python_version < '3.3'
pytest==4.6.6
pytest-cov==2.8.1
coverage==5.5
flake8==3.9.0
pytest==6.2.3
pytest-cov==2.11.1

Просмотреть файл

@ -2,10 +2,7 @@
"""Tests for models azure"""
# stdlib imports
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
import unittest.mock as mock
# non-stdlib imports
import azure.storage.blob
import azure.storage.file

Просмотреть файл

@ -5,15 +5,9 @@
import datetime
import hashlib
import hmac
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
import unittest.mock as mock
import os
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
import pathlib
import time
import unittest
# non-stdlib imports
@ -308,8 +302,7 @@ def test_set_final_path_view():
assert total_size == ase._size
@unittest.skipIf(
util.on_python2() or not util.on_linux(), 'fallocate does not exist')
@unittest.skipIf(not util.on_linux(), 'fallocate does not exist')
def test_downloaddescriptor_allocate_disk_space_via_seek(tmpdir):
fp = pathlib.Path(str(tmpdir.join('fp')))
opts = mock.MagicMock()

Просмотреть файл

@ -3,10 +3,7 @@
# stdlib imports
import json
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
import unittest.mock as mock
# non-stdlib imports
import pytest
# module under test

Просмотреть файл

@ -2,10 +2,7 @@
"""Tests for offload"""
# stdlib imports
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
import unittest.mock as mock
# non-stdlib imports
import pytest
# local imports
@ -23,10 +20,10 @@ def test_multiprocess_offload():
with pytest.raises(ValueError):
a = offload._MultiprocessOffload(None, None)
if util.on_windows():
target = PicklableMagicMock()
else:
if util.on_linux():
target = mock.MagicMock()
else:
target = PicklableMagicMock()
a = offload._MultiprocessOffload(target, 1, 'test')
assert len(a._procs) == 1
assert not a.terminated

Просмотреть файл

@ -2,14 +2,8 @@
"""Tests for models options"""
# stdlib imports
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
import unittest.mock as mock
import pathlib
# non-stdlib imports
import pytest
# module under test

Просмотреть файл

@ -2,10 +2,7 @@
"""Tests for models synccopy"""
# stdlib imports
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
import unittest.mock as mock
# non-stdlib imports
import bitstring
import pytest

Просмотреть файл

@ -3,14 +3,8 @@
# stdlib imports
import hashlib
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
import unittest.mock as mock
import pathlib
# non-stdlib imports
import bitstring
import pytest

Просмотреть файл

@ -2,14 +2,8 @@
"""Tests for operations azure"""
# stdlib imports
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
import unittest.mock as mock
import pathlib
# non-stdlib imports
import azure.storage.blob
import azure.storage.file

Просмотреть файл

@ -2,10 +2,7 @@
"""Tests for general blob operations"""
# stdlib imports
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
import unittest.mock as mock
# non-stdlib imports
import azure.common
import azure.storage.blob

Просмотреть файл

@ -2,10 +2,7 @@
"""Tests for operations: blob append"""
# stdlib imports
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
import unittest.mock as mock
# non-stdlib imports
import azure.storage.common
# local imports

Просмотреть файл

@ -2,10 +2,7 @@
"""Tests for operations: block blob"""
# stdlib imports
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
import unittest.mock as mock
# non-stdlib imports
import azure.storage.common
# local imports

Просмотреть файл

@ -2,10 +2,7 @@
"""Tests for models"""
# stdlib imports
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
import unittest.mock as mock
# non-stdlib imports
import azure.storage.common
# local imports

Просмотреть файл

@ -2,14 +2,8 @@
"""Tests for file operations"""
# stdlib imports
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
import unittest.mock as mock
import pathlib
# non-stdlib imports
import azure.common
import azure.storage.common

Просмотреть файл

@ -2,10 +2,7 @@
"""Tests for crypto operations"""
# stdlib imports
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
import unittest.mock as mock
import os
import time
# non-stdlib imports
@ -17,8 +14,8 @@ import blobxfer.operations.crypto as ops
_RSAKEY = cryptography.hazmat.primitives.asymmetric.rsa.generate_private_key(
public_exponent=65537, key_size=2048,
backend=cryptography.hazmat.backends.default_backend())
public_exponent=65537, key_size=2048,
backend=cryptography.hazmat.backends.default_backend())
@mock.patch(

Просмотреть файл

@ -3,19 +3,10 @@
# stdlib imports
import datetime
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
import unittest.mock as mock
import multiprocessing
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
try:
import queue
except ImportError: # noqa
import Queue as queue
import pathlib
import queue
# non-stdlib imports
import azure.storage.blob
import dateutil.tz
@ -626,7 +617,7 @@ def test_add_to_download_queue(tmpdir):
d._add_to_download_queue(lpath, ase)
assert d._transfer_queue.qsize() == 1
assert path in d._dd_map
assert str(path) in d._dd_map
def test_initialize_and_terminate_threads():

Просмотреть файл

@ -2,10 +2,7 @@
"""Tests for progress operations"""
# stdlib imports
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
import unittest.mock as mock
# non-stdlib imports
# local imports
import blobxfer.models.azure as azmodels

Просмотреть файл

@ -2,14 +2,8 @@
"""Tests for operations resume"""
# stdlib imports
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
import unittest.mock as mock
import pathlib
# non-stdlib imports
# local imports
# module under test
@ -21,11 +15,7 @@ def test_generate_record_key():
ase._client.primary_endpoint = 'ep'
ase.path = 'abc'
with mock.patch('blobxfer.util.on_python2', return_value=True):
assert ops._BaseResumeManager.generate_record_key(ase) == b'ep:abc'
with mock.patch('blobxfer.util.on_python2', return_value=False):
assert ops._BaseResumeManager.generate_record_key(ase) == 'ep:abc'
assert ops._BaseResumeManager.generate_record_key(ase) == 'ep:abc'
def test_download_resume_manager(tmpdir):
@ -94,12 +84,6 @@ def test_download_resume_manager(tmpdir):
# oserror path
with mock.patch('blobxfer.util.on_windows', return_value=True):
with mock.patch('blobxfer.util.on_python2', return_value=False):
drm.delete()
assert drm._data is None
# oserror path
with mock.patch('blobxfer.util.on_python2', return_value=True):
drm.delete()
assert drm._data is None

Просмотреть файл

@ -3,14 +3,8 @@
# stdlib imports
import datetime
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
import unittest.mock as mock
import pathlib
# non-stdlib imports
import pytest
# local imports

Просмотреть файл

@ -3,14 +3,8 @@
# stdlib imports
import datetime
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
try:
import pathlib2 as pathlib
except ImportError: # noqa
import pathlib
import unittest.mock as mock
import pathlib
# non-stdlib imports
import azure.storage.blob
import pytest

Просмотреть файл

@ -2,10 +2,7 @@
"""Tests for retry"""
# stdlib imports
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
import unittest.mock as mock
import ssl
# non-stdlib imports
import azure.storage.common.models

Просмотреть файл

@ -3,11 +3,6 @@
# stdlib imports
import datetime
try:
import unittest.mock as mock
except ImportError: # noqa
import mock
import sys
import time
# non-stdlib imports
import dateutil.tz
@ -16,11 +11,6 @@ import pytest
import blobxfer.util
def test_on_python2():
py2 = sys.version_info.major == 2
assert py2 == blobxfer.util.on_python2()
def test_is_none_or_empty():
a = None
assert blobxfer.util.is_none_or_empty(a)
@ -59,21 +49,6 @@ def test_is_not_empty():
assert blobxfer.util.is_not_empty(a)
def test_join_thread():
with mock.patch('blobxfer.util.on_python2', return_value=True):
thr = mock.MagicMock()
thr.isAlive.side_effect = [True, False]
blobxfer.util.join_thread(thr)
assert thr.isAlive.call_count == 2
with mock.patch('blobxfer.util.on_python2', return_value=False):
thr = mock.MagicMock()
thr.isAlive.side_effect = [True, False]
blobxfer.util.join_thread(thr)
thr.join.assert_called_once_with()
thr.isAlive.assert_not_called()
def test_merge_dict():
with pytest.raises(ValueError):
blobxfer.util.merge_dict(1, 2)
@ -138,10 +113,7 @@ def test_get_mime_type():
def test_base64_encode_as_string():
a = b'abc'
enc = blobxfer.util.base64_encode_as_string(a)
if blobxfer.util.on_python2():
assert type(enc) == str
else:
assert type(enc) != bytes
assert type(enc) != bytes
dec = blobxfer.util.base64_decode_string(enc)
assert a == dec

Просмотреть файл

@ -1,13 +1,13 @@
[tox]
envlist = py27, py34, py35, py36, py37
envlist = py36, py37, py38, py39
[testenv]
deps = -rtest_requirements.txt
passenv = CI TRAVIS TRAVIS_*
commands =
pip list --format=columns --outdated
flake8 {envsitepackagesdir}/blobxfer_cli/
flake8 {envsitepackagesdir}/blobxfer/
flake8 --exit-zero {envsitepackagesdir}/blobxfer_cli/
flake8 --exit-zero {envsitepackagesdir}/blobxfer/
py.test \
-x -l -s \
--ignore venv/ \