Migrate to VSTS for build

- Update dependencies
- AppVeyor and Travis changes
  - Disable deployments
  - Enable test coverage on non-VSTS Python versions
- Add MacOS build

.travis.yml

@@ -7,98 +7,11 @@ matrix:
   - python: 2.7
   - python: 3.4
   - python: 3.5
-  - python: 3.6
-    addons:
-      apt:
-        packages:
-        - pandoc
-  # disable python3.7 for now, pep479 stopiteration behavior change
-  # breaks tests
-  # use workaround below until travis supports 3.7
-#  - python: 3.7
-#    dist: xenial
-#    sudo: true
-#  - language: generic
-#    os: osx
-#    before_install:
-#      - brew update
-#      - brew install python3
-#      - virtualenv -p python3 py3
-#      - source py3/bin/activate
-
-env:
-  global:
-  - BLOBXFER_ARTIFACT=blobxfer-$TRAVIS_TAG-$TRAVIS_OS_NAME-x86_64
-  - secure: XqrQXqxLRRFvhD9BA5AKoH3nPBA22xihQ1f4Q16bumW9WlpWv/zbbOa3eWIbCm2KqT+yOaXQN8vhDiBsGdJo6tvV4aZtPu2H5v4AsE5lmYsqeTZKh6w8EIfWeeKIzF5HoTKF35JK84+cpWRh42d0Shr2Awf1YlAu1pH6BcVkFQ5qV42rD+XH3TJJAwT97I0bARhkv/UJdf6gyIIw2G4FIDEyXfH3Lvxlmbnq85OL5p8VzTda70/7Mo2T3/rQofwVyRmMlWLLEuZuUzyo7R9KGpKdYMTijkLdFw9tSHOlcuW46iQimfWPsvdXCv8FUiDmulPG2a7fF5w7f89CDmupOhK0xd20C1v71uVa+f/k28qskA5PXaG0nzPQLWF8Avd/uehgXsaBNvLrC7PQZHOea/Ce4J5Yhdd6eigUmduP0GuOqZPm71Z8CxdOxNpOfUEE6Jx0eKW8hQXDt4N7ZwTU+kP+bOEKxPjky1Reg6DsiuRcrVHwYgk7RjidRr4kHLNam2h11NBDPC+8sIhTv4UBhtjspnrk2wT236uVBQWpGk5niPaMPCcdnCIghSMIQQdBSbv7huwv5MJOsjLcjbGtT1u0JgMtXI/TOjNsAZP5YzPPFQpzg8LwJ2qmEVTLOy6EJ5bJdWFCN1GPlupKbwURmpKyIqCdZoSnYnsgkEF0PFA=
-  - secure: F15Weue/tcIB7hcZFTOEVxQSwy5D0124Stq185bEudE3orpsscNp97FCNUczQHqt38OrRrswTU/HwpuJQF0G54RxLWQspmef7dgRIJSboQUd9j5Se/bmRmSsBGvffOCLyEDU8Lq0tT8xb2o30xclcqX9fb9EdXVlbHlIqeGnmkEEmgnCxUkfHEQW+kYN2CRVeVwKjnYnyByFtXErTjwc1Mp4IOpWjsclS+AIe2S0gAU5jx8ZqImXKUmhkm7OOIyOYaJjT+0r/MrpSJgwjjb+yPazhOL6tOD1gCFGds5NwvZd8B7cpD/DqgakEdQWz+pvedJZQTdfxSQPK7qYz43tjSwQhxIHO02hpSxq5T5feUuXOiLKlrona6aTLnzsr/ftJ0KlMElihwEJPCzrb9I5oxbCMz/ZH2a1yzGVUaeob3we1vIFN66dB27n0zEE3wxhJOcul5Y8SIQRrrMVLPYZmJr8HxV220lbadUOJInicyM5EsO+S7jLlb/b2n4i2mr3J0eNif1MaxLrj1NcbQOVjyjg/n9Zkw8fNB7x9qfsWmIMovKYQKFQpNW1sv5T5CR5zqV0in6eKvgw6qyt6oxxMN/ViZ0vyvywDvsEcH7pTZGsnC/VDrkEj3Z62zFa8ewY4W1AGQDWZtJBnPJXx2wmFWgnMG8mSpFuI/gJDkgprEI=
 
 install:
 - travis_retry pip install --upgrade pip
 - travis_retry pip install --upgrade setuptools wheel
-- travis_retry pip install --upgrade codecov tox-travis
+- travis_retry pip install --upgrade tox-travis
 
 script:
 - tox
-
-after_success:
-- codecov
-- |
-    if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then
-      echo "Not building binary with PR $TRAVIS_PULL_REQUEST"
-      return
-    fi
-    if [[ "$TRAVIS_PYTHON_VERSION" != "3.6" ]]; then
-      echo "Not building binary without Python 3.6"
-      return
-    fi
-    if [[ ! -z "$TRAVIS_TAG" ]]; then
-      export UPLOAD_PATH="releases/$TRAVIS_TAG"
-      if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-        brew install pandoc
-      else
-        travis_retry pip install pypandoc
-      fi
-    elif [[ "$TRAVIS_BRANCH" == "master" ]] || [[ "$TRAVIS_BRANCH" == "develop" ]]; then
-      export BLOBXFER_ARTIFACT="blobxfer-${TRAVIS_BRANCH}-${TRAVIS_BUILD_NUMBER}-$TRAVIS_OS_NAME-x86_64"
-      export UPLOAD_PATH="builds/$TRAVIS_BRANCH"
-    else
-      echo "Invalid tag=$TRAVIS_TAG or branch=$TRAVIS_BRANCH to build binary"
-      return
-    fi
-    echo "BLOBXFER_ARTIFACT=$BLOBXFER_ARTIFACT UPLOAD_PATH=$UPLOAD_PATH"
-    if [[ ! -e "bin/$BLOBXFER_ARTIFACT" ]]; then
-        virtualenv -p python3 pyi
-        $SHELL -c "set -e; source pyi/bin/activate; \
-                   pip install pyinstaller; \
-                   pip install --no-cache-dir -e .; \
-                   pyinstaller -F -n $BLOBXFER_ARTIFACT -p blobxfer:cli --additional-hooks-dir blobxfer --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --distpath bin cli/cli.py; \
-                   deactivate"
-        chmod +x bin/$BLOBXFER_ARTIFACT
-        sha256sum bin/$BLOBXFER_ARTIFACT | cut -d' ' -f1 > bin/${BLOBXFER_ARTIFACT}.sha256
-        cat bin/${BLOBXFER_ARTIFACT}.sha256
-        bin/$BLOBXFER_ARTIFACT upload --remote-path $UPLOAD_PATH --local-path bin/${BLOBXFER_ARTIFACT} --strip-components 1 --file-md5 --overwrite
-    fi
-
-deploy:
-  - provider: pypi
-    distributions: "sdist bdist_wheel"
-    skip_cleanup: true
-    on:
-      tags: true
-      condition: "$TRAVIS_PYTHON_VERSION == 3.6"
-    user: alfpark
-    password:
-      secure: p6i4ircy+h1HaIlVz+SsdA3yoyG86/X12QWVDyC2Vw9M53/rPjQKn/dbhQSTcvWZ90EmYNOnmkBT9tc7B9iRkixmCminic0oUX0bmEhvenscW8Bbhaa2Uh9P2AJtd3gsVcicWzaNhtnjui3/rJRljolGA1PwfXal0Ryg24NaBd2VRKCwOU4xsZZ2fCDZ02cotsehRvGeUa4d54CJ6eBuzN08q9AXYh9IGWCngPNo68XRRs+gDQ65kwmUYp6tIFB+zbL/0erbB1E88F+Dq6Ac/vNCZjjhXor8fDqoPJmch3JZaP091bHoPD+rWnEb8Vm6B4nBBwhFMbKg/9Kt3Y6Bmb2ifOBz9EPfeSxxXVetKOpOKxKfaGj7mcbG8w64uBxQxr/GJLmpVMbFCJRh1PeRL9J9deFwdANBvflgQzV0XwKOfv2N03dmRtJOXwmXPO/FDmr6oojRT5JHj8KpbO6/adCm9COm9mhjhD6ZOPX7ppBGeXmD5sGKrXX+rVfdtYeCLrawGMf/ZmSWHiT0q1KTPLrNaKK0Y2o5XCUKXcgcCOIPaP03Kqje++RxP3Ef5x2uB2o2ertsrY4rk0UyR93+ARgko/MvW8OlQJbz9oUh3qh87IdjOBgY6OTObTEVAGZWYRKQMKRxzTDGRuk/D1gXcXI7VfBZwylSEvHjE85rLJ4=
-  - provider: releases
-    skip_cleanup: true
-    on:
-      tags: true
-      condition: "$TRAVIS_PYTHON_VERSION == 3.6"
-    draft: true
-    overwrite: true
-    tag_name: $TRAVIS_TAG
-    file:
-      - bin/${BLOBXFER_ARTIFACT}
-      - bin/${BLOBXFER_ARTIFACT}.sha256
-    api_key:
-      secure: QCcrLa2x15RVNX42x6Ts4E/AU4QvwacuoQJ5BlFK6hdevRZYyRSQB3yBD6ZhAqi79qxd7KpRoPj8mu0N3WaZuB3XtQ2scxQLvN5vueC6c9A2Ic9venoLfVYG9Np4jOYxwx3QRtMPA6SNEmxNxBK+jjClwNi71vshTwo14z0/IW7Zi7WNEzoZW3iNj7e+zbaVvuyx+vGI+Ckgjyy7OgaulNtlHG/vYuJuAMn7WzbyGuSQTuqI2npKvykumi9rNSQpmR+2WP9gbo8D+WU+iESHJ+r0YUbZRWDwGHA7fvbCNLMwFdXwaXccc6jozxqULTmt6mNnlSpaKduYiSoru9L8+kzxR1b+xYEK0fGr6/khABt5O26Pk6tCynKz4LrurSefA3mHrNYIT3PGGUvUTJAhSVaK9O/ag2qkDTA0NGp7KV+xDG4KCWvqeuPssqh9ppDl27T1s/aWMtKeeswCRhGMlfCICg9hhPzyc97ZBzl7DLRPITUceSzc2cGp88zGYdlJEg+5yuSl1j+Al5jmWMxuLF1MLOmFYw1pA1z7zPpcq8zzC0t5eu6n/fFh6k57sZQ6VDeS4kix4q0W6XjIFx4yA/Aa4CpeEAOb+jPzZxRuQ/UvRgL4M7lnb8aIOqNcoXEcccgnrcULjJ9pcJCMHE0p5WU7JnF8mpazJude51yikx0=

.vsts/pipeline.yml

@@ -0,0 +1,319 @@
+name: $(SourceBranch)$(Rev:.r)
+
+phases:
+  - phase: Windows
+    queue:
+      name: Hosted VS2017
+      parallel: 1
+      matrix:
+        Python36:
+          python.version: '3.6'
+          PYENV_VERSION: '3.6.6'
+          PYTOX_ENV: 'py36'
+    steps:
+      - task: InstallPython@1
+        inputs:
+          version: 'python==$(PYENV_VERSION)'
+      - powershell: |
+          gci env:* | sort-object name
+          python -m pip install --upgrade pip
+          pip install --upgrade setuptools wheel
+          pip install codecov tox
+          pip list --format=columns --outdated
+        displayName: Fetch Dependencies
+      - powershell: |
+          tox -e $env:PYTOX_ENV
+        displayName: Unit Tests
+      - powershell: |
+          if($env:BUILD_SOURCEBRANCHNAME -eq "master")
+          {
+            $dockerTag = "latest"
+          }
+          else
+          {
+            $dockerTag = $env:BUILD_SOURCEBRANCHNAME
+          }
+          $artifactCli = "blobxfer-" + $env:BUILD_SOURCEBRANCHNAME + "-" + $env:BUILD_BUILDID + "-win-amd64.exe"
+          $artifactUploadPath = "builds/" + $env:BUILD_SOURCEBRANCHNAME
+          $buildverDotted = "0.0.0"
+          $bvt0,$bvt1,$bvt2 = $buildverDotted.split('.')
+          $buildverTuple = [string]::Format("({0}, {1}, {2}, {3})",$bvt0,$bvt1,$bvt2,$env:BUILD_BUILDID)
+          echo "##vso[task.setvariable variable=BUILDVER_DOTTED;]$buildverDotted"
+          echo "##vso[task.setvariable variable=BUILDVER_TUPLE;]$buildverTuple"
+          echo "##vso[task.setvariable variable=ARTIFACT_CLI;]$artifactCli"
+          echo "##vso[task.setvariable variable=ARTIFACT_UPLOAD_PATH;]$artifactUploadPath"
+          echo "##vso[task.setvariable variable=DOCKER_TAG;]$dockerTag"
+        displayName: Pre-build Environment (Branch)
+        condition: >
+          and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
+          in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.6'))
+      - powershell: |
+          git tag -l --points-at $env:BUILD_SOURCEVERSION | Tee-Object -Variable gitTag
+          $gitTag -match "^([\d\.])+"
+          $buildverDotted = $matches[0]
+          $artifactCli = "blobxfer-" + $gitTag + "-win-amd64.exe"
+          $artifactUploadPath = "releases/" + $gitTag
+          $bvt0,$bvt1,$bvt2 = $buildverDotted.split('.')
+          $buildverTuple = [string]::Format("({0}, {1}, {2}, {3})",$bvt0,$bvt1,$bvt2,$env:BUILD_BUILDID)
+          echo "##vso[task.setvariable variable=BUILDVER_DOTTED;]$buildverDotted"
+          echo "##vso[task.setvariable variable=BUILDVER_TUPLE;]$buildverTuple"
+          echo "##vso[task.setvariable variable=GIT_TAG;]$gitTag"
+          echo "##vso[task.setvariable variable=ARTIFACT_CLI;]$artifactCli"
+          echo "##vso[task.setvariable variable=ARTIFACT_UPLOAD_PATH;]$artifactUploadPath"
+          echo "##vso[task.setvariable variable=DOCKER_TAG;]$gitTag"
+        displayName: Pre-build Environment (Tagged Release)
+        condition: >
+          and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
+          startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.6'))
+      - powershell: |
+          $artifactCliPath = "bin\\" + $env:ARTIFACT_CLI
+          echo "##vso[task.setvariable variable=ARTIFACT_CLI_PATH;]$artifactCliPath"
+          $branchGitSha1 = [string]::Format("{0}@{1}",$env:BUILD_SOURCEBRANCHNAME,$env:BUILD_SOURCEVERSION.Substring(0,7))
+          echo "$env:ARTIFACT_CLI $env:BUILDVER_TUPLE $branchGitSha1"
+          $fileverInfo = "cli\\file_version_info.txt"
+          (Get-Content $fileverInfo).replace('{BUILDVER_TUPLE}', $env:BUILDVER_TUPLE) | Set-Content $fileverInfo
+          (Get-Content $fileverInfo).replace('{BUILDVER_DOTTED}', $env:BUILDVER_DOTTED) | Set-Content $fileverInfo
+          (Get-Content $fileverInfo).replace('{BRANCH_GITSHA1}', $branchGitSha1) | Set-Content $fileverInfo
+          (Get-Content $fileverInfo).replace('{EXE}', $env:ARTIFACT_CLI) | Set-Content $fileverInfo
+          pip install virtualenv
+          python -m virtualenv pyi
+        displayName: Pre-build
+        condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], ''))
+      - script: |
+          echo "ARTIFACT_CLI=%ARTIFACT_CLI% ARTIFACT_CLI_PATH=%ARTIFACT_CLI_PATH% ARTIFACT_UPLOAD_PATH=%ARTIFACT_UPLOAD_PATH%"
+          call pyi\\Scripts\\activate.bat
+          pip -V
+          pip install --no-cache-dir pyinstaller
+          pip install --no-cache-dir -e .
+          pyinstaller -F -n "%ARTIFACT_CLI%" -p blobxfer -p cli --additional-hooks-dir blobxfer --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --icon cli\\azure.ico --version-file cli\\file_version_info.txt --distpath bin cli\\cli.py
+          call pyi\\Scripts\\deactivate.bat
+          "%ARTIFACT_CLI_PATH%" upload --storage-account "$(blobxfer.storageAccount)" --storage-account-key "$(blobxfer.storageAccountKey)" --remote-path "%ARTIFACT_UPLOAD_PATH%" --local-path "%ARTIFACT_CLI_PATH%" --strip-components 1 --file-md5 --overwrite
+        displayName: Build
+        condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], ''))
+      - powershell: |
+          $versionTag = "bin\\version_tag.txt"
+          $env:GIT_TAG | Out-File $versionTag -Force -NoNewline
+          Get-Content -Path $versionTag
+          copy .vsts\\populate_tag.ps1 bin\\populate_tag.ps1
+          dir bin
+        displayName: Generate Version Tag File
+        condition: and(succeeded(), ne(variables['GIT_TAG'], ''))
+      - task: PublishBuildArtifacts@1
+        inputs:
+          pathtoPublish: bin
+          artifactName: Blobxfer-Windows
+        condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], ''))
+      - powershell: |
+          docker version
+          docker login -u="$(docker.username)" -p="$(docker.password)"
+          $cliImage = "alfpark/blobxfer:" + $env:DOCKER_TAG + "-windows"
+          echo "Building Docker image: $cliImage"
+          pushd docker\\win
+          docker build --build-arg GIT_BRANCH=$env:BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$env:BUILD_SOURCEVERSION -t $cliImage .
+          docker push $cliImage
+          popd
+        displayName: Docker build
+        condition: and(succeeded(), ne(variables['DOCKER_TAG'], ''))
+
+  - phase: Linux
+    queue:
+      name: Hosted Linux Preview
+      parallel: 1
+      matrix:
+        Python36:
+          python.version: '3.6'
+          PYENV_VERSION: '3.6.6'
+          PYTOX_ENV: 'py36'
+    steps:
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: $(python.version)
+          architecture: x64
+      - script: |
+          set -e
+          set -o pipefail
+          env
+          if [[ "$PYENV_VERSION" == 3.* ]]; then
+            which python3
+            python3 --version
+            echo "##vso[task.setvariable variable=PYTHON;]python3"
+            echo "##vso[task.setvariable variable=PIP;]pip3"
+          else
+            which python
+            python --version
+            echo "##vso[task.setvariable variable=PYTHON;]python"
+            echo "##vso[task.setvariable variable=PIP;]pip"
+          fi
+          ls -alFR
+        displayName: Initialize Build
+      - script: |
+          set -e
+          set -o pipefail
+          echo "PATH=$PATH"
+          echo "PYTHON=$PYTHON PIP=$PIP"
+          $PYTHON -m pip install --upgrade pip
+          $PIP install --upgrade setuptools wheel
+          $PIP install codecov tox
+          $PIP list --format=columns --outdated
+        displayName: Fetch Dependencies
+      - script: |
+          set -e
+          set -o pipefail
+          tox -e $PYTOX_ENV
+          codecov -t $(codecov.uploadToken)
+        displayName: Unit Tests
+      - script: |
+          set -e
+          set -o pipefail
+          ARTIFACT_CLI="blobxfer-${BUILD_SOURCEBRANCHNAME}-${BUILD_BUILDID}-linux-x86_64"
+          ARTIFACT_UPLOAD_PATH="builds/${BUILD_SOURCEBRANCHNAME}"
+          echo "##vso[task.setvariable variable=ARTIFACT_CLI;]${ARTIFACT_CLI}"
+          echo "##vso[task.setvariable variable=ARTIFACT_UPLOAD_PATH;]${ARTIFACT_UPLOAD_PATH}"
+          echo "ARTIFACT_CLI=$ARTIFACT_CLI ARTIFACT_UPLOAD_PATH=$ARTIFACT_UPLOAD_PATH"
+        displayName: Pre-build Environment (Branch)
+        condition: >
+          and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
+          in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.6'))
+      - script: |
+          GIT_TAG=$(git tag -l --points-at $BUILD_SOURCEVERSION)
+          ARTIFACT_CLI="blobxfer-${GIT_TAG}-linux-x86_64"
+          ARTIFACT_UPLOAD_PATH="releases/${GIT_TAG}"
+          echo "##vso[task.setvariable variable=GIT_TAG;]${GIT_TAG}"
+          echo "##vso[task.setvariable variable=ARTIFACT_CLI;]${ARTIFACT_CLI}"
+          echo "##vso[task.setvariable variable=ARTIFACT_UPLOAD_PATH;]${ARTIFACT_UPLOAD_PATH}"
+          apt-get install -y pandoc
+          pip install pypandoc
+        displayName: Pre-build Environment (Tagged Release)
+        condition: >
+          and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
+          startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.6'))
+      - template: ./pyenv.yml
+      - script: |
+          set -e
+          set -o pipefail
+          echo "ARTIFACT_CLI=$ARTIFACT_CLI ARTIFACT_UPLOAD_PATH=$ARTIFACT_UPLOAD_PATH GIT_TAG=$GIT_TAG"
+          ARTIFACT_CLI_PATH="bin/${ARTIFACT_CLI}"
+          $PIP install virtualenv
+          $PYTHON -m virtualenv -p $PYTHON pyi
+          /bin/bash -c \
+          "set -e; source pyi/bin/activate; \
+           pip install pyinstaller; \
+           pip install --no-cache-dir -e .; \
+           pyinstaller -F -n ${ARTIFACT_CLI} -p blobxfer -p cli --additional-hooks-dir blobxfer --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --distpath bin cli/cli.py; \
+           deactivate"
+          chmod +x ${ARTIFACT_CLI_PATH}
+          sha256sum ${ARTIFACT_CLI_PATH} | cut -d' ' -f1 > ${ARTIFACT_CLI_PATH}.sha256
+          cat ${ARTIFACT_CLI_PATH}.sha256
+          export BLOBXFER_STORAGE_ACCOUNT=$(blobxfer.storageAccount)
+          export BLOBXFER_STORAGE_ACCOUNT_KEY="$(blobxfer.storageAccountKey)"
+          ${ARTIFACT_CLI_PATH} upload --remote-path ${ARTIFACT_UPLOAD_PATH} --local-path ${ARTIFACT_CLI_PATH} --strip-components 1 --file-md5 --file-attributes --overwrite
+        displayName: Build
+        condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], ''))
+      - task: PublishBuildArtifacts@1
+        inputs:
+          pathtoPublish: bin
+          artifactName: Blobxfer-Linux
+        condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], ''))
+      - task: PyPIPublisher@0
+        inputs:
+          pypiConnection: PyPI
+          packageDirectory: .
+          alsoPublishWheel: true
+        condition: and(succeeded(), ne(variables['GIT_TAG'], ''))
+
+  - phase: MacOS
+    queue:
+      name: Hosted macOS Preview
+      parallel: 1
+      matrix:
+        Python36:
+          python.version: '3.6'
+          PYENV_VERSION: '3.6.6'
+          PYTOX_ENV: 'py36'
+    steps:
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: $(python.version)
+          architecture: x64
+      - script: |
+          set -e
+          set -o pipefail
+          env
+          if [[ "$PYENV_VERSION" == 3.* ]]; then
+            which python3
+            python3 --version
+            echo "##vso[task.setvariable variable=PYTHON;]python3"
+            echo "##vso[task.setvariable variable=PIP;]pip3"
+          else
+            which python
+            python --version
+            echo "##vso[task.setvariable variable=PYTHON;]python"
+            echo "##vso[task.setvariable variable=PIP;]pip"
+          fi
+          ls -alFR
+        displayName: Initialize Build
+      - script: |
+          set -e
+          set -o pipefail
+          echo "PATH=$PATH"
+          echo "PYTHON=$PYTHON PIP=$PIP"
+          $PYTHON -m pip install --upgrade pip
+          $PIP install --upgrade setuptools wheel
+          $PIP install codecov tox
+          $PIP list --format=columns --outdated
+        displayName: Fetch Dependencies
+      - script: |
+          set -e
+          set -o pipefail
+          tox -e $PYTOX_ENV
+        displayName: Unit Tests
+      - script: |
+          set -e
+          set -o pipefail
+          ARTIFACT_CLI="blobxfer-${BUILD_SOURCEBRANCHNAME}-${BUILD_BUILDID}-mac-x86_64"
+          ARTIFACT_UPLOAD_PATH="builds/${BUILD_SOURCEBRANCHNAME}"
+          echo "##vso[task.setvariable variable=ARTIFACT_CLI;]${ARTIFACT_CLI}"
+          echo "##vso[task.setvariable variable=ARTIFACT_UPLOAD_PATH;]${ARTIFACT_UPLOAD_PATH}"
+          echo "ARTIFACT_CLI=$ARTIFACT_CLI ARTIFACT_UPLOAD_PATH=$ARTIFACT_UPLOAD_PATH"
+        displayName: Pre-build Environment (Branch)
+        condition: >
+          and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
+          in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.6'))
+      - script: |
+          GIT_TAG=$(git tag -l --points-at $BUILD_SOURCEVERSION)
+          ARTIFACT_CLI="blobxfer-${GIT_TAG}-mac-x86_64"
+          ARTIFACT_UPLOAD_PATH="releases/${GIT_TAG}"
+          echo "##vso[task.setvariable variable=GIT_TAG;]${GIT_TAG}"
+          echo "##vso[task.setvariable variable=ARTIFACT_CLI;]${ARTIFACT_CLI}"
+          echo "##vso[task.setvariable variable=ARTIFACT_UPLOAD_PATH;]${ARTIFACT_UPLOAD_PATH}"
+        displayName: Pre-build Environment (Tagged Release)
+        condition: >
+          and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
+          startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.6'))
+      - template: ./pyenv.yml
+      - script: |
+          set -e
+          set -o pipefail
+          echo "ARTIFACT_CLI=$ARTIFACT_CLI ARTIFACT_UPLOAD_PATH=$ARTIFACT_UPLOAD_PATH GIT_TAG=$GIT_TAG"
+          ARTIFACT_CLI_PATH="bin/${ARTIFACT_CLI}"
+          $PIP install virtualenv
+          $PYTHON -m virtualenv -p $PYTHON pyi
+          /bin/bash -c \
+          "set -e; source pyi/bin/activate; \
+           pip install pyinstaller; \
+           pip install --no-cache-dir -e .; \
+           pyinstaller -F -n ${ARTIFACT_CLI} -p blobxfer -p cli --additional-hooks-dir blobxfer --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --distpath bin cli/cli.py; \
+           deactivate"
+          chmod +x ${ARTIFACT_CLI_PATH}
+          shasum -a 256 ${ARTIFACT_CLI_PATH} | cut -d' ' -f1 > ${ARTIFACT_CLI_PATH}.sha256
+          cat ${ARTIFACT_CLI_PATH}.sha256
+          export BLOBXFER_STORAGE_ACCOUNT=$(blobxfer.storageAccount)
+          export BLOBXFER_STORAGE_ACCOUNT_KEY="$(blobxfer.storageAccountKey)"
+          ${ARTIFACT_CLI_PATH} upload --remote-path ${ARTIFACT_UPLOAD_PATH} --local-path ${ARTIFACT_CLI_PATH} --strip-components 1 --file-md5 --file-attributes --overwrite
+        displayName: Build
+        condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], ''))
+      - task: PublishBuildArtifacts@1
+        inputs:
+          pathtoPublish: bin
+          artifactName: Blobxfer-MacOS
+        condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], ''))

.vsts/populate_tag.ps1

@@ -0,0 +1,7 @@
+try {
+    $buildver = Get-Content "version_tag.txt"
+    echo "##vso[task.setvariable variable=VERSION_TAG;]$buildver"
+    Write-Host "tag version: $buildver"
+} catch {
+    Write-Host "version.txt file not found"
+}

.vsts/pyenv.yml

@@ -0,0 +1,20 @@
+steps:
+  - script: |
+      set -e
+      set -o pipefail
+      echo "ARTIFACT_CLI=$ARTIFACT_CLI"
+      if [ "${SYSTEM_PHASENAME}" == "Linux" ]; then
+        apt-get update
+        apt-get install -y libbz2-dev libreadline-dev libsqlite3-dev
+      fi
+      export PYENV_ROOT="$(Agent.WorkFolder)/.pyenv"
+      git clone https://github.com/pyenv/pyenv.git $PYENV_ROOT
+      export PATH="$PYENV_ROOT/bin:$PATH"
+      pyenv --version
+      PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install $PYENV_VERSION
+      pyenv global $PYENV_VERSION
+      echo $PATH
+      echo "##vso[task.prependpath]$PYENV_ROOT/bin"
+      echo "##vso[task.prependpath]$PYENV_ROOT/shims"
+    displayName: Install Python
+    condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], ''))

README.md

@@ -1,3 +1,4 @@
+[![Build Status](https://azurebatch.visualstudio.com/blobxfer/_apis/build/status/blobxfer-CI)](https://azurebatch.visualstudio.com/blobxfer/_build/latest?definitionId=12)
 [![Build Status](https://travis-ci.org/Azure/blobxfer.svg?branch=master)](https://travis-ci.org/Azure/blobxfer)
 [![Build status](https://ci.appveyor.com/api/projects/status/qgth9p7jlessgp5i/branch/master?svg=true)](https://ci.appveyor.com/project/alfpark/blobxfer)
 [![codecov](https://codecov.io/gh/Azure/blobxfer/branch/master/graph/badge.svg)](https://codecov.io/gh/Azure/blobxfer)

appveyor.yml

@@ -12,30 +12,18 @@ environment:
     PYTHON_VERSION: "2.7"
     PYTHON_ARCH: "64"
     TOX_ENV: "py27"
-  - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
-    PYTHON: "C:\\Python36-x64"
-    PYTHON_VERSION: "3.6"
+  - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
+    PYTHON: "C:\\Python34-x64"
+    PYTHON_VERSION: "3.4"
     PYTHON_ARCH: "64"
-    TOX_ENV: "py36"
-  # disable python3.7 for now, pep479 stopiteration behavior change
-  # breaks tests
-#  - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
-#    PYTHON: "C:\\Python37-x64"
-#    PYTHON_VERSION: "3.7"
-#    PYTHON_ARCH: "64"
-#    TOX_ENV: "py37"
-  BLOBXFER_STORAGE_ACCOUNT_KEY:
-    secure: q5RpDPsMjUHNmZo14GkE08in19iEE/pjEhR+n5AkNmzf9T6Wy0RQY3npxpAWFVUQDOARMSai+uJM26J9Hz1B9iSHwJhg9cRd0Zit/3jPTmbj1UollNgfT658PPGqATmb
-  BLOBXFER_STORAGE_ACCOUNT:
-    secure: qbpOW+pGmGfG7DFFC7LozA==
-  DOCKER_USERNAME:
-    secure: S8n3Geq7JUkN7ZQKXo8CLg==
-  DOCKER_PASSWORD:
-    secure: PFjy27XjnoH8YI46VpXUi13ZJLa8G6EJ6YzR9P3ualY=
-  DOCKER_IMAGE_TAG_PREFIX: "alfpark/blobxfer:"
+    TOX_ENV: "py34"
+  - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+    PYTHON: "C:\\Python35-x64"
+    PYTHON_VERSION: "3.5"
+    PYTHON_ARCH: "64"
+    TOX_ENV: "py35"
 
 init:
-- choco install checksum
 - echo %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%
 
 install:
@@ -49,114 +37,3 @@ build: off
 
 test_script:
 - tox -e "%TOX_ENV%"
-
-after_test:
-- echo is pr %APPVEYOR_PULL_REQUEST_NUMBER% is commit tag %APPVEYOR_REPO_TAG% name %APPVEYOR_REPO_TAG_NAME% branch %APPVEYOR_REPO_BRANCH%
-- ps: >-
-    if (!($env:APPVEYOR_PULL_REQUEST_NUMBER -eq $null)) {
-      Write-Host "Build is from a PR, not creating binary"
-      return
-    }
-    if (!($env:PYTHON_VERSION -eq "3.6")) {
-      Write-Host "Python environment is not 3.6, not creating binary"
-      return
-    }
-    if ($env:APPVEYOR_REPO_TAG -eq "true") {
-      $env:BLOBXFER_ARTIFACT = "blobxfer-" + $env:APPVEYOR_REPO_TAG_NAME + "-win-amd64.exe"
-      $env:UPLOAD_PATH="releases/" + $env:APPVEYOR_REPO_TAG_NAME
-      $env:APPVEYOR_REPO_TAG_NAME -match "^([\d\.])+"
-      $env:BUILDVER_DOTTED = $matches[0]
-    }
-    else {
-      if ($env:APPVEYOR_REPO_BRANCH -eq "master" -Or $env:APPVEYOR_REPO_BRANCH -eq "develop") {
-        $env:BLOBXFER_ARTIFACT = "blobxfer-" + $env:APPVEYOR_REPO_BRANCH + "-" + $env:APPVEYOR_BUILD_NUMBER + "-win-amd64.exe"
-        $env:UPLOAD_PATH="builds/" + $env:APPVEYOR_REPO_BRANCH
-        $env:BUILDVER_DOTTED = "0.0.0"
-      }
-      else {
-        Write-Host "Invalid tag or branch $env:APPVEYOR_REPO_BRANCH to build binary"
-        return
-      }
-    }
-
-    $bvt0,$bvt1,$bvt2 = $env:BUILDVER_DOTTED.split('.')
-
-    $env:BUILDVER_TUPLE = [string]::Format("({0}, {1}, {2}, {3})",$bvt0,$bvt1,$bvt2,$env:APPVEYOR_BUILD_NUMBER)
-
-    $env:BRANCH_GITSHA1 = [string]::Format("{0}@{1}",$env:APPVEYOR_REPO_BRANCH,$env:APPVEYOR_REPO_COMMIT.Substring(0,7))
-
-- IF "%BLOBXFER_ARTIFACT%"=="" (
-    echo "blobxfer artifact not defined"
-  ) ELSE (
-    echo "blobxfer artifact is %BLOBXFER_ARTIFACT% upload path %UPLOAD_PATH%" &&
-    virtualenv -p "%PYTHON%\\python.exe" pyi &&
-    pyi\\Scripts\\activate.bat &&
-    pip install --upgrade pyinstaller &&
-    pip install -e . &&
-    sed -i -e "s/{BUILDVER_TUPLE}/%BUILDVER_TUPLE%/g" cli\\file_version_info.txt &&
-    sed -i -e "s/{BUILDVER_DOTTED}/%BUILDVER_DOTTED%/g" cli\\file_version_info.txt &&
-    sed -i -e "s/{BRANCH_GITSHA1}/%BRANCH_GITSHA1%/g" cli\\file_version_info.txt &&
-    sed -i -e "s/{EXE}/%BLOBXFER_ARTIFACT%/g" cli\\file_version_info.txt &&
-    pyinstaller -F -n "%BLOBXFER_ARTIFACT%" -p blobxfer:cli --additional-hooks-dir blobxfer --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --icon cli\\azure.ico --version-file cli\\file_version_info.txt --distpath bin cli\\cli.py &&
-    pyi\\Scripts\\deactivate.bat &&
-    checksum "bin\\%BLOBXFER_ARTIFACT%" -t=sha256 > "bin\\%BLOBXFER_ARTIFACT%.sha256" &&
-    cat "bin\\%BLOBXFER_ARTIFACT%.sha256" &&
-    appveyor PushArtifact "bin\\%BLOBXFER_ARTIFACT%" &&
-    appveyor PushArtifact "bin\\%BLOBXFER_ARTIFACT%.sha256" &&
-    bin\\%BLOBXFER_ARTIFACT% upload --remote-path %UPLOAD_PATH% --local-path bin\\%BLOBXFER_ARTIFACT% --strip-components 1 --file-md5 --overwrite
-  )
-- ps: >-
-    if (!($env:APPVEYOR_PULL_REQUEST_NUMBER -eq $null)) {
-      Write-Host "Build is from a PR, not creating a Docker image"
-      return
-    }
-    if (!($env:PYTHON_VERSION -eq "3.6")) {
-      Write-Host "Python environment is not 3.6, not creating a Docker image"
-      return
-    }
-
-    $DOCKER_IMAGE_TAG_SUFFIX = $null
-
-    if ($env:APPVEYOR_REPO_TAG -eq "true") {
-      $DOCKER_IMAGE_TAG_SUFFIX = $env:APPVEYOR_REPO_TAG_NAME + '-windows'
-    }
-    elseif ($env:APPVEYOR_REPO_BRANCH -eq "master") {
-      $DOCKER_IMAGE_TAG_SUFFIX = 'latest-windows'
-    }
-    elseif ($env:APPVEYOR_REPO_BRANCH -eq "develop") {
-      $DOCKER_IMAGE_TAG_SUFFIX = 'develop-windows'
-    }
-    if ($DOCKER_IMAGE_TAG_SUFFIX -eq $null) {
-      Write-Host "Image tag suffix is null, not creating a Docker image"
-      return
-    }
-
-    $DOCKER_IMAGE_TAG = $env:DOCKER_IMAGE_TAG_PREFIX + $DOCKER_IMAGE_TAG_SUFFIX
-
-    Write-Host "Creating image with tag: $DOCKER_IMAGE_TAG"
-
-    docker version
-
-    pushd docker\\win
-
-    docker build --build-arg GIT_BRANCH=$env:APPVEYOR_REPO_BRANCH --build-arg GIT_COMMIT=$env:APPVEYOR_REPO_COMMIT -t $DOCKER_IMAGE_TAG .
-
-    docker login -u="$env:DOCKER_USERNAME" -p="$env:DOCKER_PASSWORD"
-
-    docker push $DOCKER_IMAGE_TAG
-
-    popd
-
-deploy:
-- provider: GitHub
-  tag: $(APPVEYOR_REPO_TAG_NAME)
-  description: 'blobxfer release'
-  auth_token:
-    secure: +f4N6Qsv3HvJyii0Bs+8qBx3YS7+7FJUWbFSiAdEIUDubFQnNkJgFnBw0Ew2SLkv
-  artifact: /.*\.exe.*/
-  draft: true
-  prerelease: true
-  force_update: true
-  on:
-    appveyor_repo_tag: true
-    PYTHON_VERSION: "3.6"

blobxfer/util.py

@@ -50,6 +50,7 @@ import future.utils
 
 # global defines
 MEGABYTE = 1048576
+_ON_LINUX = platform.system() == 'Linux'
 _ON_WINDOWS = platform.system() == 'Windows'
 _REGISTERED_LOGGER_HANDLERS = []
 _PAGEBLOB_BOUNDARY = 512
@@ -64,6 +65,15 @@ def on_python2():
     return future.utils.PY2
 
 
+def on_linux():  # noqa
+    # type: (None) -> bool
+    """Execution on Linux
+    :rtype: bool
+    :return: if on Linux
+    """
+    return _ON_LINUX
+
+
 def on_windows():  # noqa
     # type: (None) -> bool
     """Execution on Windows

setup.py

@@ -43,8 +43,8 @@ install_requires = [
     'pathlib2==2.3.2;python_version<"3.5"',
     'python-dateutil==2.7.3',
     'requests==2.19.1',
-    'ruamel.yaml==0.15.64',
-    'scandir==1.8;python_version<"3.5"',
+    'ruamel.yaml==0.15.66',
+    'scandir==1.9.0;python_version<"3.5"',
 ]
 
 setup(

tests/test_blobxfer_models_download.py

@@ -124,31 +124,37 @@ def test_downloadspecification():
             local_destination_path=models.LocalDestinationPath('dest'),
         )
 
-    with pytest.raises(ValueError):
-        ds = models.Specification(
-            download_options=options.Download(
-                check_file_md5=True,
-                chunk_size_bytes=-1,
-                delete_extraneous_destination=False,
-                mode=azmodels.StorageModes.Auto,
-                overwrite=True,
-                recursive=True,
-                rename=False,
-                restore_file_properties=options.FileProperties(
-                    attributes=True,
-                    lmt=False,
-                    md5=None,
+    if util.on_windows():
+        patch_func = 'time.sleep'
+    else:
+        patch_func = 'os.getuid'
+    with mock.patch(patch_func) as patched_getuid:
+        patched_getuid.return_value = 1
+        with pytest.raises(ValueError):
+            ds = models.Specification(
+                download_options=options.Download(
+                    check_file_md5=True,
+                    chunk_size_bytes=-1,
+                    delete_extraneous_destination=False,
+                    mode=azmodels.StorageModes.Auto,
+                    overwrite=True,
+                    recursive=True,
+                    rename=False,
+                    restore_file_properties=options.FileProperties(
+                        attributes=True,
+                        lmt=False,
+                        md5=None,
+                    ),
+                    rsa_private_key=None,
+                    strip_components=0,
                 ),
-                rsa_private_key=None,
-                strip_components=0,
-            ),
-            skip_on_options=options.SkipOn(
-                filesize_match=True,
-                lmt_ge=False,
-                md5_match=True,
-            ),
-            local_destination_path=models.LocalDestinationPath('dest'),
-        )
+                skip_on_options=options.SkipOn(
+                    filesize_match=True,
+                    lmt_ge=False,
+                    md5_match=True,
+                ),
+                local_destination_path=models.LocalDestinationPath('dest'),
+            )
 
 
 def test_downloaddescriptor(tmpdir):
@@ -261,7 +267,7 @@ def test_set_final_path_view():
 
 
 @unittest.skipIf(
-    util.on_python2() or util.on_windows(), 'fallocate does not exist')
+    util.on_python2() or not util.on_linux(), 'fallocate does not exist')
 def test_downloaddescriptor_allocate_disk_space_via_seek(tmpdir):
     fp = pathlib.Path(str(tmpdir.join('fp')))
     opts = mock.MagicMock()

tests/test_blobxfer_models_upload.py

@@ -73,6 +73,10 @@ def test_localpath(tmpdir):
     assert lp.gid == stat.st_gid
 
 
+def _resolve_pypath(path):
+    return str(pathlib.Path(str(path)).resolve())
+
+
 def test_localsourcepaths_files(tmpdir):
     tmpdir.mkdir('abc')
     tmpdir.join('moo.cow').write('z')
@@ -108,9 +112,9 @@ def test_localsourcepaths_files(tmpdir):
 
     assert not a.can_rename()
     assert len(a.paths) == 1
-    assert str(abcpath.join('blah.x')) in a_set
-    assert str(defpath.join('world.txt')) in a_set
-    assert str(defpath.join('moo.cow')) not in a_set
+    assert _resolve_pypath(abcpath.join('blah.x')) in a_set
+    assert _resolve_pypath(defpath.join('world.txt')) in a_set
+    assert _resolve_pypath(defpath.join('moo.cow')) not in a_set
 
     b = upload.LocalSourcePath()
     b.add_includes(['moo.cow', '*blah*'])