From 0cb5e788769e112fa8eb2fc10bcb90968f73ec5b Mon Sep 17 00:00:00 2001 From: Fred Park Date: Wed, 5 Sep 2018 09:19:26 -0700 Subject: [PATCH] Migrate to VSTS for build - Update dependencies - AppVeyor and Travis changes - Disable deployments - Enable test coverage on non-VSTS Python versions - Add MacOS build --- .travis.yml | 89 +------ .vsts/pipeline.yml | 319 +++++++++++++++++++++++++ .vsts/populate_tag.ps1 | 7 + .vsts/pyenv.yml | 20 ++ README.md | 1 + appveyor.yml | 141 +---------- blobxfer/util.py | 10 + setup.py | 4 +- tests/test_blobxfer_models_download.py | 56 +++-- tests/test_blobxfer_models_upload.py | 10 +- 10 files changed, 407 insertions(+), 250 deletions(-) create mode 100644 .vsts/pipeline.yml create mode 100644 .vsts/populate_tag.ps1 create mode 100644 .vsts/pyenv.yml diff --git a/.travis.yml b/.travis.yml index ccbe171..0f34865 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,98 +7,11 @@ matrix: - python: 2.7 - python: 3.4 - python: 3.5 - - python: 3.6 - addons: - apt: - packages: - - pandoc - # disable python3.7 for now, pep479 stopiteration behavior change - # breaks tests - # use workaround below until travis supports 3.7 -# - python: 3.7 -# dist: xenial -# sudo: true -# - language: generic -# os: osx -# before_install: -# - brew update -# - brew install python3 -# - virtualenv -p python3 py3 -# - source py3/bin/activate - -env: - global: - - BLOBXFER_ARTIFACT=blobxfer-$TRAVIS_TAG-$TRAVIS_OS_NAME-x86_64 - - secure: XqrQXqxLRRFvhD9BA5AKoH3nPBA22xihQ1f4Q16bumW9WlpWv/zbbOa3eWIbCm2KqT+yOaXQN8vhDiBsGdJo6tvV4aZtPu2H5v4AsE5lmYsqeTZKh6w8EIfWeeKIzF5HoTKF35JK84+cpWRh42d0Shr2Awf1YlAu1pH6BcVkFQ5qV42rD+XH3TJJAwT97I0bARhkv/UJdf6gyIIw2G4FIDEyXfH3Lvxlmbnq85OL5p8VzTda70/7Mo2T3/rQofwVyRmMlWLLEuZuUzyo7R9KGpKdYMTijkLdFw9tSHOlcuW46iQimfWPsvdXCv8FUiDmulPG2a7fF5w7f89CDmupOhK0xd20C1v71uVa+f/k28qskA5PXaG0nzPQLWF8Avd/uehgXsaBNvLrC7PQZHOea/Ce4J5Yhdd6eigUmduP0GuOqZPm71Z8CxdOxNpOfUEE6Jx0eKW8hQXDt4N7ZwTU+kP+bOEKxPjky1Reg6DsiuRcrVHwYgk7RjidRr4kHLNam2h11NBDPC+8sIhTv4UBhtjspnrk2wT236uVBQWpGk5niPaMPCcdnCIghSMIQQdBSbv7huwv5MJOsjLcjbGtT1u0JgMtXI/TOjNsAZP5YzPPFQpzg8LwJ2qmEVTLOy6EJ5bJdWFCN1GPlupKbwURmpKyIqCdZoSnYnsgkEF0PFA= - - secure: F15Weue/tcIB7hcZFTOEVxQSwy5D0124Stq185bEudE3orpsscNp97FCNUczQHqt38OrRrswTU/HwpuJQF0G54RxLWQspmef7dgRIJSboQUd9j5Se/bmRmSsBGvffOCLyEDU8Lq0tT8xb2o30xclcqX9fb9EdXVlbHlIqeGnmkEEmgnCxUkfHEQW+kYN2CRVeVwKjnYnyByFtXErTjwc1Mp4IOpWjsclS+AIe2S0gAU5jx8ZqImXKUmhkm7OOIyOYaJjT+0r/MrpSJgwjjb+yPazhOL6tOD1gCFGds5NwvZd8B7cpD/DqgakEdQWz+pvedJZQTdfxSQPK7qYz43tjSwQhxIHO02hpSxq5T5feUuXOiLKlrona6aTLnzsr/ftJ0KlMElihwEJPCzrb9I5oxbCMz/ZH2a1yzGVUaeob3we1vIFN66dB27n0zEE3wxhJOcul5Y8SIQRrrMVLPYZmJr8HxV220lbadUOJInicyM5EsO+S7jLlb/b2n4i2mr3J0eNif1MaxLrj1NcbQOVjyjg/n9Zkw8fNB7x9qfsWmIMovKYQKFQpNW1sv5T5CR5zqV0in6eKvgw6qyt6oxxMN/ViZ0vyvywDvsEcH7pTZGsnC/VDrkEj3Z62zFa8ewY4W1AGQDWZtJBnPJXx2wmFWgnMG8mSpFuI/gJDkgprEI= install: - travis_retry pip install --upgrade pip - travis_retry pip install --upgrade setuptools wheel -- travis_retry pip install --upgrade codecov tox-travis +- travis_retry pip install --upgrade tox-travis script: - tox - -after_success: -- codecov -- | - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then - echo "Not building binary with PR $TRAVIS_PULL_REQUEST" - return - fi - if [[ "$TRAVIS_PYTHON_VERSION" != "3.6" ]]; then - echo "Not building binary without Python 3.6" - return - fi - if [[ ! -z "$TRAVIS_TAG" ]]; then - export UPLOAD_PATH="releases/$TRAVIS_TAG" - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - brew install pandoc - else - travis_retry pip install pypandoc - fi - elif [[ "$TRAVIS_BRANCH" == "master" ]] || [[ "$TRAVIS_BRANCH" == "develop" ]]; then - export BLOBXFER_ARTIFACT="blobxfer-${TRAVIS_BRANCH}-${TRAVIS_BUILD_NUMBER}-$TRAVIS_OS_NAME-x86_64" - export UPLOAD_PATH="builds/$TRAVIS_BRANCH" - else - echo "Invalid tag=$TRAVIS_TAG or branch=$TRAVIS_BRANCH to build binary" - return - fi - echo "BLOBXFER_ARTIFACT=$BLOBXFER_ARTIFACT UPLOAD_PATH=$UPLOAD_PATH" - if [[ ! -e "bin/$BLOBXFER_ARTIFACT" ]]; then - virtualenv -p python3 pyi - $SHELL -c "set -e; source pyi/bin/activate; \ - pip install pyinstaller; \ - pip install --no-cache-dir -e .; \ - pyinstaller -F -n $BLOBXFER_ARTIFACT -p blobxfer:cli --additional-hooks-dir blobxfer --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --distpath bin cli/cli.py; \ - deactivate" - chmod +x bin/$BLOBXFER_ARTIFACT - sha256sum bin/$BLOBXFER_ARTIFACT | cut -d' ' -f1 > bin/${BLOBXFER_ARTIFACT}.sha256 - cat bin/${BLOBXFER_ARTIFACT}.sha256 - bin/$BLOBXFER_ARTIFACT upload --remote-path $UPLOAD_PATH --local-path bin/${BLOBXFER_ARTIFACT} --strip-components 1 --file-md5 --overwrite - fi - -deploy: - - provider: pypi - distributions: "sdist bdist_wheel" - skip_cleanup: true - on: - tags: true - condition: "$TRAVIS_PYTHON_VERSION == 3.6" - user: alfpark - password: - secure: p6i4ircy+h1HaIlVz+SsdA3yoyG86/X12QWVDyC2Vw9M53/rPjQKn/dbhQSTcvWZ90EmYNOnmkBT9tc7B9iRkixmCminic0oUX0bmEhvenscW8Bbhaa2Uh9P2AJtd3gsVcicWzaNhtnjui3/rJRljolGA1PwfXal0Ryg24NaBd2VRKCwOU4xsZZ2fCDZ02cotsehRvGeUa4d54CJ6eBuzN08q9AXYh9IGWCngPNo68XRRs+gDQ65kwmUYp6tIFB+zbL/0erbB1E88F+Dq6Ac/vNCZjjhXor8fDqoPJmch3JZaP091bHoPD+rWnEb8Vm6B4nBBwhFMbKg/9Kt3Y6Bmb2ifOBz9EPfeSxxXVetKOpOKxKfaGj7mcbG8w64uBxQxr/GJLmpVMbFCJRh1PeRL9J9deFwdANBvflgQzV0XwKOfv2N03dmRtJOXwmXPO/FDmr6oojRT5JHj8KpbO6/adCm9COm9mhjhD6ZOPX7ppBGeXmD5sGKrXX+rVfdtYeCLrawGMf/ZmSWHiT0q1KTPLrNaKK0Y2o5XCUKXcgcCOIPaP03Kqje++RxP3Ef5x2uB2o2ertsrY4rk0UyR93+ARgko/MvW8OlQJbz9oUh3qh87IdjOBgY6OTObTEVAGZWYRKQMKRxzTDGRuk/D1gXcXI7VfBZwylSEvHjE85rLJ4= - - provider: releases - skip_cleanup: true - on: - tags: true - condition: "$TRAVIS_PYTHON_VERSION == 3.6" - draft: true - overwrite: true - tag_name: $TRAVIS_TAG - file: - - bin/${BLOBXFER_ARTIFACT} - - bin/${BLOBXFER_ARTIFACT}.sha256 - api_key: - secure: QCcrLa2x15RVNX42x6Ts4E/AU4QvwacuoQJ5BlFK6hdevRZYyRSQB3yBD6ZhAqi79qxd7KpRoPj8mu0N3WaZuB3XtQ2scxQLvN5vueC6c9A2Ic9venoLfVYG9Np4jOYxwx3QRtMPA6SNEmxNxBK+jjClwNi71vshTwo14z0/IW7Zi7WNEzoZW3iNj7e+zbaVvuyx+vGI+Ckgjyy7OgaulNtlHG/vYuJuAMn7WzbyGuSQTuqI2npKvykumi9rNSQpmR+2WP9gbo8D+WU+iESHJ+r0YUbZRWDwGHA7fvbCNLMwFdXwaXccc6jozxqULTmt6mNnlSpaKduYiSoru9L8+kzxR1b+xYEK0fGr6/khABt5O26Pk6tCynKz4LrurSefA3mHrNYIT3PGGUvUTJAhSVaK9O/ag2qkDTA0NGp7KV+xDG4KCWvqeuPssqh9ppDl27T1s/aWMtKeeswCRhGMlfCICg9hhPzyc97ZBzl7DLRPITUceSzc2cGp88zGYdlJEg+5yuSl1j+Al5jmWMxuLF1MLOmFYw1pA1z7zPpcq8zzC0t5eu6n/fFh6k57sZQ6VDeS4kix4q0W6XjIFx4yA/Aa4CpeEAOb+jPzZxRuQ/UvRgL4M7lnb8aIOqNcoXEcccgnrcULjJ9pcJCMHE0p5WU7JnF8mpazJude51yikx0= diff --git a/.vsts/pipeline.yml b/.vsts/pipeline.yml new file mode 100644 index 0000000..605ee12 --- /dev/null +++ b/.vsts/pipeline.yml @@ -0,0 +1,319 @@ +name: $(SourceBranch)$(Rev:.r) + +phases: + - phase: Windows + queue: + name: Hosted VS2017 + parallel: 1 + matrix: + Python36: + python.version: '3.6' + PYENV_VERSION: '3.6.6' + PYTOX_ENV: 'py36' + steps: + - task: InstallPython@1 + inputs: + version: 'python==$(PYENV_VERSION)' + - powershell: | + gci env:* | sort-object name + python -m pip install --upgrade pip + pip install --upgrade setuptools wheel + pip install codecov tox + pip list --format=columns --outdated + displayName: Fetch Dependencies + - powershell: | + tox -e $env:PYTOX_ENV + displayName: Unit Tests + - powershell: | + if($env:BUILD_SOURCEBRANCHNAME -eq "master") + { + $dockerTag = "latest" + } + else + { + $dockerTag = $env:BUILD_SOURCEBRANCHNAME + } + $artifactCli = "blobxfer-" + $env:BUILD_SOURCEBRANCHNAME + "-" + $env:BUILD_BUILDID + "-win-amd64.exe" + $artifactUploadPath = "builds/" + $env:BUILD_SOURCEBRANCHNAME + $buildverDotted = "0.0.0" + $bvt0,$bvt1,$bvt2 = $buildverDotted.split('.') + $buildverTuple = [string]::Format("({0}, {1}, {2}, {3})",$bvt0,$bvt1,$bvt2,$env:BUILD_BUILDID) + echo "##vso[task.setvariable variable=BUILDVER_DOTTED;]$buildverDotted" + echo "##vso[task.setvariable variable=BUILDVER_TUPLE;]$buildverTuple" + echo "##vso[task.setvariable variable=ARTIFACT_CLI;]$artifactCli" + echo "##vso[task.setvariable variable=ARTIFACT_UPLOAD_PATH;]$artifactUploadPath" + echo "##vso[task.setvariable variable=DOCKER_TAG;]$dockerTag" + displayName: Pre-build Environment (Branch) + condition: > + and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'), + in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.6')) + - powershell: | + git tag -l --points-at $env:BUILD_SOURCEVERSION | Tee-Object -Variable gitTag + $gitTag -match "^([\d\.])+" + $buildverDotted = $matches[0] + $artifactCli = "blobxfer-" + $gitTag + "-win-amd64.exe" + $artifactUploadPath = "releases/" + $gitTag + $bvt0,$bvt1,$bvt2 = $buildverDotted.split('.') + $buildverTuple = [string]::Format("({0}, {1}, {2}, {3})",$bvt0,$bvt1,$bvt2,$env:BUILD_BUILDID) + echo "##vso[task.setvariable variable=BUILDVER_DOTTED;]$buildverDotted" + echo "##vso[task.setvariable variable=BUILDVER_TUPLE;]$buildverTuple" + echo "##vso[task.setvariable variable=GIT_TAG;]$gitTag" + echo "##vso[task.setvariable variable=ARTIFACT_CLI;]$artifactCli" + echo "##vso[task.setvariable variable=ARTIFACT_UPLOAD_PATH;]$artifactUploadPath" + echo "##vso[task.setvariable variable=DOCKER_TAG;]$gitTag" + displayName: Pre-build Environment (Tagged Release) + condition: > + and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'), + startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.6')) + - powershell: | + $artifactCliPath = "bin\\" + $env:ARTIFACT_CLI + echo "##vso[task.setvariable variable=ARTIFACT_CLI_PATH;]$artifactCliPath" + $branchGitSha1 = [string]::Format("{0}@{1}",$env:BUILD_SOURCEBRANCHNAME,$env:BUILD_SOURCEVERSION.Substring(0,7)) + echo "$env:ARTIFACT_CLI $env:BUILDVER_TUPLE $branchGitSha1" + $fileverInfo = "cli\\file_version_info.txt" + (Get-Content $fileverInfo).replace('{BUILDVER_TUPLE}', $env:BUILDVER_TUPLE) | Set-Content $fileverInfo + (Get-Content $fileverInfo).replace('{BUILDVER_DOTTED}', $env:BUILDVER_DOTTED) | Set-Content $fileverInfo + (Get-Content $fileverInfo).replace('{BRANCH_GITSHA1}', $branchGitSha1) | Set-Content $fileverInfo + (Get-Content $fileverInfo).replace('{EXE}', $env:ARTIFACT_CLI) | Set-Content $fileverInfo + pip install virtualenv + python -m virtualenv pyi + displayName: Pre-build + condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], '')) + - script: | + echo "ARTIFACT_CLI=%ARTIFACT_CLI% ARTIFACT_CLI_PATH=%ARTIFACT_CLI_PATH% ARTIFACT_UPLOAD_PATH=%ARTIFACT_UPLOAD_PATH%" + call pyi\\Scripts\\activate.bat + pip -V + pip install --no-cache-dir pyinstaller + pip install --no-cache-dir -e . + pyinstaller -F -n "%ARTIFACT_CLI%" -p blobxfer -p cli --additional-hooks-dir blobxfer --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --icon cli\\azure.ico --version-file cli\\file_version_info.txt --distpath bin cli\\cli.py + call pyi\\Scripts\\deactivate.bat + "%ARTIFACT_CLI_PATH%" upload --storage-account "$(blobxfer.storageAccount)" --storage-account-key "$(blobxfer.storageAccountKey)" --remote-path "%ARTIFACT_UPLOAD_PATH%" --local-path "%ARTIFACT_CLI_PATH%" --strip-components 1 --file-md5 --overwrite + displayName: Build + condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], '')) + - powershell: | + $versionTag = "bin\\version_tag.txt" + $env:GIT_TAG | Out-File $versionTag -Force -NoNewline + Get-Content -Path $versionTag + copy .vsts\\populate_tag.ps1 bin\\populate_tag.ps1 + dir bin + displayName: Generate Version Tag File + condition: and(succeeded(), ne(variables['GIT_TAG'], '')) + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: bin + artifactName: Blobxfer-Windows + condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], '')) + - powershell: | + docker version + docker login -u="$(docker.username)" -p="$(docker.password)" + $cliImage = "alfpark/blobxfer:" + $env:DOCKER_TAG + "-windows" + echo "Building Docker image: $cliImage" + pushd docker\\win + docker build --build-arg GIT_BRANCH=$env:BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$env:BUILD_SOURCEVERSION -t $cliImage . + docker push $cliImage + popd + displayName: Docker build + condition: and(succeeded(), ne(variables['DOCKER_TAG'], '')) + + - phase: Linux + queue: + name: Hosted Linux Preview + parallel: 1 + matrix: + Python36: + python.version: '3.6' + PYENV_VERSION: '3.6.6' + PYTOX_ENV: 'py36' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: $(python.version) + architecture: x64 + - script: | + set -e + set -o pipefail + env + if [[ "$PYENV_VERSION" == 3.* ]]; then + which python3 + python3 --version + echo "##vso[task.setvariable variable=PYTHON;]python3" + echo "##vso[task.setvariable variable=PIP;]pip3" + else + which python + python --version + echo "##vso[task.setvariable variable=PYTHON;]python" + echo "##vso[task.setvariable variable=PIP;]pip" + fi + ls -alFR + displayName: Initialize Build + - script: | + set -e + set -o pipefail + echo "PATH=$PATH" + echo "PYTHON=$PYTHON PIP=$PIP" + $PYTHON -m pip install --upgrade pip + $PIP install --upgrade setuptools wheel + $PIP install codecov tox + $PIP list --format=columns --outdated + displayName: Fetch Dependencies + - script: | + set -e + set -o pipefail + tox -e $PYTOX_ENV + codecov -t $(codecov.uploadToken) + displayName: Unit Tests + - script: | + set -e + set -o pipefail + ARTIFACT_CLI="blobxfer-${BUILD_SOURCEBRANCHNAME}-${BUILD_BUILDID}-linux-x86_64" + ARTIFACT_UPLOAD_PATH="builds/${BUILD_SOURCEBRANCHNAME}" + echo "##vso[task.setvariable variable=ARTIFACT_CLI;]${ARTIFACT_CLI}" + echo "##vso[task.setvariable variable=ARTIFACT_UPLOAD_PATH;]${ARTIFACT_UPLOAD_PATH}" + echo "ARTIFACT_CLI=$ARTIFACT_CLI ARTIFACT_UPLOAD_PATH=$ARTIFACT_UPLOAD_PATH" + displayName: Pre-build Environment (Branch) + condition: > + and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'), + in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.6')) + - script: | + GIT_TAG=$(git tag -l --points-at $BUILD_SOURCEVERSION) + ARTIFACT_CLI="blobxfer-${GIT_TAG}-linux-x86_64" + ARTIFACT_UPLOAD_PATH="releases/${GIT_TAG}" + echo "##vso[task.setvariable variable=GIT_TAG;]${GIT_TAG}" + echo "##vso[task.setvariable variable=ARTIFACT_CLI;]${ARTIFACT_CLI}" + echo "##vso[task.setvariable variable=ARTIFACT_UPLOAD_PATH;]${ARTIFACT_UPLOAD_PATH}" + apt-get install -y pandoc + pip install pypandoc + displayName: Pre-build Environment (Tagged Release) + condition: > + and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'), + startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.6')) + - template: ./pyenv.yml + - script: | + set -e + set -o pipefail + echo "ARTIFACT_CLI=$ARTIFACT_CLI ARTIFACT_UPLOAD_PATH=$ARTIFACT_UPLOAD_PATH GIT_TAG=$GIT_TAG" + ARTIFACT_CLI_PATH="bin/${ARTIFACT_CLI}" + $PIP install virtualenv + $PYTHON -m virtualenv -p $PYTHON pyi + /bin/bash -c \ + "set -e; source pyi/bin/activate; \ + pip install pyinstaller; \ + pip install --no-cache-dir -e .; \ + pyinstaller -F -n ${ARTIFACT_CLI} -p blobxfer -p cli --additional-hooks-dir blobxfer --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --distpath bin cli/cli.py; \ + deactivate" + chmod +x ${ARTIFACT_CLI_PATH} + sha256sum ${ARTIFACT_CLI_PATH} | cut -d' ' -f1 > ${ARTIFACT_CLI_PATH}.sha256 + cat ${ARTIFACT_CLI_PATH}.sha256 + export BLOBXFER_STORAGE_ACCOUNT=$(blobxfer.storageAccount) + export BLOBXFER_STORAGE_ACCOUNT_KEY="$(blobxfer.storageAccountKey)" + ${ARTIFACT_CLI_PATH} upload --remote-path ${ARTIFACT_UPLOAD_PATH} --local-path ${ARTIFACT_CLI_PATH} --strip-components 1 --file-md5 --file-attributes --overwrite + displayName: Build + condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], '')) + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: bin + artifactName: Blobxfer-Linux + condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], '')) + - task: PyPIPublisher@0 + inputs: + pypiConnection: PyPI + packageDirectory: . + alsoPublishWheel: true + condition: and(succeeded(), ne(variables['GIT_TAG'], '')) + + - phase: MacOS + queue: + name: Hosted macOS Preview + parallel: 1 + matrix: + Python36: + python.version: '3.6' + PYENV_VERSION: '3.6.6' + PYTOX_ENV: 'py36' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: $(python.version) + architecture: x64 + - script: | + set -e + set -o pipefail + env + if [[ "$PYENV_VERSION" == 3.* ]]; then + which python3 + python3 --version + echo "##vso[task.setvariable variable=PYTHON;]python3" + echo "##vso[task.setvariable variable=PIP;]pip3" + else + which python + python --version + echo "##vso[task.setvariable variable=PYTHON;]python" + echo "##vso[task.setvariable variable=PIP;]pip" + fi + ls -alFR + displayName: Initialize Build + - script: | + set -e + set -o pipefail + echo "PATH=$PATH" + echo "PYTHON=$PYTHON PIP=$PIP" + $PYTHON -m pip install --upgrade pip + $PIP install --upgrade setuptools wheel + $PIP install codecov tox + $PIP list --format=columns --outdated + displayName: Fetch Dependencies + - script: | + set -e + set -o pipefail + tox -e $PYTOX_ENV + displayName: Unit Tests + - script: | + set -e + set -o pipefail + ARTIFACT_CLI="blobxfer-${BUILD_SOURCEBRANCHNAME}-${BUILD_BUILDID}-mac-x86_64" + ARTIFACT_UPLOAD_PATH="builds/${BUILD_SOURCEBRANCHNAME}" + echo "##vso[task.setvariable variable=ARTIFACT_CLI;]${ARTIFACT_CLI}" + echo "##vso[task.setvariable variable=ARTIFACT_UPLOAD_PATH;]${ARTIFACT_UPLOAD_PATH}" + echo "ARTIFACT_CLI=$ARTIFACT_CLI ARTIFACT_UPLOAD_PATH=$ARTIFACT_UPLOAD_PATH" + displayName: Pre-build Environment (Branch) + condition: > + and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'), + in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.6')) + - script: | + GIT_TAG=$(git tag -l --points-at $BUILD_SOURCEVERSION) + ARTIFACT_CLI="blobxfer-${GIT_TAG}-mac-x86_64" + ARTIFACT_UPLOAD_PATH="releases/${GIT_TAG}" + echo "##vso[task.setvariable variable=GIT_TAG;]${GIT_TAG}" + echo "##vso[task.setvariable variable=ARTIFACT_CLI;]${ARTIFACT_CLI}" + echo "##vso[task.setvariable variable=ARTIFACT_UPLOAD_PATH;]${ARTIFACT_UPLOAD_PATH}" + displayName: Pre-build Environment (Tagged Release) + condition: > + and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'), + startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.6')) + - template: ./pyenv.yml + - script: | + set -e + set -o pipefail + echo "ARTIFACT_CLI=$ARTIFACT_CLI ARTIFACT_UPLOAD_PATH=$ARTIFACT_UPLOAD_PATH GIT_TAG=$GIT_TAG" + ARTIFACT_CLI_PATH="bin/${ARTIFACT_CLI}" + $PIP install virtualenv + $PYTHON -m virtualenv -p $PYTHON pyi + /bin/bash -c \ + "set -e; source pyi/bin/activate; \ + pip install pyinstaller; \ + pip install --no-cache-dir -e .; \ + pyinstaller -F -n ${ARTIFACT_CLI} -p blobxfer -p cli --additional-hooks-dir blobxfer --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --distpath bin cli/cli.py; \ + deactivate" + chmod +x ${ARTIFACT_CLI_PATH} + shasum -a 256 ${ARTIFACT_CLI_PATH} | cut -d' ' -f1 > ${ARTIFACT_CLI_PATH}.sha256 + cat ${ARTIFACT_CLI_PATH}.sha256 + export BLOBXFER_STORAGE_ACCOUNT=$(blobxfer.storageAccount) + export BLOBXFER_STORAGE_ACCOUNT_KEY="$(blobxfer.storageAccountKey)" + ${ARTIFACT_CLI_PATH} upload --remote-path ${ARTIFACT_UPLOAD_PATH} --local-path ${ARTIFACT_CLI_PATH} --strip-components 1 --file-md5 --file-attributes --overwrite + displayName: Build + condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], '')) + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: bin + artifactName: Blobxfer-MacOS + condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], '')) diff --git a/.vsts/populate_tag.ps1 b/.vsts/populate_tag.ps1 new file mode 100644 index 0000000..c8e9df3 --- /dev/null +++ b/.vsts/populate_tag.ps1 @@ -0,0 +1,7 @@ +try { + $buildver = Get-Content "version_tag.txt" + echo "##vso[task.setvariable variable=VERSION_TAG;]$buildver" + Write-Host "tag version: $buildver" +} catch { + Write-Host "version.txt file not found" +} diff --git a/.vsts/pyenv.yml b/.vsts/pyenv.yml new file mode 100644 index 0000000..0ac9c51 --- /dev/null +++ b/.vsts/pyenv.yml @@ -0,0 +1,20 @@ +steps: + - script: | + set -e + set -o pipefail + echo "ARTIFACT_CLI=$ARTIFACT_CLI" + if [ "${SYSTEM_PHASENAME}" == "Linux" ]; then + apt-get update + apt-get install -y libbz2-dev libreadline-dev libsqlite3-dev + fi + export PYENV_ROOT="$(Agent.WorkFolder)/.pyenv" + git clone https://github.com/pyenv/pyenv.git $PYENV_ROOT + export PATH="$PYENV_ROOT/bin:$PATH" + pyenv --version + PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install $PYENV_VERSION + pyenv global $PYENV_VERSION + echo $PATH + echo "##vso[task.prependpath]$PYENV_ROOT/bin" + echo "##vso[task.prependpath]$PYENV_ROOT/shims" + displayName: Install Python + condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], '')) diff --git a/README.md b/README.md index 0d2e807..c6b1dfa 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +[![Build Status](https://azurebatch.visualstudio.com/blobxfer/_apis/build/status/blobxfer-CI)](https://azurebatch.visualstudio.com/blobxfer/_build/latest?definitionId=12) [![Build Status](https://travis-ci.org/Azure/blobxfer.svg?branch=master)](https://travis-ci.org/Azure/blobxfer) [![Build status](https://ci.appveyor.com/api/projects/status/qgth9p7jlessgp5i/branch/master?svg=true)](https://ci.appveyor.com/project/alfpark/blobxfer) [![codecov](https://codecov.io/gh/Azure/blobxfer/branch/master/graph/badge.svg)](https://codecov.io/gh/Azure/blobxfer) diff --git a/appveyor.yml b/appveyor.yml index 723f183..e25cd2e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -12,30 +12,18 @@ environment: PYTHON_VERSION: "2.7" PYTHON_ARCH: "64" TOX_ENV: "py27" - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - PYTHON: "C:\\Python36-x64" - PYTHON_VERSION: "3.6" + - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 + PYTHON: "C:\\Python34-x64" + PYTHON_VERSION: "3.4" PYTHON_ARCH: "64" - TOX_ENV: "py36" - # disable python3.7 for now, pep479 stopiteration behavior change - # breaks tests -# - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 -# PYTHON: "C:\\Python37-x64" -# PYTHON_VERSION: "3.7" -# PYTHON_ARCH: "64" -# TOX_ENV: "py37" - BLOBXFER_STORAGE_ACCOUNT_KEY: - secure: q5RpDPsMjUHNmZo14GkE08in19iEE/pjEhR+n5AkNmzf9T6Wy0RQY3npxpAWFVUQDOARMSai+uJM26J9Hz1B9iSHwJhg9cRd0Zit/3jPTmbj1UollNgfT658PPGqATmb - BLOBXFER_STORAGE_ACCOUNT: - secure: qbpOW+pGmGfG7DFFC7LozA== - DOCKER_USERNAME: - secure: S8n3Geq7JUkN7ZQKXo8CLg== - DOCKER_PASSWORD: - secure: PFjy27XjnoH8YI46VpXUi13ZJLa8G6EJ6YzR9P3ualY= - DOCKER_IMAGE_TAG_PREFIX: "alfpark/blobxfer:" + TOX_ENV: "py34" + - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + PYTHON: "C:\\Python35-x64" + PYTHON_VERSION: "3.5" + PYTHON_ARCH: "64" + TOX_ENV: "py35" init: -- choco install checksum - echo %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH% install: @@ -49,114 +37,3 @@ build: off test_script: - tox -e "%TOX_ENV%" - -after_test: -- echo is pr %APPVEYOR_PULL_REQUEST_NUMBER% is commit tag %APPVEYOR_REPO_TAG% name %APPVEYOR_REPO_TAG_NAME% branch %APPVEYOR_REPO_BRANCH% -- ps: >- - if (!($env:APPVEYOR_PULL_REQUEST_NUMBER -eq $null)) { - Write-Host "Build is from a PR, not creating binary" - return - } - if (!($env:PYTHON_VERSION -eq "3.6")) { - Write-Host "Python environment is not 3.6, not creating binary" - return - } - if ($env:APPVEYOR_REPO_TAG -eq "true") { - $env:BLOBXFER_ARTIFACT = "blobxfer-" + $env:APPVEYOR_REPO_TAG_NAME + "-win-amd64.exe" - $env:UPLOAD_PATH="releases/" + $env:APPVEYOR_REPO_TAG_NAME - $env:APPVEYOR_REPO_TAG_NAME -match "^([\d\.])+" - $env:BUILDVER_DOTTED = $matches[0] - } - else { - if ($env:APPVEYOR_REPO_BRANCH -eq "master" -Or $env:APPVEYOR_REPO_BRANCH -eq "develop") { - $env:BLOBXFER_ARTIFACT = "blobxfer-" + $env:APPVEYOR_REPO_BRANCH + "-" + $env:APPVEYOR_BUILD_NUMBER + "-win-amd64.exe" - $env:UPLOAD_PATH="builds/" + $env:APPVEYOR_REPO_BRANCH - $env:BUILDVER_DOTTED = "0.0.0" - } - else { - Write-Host "Invalid tag or branch $env:APPVEYOR_REPO_BRANCH to build binary" - return - } - } - - $bvt0,$bvt1,$bvt2 = $env:BUILDVER_DOTTED.split('.') - - $env:BUILDVER_TUPLE = [string]::Format("({0}, {1}, {2}, {3})",$bvt0,$bvt1,$bvt2,$env:APPVEYOR_BUILD_NUMBER) - - $env:BRANCH_GITSHA1 = [string]::Format("{0}@{1}",$env:APPVEYOR_REPO_BRANCH,$env:APPVEYOR_REPO_COMMIT.Substring(0,7)) - -- IF "%BLOBXFER_ARTIFACT%"=="" ( - echo "blobxfer artifact not defined" - ) ELSE ( - echo "blobxfer artifact is %BLOBXFER_ARTIFACT% upload path %UPLOAD_PATH%" && - virtualenv -p "%PYTHON%\\python.exe" pyi && - pyi\\Scripts\\activate.bat && - pip install --upgrade pyinstaller && - pip install -e . && - sed -i -e "s/{BUILDVER_TUPLE}/%BUILDVER_TUPLE%/g" cli\\file_version_info.txt && - sed -i -e "s/{BUILDVER_DOTTED}/%BUILDVER_DOTTED%/g" cli\\file_version_info.txt && - sed -i -e "s/{BRANCH_GITSHA1}/%BRANCH_GITSHA1%/g" cli\\file_version_info.txt && - sed -i -e "s/{EXE}/%BLOBXFER_ARTIFACT%/g" cli\\file_version_info.txt && - pyinstaller -F -n "%BLOBXFER_ARTIFACT%" -p blobxfer:cli --additional-hooks-dir blobxfer --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --icon cli\\azure.ico --version-file cli\\file_version_info.txt --distpath bin cli\\cli.py && - pyi\\Scripts\\deactivate.bat && - checksum "bin\\%BLOBXFER_ARTIFACT%" -t=sha256 > "bin\\%BLOBXFER_ARTIFACT%.sha256" && - cat "bin\\%BLOBXFER_ARTIFACT%.sha256" && - appveyor PushArtifact "bin\\%BLOBXFER_ARTIFACT%" && - appveyor PushArtifact "bin\\%BLOBXFER_ARTIFACT%.sha256" && - bin\\%BLOBXFER_ARTIFACT% upload --remote-path %UPLOAD_PATH% --local-path bin\\%BLOBXFER_ARTIFACT% --strip-components 1 --file-md5 --overwrite - ) -- ps: >- - if (!($env:APPVEYOR_PULL_REQUEST_NUMBER -eq $null)) { - Write-Host "Build is from a PR, not creating a Docker image" - return - } - if (!($env:PYTHON_VERSION -eq "3.6")) { - Write-Host "Python environment is not 3.6, not creating a Docker image" - return - } - - $DOCKER_IMAGE_TAG_SUFFIX = $null - - if ($env:APPVEYOR_REPO_TAG -eq "true") { - $DOCKER_IMAGE_TAG_SUFFIX = $env:APPVEYOR_REPO_TAG_NAME + '-windows' - } - elseif ($env:APPVEYOR_REPO_BRANCH -eq "master") { - $DOCKER_IMAGE_TAG_SUFFIX = 'latest-windows' - } - elseif ($env:APPVEYOR_REPO_BRANCH -eq "develop") { - $DOCKER_IMAGE_TAG_SUFFIX = 'develop-windows' - } - if ($DOCKER_IMAGE_TAG_SUFFIX -eq $null) { - Write-Host "Image tag suffix is null, not creating a Docker image" - return - } - - $DOCKER_IMAGE_TAG = $env:DOCKER_IMAGE_TAG_PREFIX + $DOCKER_IMAGE_TAG_SUFFIX - - Write-Host "Creating image with tag: $DOCKER_IMAGE_TAG" - - docker version - - pushd docker\\win - - docker build --build-arg GIT_BRANCH=$env:APPVEYOR_REPO_BRANCH --build-arg GIT_COMMIT=$env:APPVEYOR_REPO_COMMIT -t $DOCKER_IMAGE_TAG . - - docker login -u="$env:DOCKER_USERNAME" -p="$env:DOCKER_PASSWORD" - - docker push $DOCKER_IMAGE_TAG - - popd - -deploy: -- provider: GitHub - tag: $(APPVEYOR_REPO_TAG_NAME) - description: 'blobxfer release' - auth_token: - secure: +f4N6Qsv3HvJyii0Bs+8qBx3YS7+7FJUWbFSiAdEIUDubFQnNkJgFnBw0Ew2SLkv - artifact: /.*\.exe.*/ - draft: true - prerelease: true - force_update: true - on: - appveyor_repo_tag: true - PYTHON_VERSION: "3.6" diff --git a/blobxfer/util.py b/blobxfer/util.py index e1118d8..714a1d4 100644 --- a/blobxfer/util.py +++ b/blobxfer/util.py @@ -50,6 +50,7 @@ import future.utils # global defines MEGABYTE = 1048576 +_ON_LINUX = platform.system() == 'Linux' _ON_WINDOWS = platform.system() == 'Windows' _REGISTERED_LOGGER_HANDLERS = [] _PAGEBLOB_BOUNDARY = 512 @@ -64,6 +65,15 @@ def on_python2(): return future.utils.PY2 +def on_linux(): # noqa + # type: (None) -> bool + """Execution on Linux + :rtype: bool + :return: if on Linux + """ + return _ON_LINUX + + def on_windows(): # noqa # type: (None) -> bool """Execution on Windows diff --git a/setup.py b/setup.py index dffcb89..038b82d 100644 --- a/setup.py +++ b/setup.py @@ -43,8 +43,8 @@ install_requires = [ 'pathlib2==2.3.2;python_version<"3.5"', 'python-dateutil==2.7.3', 'requests==2.19.1', - 'ruamel.yaml==0.15.64', - 'scandir==1.8;python_version<"3.5"', + 'ruamel.yaml==0.15.66', + 'scandir==1.9.0;python_version<"3.5"', ] setup( diff --git a/tests/test_blobxfer_models_download.py b/tests/test_blobxfer_models_download.py index 2b6e6aa..c7f79af 100644 --- a/tests/test_blobxfer_models_download.py +++ b/tests/test_blobxfer_models_download.py @@ -124,31 +124,37 @@ def test_downloadspecification(): local_destination_path=models.LocalDestinationPath('dest'), ) - with pytest.raises(ValueError): - ds = models.Specification( - download_options=options.Download( - check_file_md5=True, - chunk_size_bytes=-1, - delete_extraneous_destination=False, - mode=azmodels.StorageModes.Auto, - overwrite=True, - recursive=True, - rename=False, - restore_file_properties=options.FileProperties( - attributes=True, - lmt=False, - md5=None, + if util.on_windows(): + patch_func = 'time.sleep' + else: + patch_func = 'os.getuid' + with mock.patch(patch_func) as patched_getuid: + patched_getuid.return_value = 1 + with pytest.raises(ValueError): + ds = models.Specification( + download_options=options.Download( + check_file_md5=True, + chunk_size_bytes=-1, + delete_extraneous_destination=False, + mode=azmodels.StorageModes.Auto, + overwrite=True, + recursive=True, + rename=False, + restore_file_properties=options.FileProperties( + attributes=True, + lmt=False, + md5=None, + ), + rsa_private_key=None, + strip_components=0, ), - rsa_private_key=None, - strip_components=0, - ), - skip_on_options=options.SkipOn( - filesize_match=True, - lmt_ge=False, - md5_match=True, - ), - local_destination_path=models.LocalDestinationPath('dest'), - ) + skip_on_options=options.SkipOn( + filesize_match=True, + lmt_ge=False, + md5_match=True, + ), + local_destination_path=models.LocalDestinationPath('dest'), + ) def test_downloaddescriptor(tmpdir): @@ -261,7 +267,7 @@ def test_set_final_path_view(): @unittest.skipIf( - util.on_python2() or util.on_windows(), 'fallocate does not exist') + util.on_python2() or not util.on_linux(), 'fallocate does not exist') def test_downloaddescriptor_allocate_disk_space_via_seek(tmpdir): fp = pathlib.Path(str(tmpdir.join('fp'))) opts = mock.MagicMock() diff --git a/tests/test_blobxfer_models_upload.py b/tests/test_blobxfer_models_upload.py index 9d7db9d..b9726be 100644 --- a/tests/test_blobxfer_models_upload.py +++ b/tests/test_blobxfer_models_upload.py @@ -73,6 +73,10 @@ def test_localpath(tmpdir): assert lp.gid == stat.st_gid +def _resolve_pypath(path): + return str(pathlib.Path(str(path)).resolve()) + + def test_localsourcepaths_files(tmpdir): tmpdir.mkdir('abc') tmpdir.join('moo.cow').write('z') @@ -108,9 +112,9 @@ def test_localsourcepaths_files(tmpdir): assert not a.can_rename() assert len(a.paths) == 1 - assert str(abcpath.join('blah.x')) in a_set - assert str(defpath.join('world.txt')) in a_set - assert str(defpath.join('moo.cow')) not in a_set + assert _resolve_pypath(abcpath.join('blah.x')) in a_set + assert _resolve_pypath(defpath.join('world.txt')) in a_set + assert _resolve_pypath(defpath.join('moo.cow')) not in a_set b = upload.LocalSourcePath() b.add_includes(['moo.cow', '*blah*'])