- Update Windows Docker images to Python 3.7.1
- Fix flake8 errors
- Fix shellcheck errors
- Various build updates and fixes
This commit is contained in:
Fred Park 2018-10-26 13:36:33 -07:00
Родитель 2624ba3d87
Коммит ab9cc70828
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 3C4D545F457737EB
17 изменённых файлов: 83 добавлений и 83 удалений

Просмотреть файл

@ -6,15 +6,7 @@ python:
- 2.7
- 3.4
- 3.5
matrix:
include:
- python: 3.7
dist: xenial
sudo: true
before_install:
- wget -c https://goo.gl/ZzKHFv -O - | tar -xvJ -C /tmp/
- PATH="/tmp/shellcheck-latest:$PATH"
- 3.6
install:
- travis_retry pip install --upgrade pip
@ -24,6 +16,6 @@ install:
- pip list --format=columns --outdated
script:
- flake8 --statistics shipyard.py convoy/*.py
- if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then flake8 --statistics cascade/*.py cargo/*.py heimdall/*.py; fi
- flake8 --select F,E,W --ignore W504 --statistics shipyard.py convoy/*.py
- if [[ $TRAVIS_PYTHON_VERSION != '2.7' ]] && [[ $TRAVIS_PYTHON_VERSION != '3.4' ]]; then flake8 --select F,E,W --ignore W504 --statistics cascade/*.py cargo/*.py federation/*.py heimdall/*.py; fi
- shellcheck ./*.sh cargo/*.sh cascade/*.sh docker/*.sh scripts/*.sh

Просмотреть файл

@ -7,9 +7,9 @@ jobs:
strategy:
maxParallel: 1
matrix:
Python36:
python.version: '3.6'
PYENV_VERSION: '3.6.6'
Python37:
python.version: '3.7'
PYENV_VERSION: '3.7.1'
steps:
- task: InstallPython@1
inputs:
@ -23,9 +23,9 @@ jobs:
pip list --format=columns --outdated
displayName: Fetch Dependencies
- powershell: |
flake8 --statistics shipyard.py convoy\*.py
if ($env:PYTHON_VERSION -eq "3.6") {
flake8 --statistics cascade\*.py cargo\*.py heimdall\*.py
flake8 --select F,E,W --ignore W504 --statistics shipyard.py convoy\*.py
if ($env:PYTHON_VERSION -eq "3.7") {
flake8 --select F,E,W --ignore W504 --statistics cascade\*.py cargo\*.py federation\*.py heimdall\*.py
}
displayName: Static Analysis
- powershell: |
@ -41,7 +41,7 @@ jobs:
displayName: Pre-build Environment (Branch)
condition: >
and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.6'))
in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.7'))
- powershell: |
git tag -l --points-at $env:BUILD_SOURCEVERSION | Tee-Object -Variable gitTag
$gitTag -match "^([\d\.])+"
@ -64,7 +64,7 @@ jobs:
displayName: Pre-build Environment (Tagged Release)
condition: >
and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.6'))
startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.7'))
- powershell: |
$artifactCliPath = "bin\\" + $env:ARTIFACT_CLI
echo "##vso[task.setvariable variable=ARTIFACT_CLI_PATH;]$artifactCliPath"
@ -143,9 +143,9 @@ jobs:
strategy:
maxParallel: 1
matrix:
Python36:
python.version: '3.6'
PYENV_VERSION: '3.6.6'
Python37:
python.version: '3.7'
PYENV_VERSION: '3.7.1'
steps:
- task: UsePythonVersion@0
inputs:
@ -155,8 +155,8 @@ jobs:
set -e
set -o pipefail
env
curl -fSsL https://goo.gl/ZzKHFv | tar -xJvp -C /tmp/
echo "##vso[task.prependpath]/tmp/shellcheck-latest"
curl -fSsL "https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.x86_64.tar.xz" | tar -xJvp -C /tmp/
echo "##vso[task.prependpath]/tmp/shellcheck-stable"
if [[ "$PYENV_VERSION" == 3.* ]]; then
which python3
python3 --version
@ -184,9 +184,9 @@ jobs:
- script: |
set -e
set -o pipefail
flake8 --statistics shipyard.py convoy/*.py
if [[ "$PYENV_VERSION" == 3.6* ]]; then
flake8 --statistics cascade/*.py cargo/*.py heimdall/*.py
flake8 --select F,E,W --ignore W504 --statistics shipyard.py convoy/*.py
if [[ "$PYENV_VERSION" == 3.7* ]]; then
flake8 --select F,E,W --ignore W504 --statistics cascade/*.py cargo/*.py federation/*.py heimdall/*.py
fi
shellcheck ./*.sh cargo/*.sh cascade/*.sh docker/*.sh scripts/*.sh
displayName: Static Analysis
@ -201,7 +201,7 @@ jobs:
displayName: Pre-build Environment (Branch)
condition: >
and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.6'))
in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.7'))
- script: |
GIT_TAG=$(git tag -l --points-at $BUILD_SOURCEVERSION)
ARTIFACT_CLI="batch-shipyard-${GIT_TAG}-cli-linux-x86_64"
@ -212,7 +212,7 @@ jobs:
displayName: Pre-build Environment (Tagged Release)
condition: >
and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.6'))
startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.7'))
- template: ./pyenv.yml
- script: |
set -e
@ -248,9 +248,9 @@ jobs:
strategy:
maxParallel: 1
matrix:
Python36:
python.version: '3.6'
PYENV_VERSION: '3.6.6'
Python37:
python.version: '3.7'
PYENV_VERSION: '3.7.1'
steps:
- task: UsePythonVersion@0
inputs:
@ -286,9 +286,9 @@ jobs:
- script: |
set -e
set -o pipefail
flake8 --statistics shipyard.py convoy/*.py
if [[ "$PYENV_VERSION" == 3.6* ]]; then
flake8 --statistics cascade/*.py cargo/*.py heimdall/*.py
flake8 --select F,E,W --ignore W504 --statistics shipyard.py convoy/*.py
if [[ "$PYENV_VERSION" == 3.7* ]]; then
flake8 --select F,E,W --ignore W504 --statistics cascade/*.py cargo/*.py federation/*.py heimdall/*.py
fi
displayName: Static Analysis
- script: |
@ -302,7 +302,7 @@ jobs:
displayName: Pre-build Environment (Branch)
condition: >
and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.6'))
in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.7'))
- script: |
GIT_TAG=$(git tag -l --points-at $BUILD_SOURCEVERSION)
ARTIFACT_CLI="batch-shipyard-${GIT_TAG}-cli-mac-x86_64"
@ -313,7 +313,7 @@ jobs:
displayName: Pre-build Environment (Tagged Release)
condition: >
and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.6'))
startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.7'))
- template: ./pyenv.yml
- script: |
set -e

Просмотреть файл

@ -2,18 +2,22 @@ steps:
- script: |
set -e
set -o pipefail
echo "ARTIFACT_CLI=$ARTIFACT_CLI"
echo "SYSTEM_PHASENAME=$SYSTEM_PHASENAME ARTIFACT_CLI=$ARTIFACT_CLI"
PYTHON_CONFIGURE_OPTS="--enable-shared"
if [ "${SYSTEM_PHASENAME}" == "Linux" ]; then
sudo apt-get update
sudo apt-get install -y libbz2-dev libreadline-dev libsqlite3-dev
elif [ "${SYSTEM_PHASENAME}" == "MacOS" ]; then
PYTHON_CONFIGURE_OPTS="$PYTHON_CONFIGURE_OPTS --with-openssl=$(brew --prefix openssl)"
fi
export PYENV_ROOT="$(Agent.WorkFolder)/.pyenv"
git clone https://github.com/pyenv/pyenv.git $PYENV_ROOT
export PATH="$PYENV_ROOT/bin:$PATH"
pyenv --version
PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install $PYENV_VERSION
export PYTHON_CONFIGURE_OPTS
pyenv install $PYENV_VERSION
unset PYTHON_CONFIGURE_OPTS
pyenv global $PYENV_VERSION
echo $PATH
echo "##vso[task.prependpath]$PYENV_ROOT/bin"
echo "##vso[task.prependpath]$PYENV_ROOT/shims"
displayName: Install Python

Просмотреть файл

@ -11,17 +11,17 @@ environment:
PYTHON: "C:\\Python27-x64"
PYTHON_VERSION: "2.7"
PYTHON_ARCH: "64"
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
PYTHON: "C:\\Python34-x64"
PYTHON_VERSION: "3.4"
PYTHON_ARCH: "64"
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
PYTHON: "C:\\Python35-x64"
PYTHON_VERSION: "3.5"
PYTHON_ARCH: "64"
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
PYTHON: "C:\\Python37-x64"
PYTHON_VERSION: "3.7"
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
PYTHON: "C:\\Python36-x64"
PYTHON_VERSION: "3.6"
PYTHON_ARCH: "64"
init:
@ -40,7 +40,7 @@ install:
build: off
test_script:
- flake8 --statistics shipyard.py convoy\\*.py
- IF "%PYTHON_VERSION%"=="3.7" (
flake8 --statistics cascade\\*.py cargo\\*.py
- flake8 --select F,E,W --ignore W504 --statistics shipyard.py convoy\\*.py
- IF "%PYTHON_VERSION%" GEQ "3.5" (
flake8 --select F,E,W --ignore W504 --statistics cascade\\*.py cargo\\*.py federation\\*.py heimdall\\*.py
)

Просмотреть файл

@ -1163,7 +1163,7 @@ def _get_torrent_num_seeds(
def _start_torrent_via_storage(
blob_client: azureblob.BlockBlobService,
table_client: azuretable.TableService,
resource: str, entity: dict=None) -> None:
resource: str, entity: dict = None) -> None:
"""Start a torrent via storage entity
:param azureblob.BlockBlobService blob_client: blob client
:param azuretable.TableService table_client: table client

Просмотреть файл

@ -122,7 +122,7 @@ def _parse_message(event: str, msg: str) -> dict:
def _diff_events(
data: dict, nodeid: str, event: str, end_event: str, timing: dict,
prefix: str, sizes: dict=None) -> None:
prefix: str, sizes: dict = None) -> None:
"""Diff start and end event
:param dict data: data
:param str nodeid: node id

Просмотреть файл

@ -640,14 +640,16 @@ def _setup_azurefile_mounts(blob_client, config, bc, is_windows):
sa.account, share, is_windows)
if is_windows:
cmd = (
'net use \\\\{sa}.file.{ep}\{share} {sakey} '
'net use \\\\{sa}.file.{ep}\{share} {sakey} ' # noqa
'/user:Azure\{sa}'
).format(
sa=sa.account, ep=sa.endpoint, share=share,
sakey=sa.account_key)
cmds.append(cmd)
cmd = 'mklink /d {hmp} \\\\{sa}.file.{ep}\{share}'.format(
hmp=hmp, sa=sa.account, ep=sa.endpoint, share=share)
cmd = (
'mklink /d {hmp} \\\\{sa}.file.{ep}\{share}'.format( # noqa
hmp=hmp, sa=sa.account, ep=sa.endpoint, share=share)
)
else:
cmd = (
'mount -t cifs //{sa}.file.{ep}/{share} {hmp} -o '

Просмотреть файл

@ -495,7 +495,7 @@ def create_storage_cluster(
for disk_id, sat in disk_names:
disk_map[disk_id.split('/')[-1]] = (disk_id, sat)
del disk_names
except msrestazure.azure_exceptions.CloudError as e:
except msrestazure.azure_exceptions.CloudError:
logger.error(
'could not enumerate required disks for storage cluster {}'.format(
sc_id))

Просмотреть файл

@ -2151,7 +2151,7 @@ def delete_storage_containers_boot_diagnostics(
:param str vm_name: vm name
:param str vm_id: vm id
"""
name = re.sub('[\W_]+', '', vm_name)
name = re.sub('[\W_]+', '', vm_name) # noqa
contname = 'bootdiagnostics-{}-{}'.format(
name[0:min((9, len(name)))], vm_id)
logger.info('deleting container: {}'.format(contname))

Просмотреть файл

@ -1,11 +1,12 @@
# Dockerfile for Azure/batch-shipyard Cargo (Windows)
# Adapted from: https://github.com/StefanScherer/dockerfiles-windows/blob/master/python/Dockerfile
FROM python:3.6.6-windowsservercore-ltsc2016
FROM python:3.7.1-windowsservercore-ltsc2016
MAINTAINER Fred Park <https://github.com/Azure/batch-shipyard>
ENV chocolateyUseWindowsCompression false
RUN iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \
RUN [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; \
iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \
choco install --no-progress -y git -params "/GitAndUnixToolsOnPath"
ARG GIT_BRANCH
@ -19,23 +20,23 @@ RUN git clone -b $Env:GIT_BRANCH --single-branch https://github.com/Azure/batch-
del C:\batch-shipyard\cargo\requirements.txt ; \
del C:\batch-shipyard\cargo\Dockerfile
RUN python -m compileall C:\Python\Lib ; \
RUN python -m compileall C:\Python\Lib\site-packages ; \
python -m compileall C:\batch-shipyard\cargo ; \
exit 0
FROM microsoft/nanoserver
FROM microsoft/nanoserver:sac2016
COPY --from=0 /Python /Python
COPY --from=0 /batch-shipyard/cargo /batch-shipyard
SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
ENV PYTHON_VERSION 3.6.6
ENV PYTHON_PIP_VERSION 10.0.1
ENV PYTHON_VERSION 3.7.1
ENV PYTHON_PIP_VERSION 18.1
RUN $env:PATH = 'C:\Python;C:\Python\Scripts;{0}' -f $env:PATH ; \
Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment\' -Name Path -Value $env:PATH ; \
mkdir $env:APPDATA\Python\Python36\site-packages ; \
mkdir $env:APPDATA\Python\Python37\site-packages ; \
Invoke-WebRequest 'https://bootstrap.pypa.io/get-pip.py' -OutFile 'get-pip.py' -UseBasicParsing ; \
$replace = ('import tempfile{0}import site{0}site.getusersitepackages()' -f [char][int]10) ; \
Get-Content get-pip.py | Foreach-Object { $_ -replace 'import tempfile', $replace } | Out-File -Encoding Ascii getpip.py ; \

Просмотреть файл

@ -1,11 +1,12 @@
# Dockerfile for Azure/batch-shipyard CLI (Windows)
# Adapted from: https://github.com/StefanScherer/dockerfiles-windows/blob/master/python/Dockerfile
FROM python:3.6.6-windowsservercore-ltsc2016
FROM python:3.7.1-windowsservercore-ltsc2016
MAINTAINER Fred Park <https://github.com/Azure/batch-shipyard>
ENV chocolateyUseWindowsCompression false
RUN iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \
RUN [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; \
iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \
choco install --no-progress -y git -params "/GitAndUnixToolsOnPath"
ARG GIT_BRANCH
@ -22,7 +23,7 @@ RUN git clone -b $Env:GIT_BRANCH --single-branch https://github.com/Azure/batch-
Remove-Item *.yml -Force -Recurse ; \
Remove-Item install* -Force -Recurse
RUN python -m compileall C:\Python\Lib ; \
RUN python -m compileall C:\Python\Lib\site-packages ; \
python -m compileall C:\batch-shipyard\shipyard.py ; \
python -m compileall C:\batch-shipyard\convoy ; \
exit 0
@ -34,12 +35,12 @@ COPY --from=0 /batch-shipyard /batch-shipyard
SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
ENV PYTHON_VERSION 3.6.6
ENV PYTHON_PIP_VERSION 10.0.1
ENV PYTHON_VERSION 3.7.1
ENV PYTHON_PIP_VERSION 18.1
RUN $env:PATH = 'C:\Python;C:\Python\Scripts;{0}' -f $env:PATH ; \
Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment\' -Name Path -Value $env:PATH ; \
mkdir $env:APPDATA\Python\Python36\site-packages ; \
mkdir $env:APPDATA\Python\Python37\site-packages ; \
Invoke-WebRequest 'https://bootstrap.pypa.io/get-pip.py' -OutFile 'get-pip.py' -UseBasicParsing ; \
$replace = ('import tempfile{0}import site{0}site.getusersitepackages()' -f [char][int]10) ; \
Get-Content get-pip.py | Foreach-Object { $_ -replace 'import tempfile', $replace } | Out-File -Encoding Ascii getpip.py ; \

Просмотреть файл

@ -118,7 +118,7 @@ def is_not_empty(obj: Any) -> bool:
return obj is not None and len(obj) > 0
def datetime_utcnow(as_string: bool=False) -> datetime.datetime:
def datetime_utcnow(as_string: bool = False) -> datetime.datetime:
"""Returns a datetime now with UTC timezone
:param as_string: return as ISO8601 extended string
:return: datetime object representing now with UTC timezone
@ -309,7 +309,7 @@ class Credentials():
def create_msi_credentials(
self,
resource_id: str=None
resource_id: str = None
) -> msrestazure.azure_active_directory.MSIAuthentication:
"""Create MSI credentials
:param resource_id: resource id to auth against
@ -544,7 +544,7 @@ class BatchServiceHandler():
client = self.service_proxy.batch_client(batch_account, service_url)
try:
return client.pool.get(pool_id)
except batchmodels.BatchErrorException as e:
except batchmodels.BatchErrorException:
pass
return None
@ -569,7 +569,7 @@ class BatchServiceHandler():
'service_url={})'.format(
pool_id, batch_account, service_url))
return nc[0]
except batchmodels.BatchErrorException as e:
except batchmodels.BatchErrorException:
logger.error(
'could not retrieve pool {} node counts (account={} '
'service_url={})'.format(pool_id, batch_account, service_url))
@ -640,7 +640,7 @@ class BatchServiceHandler():
job_id: str,
delete: bool,
is_job_schedule: bool,
wait: bool=False,
wait: bool = False,
) -> None:
action = 'delete' if delete else 'terminate'
cstate = (
@ -698,9 +698,9 @@ class BatchServiceHandler():
job_id: str,
naming: TaskNaming,
current_task_id: str,
last_task_id: Optional[str]=None,
tasklist: Optional[List[str]]=None,
is_merge_task: Optional[bool]=False
last_task_id: Optional[str] = None,
tasklist: Optional[List[str]] = None,
is_merge_task: Optional[bool] = False
) -> Tuple[List[str], str]:
"""Regenerate the next generic task id
:param batch_account: batch account
@ -3124,7 +3124,7 @@ class FederationProcessor():
async def check_global_lock(
self,
backoff: bool=True
backoff: bool = True
) -> Generator[None, None, None]:
if not self.fdh.has_global_lock:
if backoff:

Просмотреть файл

@ -119,7 +119,7 @@ def convert_cloud_type(cloud_type: str) -> msrestazure.azure_cloud.Cloud:
def create_msi_credentials(
cloud: msrestazure.azure_cloud.Cloud,
resource_id: str=None
resource_id: str = None
) -> msrestazure.azure_active_directory.MSIAuthentication:
"""Create MSI credentials
:param cloud: cloud kind

Просмотреть файл

@ -135,13 +135,13 @@ else
DISTRIB_RELEASE=$VERSION_ID
fi
# check for OS X
if [ -z ${DISTRIB_ID+x} ] && [ "$(uname)" == "Darwin" ]; then
if [ -z "${DISTRIB_ID+x}" ] && [ "$(uname)" == "Darwin" ]; then
DISTRIB_ID=$(uname)
DISTRIB_RELEASE=$(uname -a | cut -d' ' -f3)
fi
fi
if [ -z ${DISTRIB_ID+x} ] || [ -z ${DISTRIB_RELEASE+x} ]; then
if [ -z "${DISTRIB_ID+x}" ] || [ -z "${DISTRIB_RELEASE+x}" ]; then
echo "Unknown DISTRIB_ID or DISTRIB_RELEASE."
echo "Please refer to the Installation documentation for manual installation steps."
exit 1

Просмотреть файл

@ -32,7 +32,7 @@ else
DISTRIB_RELEASE=$VERSION_ID
fi
fi
if [ -z ${DISTRIB_ID+x} ] || [ -z ${DISTRIB_RELEASE+x} ]; then
if [ -z "${DISTRIB_ID+x}" ] || [ -z "${DISTRIB_RELEASE+x}" ]; then
log ERROR "Unknown DISTRIB_ID or DISTRIB_RELEASE."
exit 1
fi

Просмотреть файл

@ -36,7 +36,7 @@ else
DISTRIB_RELEASE=$VERSION_ID
fi
fi
if [ -z ${DISTRIB_ID+x} ] || [ -z ${DISTRIB_RELEASE+x} ]; then
if [ -z "${DISTRIB_ID+x}" ] || [ -z "${DISTRIB_RELEASE+x}" ]; then
log ERROR "Unknown DISTRIB_ID or DISTRIB_RELEASE."
exit 1
fi

Просмотреть файл

@ -49,7 +49,7 @@ else
DISTRIB_RELEASE=$VERSION_ID
fi
fi
if [ -z ${DISTRIB_ID+x} ] || [ -z ${DISTRIB_RELEASE+x} ]; then
if [ -z "${DISTRIB_ID+x}" ] || [ -z "${DISTRIB_RELEASE+x}" ]; then
log ERROR "Unknown DISTRIB_ID or DISTRIB_RELEASE."
exit 1
fi