Update dependencies

- Update blobxfer to 1.5.4
- Resolves #243

.vsts/pipeline.yml

@@ -17,6 +17,7 @@ jobs:
       - powershell: |
           gci env:* | sort-object name
           python -m pip install --upgrade pip
+          pip install --upgrade setuptools wheel
           pip install -r requirements.txt
           pip install --no-deps -r req_nodeps.txt
           pip install flake8
@@ -176,6 +177,7 @@ jobs:
           echo "PYTHON=$PYTHON PIP=$PIP"
           which shellcheck
           $PYTHON -m pip install --upgrade pip
+          $PIP install --upgrade setuptools wheel
           $PIP install -r requirements.txt
           $PIP install --no-deps -r req_nodeps.txt
           $PIP install flake8
@@ -278,6 +280,7 @@ jobs:
           echo "PATH=$PATH"
           echo "PYTHON=$PYTHON PIP=$PIP"
           $PYTHON -m pip install --upgrade pip
+          $PIP install --upgrade setuptools wheel
           $PIP install -r requirements.txt
           $PIP install --no-deps -r req_nodeps.txt
           $PIP install flake8

cargo/requirements.txt

@@ -1,3 +1,3 @@
-azure-batch==5.1.0
+azure-batch==5.1.1
 msrest==0.5.5
-requests==2.19.1
+requests==2.20.0

convoy/data.py

@@ -59,7 +59,7 @@ from .version import __version__
 logger = logging.getLogger(__name__)
 util.setup_logger(logger)
 # global defines
-_BLOBXFER_VERSION = '1.5.0'
+_BLOBXFER_VERSION = '1.5.4'
 _MEGABYTE = 1048576
 _MAX_READ_BLOCKSIZE_BYTES = 4194304
 _FILE_SPLIT_PREFIX = '_shipyard-'

convoy/fleet.py

@@ -69,20 +69,20 @@ _NVIDIA_DRIVER = {
     'compute_cc37': {
         'url': (
             'http://us.download.nvidia.com/tesla/'
-            '396.44/NVIDIA-Linux-x86_64-396.44.run'
+            '410.72/NVIDIA-Linux-x86_64-410.72.run'
         ),
         'sha256': (
-            '8f3473b9b1777fefaec35676212eac64b0ab862d3eaee777a29ccff3d27ad485'
+            '1f1ef7a1eca295f9004317f540c05d78aaed13b28021e1e28fc3b9011a853d9e'
         ),
         'target': 'nvidia-driver_cc37.run'
     },
     'compute_cc6-7': {
         'url': (
             'http://us.download.nvidia.com/tesla/'
-            '396.44/NVIDIA-Linux-x86_64-396.44.run'
+            '410.72/NVIDIA-Linux-x86_64-410.72.run'
         ),
         'sha256': (
-            '8f3473b9b1777fefaec35676212eac64b0ab862d3eaee777a29ccff3d27ad485'
+            '1f1ef7a1eca295f9004317f540c05d78aaed13b28021e1e28fc3b9011a853d9e'
         ),
         'target': 'nvidia-driver_cc6-7.run'
     },
@@ -121,10 +121,10 @@ _LIS_PACKAGE = {
     'url': (
         'http://download.microsoft.com/download/6/8/F/'
         '68FE11B8-FAA4-4F8D-8C7D-74DA7F2CFC8C/'
-        'lis-rpms-4.2.6.tar.gz'
+        'lis-rpms-4.2.6-1.tar.gz'
     ),
     'sha256': (
-        '19cf0bc2fa936b226b792f42d36589aa7b2a57fed4235843f16bc242b7690dd8'
+        'daab0218a6631704884a3bb865570358edda64282a01dfc7367534beb1228a17'
     ),
     'target': 'lis.tar.gz',
     'intermediate': 'lis_compact.tar',

federation/requirements.txt

@@ -1,10 +1,10 @@
-azure-batch==5.1.0
+azure-batch==5.1.1
 azure-cosmosdb-table==1.0.5
-azure-mgmt-compute==4.1.0
+azure-mgmt-compute==4.3.1
 azure-mgmt-resource==2.0.0
-azure-mgmt-storage==2.0.0
+azure-mgmt-storage==3.0.0
 azure-storage-blob==1.3.1
 azure-storage-queue==1.3.0
-msrestazure==0.5.0
+msrestazure==0.5.1
-python-dateutil==2.7.3
+python-dateutil==2.7.5
-requests==2.19.1
+requests==2.20.0

heimdall/docker-compose-nonginx.yml

@@ -23,7 +23,7 @@ services:
       - /etc/heimdall.json
 
   prometheus:
-    image: prom/prometheus:v2.4.0
+    image: prom/prometheus:v2.4.3
     container_name: prometheus
     restart: unless-stopped
     ports:
@@ -42,7 +42,7 @@ services:
       - "--web.enable-lifecycle"
 
   grafana:
-    image: grafana/grafana:5.2.4
+    image: grafana/grafana:5.3.2
     container_name: grafana
     restart: unless-stopped
     ports:

heimdall/docker-compose.yml

@@ -41,7 +41,7 @@ services:
       - /var/batch-shipyard/letsencrypt/etc:/etc/letsencrypt
 
   prometheus:
-    image: prom/prometheus:v2.4.0
+    image: prom/prometheus:v2.4.3
     container_name: prometheus
     restart: unless-stopped
     ports:
@@ -60,7 +60,7 @@ services:
       - "--web.enable-lifecycle"
 
   grafana:
-    image: grafana/grafana:5.2.4
+    image: grafana/grafana:5.3.2
     container_name: grafana
     restart: unless-stopped
     ports:

heimdall/requirements.txt

@@ -1,8 +1,8 @@
-azure-batch==5.1.0
+azure-batch==5.1.1
 azure-cosmosdb-table==1.0.5
-azure-mgmt-compute==4.1.0
+azure-mgmt-compute==4.3.1
 azure-mgmt-network==2.2.1
 azure-mgmt-resource==2.0.0
-azure-mgmt-storage==2.0.0
+azure-mgmt-storage==3.0.0
-msrestazure==0.5.0
+msrestazure==0.5.1
-requests==2.19.1
+requests==2.20.0

install.cmd

@@ -2,32 +2,32 @@
 
 REM check for argument
 IF [%1] EQU [] (
-	echo Usage: install.cmd [virtual env name] [optional: path to python.exe]
+    echo Usage: install.cmd [virtual env name] [optional: path to python.exe]
-	exit /b 1
+    exit /b 1
 )
 IF %~1 == shipyard.cmd (
-	echo "shipyard.cmd" cannot be specified as the virtual environment name.
+    echo "shipyard.cmd" cannot be specified as the virtual environment name.
-	exit /b 1
+    exit /b 1
 )
 SET VENVNAME=%~1
 
 REM set python to use
 IF [%2] NEQ [] (
-	SET PYTHON=%~2
+    SET PYTHON=%~2
 ) ELSE (
-	FOR /f %%i in ('where python.exe') do SET PYTHON="%%i"
+    FOR /f %%i in ('where python.exe') do SET PYTHON="%%i"
 )
 IF NOT DEFINED PYTHON (
-	echo Python not found. Please ensure python.exe is in your PATH.
+    echo Python not found. Please ensure python.exe is in your PATH.
-	exit /b 1
+    exit /b 1
 )
 echo Using python from %PYTHON%
 
 REM check that shipyard.py is in cwd
 SET SHIPYARDFILE="%cd%\shipyard.py"
 IF NOT EXIST %SHIPYARDFILE% (
-	echo shipyard.py does not exist in current working directory. Please run installer from Batch Shipyard root.
+    echo shipyard.py does not exist in current working directory. Please run installer from Batch Shipyard root.
-	exit /b 1
+    exit /b 1
 )
 
 REM check for anaconda
@@ -36,43 +36,47 @@ SET /P PYTHONVER=<.pyver.txt
 del .pyver.txt
 SET ANACONDA=0
 IF NOT "%PYTHONVER%"=="%PYTHONVER:anaconda=%" (
-	echo Anaconda detected.
+    echo Anaconda detected.
-	SET ANACONDA=1
+    SET ANACONDA=1
 )
 IF NOT "%PYTHONVER%"=="%PYTHONVER:continuum=%" (
-	echo Anaconda detected.
+    echo Anaconda detected.
-	SET ANACONDA=1
+    SET ANACONDA=1
 )
 IF NOT "%PYTHONVER%"=="%PYTHONVER:conda-forge=%" (
-	echo Anaconda detected.
+    echo Anaconda detected.
-	SET ANACONDA=1
+    SET ANACONDA=1
 )
 
 REM install env and requirements
 IF %ANACONDA% EQU 1 (
-	echo Performing install for Anaconda.
+    echo Performing install for Anaconda.
-	conda create --yes --name %VENVNAME%
+    conda create --yes --name %VENVNAME%
-	cmd.exe /c "activate %VENVNAME% & conda install --yes pip & pip install --upgrade -r requirements.txt & deactivate %VENVNAME%"
+    cmd.exe /c "activate %VENVNAME% & conda install --yes pip & pip install --upgrade -r requirements.txt & deactivate %VENVNAME%"
 ) ELSE (
-	echo Performing install for Python.
+    echo Performing install for Python.
-	pip install --upgrade virtualenv
+    pip install --upgrade virtualenv
-	IF ERRORLEVEL 1 (
+    IF ERRORLEVEL 1 (
-		echo "pip install virtualenv failed"
+        echo "pip install virtualenv failed"
-		exit /b 1
+        exit /b 1
-	)
+    )
-	virtualenv -p %PYTHON% %VENVNAME%
+    virtualenv -p %PYTHON% %VENVNAME%
-	cmd.exe /c "%VENVNAME%\Scripts\activate & pip uninstall -y azure-storage & deactivate"
+    call "%VENVNAME%\Scripts\activate.bat"
-	cmd.exe /c "%VENVNAME%\Scripts\activate & pip install --upgrade -r requirements.txt & deactivate"
+    pip uninstall -y azure-storage
-	cmd.exe /c "%VENVNAME%\Scripts\activate & pip install --upgrade --no-deps -r req_nodeps.txt & deactivate"
+    pip install --upgrade -r requirements.txt
+    pip install --upgrade --no-deps -r req_nodeps.txt
+    call "%VENVNAME%\Scripts\deactivate.bat"
 )
 
 REM create launcher cmd
 SET CMDFILE=shipyard.cmd
 (echo @echo off) > %CMDFILE%
 IF %ANACONDA% EQU 1 (
-	(echo activate %VENVNAME% ^& python %SHIPYARDFILE% %%* ^& deactivate %VENVNAME%) >> %CMDFILE%
+    (echo activate %VENVNAME% ^& python %SHIPYARDFILE% %%* ^& deactivate %VENVNAME%) >> %CMDFILE%
 ) ELSE (
-	(echo "%cd%\%VENVNAME%\Scripts\activate" ^& python %SHIPYARDFILE% %%* ^& deactivate) >> %CMDFILE%
+    echo call "%cd%\%VENVNAME%\Scripts\activate.bat" >> %CMDFILE%
+    echo python %SHIPYARDFILE% %%* >> %CMDFILE%
+    echo deactivate >> %CMDFILE%
 )
 
 echo Installation complete. Run Batch Shipyard as: %cd%\shipyard.cmd

req_nodeps.txt

@@ -1 +1 @@
-pykwalify==1.6.1
+pykwalify==1.7.0

requirements.txt

@@ -1,25 +1,25 @@
-adal==1.1.0
+adal==1.2.0
-azure-batch==5.1.0
+azure-batch==5.1.1
 azure-cosmosdb-table==1.0.5
 azure-keyvault==1.1.0
 azure-mgmt-authorization==0.50.0
 azure-mgmt-batch==5.0.1
-azure-mgmt-compute==4.1.0
+azure-mgmt-compute==4.3.1
 azure-mgmt-network==2.2.1
 azure-mgmt-resource==2.0.0
-azure-mgmt-storage==2.0.0
+azure-mgmt-storage==3.0.0
 azure-storage-blob==1.3.1
 azure-storage-file==1.3.1
 azure-storage-queue==1.3.0
-blobxfer==1.5.0
+blobxfer==1.5.4
-click==6.7
+click==7.0
-future==0.16.0
+future==0.17.1
 futures==3.2.0; python_version < '3'
 keyrings.alt==3.1
 msrest==0.5.5
-msrestazure==0.5.0
+msrestazure==0.5.1
 pathlib2==2.3.2; python_version < '3.5'
-python-dateutil==2.7.3
+python-dateutil==2.7.5
-requests==2.19.1
+requests==2.20.0
-ruamel.yaml==0.15.66
+ruamel.yaml==0.15.76
 scandir==1.9.0; python_version < '3.5'