Add dependency check to CI (#6828)

.gitignore

@@ -99,5 +99,8 @@ artifacts/
 .cache/
 .pytest_cache/
 
+# Logs
+pip.log
+
 
 az_command_coverage.txt

.travis.yml

@@ -23,6 +23,10 @@ jobs:
       env: PURPOSE='Headers'
       script: ./scripts/ci/precheck_header.sh
       python: 3.6
+    - stage: precheck
+      env: PURPOSE='Dependency Check'
+      script: ./scripts/dependency/check.sh
+      python: 3.6
     - stage: verify
       env: PURPOSE='Lint Command Table and Help'
       script: ./scripts/ci/verify_linter.sh

scripts/ci/test_integration.sh

@@ -22,6 +22,3 @@ azdev verify package $share_folder/build/
 export AZURE_CORE_COLLECT_TELEMETRY=False
 echo '= Verify Commands with -h'
 azdev verify commands
-
-echo '= Verify Dependencies'
-azdev verify dependencies

scripts/dependency/check.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+set -e
+
+cd `cd $(dirname $0); cd ../..; pwd`
+
+git status >/dev/null 2>&1 || (echo "This command is expected to run in a git environment." >&2; exit 1)
+
+VENV_DIR=`mktemp -d /tmp/azurecli.dep.check.XXXXXX`
+echo "Created temporary folder $VENV_DIR" >&2
+
+REQU_NAME="`dirname $0`/requirements.`uname`.external.txt"
+echo "Baseline requirements file $REQU_NAME"
+
+python -m pip install virtualenv
+python -m virtualenv $VENV_DIR
+
+. $VENV_DIR/bin/activate
+
+echo "Virtual environemtn established at $VENV_DIR." >&2
+echo "   Python path `which python`" >&2
+echo "   `python --version`" >&2
+
+echo "Install CORE" >&2
+pip install -e src/azure-cli-core -q --log pip.log
+
+echo "Install Namespace Package" >&2
+pip install -e src/azure-cli-nspkg -q --log pip.log
+
+for setups in `find src/command_modules -name setup.py`; do
+    dir_path=`dirname $setups`
+    echo "Install $dir_path" >&2
+    pip install -e $dir_path -q --log pip.log
+done
+
+echo "Install CLI Main Package" >&2
+pip install -e src/azure-cli -q --log pip.log
+
+pip check
+
+pip freeze | sed '/-e/d' > $REQU_NAME
+
+
+if ! git diff --exit-code $REQU_NAME; then
+
+    cat << EOM >&2
+
+
+The check failed because a change to the external dependencies has been detected. You can find the
+exact difference in the git-diff command output (which is above this message.)
+
+The failure is caused by:
+
+1) Your commit alters one or multiple 'setup.py'; or
+
+2) One or multiple dependencies have released newer version.
+
+To resolve the failures:
+
+1) To confirm that the change to the external dependency is expected, edit the $REQU_NAME file
+    to reflect the change. The file can be also edited automatically by run script $0.
+
+2) To avoid the change to the external dependency, edit the impacted setup.py to cap the external
+    dependencies.
+
+EOM
+
+    echo $ERROR_MESSAGE
+fi
+
+rm -rf $VENV_DIR
+exit 0

scripts/dependency/requirements.Darwin.external.txt

@@ -0,0 +1,108 @@
+adal==1.0.2
+applicationinsights==0.11.5
+argcomplete==1.9.4
+asn1crypto==0.24.0
+azure-batch==4.1.3
+azure-common==1.1.14
+azure-datalake-store==0.0.22
+azure-graphrbac==0.40.0
+azure-keyvault==0.3.7
+azure-mgmt-advisor==1.0.1
+azure-mgmt-authorization==0.40.0
+azure-mgmt-batch==4.1.0
+azure-mgmt-batchai==2.0.0
+azure-mgmt-billing==0.2.0
+azure-mgmt-cdn==1.0.0
+azure-mgmt-cognitiveservices==1.0.0
+azure-mgmt-compute==4.0.0rc2
+azure-mgmt-consumption==2.0.0
+azure-mgmt-containerinstance==1.0.0
+azure-mgmt-containerregistry==2.0.0
+azure-mgmt-containerservice==4.1.0
+azure-mgmt-cosmosdb==0.4.0
+azure-mgmt-datalake-analytics==0.2.0
+azure-mgmt-datalake-nspkg==2.0.0
+azure-mgmt-datalake-store==0.2.0
+azure-mgmt-datamigration==0.1.0
+azure-mgmt-devtestlabs==2.0.0
+azure-mgmt-dns==2.0.0rc1
+azure-mgmt-iothub==0.5.0
+azure-mgmt-iothubprovisioningservices==0.1.0
+azure-mgmt-keyvault==0.40.0
+azure-mgmt-loganalytics==0.2.0
+azure-mgmt-managementgroups==0.1.0
+azure-mgmt-maps==0.1.0
+azure-mgmt-marketplaceordering==0.1.0
+azure-mgmt-media==1.0.0rc1
+azure-mgmt-monitor==0.5.0
+azure-mgmt-msi==0.1.0
+azure-mgmt-network==2.0.0rc3
+azure-mgmt-nspkg==2.0.0
+azure-mgmt-policyinsights==0.1.0
+azure-mgmt-rdbms==1.2.0
+azure-mgmt-recoveryservices==0.1.0
+azure-mgmt-recoveryservicesbackup==0.1.1
+azure-mgmt-redis==4.1.0
+azure-mgmt-reservations==0.2.1
+azure-mgmt-resource==2.0.0rc2
+azure-mgmt-servicebus==0.4.0
+azure-mgmt-servicefabric==0.1.0
+azure-mgmt-sql==0.9.1
+azure-mgmt-storage==1.5.0
+azure-mgmt-trafficmanager==0.40.0
+azure-mgmt-web==0.35.0
+azure-multiapi-storage==0.2.2
+azure-nspkg==2.0.0
+azure-storage-blob==1.1.0
+azure-storage-common==1.1.0
+azure-storage-nspkg==3.0.0
+bcrypt==3.1.4
+certifi==2018.4.16
+cffi==1.11.5
+chardet==3.0.4
+colorama==0.3.9
+configparser==3.5.0
+cryptography==2.3
+entrypoints==0.2.3
+enum34==1.1.6
+funcsigs==1.0.2
+futures==3.2.0
+humanfriendly==4.15.1
+idna==2.7
+ipaddress==1.0.22
+isodate==0.6.0
+jmespath==0.9.3
+keyring==13.2.1
+knack==0.4.1
+mock==2.0.0
+monotonic==1.5
+msrest==0.5.4
+msrestazure==0.4.34
+oauthlib==2.1.0
+paramiko==2.4.1
+pathlib2==2.3.2
+pbr==4.1.1
+prompt-toolkit==1.0.15
+pyasn1==0.4.3
+pycparser==2.18
+pydocumentdb==2.3.2
+Pygments==2.2.0
+PyJWT==1.6.4
+PyNaCl==1.2.1
+pyOpenSSL==18.0.0
+python-dateutil==2.7.3
+PyYAML==3.13
+requests==2.19.1
+requests-oauthlib==1.0.0
+scandir==1.7
+scp==0.11.0
+six==1.11.0
+sshtunnel==0.1.4
+tabulate==0.8.2
+typing==3.6.4
+urllib3==1.23
+vsts-cd-manager==1.0.2
+wcwidth==0.1.7
+websocket-client==0.48.0
+Whoosh==2.7.4
+xmltodict==0.11.0

scripts/dependency/requirements.Linux.external.txt

@@ -0,0 +1,101 @@
+adal==1.0.2
+applicationinsights==0.11.5
+argcomplete==1.9.4
+asn1crypto==0.24.0
+azure-batch==4.1.3
+azure-common==1.1.14
+azure-datalake-store==0.0.22
+azure-graphrbac==0.40.0
+azure-keyvault==0.3.7
+azure-mgmt-advisor==1.0.1
+azure-mgmt-authorization==0.40.0
+azure-mgmt-batch==4.1.0
+azure-mgmt-batchai==2.0.0
+azure-mgmt-billing==0.2.0
+azure-mgmt-cdn==1.0.0
+azure-mgmt-cognitiveservices==1.0.0
+azure-mgmt-compute==4.0.0rc2
+azure-mgmt-consumption==2.0.0
+azure-mgmt-containerinstance==1.0.0
+azure-mgmt-containerregistry==2.0.0
+azure-mgmt-containerservice==4.1.0
+azure-mgmt-cosmosdb==0.4.0
+azure-mgmt-datalake-analytics==0.2.0
+azure-mgmt-datalake-nspkg==2.0.0
+azure-mgmt-datalake-store==0.2.0
+azure-mgmt-datamigration==0.1.0
+azure-mgmt-devtestlabs==2.0.0
+azure-mgmt-dns==2.0.0rc1
+azure-mgmt-iothub==0.5.0
+azure-mgmt-iothubprovisioningservices==0.1.0
+azure-mgmt-keyvault==0.40.0
+azure-mgmt-loganalytics==0.2.0
+azure-mgmt-managementgroups==0.1.0
+azure-mgmt-maps==0.1.0
+azure-mgmt-marketplaceordering==0.1.0
+azure-mgmt-media==1.0.0rc1
+azure-mgmt-monitor==0.5.0
+azure-mgmt-msi==0.1.0
+azure-mgmt-network==2.0.0rc3
+azure-mgmt-nspkg==2.0.0
+azure-mgmt-policyinsights==0.1.0
+azure-mgmt-rdbms==1.2.0
+azure-mgmt-recoveryservices==0.1.0
+azure-mgmt-recoveryservicesbackup==0.1.1
+azure-mgmt-redis==4.1.0
+azure-mgmt-reservations==0.2.1
+azure-mgmt-resource==2.0.0rc2
+azure-mgmt-servicebus==0.4.0
+azure-mgmt-servicefabric==0.1.0
+azure-mgmt-sql==0.9.1
+azure-mgmt-storage==1.5.0
+azure-mgmt-trafficmanager==0.40.0
+azure-mgmt-web==0.35.0
+azure-multiapi-storage==0.2.1
+azure-nspkg==2.0.0
+azure-storage-blob==1.1.0
+azure-storage-common==1.1.0
+azure-storage-nspkg==3.0.0
+bcrypt==3.1.4
+certifi==2018.4.16
+cffi==1.11.5
+chardet==3.0.4
+colorama==0.3.9
+cryptography==2.2.2
+entrypoints==0.2.3
+humanfriendly==4.15.1
+idna==2.7
+isodate==0.6.0
+jeepney==0.3.1
+jmespath==0.9.3
+keyring==13.2.1
+knack==0.4.1
+mock==2.0.0
+msrest==0.5.4
+msrestazure==0.4.34
+oauthlib==2.1.0
+paramiko==2.4.1
+pbr==4.1.0
+prompt-toolkit==1.0.15
+pyasn1==0.4.3
+pycparser==2.18
+pydocumentdb==2.3.2
+Pygments==2.2.0
+PyJWT==1.6.4
+PyNaCl==1.2.1
+pyOpenSSL==18.0.0
+python-dateutil==2.7.3
+PyYAML==3.13
+requests==2.19.1
+requests-oauthlib==1.0.0
+scp==0.11.0
+SecretStorage==3.0.1
+six==1.11.0
+sshtunnel==0.1.4
+tabulate==0.8.2
+urllib3==1.23
+vsts-cd-manager==1.0.2
+wcwidth==0.1.7
+websocket-client==0.48.0
+Whoosh==2.7.4
+xmltodict==0.11.0

src/__init__.py

@@ -1,4 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------

src/command_modules/__init__.py

@@ -1,4 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------

tools/automation/verify/__init__.py

@@ -8,7 +8,6 @@ from automation.verify.doc_source_map import verify_doc_source_map
 from automation.verify.default_modules import verify_default_modules
 import automation.verify.verify_packages
 import automation.verify.verify_commands
-import automation.verify.verify_dependencies
 import automation.verify.verify_module_load_times
 import automation.verify.verify_load_all
 
@@ -63,6 +62,5 @@ def init_args(root):
 
     automation.verify.verify_packages.init(sub_parser)
     automation.verify.verify_commands.init(sub_parser)
-    automation.verify.verify_dependencies.init(sub_parser)
     automation.verify.verify_module_load_times.init(sub_parser)
     automation.verify.verify_load_all.init(sub_parser)

tools/automation/verify/verify_dependencies.py

@@ -1,35 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
-"""Verify package dependency compatibility (with support for allowed exceptions). """
-
-import subprocess
-import sys
-
-ALLOWED_ERRORS = []
-
-
-def init(root):
-    parser = root.add_parser('dependencies', help='Run pip check')
-    parser.set_defaults(func=verify_dependencies)
-
-
-def verify_dependencies(_):
-    try:
-        subprocess.check_output(['pip', 'check'], stderr=subprocess.STDOUT, universal_newlines=True)
-    except subprocess.CalledProcessError as err:
-        pip_messages = err.output.splitlines()
-        errors = []
-        for msg in pip_messages:
-            if not any(a in msg for a in ALLOWED_ERRORS):
-                errors.append(msg)
-        if errors:
-            sys.stderr.write('Dependency compatibility errors found!\n')
-            sys.stderr.write('\n'.join(errors))
-            sys.exit(1)
-        else:
-            sys.stderr.write("'pip check' returned exit code {} but the errors are allowable.\n".format(err.returncode))
-            sys.stderr.write("Full output from pip follows:\n")
-            sys.stderr.write(err.output + '\n')