Smoke Tests use new workflow for package install (#11438)

* smoke test uses new workflow for installing dev feed packages and resolving dependencies
* smoke test dependencies packaging and pkginfo are not required
* add pip freeze to show installed packages
* first cut at grouping dependencies
* describe how dependency version specifiers are combined
* remove requirements.txt work from install_dev_build_dependency.py
* better task name, output requirements_dependencies.txt

common/smoketest/dependencies.py

@@ -0,0 +1,71 @@
+import argparse
+import pkg_resources
+
+try:
+    # pip < 20
+    from pip._internal.req import parse_requirements
+    from pip._internal.download import PipSession
+except:
+    # pip >= 20
+    from pip._internal.req import parse_requirements
+    from pip._internal.network.session import PipSession
+
+def combine_requirements(requirements):
+    name = requirements[0].project_name
+    specs = []
+    for req in requirements:
+        if len(req.specs) == 0:
+            continue
+
+        specs.extend([s[0] + s[1] for s in req.specs])
+
+    return name + ",".join(specs)
+
+def get_dependencies(packages):
+    requirements = []
+    for package in packages:
+        package_info = pkg_resources.working_set.by_key[package]
+
+        applicable_requirements = [r for r in package_info.requires() if r.marker is None or r.marker.evaluate()]
+        requirements.extend(applicable_requirements)
+
+    return requirements
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="List dependencies for a given requirements.txt file"
+    )
+
+    parser.add_argument(
+        "-r",
+        "--requirements",
+        dest="requirements_file",
+        help="File containing list of packages for which to find dependencies",
+        required=True
+    )
+
+    args = parser.parse_args()
+    # Get package names from requirements.txt
+    requirements = parse_requirements(args.requirements_file, session=PipSession())
+    package_names = [item.req.name for item in requirements]
+
+    dependencies = get_dependencies(package_names)
+
+    # It may be the case that packages have multiple sets of dependency
+    # requirements, for example:
+    # Package A requires Foo>=1.0.0,<2.0.0
+    # Package B requires Foo>=1.0.0,<1.2.3
+    # This combines all required versions into one string for pip to resolve
+    # Output: Foo>=1.0.0,<2.0.0,>=1.0.0,<1.2.3
+    # Pip parses this value using the Requirement object (https://setuptools.readthedocs.io/en/latest/pkg_resources.html#requirement-objects)
+    # According to https://packaging.python.org/glossary/#term-requirement-specifier
+    grouped_dependencies = {}
+    for dep in dependencies:
+        if dep.key in grouped_dependencies:
+            grouped_dependencies[dep.key].append(dep)
+        else:
+            grouped_dependencies[dep.key] = [dep]
+
+    final_dependencies = [combine_requirements(r) for r in grouped_dependencies.values()]
+
+    print("\n".join(final_dependencies))

common/smoketest/smoke-test.yml

@@ -84,18 +84,21 @@ jobs:
           pip --version
         displayName: pip --version
 
-      - script: pip install packaging pkginfo
-        displayName: Install requirements for dev tools
-
-      - script: pip install -r ./common/smoketest/requirements.txt
-        displayName: "Install requirements.txt"
+      - script: pip install -r ./common/smoketest/requirements.txt --pre --no-deps --index-url https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple
+        displayName: Install requirements from dev feed without dependencies
 
       - script: pip install -r ./common/smoketest/requirements_async.txt
         displayName: "Install requirements_async.txt"
         condition: and(succeeded(), eq(variables['InstallAsyncRequirements'], 'true'))
 
-      - script: python ./eng/tox/install_dev_build_dependency.py -r ./common/smoketest/requirements.txt
-        displayName: "Install dev dependencies from feed"
+      - script: python ./common/smoketest/dependencies.py -r ./common/smoketest/requirements.txt | tee ./common/smoketest/requirements_dependencies.txt
+        displayName: Create dependency list from installed dev packages
+
+      - script: pip install -r ./common/smoketest/requirements_dependencies.txt
+        displayName: Install dev package dependencies from PyPI
+
+      - script: pip freeze
+        displayName: Show installed packages (pip freeze)
 
       - template: /eng/common/TestResources/deploy-test-resources.yml
         parameters:
@@ -105,7 +108,7 @@ jobs:
           ArmTemplateParameters: $(ArmTemplateParameters)
 
       - script: python ./common/smoketest/program.py
-        displayName: "Run Smoke Test"
+        displayName: Run Smoke Test
 
       - template: /eng/common/TestResources/remove-test-resources.yml
         parameters:

eng/tox/install_dev_build_dependency.py

@@ -15,16 +15,6 @@ from subprocess import check_call
 
 from pip._internal.operations import freeze
 
-try:
-    # pip < 20
-    from pip._internal.req import parse_requirements
-    from pip._internal.download import PipSession
-except:
-    # pip >= 20
-    from pip._internal.req import parse_requirements
-    from pip._internal.network.session import PipSession
-
-
 # import common_task module
 root_dir = path.abspath(path.join(path.abspath(__file__), "..", "..", ".."))
 common_task_path = path.abspath(path.join(root_dir, "scripts", "devops_tasks"))
@@ -124,36 +114,15 @@ if __name__ == "__main__":
         "--target",
         dest="target_package",
         help="The target package directory on disk.",
-        required=False,
-    )
-
-    parser.add_argument(
-        "-r",
-        "--requirements",
-        dest="requirements_file",
-        help="Use dev builds of all installed azure-* packages",
-        required=False
+        required=True,
     )
 
     args = parser.parse_args()
 
-    if not args.target_package and not args.requirements_file:
-        raise "Must specify -t or -r"
-
     if args.target_package:
         # get target package name from target package path
         pkg_dir = path.abspath(args.target_package)
         pkg_name, _, ver = get_package_details(path.join(pkg_dir, "setup.py"))
         install_dev_build_packages(pkg_name)
 
-    elif args.requirements_file:
-        # Get package names from requirements.txt
-        requirements = parse_requirements(args.requirements_file, session=PipSession())
-        package_names = [item.req.name for item in requirements]
-
-        # Remove existing packages (that came from the public feed) and install
-        # from dev feed
-        uninstall_packages(package_names)
-        install_packages(package_names)
-