Add distros and location to the test suites definition files (#2756)

* Add distro, location, and VM size configuration to test suites

---------

Co-authored-by: narrieta <narrieta>

tests_e2e/orchestrator/lib/agent_test_loader.py

@@ -15,77 +15,178 @@
 # limitations under the License.
 #
 import importlib.util
-import json
+# E0401: Unable to import 'yaml' (import-error)
+import yaml  # pylint: disable=E0401
 
 from pathlib import Path
 from typing import Any, Dict, List, Type
 
+import tests_e2e
 from tests_e2e.tests.lib.agent_test import AgentTest
 
 
-class TestSuiteDescription(object):
+class TestSuiteInfo(object):
     """
-    Description of the test suite loaded from its JSON file.
+    Description of a test suite
     """
+    # The name of the test suite
     name: str
+    # The tests that comprise the suite
     tests: List[Type[AgentTest]]
+    # An image or image set (as defined in images.yml) specifying the images the suite must run on.
+    images: str
+    # The location (region) on which the suite must run; if empty, the suite can run on any location
+    location: str
+    # Whether this suite must run on its own test VM
+    owns_vm: bool
+
+    def __str__(self):
+        return self.name
+
+
+class VmImageInfo(object):
+    # The URN of the image (publisher, offer, version separated by spaces)
+    urn: str
+    # Indicates that the image is available only on those locations. If empty, the image should be available in all locations
+    locations: List[str]
+    # Indicates that the image is available only for those VM sizes. If empty, the image should be available for all VM sizes
+    vm_sizes: List[str]
+
+    def __str__(self):
+        return self.urn
 
 
 class AgentTestLoader(object):
     """
-    Loads the description of a set of test suites
+    Loads a given set of test suites from the YAML configuration files.
     """
-    def __init__(self, test_source_directory: Path):
+    def __init__(self, test_suites: str):
         """
-        The test_source_directory parameter must be the root directory of the end-to-end tests (".../WALinuxAgent/tests_e2e")
-        """
-        self._root: Path = test_source_directory
-
-    def load(self, test_suites: str) -> List[TestSuiteDescription]:
-        """
-        Loads the specified 'test_suites', which are given as a string of comma-separated suite names or a JSON description
+        Loads the specified 'test_suites', which are given as a string of comma-separated suite names or a YAML description
         of a single test_suite.
 
-        When given as a comma-separated list, each item must correspond to the name of the JSON files describing s suite (those
-        files are located under the .../WALinuxAgent/tests_e2e/test_suites directory). For example, if test_suites == "agent_bvt, fast-track"
-        then this method will load files agent_bvt.json and fast-track.json.
+        When given as a comma-separated list, each item must correspond to the name of the YAML files describing s suite (those
+        files are located under the .../WALinuxAgent/tests_e2e/test_suites directory). For example, if test_suites == "agent_bvt, fast_track"
+        then this method will load files agent_bvt.yml and fast_track.yml.
 
-        When given as a JSON string, the value must correspond to the description a single test suite, for example
+        When given as a YAML string, the value must correspond to the description a single test suite, for example
 
-            {
-              "name": "AgentBvt",
-
-              "tests": [
-                "bvts/extension_operations.py",
-                "bvts/run_command.py",
-                "bvts/vm_access.py"
-              ]
-            }
+            name: "AgentBvt"
+            tests:
+              - "bvts/extension_operations.py"
+              - "bvts/run_command.py"
+              - "bvts/vm_access.py"
         """
-        # Attempt to parse 'test_suites' as the JSON description for a single suite
-        try:
-            return [self._load_test_suite(json.loads(test_suites))]
-        except json.decoder.JSONDecodeError:
-            pass
+        self.__test_suites: List[TestSuiteInfo] = self._load_test_suites(test_suites)
+        self.__images: Dict[str, List[VmImageInfo]] = self._load_images()
+        self._validate()
 
-        # Else, it should be a comma-separated list of description files
-        description_files: List[Path] = [self._root/"test_suites"/f"{t.strip()}.json" for t in test_suites.split(',')]
-        return [self._load_test_suite(AgentTestLoader._load_file(s)) for s in description_files]
+    _SOURCE_CODE_ROOT: Path = Path(tests_e2e.__path__[0])
 
-    def _load_test_suite(self, test_suite: Dict[str, Any]) -> TestSuiteDescription:
+    @property
+    def test_suites(self) -> List[TestSuiteInfo]:
+        return self.__test_suites
+
+    @property
+    def images(self) -> Dict[str, List[VmImageInfo]]:
         """
-        Creates a TestSuiteDescription from its JSON representation, which has been loaded by JSON.loads and is passed
-        to this method as a dictionary
+        A dictionary where, for each item, the key is the name of an image or image set and the value is a list of VmImageInfos for
+        the corresponding images.
         """
-        suite = TestSuiteDescription()
-        suite.name = test_suite["name"]
-        suite.tests = []
-        for source_file in [self._root/"tests"/t for t in test_suite["tests"]]:
-            suite.tests.extend(AgentTestLoader._load_tests(source_file))
-        return suite
+        return self.__images
+
+    def _validate(self):
+        """
+        Performs some basic validations on the data loaded from the YAML description files
+        """
+        for suite in self.test_suites:
+            # Validate that the images the suite must run on are in images.yml
+            if suite.images not in self.images:
+                raise Exception(f"Invalid image reference in test suite {suite.name}: Can't find {suite.images} in images.yml")
+
+            # If the suite specifies a location, validate that the images are available in that location
+            if suite.location != '':
+                if not any(suite.location in i.locations for i in self.images[suite.images]):
+                    raise Exception(f"Test suite {suite.name} must be executed in {suite.location}, but no images in {suite.images} are available in that location")
 
     @staticmethod
-    def _load_tests(source_file: Path) -> List[Type[AgentTest]]:
+    def _load_test_suites(test_suites: str) -> List[TestSuiteInfo]:
+        #
+        # Attempt to parse 'test_suites' as the YML description of a single suite
+        #
+        parsed = yaml.safe_load(test_suites)
+
+        #
+        # A comma-separated list (e.g. "foo", "foo, bar", etc.) is valid YAML, but it is parsed as a string. An actual test suite would
+        # be parsed as a dictionary. If it is a dict, take is as the YML description of a single test suite
+        #
+        if isinstance(parsed, dict):
+            return [AgentTestLoader._load_test_suite(parsed)]
+
+        #
+        # If test_suites is not YML, then it should be a comma-separated list of description files
+        #
+        description_files: List[Path] = [AgentTestLoader._SOURCE_CODE_ROOT/"test_suites"/f"{t.strip()}.yml" for t in test_suites.split(',')]
+        return [AgentTestLoader._load_test_suite(f) for f in description_files]
+
+    @staticmethod
+    def _load_test_suite(description_file: Path) -> TestSuiteInfo:
+        """
+        Loads the description of a TestSuite from its YAML file.
+
+        A test suite has 5 properties: name, tests, images, location, and owns-vm. For example:
+
+            name: "AgentBvt"
+            tests:
+              - "bvts/extension_operations.py"
+              - "bvts/run_command.py"
+              - "bvts/vm_access.py"
+            images: "endorsed"
+            location: "eastuseaup"
+            owns-vm: true
+
+        * name     - A string used to identify the test suite
+        * tests    - A list of the tests in the suite. Each test is specified by the path for its source code relative to
+                     WALinuxAgent/tests_e2e/tests.
+        * images   - A string specifying the images on which the test suite must be executed. The value can be the name
+                     of a single image (e.g."ubuntu_2004"), or the name of an image set (e.g. "endorsed"). The names for
+                     images and image sets are defined in WALinuxAgent/tests_e2e/tests_suites/images.yml.
+        * location - [Optional; string] If given, the test suite must be executed on that location. If not specified,
+                     or set to an empty string, the test suite will be executed in the default location. This is useful
+                     for test suites that exercise a feature that is enabled only in certain regions.
+        * owns-vm - [Optional; boolean] By default all suites in a test run are executed on the same test VMs; if this
+                    value is set to True, new test VMs will be created and will be used exclusively for this test suite.
+                    This is useful for suites that modify the test VMs in such a way that the setup may cause problems
+                    in other test suites (for example, some tests targeted to the HGAP block internet access in order to
+                    force the agent to use the HGAP).
+
+        """
+        test_suite: Dict[str, Any] = AgentTestLoader._load_file(description_file)
+
+        if any([test_suite.get(p) is None for p in ["name", "tests", "images"]]):
+            raise Exception(f"Invalid test suite: {description_file}. 'name', 'tests', and 'images' are required properties")
+
+        test_suite_info = TestSuiteInfo()
+
+        test_suite_info.name = test_suite["name"]
+
+        test_suite_info.tests = []
+        source_files = [AgentTestLoader._SOURCE_CODE_ROOT/"tests"/t for t in test_suite["tests"]]
+        for f in source_files:
+            test_suite_info.tests.extend(AgentTestLoader._load_test_classes(f))
+
+        test_suite_info.images = test_suite["images"]
+
+        test_suite_info.location = test_suite.get("location")
+        if test_suite_info.location is None:
+            test_suite_info.location = ""
+
+        test_suite_info.owns_vm = "owns-vm" in test_suite and test_suite["owns-vm"]
+
+        return test_suite_info
+
+    @staticmethod
+    def _load_test_classes(source_file: Path) -> List[Type[AgentTest]]:
         """
         Takes a 'source_file', which must be a Python module, and returns a list of all the classes derived from AgentTest.
         """
@@ -96,12 +197,53 @@ class AgentTestLoader(object):
         return [v for v in module.__dict__.values() if isinstance(v, type) and issubclass(v, AgentTest) and v != AgentTest]
 
     @staticmethod
-    def _load_file(file: Path):
-        """Helper to load a JSON file"""
+    def _load_images() -> Dict[str, List[VmImageInfo]]:
+        """
+        Loads images.yml into a dictionary where, for each item, the key is an image or image set and the value is a list of VmImageInfos
+        for the corresponding images.
+
+        See the comments in image.yml for a description of the structure of each item.
+        """
+        image_descriptions = AgentTestLoader._load_file(AgentTestLoader._SOURCE_CODE_ROOT/"test_suites"/"images.yml")
+        if "images" not in image_descriptions:
+            raise Exception("images.yml is missing the 'images' item")
+
+        images = {}
+
+        # first load the images as 1-item lists
+        for name, description in image_descriptions["images"].items():
+            i = VmImageInfo()
+            if isinstance(description, str):
+                i.urn = description
+                i.locations = []
+                i.vm_sizes = []
+            else:
+                if "urn" not in description:
+                    raise Exception(f"Image {name} is missing the 'urn' property: {description}")
+                i.urn = description["urn"]
+                i.locations = description["locations"] if "locations" in description else []
+                i.vm_sizes = description["vm_sizes"] if "vm_sizes" in description else []
+            images[name] = [i]
+
+        # now load the image-sets, mapping them to the images that we just computed
+        for image_set_name, image_list in image_descriptions["image-sets"].items():
+            # the same name cannot denote an image and an image-set
+            if image_set_name in images:
+                raise Exception(f"Invalid image-set in images.yml: {image_set_name}. The name is used by an existing image")
+            images_in_set = []
+            for i in image_list:
+                if i not in images:
+                    raise Exception(f"Can't find image {i} (referenced by image-set {image_set_name}) in images.yml")
+                images_in_set.extend(images[i])
+            images[image_set_name] = images_in_set
+
+        return images
+
+    @staticmethod
+    def _load_file(file: Path) -> Dict[str, Any]:
+        """Helper to load a YML file"""
         try:
             with file.open() as f:
-                return json.load(f)
+                return yaml.safe_load(f)
         except Exception as e:
             raise Exception(f"Can't load {file}: {e}")
-
-

tests_e2e/orchestrator/lib/agent_test_suite.py

@@ -35,7 +35,7 @@ from lisa import (  # pylint: disable=E0401
     Node,
     notifier,
     TestCaseMetadata,
-    TestSuite,
+    TestSuite as LisaTestSuite,
     TestSuiteMetadata,
 )
 from lisa.messages import TestStatus, TestResultMessage  # pylint: disable=E0401
@@ -44,7 +44,7 @@ from lisa.sut_orchestrator.azure.common import get_node_context, AzureNodeSchema
 
 import makepkg
 from azurelinuxagent.common.version import AGENT_VERSION
-from tests_e2e.orchestrator.lib.agent_test_loader import AgentTestLoader, TestSuiteDescription
+from tests_e2e.orchestrator.lib.agent_test_loader import TestSuiteInfo
 from tests_e2e.tests.lib.agent_test_context import AgentTestContext
 from tests_e2e.tests.lib.identifiers import VmIdentifier
 from tests_e2e.tests.lib.logging import log as agent_test_logger  # Logger used by the tests
@@ -98,7 +98,7 @@ class CollectLogs(Enum):
 
 
 @TestSuiteMetadata(area="waagent", category="", description="")
-class AgentTestSuite(TestSuite):
+class AgentTestSuite(LisaTestSuite):
     """
     Manages the setup of test VMs and execution of Agent test suites. This class acts as the interface with the LISA framework, which
     will invoke the execute() method when a runbook is executed.
@@ -112,7 +112,7 @@ class AgentTestSuite(TestSuite):
             self.node: Node = None
             self.runbook_name: str = None
             self.image_name: str = None
-            self.test_suites: List[str] = None
+            self.test_suites: List[AgentTestSuite] = None
             self.collect_logs: str = None
             self.skip_setup: bool = None
 
@@ -128,7 +128,7 @@ class AgentTestSuite(TestSuite):
         # Remove the resource group and node suffix, e.g. "e1-n0" in "lisa-20230110-162242-963-e1-n0"
         runbook_name = re.sub(r"-\w+-\w+$", "", runbook.name)
 
-        self.__context = AgentTestSuite._Context(
+        self.__context = self._Context(
             vm=VmIdentifier(
                 location=runbook.location,
                 subscription=node.features._platform.subscription_id,
@@ -147,15 +147,15 @@ class AgentTestSuite(TestSuite):
         self.__context.log = log
         self.__context.node = node
         self.__context.image_name = f"{runbook.marketplace.offer}-{runbook.marketplace.sku}"
-        self.__context.test_suites = AgentTestSuite._get_required_parameter(variables, "test_suites")
-        self.__context.collect_logs = AgentTestSuite._get_required_parameter(variables, "collect_logs")
-        self.__context.skip_setup = AgentTestSuite._get_required_parameter(variables, "skip_setup")
+        self.__context.test_suites = self._get_required_parameter(variables, "test_suites_info")
+        self.__context.collect_logs = self._get_required_parameter(variables, "collect_logs")
+        self.__context.skip_setup = self._get_required_parameter(variables, "skip_setup")
 
         self._log.info(
             "Test suite parameters: [skip_setup: %s] [collect_logs: %s] [test_suites: %s]",
             self.context.skip_setup,
             self.context.collect_logs,
-            self.context.test_suites)
+            [t.name for t in self.context.test_suites])
 
     @staticmethod
     def _get_required_parameter(variables: Dict[str, Any], name: str) -> Any:
@@ -191,7 +191,7 @@ class AgentTestSuite(TestSuite):
 
         Returns the path to the agent package.
         """
-        AgentTestSuite._setup_lock.acquire()
+        self._setup_lock.acquire()
 
         try:
             self._log.info("")
@@ -211,7 +211,7 @@ class AgentTestSuite(TestSuite):
             completed.touch()
 
         finally:
-            AgentTestSuite._setup_lock.release()
+            self._setup_lock.release()
 
     def _build_agent_package(self) -> None:
         """
@@ -290,10 +290,9 @@ class AgentTestSuite(TestSuite):
             self._log.exception("Failed to collect logs from the test machine")
 
     @TestCaseMetadata(description="", priority=0)
-    def execute(self, node: Node, variables: Dict[str, Any], log: Logger) -> None:
+    def agent_test_suite(self, node: Node, variables: Dict[str, Any], log: Logger) -> None:
         """
-        Executes each of the AgentTests in the given List. Note that 'test_suite' is a list of test classes, rather than
-        instances of the test class (this method will instantiate each of these test classes).
+        Executes each of the AgentTests included in "test_suites_info" variable (which is generated by the AgentTestSuitesCombinator).
         """
         self._set_context(node, variables, log)
 
@@ -308,9 +307,9 @@ class AgentTestSuite(TestSuite):
                     if not self.context.skip_setup:
                         self._setup_node()
 
-                    test_suites: List[TestSuiteDescription] = AgentTestLoader(self.context.test_source_directory).load(self.context.test_suites)
-
-                    for suite in test_suites:
+                    # pylint seems to think self.context.test_suites is not iterable. Suppressing warning, since its type is List[AgentTestSuite]
+                    #  E1133: Non-iterable value self.context.test_suites is used in an iterating context (not-an-iterable)
+                    for suite in self.context.test_suites:  # pylint: disable=E1133
                         test_suite_success = self._execute_test_suite(suite) and test_suite_success
 
                 finally:
@@ -329,7 +328,7 @@ class AgentTestSuite(TestSuite):
             finally:
                 self._clean_up()
 
-    def _execute_test_suite(self, suite: TestSuiteDescription) -> bool:
+    def _execute_test_suite(self, suite: TestSuiteInfo) -> bool:
         """
         Executes the given test suite and returns True if all the tests in the suite succeeded.
         """

tests_e2e/orchestrator/lib/agent_test_suite_combinator.py

@@ -0,0 +1,119 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+import logging
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Type
+
+# E0401: Unable to import 'dataclasses_json' (import-error)
+from dataclasses_json import dataclass_json  # pylint: disable=E0401
+
+# Disable those warnings, since 'lisa' is an external, non-standard, dependency
+#     E0401: Unable to import 'lisa' (import-error)
+#     etc
+from lisa import schema  # pylint: disable=E0401
+from lisa.combinator import Combinator  # pylint: disable=E0401
+from lisa.util import field_metadata  # pylint: disable=E0401
+
+from tests_e2e.orchestrator.lib.agent_test_loader import AgentTestLoader
+
+
+@dataclass_json()
+@dataclass
+class AgentTestSuitesCombinatorSchema(schema.Combinator):
+    test_suites: str = field(
+        default_factory=str, metadata=field_metadata(required=True)
+    )
+
+
+class AgentTestSuitesCombinator(Combinator):
+    """
+    The "agent_test_suites" combinator returns a list of items containing five variables that specify the environments
+    that the agent test suites must be executed on:
+
+        * marketplace_image: e.g. "Canonical UbuntuServer 18.04-LTS latest",
+        * location: e.g. "westus2",
+        * vm_size: e.g. "Standard_D2pls_v5"
+        * vhd: e.g "https://rhel.blob.core.windows.net/images/RHEL_8_Standard-8.3.202006170423.vhd?se=..."
+        * test_suites_info: e.g. [AgentBvt, FastTrack]
+
+    (marketplace_image, location, vm_size) and vhd are mutually exclusive and define the environment (i.e. the test VM)
+    in which the test will be executed. test_suites_info defines the test suites that should be executed in that
+    environment.
+    """
+    def __init__(self, runbook: AgentTestSuitesCombinatorSchema) -> None:
+        super().__init__(runbook)
+        self._environments = self.create_environment_list(self.runbook.test_suites)
+        self._index = 0
+
+    @classmethod
+    def type_name(cls) -> str:
+        return "agent_test_suites"
+
+    @classmethod
+    def type_schema(cls) -> Type[schema.TypedSchema]:
+        return AgentTestSuitesCombinatorSchema
+
+    def _next(self) -> Optional[Dict[str, Any]]:
+        result: Optional[Dict[str, Any]] = None
+        if self._index < len(self._environments):
+            result = self._environments[self._index]
+            self._index += 1
+        return result
+
+    _DEFAULT_LOCATION = "westus2"
+
+    @staticmethod
+    def create_environment_list(test_suites: str) -> List[Dict[str, Any]]:
+        environment_list: List[Dict[str, Any]] = []
+        shared_environments: Dict[str, Dict[str, Any]] = {}
+
+        loader = AgentTestLoader(test_suites)
+
+        for suite_info in loader.test_suites:
+            images_info = loader.images[suite_info.images]
+            for image in images_info:
+                # If the suite specifies a location, use it. Else, if the image specifies a list of locations, use
+                # any of them. Otherwise, use the default location.
+                if suite_info.location != '':
+                    location = suite_info.location
+                elif len(image.locations) > 0:
+                    location = image.locations[0]
+                else:
+                    location = AgentTestSuitesCombinator._DEFAULT_LOCATION
+
+                # If the image specifies a list of VM sizes, use any of them. Otherwise, set the size to empty and let LISA choose it.
+                vm_size = image.vm_sizes[0] if len(image.vm_sizes) > 0 else ""
+
+                if suite_info.owns_vm:
+                    environment_list.append({
+                        "marketplace_image": image.urn,
+                        "location": location,
+                        "vm_size": vm_size,
+                        "vhd": "",
+                        "test_suites_info": [suite_info]
+                    })
+                else:
+                    key: str = f"{image.urn}:{location}"
+                    if key in shared_environments:
+                        shared_environments[key]["test_suites_info"].append(suite_info)
+                    else:
+                        shared_environments[key] = {
+                            "marketplace_image": image.urn,
+                            "location": location,
+                            "vm_size": vm_size,
+                            "vhd": "",
+                            "test_suites_info": [suite_info]
+                        }
+
+        environment_list.extend(shared_environments.values())
+
+        log: logging.Logger = logging.getLogger("lisa")
+        log.info("******** Environments *****")
+        for e in environment_list:
+            log.info(
+                "{ marketplace_image: '%s', location: '%s', vm_size: '%s', vhd: '%s', test_suites_info: '%s' }",
+                e['marketplace_image'], e['location'], e['vm_size'], e['vhd'], [s.name for s in e['test_suites_info']])
+        log.info("***************************")
+
+        return environment_list

tests_e2e/orchestrator/runbook.yml

@@ -26,11 +26,6 @@ variable:
   #
   # These variables define parameters for the AgentTestSuite; see the test wiki for details.
   #
-  # The test suites to execute
-  - name: test_suites
-    value: "agent_bvt"
-    is_case_visible: true
-
   # Whether to collect logs from the test VM
   - name: collect_logs
     value: "failed"
@@ -42,7 +37,30 @@ variable:
     is_case_visible: true
 
   #
-  # Set these to use an SSH proxy when executing the runbook
+  # These variables parameters for the AgentTestSuitesCombinator combinator
+  #
+  # The test suites to execute
+  - name: test_suites
+    value: "agent_bvt"
+    is_case_visible: true
+
+  #
+  # These variables are set by the AgentTestSuitesCombinator combinator
+  #
+  - name: marketplace_image
+    value: ""
+  - name: vm_size
+    value: ""
+  - name: location
+    value: ""
+  - name: vhd
+    value: ""
+  - name: test_suites_info
+    value: []
+    is_case_visible: true
+
+  #
+  # Set these variables to use an SSH proxy when executing the runbook
   #
   - name: proxy
     value: False
@@ -54,18 +72,6 @@ variable:
     value: ""
     is_secret: true
 
-  #
-  # The image, vm_size, and location are set by the combinator
-  #
-  - name: marketplace_image
-    value: ""
-  - name: vm_size
-    value: ""
-  - name: location
-    value: ""
-  - name: default_location
-    value: "westus2"
-
 platform:
   - type: azure
     admin_username: $(user)
@@ -80,38 +86,14 @@ platform:
       core_count:
         min: 2
       azure:
-        marketplace: "$(marketplace_image)"
-        vhd: ""
+        marketplace: $(marketplace_image)
+        vhd: $(vhd)
         location: $(location)
         vm_size: $(vm_size)
 
 combinator:
-  type: batch
-  items:
-    - marketplace_image: "Canonical UbuntuServer 18.04-LTS latest"
-      location: $(default_location)
-      vm_size: ""
-    - marketplace_image: "Debian debian-10 10 latest"
-      location: $(default_location)
-      vm_size: ""
-    - marketplace_image: "OpenLogic CentOS 7_9 latest"
-      location: $(default_location)
-      vm_size: ""
-    - marketplace_image: "SUSE sles-15-sp2-basic gen2 latest"
-      location: $(default_location)
-      vm_size: ""
-    - marketplace_image: "RedHat RHEL 7-RAW latest"
-      location: $(default_location)
-      vm_size: ""
-    - marketplace_image: "microsoftcblmariner cbl-mariner cbl-mariner-1 latest"
-      location: $(default_location)
-      vm_size: ""
-    - marketplace_image: "microsoftcblmariner cbl-mariner cbl-mariner-2 latest"
-      location: $(default_location)
-      vm_size: ""
-    - marketplace_image: "microsoftcblmariner cbl-mariner cbl-mariner-2-arm64 latest"
-      location: "eastus"
-      vm_size: "Standard_D2pls_v5"
+  type: agent_test_suites
+  test_suites: $(test_suites)
 
 concurrency: 10
 

tests_e2e/pipeline/pipeline-cleanup.yml

@@ -4,7 +4,7 @@
 # Runs every 3 hours and deletes any resource groups that are more than a day old and contain string "lisa-WALinuxAgent-"
 #
 schedules:
-  - cron: "0 */3 * * *" # Run every 3 hours
+  - cron: "0 */12 * * *" # Run twice a day (every 12 hours)
     displayName: cleanup build
     branches:
       include:

tests_e2e/test_suites/agent_bvt.json

@@ -1,9 +0,0 @@
-{
-  "name": "AgentBvt",
-
-  "tests": [
-    "bvts/extension_operations.py",
-    "bvts/run_command.py",
-    "bvts/vm_access.py"
-  ]
-}

tests_e2e/test_suites/agent_bvt.yml

@@ -0,0 +1,6 @@
+name: "AgentBvt"
+tests:
+  - "bvts/extension_operations.py"
+  - "bvts/run_command.py"
+  - "bvts/vm_access.py"
+images: "endorsed"

tests_e2e/test_suites/fail.json

@@ -1,4 +0,0 @@
-{
-  "name": "Fail",
-  "tests": ["fail_test.py", "error_test.py"]
-}

tests_e2e/test_suites/fail.yml

@@ -0,0 +1,5 @@
+name: "Fail"
+tests:
+  - "fail_test.py"
+  - "error_test.py"
+images: "ubuntu_1804"

tests_e2e/test_suites/images.yml

@@ -0,0 +1,70 @@
+#
+# Image sets are used to group images
+#
+image-sets:
+   # Endorsed distros that are tested on the daily runs
+   endorsed:
+#
+# TODO: Add CentOS 6.10 and Debian 8
+#
+#      - "centos_610"
+      - "centos_79"
+#      - "debian_8"
+      - "debian_10"
+      - "debian_9"
+      - "suse_12"
+      - "mariner_1"
+      - "mariner_2"
+      - "mariner_2_arm64"
+      - "suse_15"
+      - "rhel_78"
+      - "rhel_82"
+      - "ubuntu_1604"
+      - "ubuntu_1804"
+      - "ubuntu_2004"
+
+#
+# An image can be specified by a string giving its urn, as in
+#
+#    ubuntu_2004: "Canonical 0001-com-ubuntu-server-focal 20_04-lts latest"
+#
+# or by an object with 3 properties: urn, locations and vm_sizes, as in
+#
+#   mariner_2_arm64:
+#      urn: "microsoftcblmariner cbl-mariner cbl-mariner-2-arm64 latest"
+#      locations:
+#         - "eastus"
+#      vm_sizes:
+#         - "Standard_D2pls_v5"
+#
+# 'urn' is required, while 'locations' and 'vm_sizes' are optional. The latter
+# two properties can be used to specify that the image is available only in
+# some locations, or that it can be used only on some VM sizes.
+#
+# URNs follow the format '<Publisher> <Offer> <Sku> <Version>' or
+# '<Publisher>:<Offer>:<Sku>:<Version>'
+#
+images:
+#
+# TODO: Add CentOS 6.10 and Debian 8
+#
+#   centos_610: "OpenLogic CentOS 6.10 latest"
+   centos_79: "OpenLogic CentOS 7_9 latest"
+#   debian_8: "credativ Debian 8 latest"
+   debian_9: "credativ Debian 9 latest"
+   debian_10: "Debian debian-10 10 latest"
+   mariner_1: "microsoftcblmariner cbl-mariner cbl-mariner-1 latest"
+   mariner_2: "microsoftcblmariner cbl-mariner cbl-mariner-2 latest"
+   mariner_2_arm64:
+      urn: "microsoftcblmariner cbl-mariner cbl-mariner-2-arm64 latest"
+      locations:
+         - "eastus"
+      vm_sizes:
+         - "Standard_D2pls_v5"
+   suse_12: "SUSE sles-12-sp5-basic gen1 latest"
+   suse_15: "SUSE sles-15-sp2-basic gen2 latest"
+   rhel_78: "RedHat RHEL 7.8 latest"
+   rhel_82: "RedHat RHEL 8.2 latest"
+   ubuntu_1604: "Canonical UbuntuServer 16.04-LTS latest"
+   ubuntu_1804: "Canonical UbuntuServer 18.04-LTS latest"
+   ubuntu_2004: "Canonical 0001-com-ubuntu-server-focal 20_04-lts latest"

tests_e2e/test_suites/pass.json

@@ -1,4 +0,0 @@
-{
-  "name": "Pass",
-  "tests": ["pass_test.py"]
-}

tests_e2e/test_suites/pass.yml

@@ -0,0 +1,4 @@
+name: "Pass"
+tests:
+  - "pass_test.py"
+images: "ubuntu_2004"