Add keyvault test to daily run + Specify tests suite as a list (#3089)

Co-authored-by: narrieta <narrieta>

tests_e2e/orchestrator/lib/agent_test_loader.py

@@ -88,7 +88,7 @@ class AgentTestLoader(object):
     """
     Loads a given set of test suites from the YAML configuration files.
     """
-    def __init__(self, test_suites: str, cloud: str):
+    def __init__(self, test_suites: List[str], cloud: str):
         """
         Loads the specified 'test_suites', which are given as a string of comma-separated suite names or a YAML description
         of a single test_suite.
@@ -175,25 +175,9 @@ class AgentTestLoader(object):
                 if suite_skip_image not in self.images:
                     raise Exception(f"Invalid image reference in test suite {suite.name}: Can't find {suite_skip_image} in images.yml")
 
-
     @staticmethod
-    def _load_test_suites(test_suites: str) -> List[TestSuiteInfo]:
-        #
-        # Attempt to parse 'test_suites' as the YML description of a single suite
-        #
-        parsed = yaml.safe_load(test_suites)
-
-        #
-        # A comma-separated list (e.g. "foo", "foo, bar", etc.) is valid YAML, but it is parsed as a string. An actual test suite would
-        # be parsed as a dictionary. If it is a dict, take is as the YML description of a single test suite
-        #
-        if isinstance(parsed, dict):
-            return [AgentTestLoader._load_test_suite(parsed)]
-
-        #
-        # If test_suites is not YML, then it should be a comma-separated list of description files
-        #
-        description_files: List[Path] = [AgentTestLoader._SOURCE_CODE_ROOT/"test_suites"/f"{t.strip()}.yml" for t in test_suites.split(',')]
+    def _load_test_suites(test_suites: List[str]) -> List[TestSuiteInfo]:
+        description_files: List[Path] = [AgentTestLoader._SOURCE_CODE_ROOT/"test_suites"/f"{t}.yml" for t in test_suites]
         return [AgentTestLoader._load_test_suite(f) for f in description_files]
 
     @staticmethod

tests_e2e/orchestrator/lib/agent_test_suite_combinator.py

@@ -46,6 +46,7 @@ class AgentTestSuitesCombinatorSchema(schema.Combinator):
     resource_group_name: str = field(default_factory=str, metadata=field_metadata(required=True))
     subscription_id: str = field(default_factory=str, metadata=field_metadata(required=True))
     test_suites: str = field(default_factory=str, metadata=field_metadata(required=True))
+    default_test_suites: List[str] = field(default_factory=list, metadata=field_metadata(required=True))
     user: str = field(default_factory=str, metadata=field_metadata(required=True))
     vm_name: str = field(default_factory=str, metadata=field_metadata(required=True))
     vm_size: str = field(default_factory=str, metadata=field_metadata(required=True))
@@ -81,20 +82,25 @@ class AgentTestSuitesCombinator(Combinator):
             if self.runbook.resource_group_name == '':
                 raise Exception("Invalid runbook parameters: The 'vmss_name' parameter indicates an existing VMSS, a 'resource_group_name' must be specified.")
 
+        if self.runbook.test_suites != "":
+            test_suites = [t.strip() for t in self.runbook.test_suites.split(',')]
+        else:
+            test_suites = self.runbook.default_test_suites
+
         self._log: logging.Logger = logging.getLogger("lisa")
 
         with set_thread_name("AgentTestSuitesCombinator"):
             if self.runbook.vm_name != '':
-                self._environments = [self.create_existing_vm_environment()]
+                self._environments = [self.create_existing_vm_environment(test_suites)]
             elif self.runbook.vmss_name != '':
-                self._environments = [self.create_existing_vmss_environment()]
+                self._environments = [self.create_existing_vmss_environment(test_suites)]
             else:
-                self._environments = self.create_environment_list()
+                self._environments = self.create_environment_list(test_suites)
             self._index = 0
 
     @classmethod
     def type_name(cls) -> str:
-        return "agent_test_suites"
+        return "agent_test_suite_combinator"
 
     @classmethod
     def type_schema(cls) -> Type[schema.TypedSchema]:
@@ -125,7 +131,7 @@ class AgentTestSuitesCombinator(Combinator):
         "AzureUSGovernment": "usgovarizona",
     }
 
-    def create_environment_list(self) -> List[Dict[str, Any]]:
+    def create_environment_list(self, test_suites: List[str]) -> List[Dict[str, Any]]:
         """
         Examines the test_suites specified in the runbook and returns a list of the environments (i.e. test VMs or scale sets) that need to be
         created in order to execute these suites.
@@ -136,7 +142,7 @@ class AgentTestSuitesCombinator(Combinator):
         environments: List[Dict[str, Any]] = []
         shared_environments: Dict[str, Dict[str, Any]] = {}  # environments shared by multiple test suites
 
-        loader = AgentTestLoader(self.runbook.test_suites, self.runbook.cloud)
+        loader = AgentTestLoader(test_suites, self.runbook.cloud)
 
         runbook_images = self._get_runbook_images(loader)
 
@@ -260,8 +266,8 @@ class AgentTestSuitesCombinator(Combinator):
 
         return environments
 
-    def create_existing_vm_environment(self) -> Dict[str, Any]:
-        loader = AgentTestLoader(self.runbook.test_suites, self.runbook.cloud)
+    def create_existing_vm_environment(self, test_suites: List[str]) -> Dict[str, Any]:
+        loader = AgentTestLoader(test_suites, self.runbook.cloud)
 
         vm: VirtualMachineClient = VirtualMachineClient(
             cloud=self.runbook.cloud,
@@ -300,8 +306,8 @@ class AgentTestSuitesCombinator(Combinator):
             "c_test_suites": loader.test_suites,
         }
 
-    def create_existing_vmss_environment(self) -> Dict[str, Any]:
-        loader = AgentTestLoader(self.runbook.test_suites, self.runbook.cloud)
+    def create_existing_vmss_environment(self, test_suites: List[str]) -> Dict[str, Any]:
+        loader = AgentTestLoader(test_suites, self.runbook.cloud)
 
         vmss = VirtualMachineScaleSetClient(
             cloud=self.runbook.cloud,

tests_e2e/orchestrator/runbook.yml

@@ -26,10 +26,33 @@ variable:
     is_case_visible: true
 
   #
-  # Test suites to execute
+  # Test suites to execute.
+  #
+  # Use "test_suites" to specify from the command-line the test suites to execute. If not specifies, the "default_test_suites" are executed.
   #
   - name: test_suites
-    value: "agent_bvt, no_outbound_connections, extensions_disabled, agent_not_provisioned, fips, agent_ext_workflow, agent_status, multi_config_ext, agent_cgroups, ext_cgroups, agent_firewall, ext_telemetry_pipeline, ext_sequencing, agent_persist_firewall, publish_hostname, agent_update, recover_network_interface"
+    value: ""
+
+  - name: default_test_suites
+    value:
+      - agent_bvt
+      - agent_cgroups
+      - agent_ext_workflow
+      - agent_firewall
+      - agent_not_provisioned
+      - agent_persist_firewall
+      - agent_status
+      - agent_update
+      - ext_cgroups
+      - extensions_disabled
+      - ext_sequencing
+      - ext_telemetry_pipeline
+      - fips
+      - keyvault_certificates
+      - multi_config_ext
+      - no_outbound_connections
+      - publish_hostname
+      - recover_network_interface
 
   #
   # Parameters used to create test VMs
@@ -183,7 +206,7 @@ environment: $(c_environment)
 platform: $(c_platform)
 
 combinator:
-  type: agent_test_suites
+  type: agent_test_suite_combinator
   allow_ssh: $(allow_ssh)
   cloud: $(cloud)
   identity_file: $(identity_file)
@@ -193,6 +216,7 @@ combinator:
   resource_group_name: $(resource_group_name)
   subscription_id: $(subscription_id)
   test_suites: $(test_suites)
+  default_test_suites: $(default_test_suites)
   user: $(user)
   vm_name: $(vm_name)
   vm_size: $(vm_size)