Implement StartStop feature for Libvirt orchestrator.

As part of this change, the libvirt connection object is now kept
around long-term. This is required so that re-attaching the console
logger works correctly. (Though in reality, it was already effectively
kept around because of the console logger.)

lisa/sut_orchestrator/libvirt/ch_platform.py

@@ -7,8 +7,6 @@ import xml.etree.ElementTree as ET  # noqa: N817
 from pathlib import Path
 from typing import List, Type
 
-import libvirt  # type: ignore
-
 from lisa import schema
 from lisa.environment import Environment
 from lisa.feature import Feature
@@ -73,7 +71,6 @@ class CloudHypervisorPlatform(BaseLibvirtPlatform):
         node_context: NodeContext,
         environment: Environment,
         log: Logger,
-        lv_conn: libvirt.virConnect,
     ) -> None:
         if node_context.firmware_source_path:
             self.host_node.shell.copy(
@@ -86,7 +83,6 @@ class CloudHypervisorPlatform(BaseLibvirtPlatform):
             node_context,
             environment,
             log,
-            lv_conn,
         )
 
     def _create_node_domain_xml(
@@ -94,7 +90,6 @@ class CloudHypervisorPlatform(BaseLibvirtPlatform):
         environment: Environment,
         log: Logger,
         node: Node,
-        lv_conn: libvirt.virConnect,
     ) -> str:
         node_context = get_node_context(node)
 
@@ -166,7 +161,6 @@ class CloudHypervisorPlatform(BaseLibvirtPlatform):
 
     def _create_domain_and_attach_logger(
         self,
-        libvirt_conn: libvirt.virConnect,
         node_context: NodeContext,
     ) -> None:
         assert node_context.domain
@@ -174,7 +168,7 @@ class CloudHypervisorPlatform(BaseLibvirtPlatform):
 
         node_context.console_logger = QemuConsoleLogger()
         node_context.console_logger.attach(
-            libvirt_conn, node_context.domain, node_context.console_log_file_path
+            node_context.domain, node_context.console_log_file_path
         )
 
     # Create the OS disk.

lisa/sut_orchestrator/libvirt/console_logger.py

@@ -21,7 +21,6 @@ class QemuConsoleLogger:
     # Attach logger to a libvirt VM.
     def attach(
         self,
-        qemu_conn: libvirt.virConnect,
         domain: libvirt.virDomain,
         log_file_path: str,
     ) -> None:
@@ -29,7 +28,7 @@ class QemuConsoleLogger:
         self._log_file = open(log_file_path, "ab")
 
         # Open the libvirt console stream.
-        console_stream = qemu_conn.newStream(libvirt.VIR_STREAM_NONBLOCK)
+        console_stream = domain.connect().newStream(libvirt.VIR_STREAM_NONBLOCK)
         domain.openConsole(
             None,
             console_stream,

lisa/sut_orchestrator/libvirt/platform.py

@@ -38,6 +38,7 @@ from .context import (
     get_environment_context,
     get_node_context,
 )
+from .platform_interface import IBaseLibvirtPlatform
 from .schema import (
     FIRMWARE_TYPE_BIOS,
     FIRMWARE_TYPE_UEFI,
@@ -46,6 +47,7 @@ from .schema import (
     DiskImageFormat,
 )
 from .serial_console import SerialConsole
+from .start_stop import StartStop
 
 # Host environment information fields
 KEY_HOST_DISTRO = "host_distro"
@@ -60,14 +62,16 @@ class _HostCapabilities:
         self.free_memory_kib = 0
 
 
-class BaseLibvirtPlatform(Platform):
+class BaseLibvirtPlatform(Platform, IBaseLibvirtPlatform):
     _supported_features: List[Type[Feature]] = [
         SerialConsole,
+        StartStop,
     ]
 
     def __init__(self, runbook: schema.Platform) -> None:
         super().__init__(runbook=runbook)
         self.libvirt_conn_str: str
+        self.libvirt_conn: libvirt.virConnect
         self.platform_runbook: BaseLibvirtPlatformSchema
         self.host_node: Node
         self.vm_disks_dir: str
@@ -110,6 +114,9 @@ class BaseLibvirtPlatform(Platform):
             self.__platform_runbook_type(), type_name=type(self).type_name()
         )
 
+        self.__init_libvirt_conn_string()
+        self.libvirt_conn = libvirt.open(self.libvirt_conn_str)
+
     def _prepare_environment(self, environment: Environment, log: Logger) -> bool:
         # Ensure environment log directory is created before connecting to any nodes.
         _ = environment.log_path
@@ -148,11 +155,9 @@ class BaseLibvirtPlatform(Platform):
                 parent_logger=log,
             )
 
-        self.__init_libvirt_conn_string()
         self._configure_environment(environment, log)
 
-        with libvirt.open(self.libvirt_conn_str) as lv_conn:
+        return self._configure_node_capabilities(environment, log)
-            return self._configure_node_capabilities(environment, log, lv_conn)
 
     def _deploy_environment(self, environment: Environment, log: Logger) -> None:
         self._deploy_nodes(environment, log)
@@ -183,12 +188,12 @@ class BaseLibvirtPlatform(Platform):
         )
 
     def _configure_node_capabilities(
-        self, environment: Environment, log: Logger, lv_conn: libvirt.virConnect
+        self, environment: Environment, log: Logger
     ) -> bool:
         if not environment.runbook.nodes_requirement:
             return True
 
-        host_capabilities = self._get_host_capabilities(lv_conn, log)
+        host_capabilities = self._get_host_capabilities(log)
         nodes_capabilities = self._create_node_capabilities(host_capabilities)
 
         nodes_requirement = []
@@ -209,12 +214,10 @@ class BaseLibvirtPlatform(Platform):
         environment.runbook.nodes_requirement = nodes_requirement
         return True
 
-    def _get_host_capabilities(
+    def _get_host_capabilities(self, log: Logger) -> _HostCapabilities:
-        self, lv_conn: libvirt.virConnect, log: Logger
-    ) -> _HostCapabilities:
         host_capabilities = _HostCapabilities()
 
-        capabilities_xml_str = lv_conn.getCapabilities()
+        capabilities_xml_str = self.libvirt_conn.getCapabilities()
         capabilities_xml = ET.fromstring(capabilities_xml_str)
 
         host_xml = capabilities_xml.find("host")
@@ -234,7 +237,9 @@ class BaseLibvirtPlatform(Platform):
 
         # Get free memory.
         # Include the disk cache size, as it will be freed if memory becomes limited.
-        memory_stats = lv_conn.getMemoryStats(libvirt.VIR_NODE_MEMORY_STATS_ALL_CELLS)
+        memory_stats = self.libvirt_conn.getMemoryStats(
+            libvirt.VIR_NODE_MEMORY_STATS_ALL_CELLS
+        )
         host_capabilities.free_memory_kib = (
             memory_stats[libvirt.VIR_NODE_MEMORY_STATS_FREE]
             + memory_stats[libvirt.VIR_NODE_MEMORY_STATS_CACHED]
@@ -309,20 +314,19 @@ class BaseLibvirtPlatform(Platform):
     def _deploy_nodes(self, environment: Environment, log: Logger) -> None:
         self._configure_nodes(environment, log)
 
-        with libvirt.open(self.libvirt_conn_str) as lv_conn:
+        try:
-            try:
+            self._create_nodes(environment, log)
-                self._create_nodes(environment, log, lv_conn)
+            self._fill_nodes_metadata(environment, log)
-                self._fill_nodes_metadata(environment, log, lv_conn)
 
-            except Exception as ex:
+        except Exception as ex:
-                assert environment.platform
+            assert environment.platform
-                if (
+            if (
-                    environment.platform.runbook.keep_environment
+                environment.platform.runbook.keep_environment
-                    == constants.ENVIRONMENT_KEEP_NO
+                == constants.ENVIRONMENT_KEEP_NO
-                ):
+            ):
-                    self._delete_nodes(environment, log)
+                self._delete_nodes(environment, log)
 
-                raise ex
+            raise ex
 
     # Pre-determine all the nodes' properties, including the name of all the resouces
     # to be created. This makes it easier to cleanup everything after the test is
@@ -454,9 +458,23 @@ class BaseLibvirtPlatform(Platform):
 
                 node_context.data_disks.append(data_disk)
 
+    def restart_domain_and_attach_logger(self, node: Node) -> None:
+        node_context = get_node_context(node)
+        domain = node_context.domain
+        assert domain
+
+        if domain.isActive():
+            # VM already running.
+            return
+
+        if node_context.console_logger is not None:
+            node_context.console_logger.wait_for_close()
+            node_context.console_logger = None
+
+        self._create_domain_and_attach_logger(node_context)
+
     def _create_domain_and_attach_logger(
         self,
-        libvirt_conn: libvirt.virConnect,
         node_context: NodeContext,
     ) -> None:
         # Start the VM in the paused state.
@@ -468,7 +486,7 @@ class BaseLibvirtPlatform(Platform):
         # Attach the console logger
         node_context.console_logger = QemuConsoleLogger()
         node_context.console_logger.attach(
-            libvirt_conn, node_context.domain, node_context.console_log_file_path
+            node_context.domain, node_context.console_log_file_path
         )
 
         # Start the VM.
@@ -479,7 +497,6 @@ class BaseLibvirtPlatform(Platform):
         self,
         environment: Environment,
         log: Logger,
-        lv_conn: libvirt.virConnect,
     ) -> None:
         self.host_node.shell.mkdir(Path(self.vm_disks_dir), exist_ok=True)
 
@@ -490,7 +507,6 @@ class BaseLibvirtPlatform(Platform):
                 node_context,
                 environment,
                 log,
-                lv_conn,
             )
 
     def _create_node(
@@ -499,7 +515,6 @@ class BaseLibvirtPlatform(Platform):
         node_context: NodeContext,
         environment: Environment,
         log: Logger,
-        lv_conn: libvirt.virConnect,
     ) -> None:
         # Create required directories and copy the required files to the host
         # node.
@@ -523,11 +538,10 @@ class BaseLibvirtPlatform(Platform):
         self._create_node_data_disks(node)
 
         # Create libvirt domain (i.e. VM).
-        xml = self._create_node_domain_xml(environment, log, node, lv_conn)
+        xml = self._create_node_domain_xml(environment, log, node)
-        node_context.domain = lv_conn.defineXML(xml)
+        node_context.domain = self.libvirt_conn.defineXML(xml)
 
         self._create_domain_and_attach_logger(
-            lv_conn,
             node_context,
         )
 
@@ -598,9 +612,7 @@ class BaseLibvirtPlatform(Platform):
             self.host_node.tools[Iptables].stop_forwarding(port, address, 22)
 
     # Retrieve the VMs' dynamic properties (e.g. IP address).
-    def _fill_nodes_metadata(
+    def _fill_nodes_metadata(self, environment: Environment, log: Logger) -> None:
-        self, environment: Environment, log: Logger, lv_conn: libvirt.virConnect
-    ) -> None:
         environment_context = get_environment_context(environment)
 
         # Give all the VMs some time to boot and then acquire an IP address.
@@ -615,9 +627,7 @@ class BaseLibvirtPlatform(Platform):
             assert isinstance(node, RemoteNode)
 
             # Get the VM's IP address.
-            local_address = self._get_node_ip_address(
+            local_address = self._get_node_ip_address(environment, log, node, timeout)
-                environment, log, lv_conn, node, timeout
-            )
 
             node_port = 22
             if self.host_node.is_remote:
@@ -770,7 +780,6 @@ class BaseLibvirtPlatform(Platform):
         environment: Environment,
         log: Logger,
         node: Node,
-        lv_conn: libvirt.virConnect,
     ) -> str:
         node_context = get_node_context(node)
 
@@ -803,7 +812,7 @@ class BaseLibvirtPlatform(Platform):
             # libvirt v7.2.0 and Ubuntu 20.04 only has libvirt v6.0.0. Therefore, we
             # have to select the firmware manually.
             firmware_config = self._get_firmware_config(
-                lv_conn, node_context.machine_type, node_context.enable_secure_boot
+                node_context.machine_type, node_context.enable_secure_boot
             )
 
             print(firmware_config)
@@ -987,14 +996,13 @@ class BaseLibvirtPlatform(Platform):
         self,
         environment: Environment,
         log: Logger,
-        lv_conn: libvirt.virConnect,
         node: Node,
         timeout: float,
     ) -> str:
         node_context = get_node_context(node)
 
         while True:
-            addr = self._try_get_node_ip_address(environment, log, lv_conn, node)
+            addr = self._try_get_node_ip_address(environment, log, node)
             if addr:
                 return addr
 
@@ -1006,12 +1014,11 @@ class BaseLibvirtPlatform(Platform):
         self,
         environment: Environment,
         log: Logger,
-        lv_conn: libvirt.virConnect,
         node: Node,
     ) -> Optional[str]:
         node_context = get_node_context(node)
 
-        domain = lv_conn.lookupByName(node_context.vm_name)
+        domain = self.libvirt_conn.lookupByName(node_context.vm_name)
 
         # Acquire IP address from libvirt's DHCP server.
         interfaces = domain.interfaceAddresses(
@@ -1031,12 +1038,11 @@ class BaseLibvirtPlatform(Platform):
 
     def _get_firmware_config(
         self,
-        lv_conn: libvirt.virConnect,
         machine_type: Optional[str],
         enable_secure_boot: bool,
     ) -> Dict[str, Any]:
         # Resolve the machine type to its full name.
-        domain_caps_str = lv_conn.getDomainCapabilities(
+        domain_caps_str = self.libvirt_conn.getDomainCapabilities(
             machine=machine_type, virttype="kvm"
         )
         domain_caps = ET.fromstring(domain_caps_str)

lisa/sut_orchestrator/libvirt/platform_interface.py

@@ -0,0 +1,9 @@
+from abc import ABC, abstractmethod
+
+from lisa.node import Node
+
+
+class IBaseLibvirtPlatform(ABC):
+    @abstractmethod
+    def restart_domain_and_attach_logger(self, node: Node) -> None:
+        pass

lisa/sut_orchestrator/libvirt/start_stop.py

@@ -0,0 +1,66 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+from typing import Any
+
+from lisa import features
+
+from .context import get_node_context
+from .platform_interface import IBaseLibvirtPlatform
+
+
+# Implements the StartStop feature.
+class StartStop(features.StartStop):
+    def _initialize(self, *args: Any, **kwargs: Any) -> None:
+        super()._initialize(*args, **kwargs)
+
+    def _stop(
+        self, wait: bool = True, state: features.StopState = features.StopState.Shutdown
+    ) -> None:
+        if state == features.StopState.Hibernate:
+            raise NotImplementedError(
+                "libvirt orchestrator does not support hibernate stop"
+            )
+
+        node_context = get_node_context(self._node)
+        domain = node_context.domain
+        assert domain
+
+        if not domain.isActive():
+            # VM is already shutdown.
+            return
+
+        if wait:
+            domain.destroy()
+
+        else:
+            domain.shutdown()
+
+    def _start(self, wait: bool = True) -> None:
+        assert isinstance(self._platform, IBaseLibvirtPlatform)
+        self._platform.restart_domain_and_attach_logger(self._node)
+
+    def _restart(self, wait: bool = True) -> None:
+        node_context = get_node_context(self._node)
+        domain = node_context.domain
+        assert domain
+
+        if wait:
+            if domain.isActive():
+                # Shutdown VM.
+                domain.destroy()
+
+            # Boot up VM and ensure console logger reattaches.
+            assert isinstance(self._platform, IBaseLibvirtPlatform)
+            self._platform.restart_domain_and_attach_logger(self._node)
+
+        else:
+            if domain.isActive():
+                # On a clean reboot, QEMU process is not torn down.
+                # So, no need to reattach the console logger.
+                domain.reboot()
+
+            else:
+                # Boot up VM and ensure console logger reattaches.
+                assert isinstance(self._platform, IBaseLibvirtPlatform)
+                self._platform.restart_domain_and_attach_logger(self._node)