init basic code and improves

1. Add basic classes like action, test runner.
2. improve init progress
3. add prototype code for testrunner.
4. update schema
5. other small changes.

examples/configs/example.yaml

@@ -1,57 +1,137 @@
-name: best-guest # run name prefix
-parent: parentconfig.yaml # share configurations for similar runs.
-logLevel: debug # specify log level, can be overwritten in lower level.
-# it uses to dispatch test suites on serveral environments.
-# to be implement in next phase.
-combinations:
-  parameters:
-    - platform:
-      - azure
-      - hyperv
-    - AzurePlatform: # Azure platform specified parameters
-        dataCenter:
-          - wu
-          - wu2
+# run name prefix to help grouping results, and put it in title.
+name: best guest
+# share configurations for similar runs.
+parent: parentconfig.yaml
+# it uses to support variables in other fields.
+# duplicate items will be overwritten one by one.
+# if a variable is not defined here, LISA can fail earlier to ask check it.
+# file path is relative to LISA command starts.
+variables:
+  - path: secrets.yaml
+    # If it's secret, it will be removed from log and other output information.
+    # secret files also need to be removed after test is done.
+    isSecret: true
+  # continue to support v2 format. it's simple.
+  - path: v2secrets.xml
+    isSecret: true
+  - path: env.yaml
+  # redefine parameters in file to reuse configurations
+  - name: username
+    value: azureuser
+    # it's not recommended highly to put secret in configurations directly.
+    isSecret: true
 # supports multiple artifacts in future.
 artifacts:
-  - name: sample # optional. artifacts can be referred by name or index.
-    type: repo
-    git:  https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git
-    ref: v5.8-rc6
-    baseDistro: ubuntu-1804 # there is a default value, but can be specified here.
+  # name is optional. artifacts can be referred by name or index.
+  - name: default
+    type: vhd
+    locations:
+      http: https://path-to-azure-storage/aaa.vhd
+  - name: dsvm
+    type: vhd
+    locations:
+      http: https://path-to-azure-storage/bbb.vhd
+# it's spec of environment and node.
+# for ready type, it's optional
+environment:
+  topology: subnet
+  template:
+    - vcpu: 4
+      # optional, if there is only one artifact
+      artifact: default
+      # sometimes, it may need special configuration for a platform
+      # we should avoid this kind of setting, it limits configuration to other platforms.
+      azure:
+        - vmSize: testSize
+  nodeCount: 
+  nodes:
+    - memoryGB: 4
+      isDefault: true
+    - memoryGB: 8
+      gpuCount: 1
+      artifact: dsvm
+  # the environment spec may not be fully supported by each platform.
+  # If so, there is a warning message.
+  # Environment spec can be forced to apply, as error is loud.
+  warnAsError: true
+# it send test progress and results to any place wanted.
+notifier:
+  - type: junit
+  - type: html
+  - type: database
+    connectionstring: $$ResultDatabaseString$$
+# it's examples for all platforms, but currently should leave single one only.
 platform:
-  - name: azure-1 # name is for reference only
+  - name: default
     type: azure
-    tag: vf # use tag to find some platform specified features.
-    secret: filepath.yaml
-    topology: crossRegion
-    totalInstances: 2 # two suites of environments can be created in same time.
-  - type: hyperv
-environment: # ready environment are list directly, no need platform.
-  - type: existing
-    proxy: 240.x.x.x
-    tag: dpdk # meet requirement of dpdk test cases.
-    username: lisa # the same on all nodes.
+    # it's platform specified parameters.
+    subscription: $$SubscriptionID$$
+    subscriptionKey: $$SubscriptionServicePrincipalKey$$
+    datacenter: wu
+    flag: "test=true"
+  # for test case development, run test cases on current machine.
+  - type: local
+  - type: ready
     nodes:
       - ip: 10.0.0.1
         port: 22
+        publicIp: 23.36.36.36
         publicPort: 1111
-        password: $PASSWORD # reference to environment variable
-        isPrimary: true
+        # reference to environment variable
+        password: $$linuxTestPassword$$
+        privateKey: $$sshPrivateKey$$
+        # If test suite doesn't specify where to run a case,
+        #  it should be run on default node.
+        isDefault: true
       - name: secondary
         ip: 10.0.0.2
         port: 22
-        publicPort: 1111
-        password: $PASSWORD2
-testSuites:
-  - name: set1
-    type: lisav2 # know the runner, then it's possible to select cases.
-    tag: azure
-  - tag: vf # match case by tags
-    force: true # force to select. If any reason test cases are dropped, fail the run.
-  - name: unstable cases
-    pattern: "*-network*" # match case by name pattern
-    retry: 2 # retry one time
-  - name: set2
-    include: false # use to exclude some test cases
-    pattern: "*-legacy"
+        publicIp: 23.36.36.36
+        publicPort: 1112
+        password: $$linuxTestPassword$$
+        privateKey: $$sshPrivateKey$$
+# rules apply ordered on previous selection.
+# The order of test cases running is not guaranteed, until it set dependencies.
+tests:
+  - criteria:
+      # all rules in same criteria are AND condition.
+      # we may support richer conditions later.
+      # match case by name pattern
+      name: "*-network*"
+      # compatible with current test cases.
+      area: network
+      category: network
+      priority: 0
+      # tag is a simple way to include test cases within same topic.
+      tag: vf
+    # force to select. If any reason test cases are dropped, fail the run.
+    forceInclude: true
+    name: network
+  # it means there are dependencies among test cases.
+  - dependedOn: network
+    # we will support some basic functionality of other test runner,
+    # for example, environment information, once it's ready.
+    # any other parameters should be handled by runner itself.
+    type: LISAv2
+    extraParameters: -TestNames "VERIFY-DEPLOYMENT-PROVISION" -ResourceCleanup Keep
+  - name: settings
+    criteria:
+      name: this group shows different settings
+    # run this group of test cases several times
+    # default is 1
+    iteration: 5
+    # Once it's set, failed test result will be rewrite to success
+    # it uses to work around some cases temporarily, don't overuse it.
+    # default is false
+    ignoreFailure: true
+  - name: excluded cases
+    criteria:
+      name: "excluded test cases"
+    # use to exclude some test cases
+    # default is true
+    include: false
+  - name: unstable test cases
+    criteria:
+      name: "we don't have unstable case"
+    # retry times. default is 0, means not retry.
+    retry: 1

examples/configs/local_dev.yaml

@@ -0,0 +1,51 @@
+# run name prefix to help grouping results, and put it in title.
+name: local try
+notifier:
+  - type: junit
+  - type: html
+# it's examples for all platforms, but currently should leave single one only.
+platform:
+  - type: local
+tests:
+  - criteria:
+      # all rules in same criteria are AND condition.
+      # we may support richer conditions later.
+      # match case by name pattern
+      name: "*-network*"
+      # compatible with current test cases.
+      area: network
+      category: network
+      priority: 0
+      # tag is a simple way to include test cases within same topic.
+      tag: vf
+    # force to select. If any reason test cases are dropped, fail the run.
+    forceInclude: true
+    name: network
+  # it means there are dependencies among test cases.
+  - dependedOn: network
+    # we will support some basic functionality of other test runner,
+    # for example, environment information, once it's ready.
+    # any other parameters should be handled by runner itself.
+    type: LISAv2
+    extraParameters: -TestNames "VERIFY-DEPLOYMENT-PROVISION" -ResourceCleanup Keep
+  - name: settings
+    criteria:
+      name: this group shows different settings
+    # run this group of test cases several times
+    # default is 1
+    iteration: 5
+    # Once it's set, failed test result will be rewrite to success
+    # it uses to work around some cases temporarily, don't overuse it.
+    # default is false
+    ignoreFailure: true
+  - name: excluded cases
+    criteria:
+      name: "excluded test cases"
+    # use to exclude some test cases
+    # default is true
+    include: false
+  - name: unstable test cases
+    criteria:
+      name: "we don't have unstable case"
+    # retry times. default is 0, means not retry.
+    retry: 1

lisa.cmd

@@ -57,8 +57,6 @@ IF NOT EXIST "!RUNTIME_PYTHON_COMMAND!" (
 REM updating Python packages
 %RUNTIME_PYTHON_PATH%\python -m pip install -q -r requirements.txt
 
-pushd lisa
 %RUNTIME_PYTHON_PATH%\python -m lisa.main %*
-popd
 
 ENDLOCAL

lisa/__init__.py

@@ -1,23 +1,4 @@
-from datetime import datetime
-import os
-from retry import retry
+from .core.action import Action, ActionStatus
+from .core.testrunner import TestRunner
 
-path_template = "../runtime/results/{0}/{0}-{1}"
-
-
-@retry(tries=10, delay=0)
-def create_result_path():
-    global index
-    date = datetime.utcnow().strftime("%Y%m%d")
-    time = datetime.utcnow().strftime("%H%M%S-%f")[:-3]
-    current_path = path_template.format(date, time)
-    if os.path.exists(current_path):
-        raise FileExistsError(
-            "%s exists, and not found an unique path." % current_path
-        )
-    return current_path
-
-
-result_path = os.path.realpath(create_result_path())
-os.makedirs(result_path)
-os.environ["RESULT_PATH"] = result_path
+__all__ = ["Action", "ActionStatus", "TestRunner"]

lisa/common/__init__.py

@@ -0,0 +1,3 @@
+from .logger import log
+
+__all__ = ["log"]

lisa/common/logger.py

@@ -3,15 +3,19 @@ import time
 import os
 
 
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s.%(msecs)03d [%(levelname)s] %(message)s",
-    datefmt="%m%d %H:%M:%S",
-    handlers=[
-        logging.FileHandler("%s/lisa-host.log" % os.environ["RESULT_PATH"]),
-        logging.StreamHandler(),
-    ],
-)
-logging.Formatter.converter = time.gmtime
+def init_log():
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s.%(msecs)03d [%(levelname)s] %(message)s",
+        datefmt="%m%d %H:%M:%S",
+        handlers=[
+            logging.FileHandler(
+                "%s/lisa-host.log" % os.environ["RESULT_PATH"]
+            ),
+            logging.StreamHandler(),
+        ],
+    )
+    logging.Formatter.converter = time.gmtime
+
 
 log = logging.getLogger("LISA")

lisa/core/action.py

@@ -0,0 +1,75 @@
+from abc import ABC, abstractmethod
+from enum import Enum
+from lisa.common import log
+
+
+class ActionStatus(Enum):
+    UNINITIALIZED = 1
+    INITIALIZING = 2
+    INITIALIZED = 3
+    WAITING = 4
+    RUNNING = 5
+    SUCCESS = 6
+    FAILED = 7
+    STOPPING = 8
+    STOPPED = 9
+    UNKNOWN = 10
+
+
+class Action(ABC):
+    def __init__(self):
+        self.__status = ActionStatus.UNINITIALIZED
+        self.__name = None
+        self.isStarted = False
+
+    def config(self, key: str, value: object):
+        pass
+
+    @property
+    def name(self):
+        if self.__name is not None:
+            name = self.__name
+        else:
+            name = self.__class__.__name__
+        return name
+
+    @abstractmethod
+    def getTypeName(self):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def start(self):
+        self.isStarted = True
+
+    @abstractmethod
+    def stop(self):
+        self.validateStarted()
+
+    @abstractmethod
+    def cleanup(self):
+        self.validateStarted()
+
+    def getStatus(self):
+        return self.__status
+
+    def setStatus(self, status: ActionStatus):
+        if self.__status != status:
+            log.info(
+                "%s status changed from %s to %s"
+                % (self.name, self.__status.name, status.name)
+            )
+        log.info("action status called")
+        self.__status = status
+
+    def validateStarted(self):
+        if not self.isStarted:
+            raise Exception("action is not started yet.")
+
+    def getPrerequisites(self):
+        return None
+
+    def validateParameters(self, parameters):
+        pass
+
+    def getPostValidation(self, parameters):
+        return None

lisa/core/lisarunner.py

@@ -0,0 +1,38 @@
+from lisa import TestRunner
+from lisa.util import Process
+from lisa import ActionStatus
+
+
+class LISARunner(TestRunner):
+    def __init__(self):
+        super().__init__()
+        self.process = None
+        self.exitCode = None
+
+    def getTypeName(self):
+        return "LISAv2"
+
+    def start(self):
+        self.process = Process()
+        self.process.start("echo hello world",)
+        self.setStatus(ActionStatus.RUNNING)
+        super().start()
+
+    def stop(self):
+        super().stop()
+        self.process.stop()
+
+    def cleanup(self):
+        super().cleanup()
+        self.process.cleanup()
+
+    def getStatus(self):
+        if self.process is not None:
+            running = self.process.isRunning()
+            if not running:
+                self.exitCode = self.process.getExitCode()
+                if self.exitCode == 0:
+                    self.setStatus(ActionStatus.SUCCESS)
+                else:
+                    self.setStatus(ActionStatus.FAILED)
+        return super().getStatus()

lisa/core/testrunner.py

@@ -0,0 +1,5 @@
+from lisa import Action
+
+
+class TestRunner(Action):
+    pass

lisa/main.py

@@ -1,19 +1,57 @@
-import sys
 import os
+import sys
+import time
 from datetime import datetime
-from lisa.common.logger import log
+
+from lisa import ActionStatus
+from lisa.common.logger import init_log, log
+from lisa.core.lisarunner import LISARunner
+from retry import retry
+
+path_template = "runtime/results/{0}/{0}-{1}"
+
+
+@retry(FileExistsError, tries=10, delay=0)
+def create_result_path():
+    date = datetime.utcnow().strftime("%Y%m%d")
+    time = datetime.utcnow().strftime("%H%M%S-%f")[:-3]
+    current_path = path_template.format(date, time)
+    if os.path.exists(current_path):
+        raise FileExistsError(
+            "%s exists, and not found an unique path." % current_path
+        )
+    return current_path
+
+
+result_path = os.path.realpath(create_result_path())
+os.makedirs(result_path)
+os.environ["RESULT_PATH"] = result_path
 
 
 def main():
+    init_log()
     log.info("Python version: %s" % sys.version)
     log.info("command line args: %s" % sys.argv)
     log.info("local time: %s", datetime.now())
     log.info("result path: %s", os.environ["RESULT_PATH"])
+    runner = LISARunner()
+    runner.start()
+    while True:
+        status = runner.getStatus()
+        log.info("main status is %s", status.name)
+        if status != ActionStatus.RUNNING:
+            break
+        time.sleep(1)
+    log.info("result is %s", runner.exitCode)
 
 
 if __name__ == "__main__":
+    exitCode = 0
     try:
         main()
     except Exception as exception:
         log.exception(exception)
-        raise
+        exitCode = -1
+    finally:
+        # force all threads end.
+        os._exit(exitCode)

lisa/util/__init__.py

@@ -0,0 +1,3 @@
+from .process import Process
+
+__all__ = ["Process"]

lisa/util/process.py

@@ -0,0 +1,88 @@
+import os
+from subprocess import Popen
+from threading import Thread
+
+from lisa.common import log
+
+
+class LogPipe(Thread):
+    def __init__(self, level):
+        """Setup the object with a logger and a loglevel
+        and start the thread
+        """
+        Thread.__init__(self)
+        self.daemon = False
+        self.level = level
+        self.fdRead, self.fdWrite = os.pipe()
+        self.pipeReader = os.fdopen(self.fdRead)
+        self.start()
+
+    def fileno(self):
+        """Return the write file descriptor of the pipe
+        """
+        return self.fdWrite
+
+    def run(self):
+        """Run the thread, logging everything.
+        """
+        for line in iter(self.pipeReader.readline, ""):
+            log.log(self.level, line.strip("\n"))
+
+        self.pipeReader.close()
+
+    def close(self):
+        """Close the write end of the pipe.
+        """
+        os.close(self.fdWrite)
+
+
+class Process:
+    def __init__(self):
+        self.process = None
+        self.exitCode = None
+        self.running = None
+        self.log_pipe = None
+
+    def start(self, command: str, cwd: str = None, new_envs: dict = None):
+        """
+            command include all parameters also.
+        """
+        environ = os.environ.copy()
+        if new_envs is not None:
+            for key, value in new_envs:
+                environ[key] = value
+        self.log_pipe = LogPipe(log.level)
+        self.process = Popen(
+            command,
+            shell=True,
+            stdout=self.log_pipe,
+            stderr=self.log_pipe,
+            cwd=cwd,
+            env=dict(environ),
+        )
+        self.running = True
+        log.info("process %s stared", self.process.pid)
+
+    def stop(self):
+        if self.process is not None:
+            self.process.terminate()
+            log.info("process %s stopped", self.process.pid)
+
+    def cleanup(self):
+        if self.log_pipe is not None:
+            self.log_pipe.close()
+
+    def isRunning(self):
+        self.exitCode = self.getExitCode()
+        if self.exitCode is not None:
+            if self.running:
+                log.info(
+                    "process %s exited: %s", self.process.pid, self.exitCode
+                )
+            self.running = False
+        return self.running
+
+    def getExitCode(self):
+        if self.process is not None:
+            self.exitCode = self.process.poll()
+        return self.exitCode