test: fix retry test case failure

Replace another retry test case in runner.

selftests/runners/test_lisa_runner.py

@@ -23,6 +23,7 @@ def generate_runner(
     env_runbook: Optional[schema.EnvironmentRoot] = None,
     case_use_new_env: bool = False,
     times: int = 1,
+    retry: int = 0,
     platform_schema: Optional[test_platform.MockPlatformSchema] = None,
 ) -> LisaRunner:
     platform_runbook = schema.Platform(
@@ -40,6 +41,7 @@ def generate_runner(
             criteria=schema.Criteria(priority=[0, 1, 2]),
             use_new_environment=case_use_new_env,
             times=times,
+            retry=retry,
         )
     ]
     runbook.wait_resource_timeout = 0
@@ -551,6 +553,47 @@ class RunnerTestCase(TestCase):
             test_results=test_results,
         )
 
+    def test_env_retry(self) -> None:
+        # env prepared, but deployment failed, so cases failed
+        platform_schema = test_platform.MockPlatformSchema()
+
+        test_testsuite.generate_cases_metadata()
+        test_testsuite.retry_failed_count = 1
+        env_runbook = generate_env_runbook()
+        runner = generate_runner(env_runbook, platform_schema=platform_schema, retry=2)
+        test_result_messages = self._run_all_tests(runner)
+
+        self.verify_env_results(
+            expected_prepared=[
+                "generated_0",
+                "generated_1",
+                "generated_2",
+                "generated_0",
+            ],
+            expected_deployed_envs=[
+                "generated_0",
+                "generated_0",
+                "generated_2",
+            ],
+            expected_deleted_envs=[
+                "generated_0",
+                "generated_0",
+                "generated_2",
+            ],
+            runner=runner,
+        )
+        self.verify_test_results(
+            expected_test_order=["mock_ut1", "mock_ut2", "mock_ut3"],
+            expected_envs=["generated_0", "generated_0", "generated_2"],
+            expected_status=[
+                TestStatus.PASSED,
+                TestStatus.PASSED,
+                TestStatus.PASSED,
+            ],
+            expected_message=["", "", ""],
+            test_results=test_result_messages,
+        )
+
     def test_env_skipped_no_case(self) -> None:
         # no case found, as not call generate_case_metadata
         # in this case, not deploy any env

selftests/test_testsuite.py

@@ -35,6 +35,7 @@ skipped = False
 queued = False
 fail_case_count = 0
 check_variable = False
+retry_failed_count = 0
 
 
 class MockTestSuite(TestSuite):
@@ -81,6 +82,10 @@ class MockTestSuite(TestSuite):
         while self.fail_case_count > 0:
             self.fail_case_count -= 1
             raise LisaException("mock_ut1 failed")
+        global retry_failed_count
+        if retry_failed_count > 0:
+            retry_failed_count -= 1
+            raise LisaException("mock_ut1 failed by retry")
 
     def mock_ut2(self, variables: Dict[str, Any], **kwargs: Any) -> None:
         if self.check_variable:
@@ -263,22 +268,6 @@ class TestSuiteTestCase(TestCase):
         self.assertEqual(TestStatus.PASSED, result.status)
         self.assertEqual("", result.message)
 
-    def test_retry_passed(self) -> None:
-        test_suite = self.generate_suite_instance()
-        test_suite.set_fail_phase(fail_case_count=1)
-        result = self.case_results[0]
-        result.runtime_data.retry = 1
-        test_suite.start(
-            environment=self.default_env,
-            case_results=self.case_results,
-            case_variables={},
-        )
-        self.assertEqual(TestStatus.PASSED, result.status)
-        self.assertEqual("", result.message)
-        result = self.case_results[1]
-        self.assertEqual(TestStatus.PASSED, result.status)
-        self.assertEqual("", result.message)
-
     def test_partial_passed(self) -> None:
         test_suite = self.generate_suite_instance()
         test_suite.set_fail_phase(partial_pass=True)