Rerun flaky tests (#5316)

dependencies/develop.txt

@@ -10,6 +10,7 @@ pylint < 2.15
 pyright == 1.1.250
 pytest
 pytest-cov
+pytest-rerunfailures
 rstcheck >= 6.0
 sphinx >= 4.5
 sphinx-argparse-nni >= 0.4.0

test/ut/tools/nnictl/test_kill_command.py

@@ -35,6 +35,7 @@ def process_patiently_kill():
     kill_command(process.pid)  # wait long enough
 
 
+@pytest.mark.flaky(reruns=1)
 def test_kill_process():
     process = multiprocessing.Process(target=process_normal)
     process.start()
@@ -47,6 +48,7 @@ def test_kill_process():
     assert end_time - start_time < 2
 
 
+@pytest.mark.flaky(reruns=2)
 def test_kill_process_slow_no_patience():
     process = subprocess.Popen([sys.executable, __file__, '--mode', 'kill_slow'])
     time.sleep(1)  # wait 1 second for the process to launch and register hooks
@@ -67,6 +69,7 @@ def test_kill_process_slow_no_patience():
             return
 
 
+@pytest.mark.flaky(reruns=2)
 def test_kill_process_slow_patiently():
     process = subprocess.Popen([sys.executable, __file__, '--mode', 'kill_slow'])
     time.sleep(1)  # wait 1 second for the process to launch and register hooks
@@ -78,6 +81,7 @@ def test_kill_process_slow_patiently():
 
 
 @pytest.mark.skipif(sys.platform != 'linux', reason='Signal issues on non-linux.')
+@pytest.mark.flaky(reruns=2)
 def test_kill_process_interrupted():
     # Launch a subprocess that launches and kills another subprocess
     process = multiprocessing.Process(target=process_patiently_kill)