From ca293232ce7f8d092ad6c5bd1710710c6a22146f Mon Sep 17 00:00:00 2001
From: Omar Zevallos <45883639+omzevall@users.noreply.github.com>
Date: Tue, 26 Feb 2019 11:06:56 -0800
Subject: [PATCH] Added some timing control for added consistency

---
 test/lib/helpers.py              |  1 +
 test/test_vfxt_cluster_status.py | 17 ++++++++++++++---
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/test/lib/helpers.py b/test/lib/helpers.py
index 0d4486a6..b12bf05e 100644
--- a/test/lib/helpers.py
+++ b/test/lib/helpers.py
@@ -31,6 +31,7 @@ def get_unused_local_port():
     sock.bind(("", 0))
     port = sock.getsockname()[1]
     sock.close()
+    sleep(2)  # for the port to fully free up
     return port
 
 
diff --git a/test/test_vfxt_cluster_status.py b/test/test_vfxt_cluster_status.py
index 56e12f96..6114e438 100644
--- a/test/test_vfxt_cluster_status.py
+++ b/test/test_vfxt_cluster_status.py
@@ -8,6 +8,7 @@
 import logging
 import os
 import sys
+from time import sleep, time
 
 # from requirements.txt
 import pytest
@@ -43,10 +44,20 @@ class TestVfxtClusterStatus:
 
     def test_node_health(self, averecmd_params):  # noqa: F811
         """Check that cluster is reporting that all nodes are up."""
+        log = logging.getLogger("test_node_health")
         for node in run_averecmd(**averecmd_params, method="node.list"):
-            result = run_averecmd(**averecmd_params,
-                                  method="node.get", args=node)
-            assert result[node]["state"] == "up"
+            timeout_secs = 60
+            time_start = time()
+            time_end = time_start + timeout_secs
+            while time() <= time_end:
+                result = run_averecmd(**averecmd_params,
+                                      method="node.get", args=node)
+                node_state = result[node]["state"]
+                log.info('Node {0} has state "{1}"'.format(node, node_state))
+                if node_state == "up":
+                    break
+                sleep(10)
+            assert node_state == "up"
 
     def test_ha_enabled(self, averecmd_params):  # noqa: F811
         """Check that high-availability (HA) is enabled."""