fix: netcat socket closing for healthcheck

Change-Id: I0e26f1435e7c6722f7d6dd7f73bb9012ee76e875
redhat-performance · Sep 4, 2020 · 04398b9 · 04398b9
1 parent 4c29197
commit 04398b9
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 4 deletions.
diff --git a/quads/tools/move_and_rebuild_hosts.py b/quads/tools/move_and_rebuild_hosts.py
@@ -247,11 +247,13 @@ async def move_and_rebuild(host, new_cloud, semaphore, rebuild=False, loop=None)
                 return False
 
         healthy = False
-        nc = Netcat(_host_obj.name)
         for i in range(RETRIES):
+            nc = Netcat(_host_obj.name)
             if nc.health_check():
                 healthy = True
+                nc.close()
                 break
+            nc.close()
 
         if not healthy:
             logger.error("Failed to recover host after changing boot order.")

diff --git a/quads/tools/netcat.py b/quads/tools/netcat.py
@@ -23,7 +23,12 @@ def health_check(self, timeout=15):
         try:
             self.socket.settimeout(timeout)
             self.connect()
-        except (TimeoutError, socket.timeout):
+        except (
+            socket.timeout,
+            TimeoutError,
+            ConnectionResetError,
+            ConnectionRefusedError,
+        ):
             return False
         return True
 
@@ -37,8 +42,8 @@ def read_until(self, data):
             self.buff += self.socket.recv(1024)
 
         pos = self.buff.find(data)
-        return_val = self.buff[:pos + len(data)]
-        self.buff = self.buff[pos + len(data):]
+        return_val = self.buff[: pos + len(data)]
+        self.buff = self.buff[pos + len(data) :]
 
         return return_val
 

diff --git a/quads/tools/validate_env.py b/quads/tools/validate_env.py
@@ -127,6 +127,7 @@ def post_system_test(self):
                             "Potential provisioning in process. SKIPPING."
                         )
                         continue
+                    nc.close()
                     badfish = None
                     try:
                         badfish = loop.run_until_complete(
@@ -182,6 +183,7 @@ def post_network_test(self):
             nc = Netcat(host.name)
             if not nc.health_check():
                 hosts_down.append(host.name)
+            nc.close()
             if len(host.interfaces) > len(test_host.interfaces):
                 test_host = host