Do a better check for hosts in build

There is a corner case in which if the foreman templates have a line such as:

sshpw --username=root my_kickstart_password

and the host is marked for build (and may be in the middle of provisioning)
quads validate_env.py mistakes the kickstart as complete because validate_env.py
only uses Netcat() to determine that the host is reachable. A more comprehensive
check is needed to know if the host is reachable using a well known password,
and a better determination can be made to know that the host may be mid-kickstart.

Also makes sure that sshhelper doesn't use keys when checking to see if
the host is accessible via ssh (in case host is foreman built and has
ssh keys and yet it's still marked for build)

Fixes: #445
Change-Id: Ib8e7c36d34c3d224f1fce405fe10101e5bf2de34

conf/quads.yml

@@ -100,6 +100,8 @@ foreman_password: password
 foreman_default_os: "RHEL 7"
 foreman_default_ptable: "generic-rhel7"
 foreman_default_medium: "RHEL Local"
+# This corresponds to your 'sshpw --username=root my_kickstart_password' foreman template
+foreman_kickstart_password: my_kickstart_password
 
 # Foreman RBAC fix ignore
 # variable to pass cloud names to be excluded from foreman_heal

quads/tools/ssh_helper.py

@@ -14,10 +14,11 @@ class SSHHelperException(Exception):
 
 
 class SSHHelper(object):
-    def __init__(self, _host, _user=None, _password=None):
+    def __init__(self, _host, _user=None, _password=None, _no_key=None):
         self.host = _host
         self.user = _user
         self.password = _password
+        self.no_key = _no_key
         try:
             self.ssh = self.connect()
         except SSHHelperException as ex:
@@ -34,13 +35,17 @@ def connect(self):
         host_config = config.lookup(self.host)
         ssh.set_missing_host_key_policy(AutoAddPolicy())
         ssh.load_system_host_keys()
+        if self.no_key:
+            key_filename=None
+        else:
+            key_filename=host_config["identityfile"][0]
 
         try:
             ssh.connect(
                 self.host,
                 username=self.user,
                 password=self.password,
-                key_filename=host_config["identityfile"][0],
+                key_filename=key_filename,
                 allow_agent=False,
                 timeout=30,
             )

quads/tools/validate_env.py

@@ -136,6 +136,23 @@ async def post_system_test(self):
                     try:
                         nc = Netcat(host)
                         healthy = await nc.health_check()
+                        if healthy:
+                            success_ssh = True
+                            try:
+                                no_key = True
+                                ssh_helper = SSHHelper(host, "root", str(Config["foreman_kickstart_password"]), no_key)
+                            except (SSHHelperException, SSHException, NoValidConnectionsError, socket.timeout) as ex:
+                                logger.error(str(ex))
+                                logger.error(
+                                    "Could not establish connection with host: %s." % host
+                                )
+                                success_ssh = False
+                            if success_ssh:
+                                # If ssh succeeds with the default kickstart password, we assume kickstart is
+                                # still in progress, and we toggle healthy to avoid inadvertantly rebooting host
+                                # during installation
+                                healthy = False
+                            ssh_helper.disconnect()
                     except OSError:
                         healthy = False
                     if not healthy: