Have nodepool scan as many ssh host keys as possible

We are seeing users try to enable fips on their test nodes. This
presents a problem because the ssh host key which we've been using is a
ed25519 key which fips disables. Fips forces the use of another key
which ansible doesn't trust and subsequent ssh connections fail.

Address this by trying to scan all available host keys on the server and
not just the first one that paramiko returns.

Change-Id: Ibb2a07a29681dcefd4017eb2fd6134ee33ab726c

nodepool/nodeutils.py

@@ -73,41 +73,79 @@ def nodescan(ip, port=22, timeout=60, gather_hostkeys=True):
 
     keys = []
     key = None
+    # First we wait for the sshd to be listening
     for count in iterate_timeout(
             timeout, exceptions.ConnectionTimeoutException,
             "connection to %s on port %s" % (ip, port)):
         sock = None
-        t = None
         try:
             sock = socket.socket(family, socket.SOCK_STREAM)
             sock.settimeout(10)
             sock.connect(sockaddr)
-            if gather_hostkeys:
-                t = paramiko.transport.Transport(sock)
-                t.start_client(timeout=timeout)
-                key = t.get_remote_server_key()
             break
         except socket.error as e:
             if e.errno not in [errno.ECONNREFUSED, errno.EHOSTUNREACH, None]:
                 log.exception(
                     'Exception connecting to %s on port %s:' % (ip, port))
-        except Exception as e:
-            log.exception("ssh-keyscan failure: %s", e)
+        except Exception:
+            log.exception("ssh socket connection failure")
         finally:
-            try:
-                if t:
-                    t.close()
-            except Exception as e:
-                log.exception('Exception closing paramiko: %s', e)
             try:
                 if sock:
                     sock.close()
-            except Exception as e:
-                log.exception('Exception closing socket: %s', e)
-
-    # Paramiko, at this time, seems to return only the ssh-rsa key, so
-    # only the single key is placed into the list.
-    if key:
-        keys.append("%s %s" % (key.get_name(), key.get_base64()))
+            except Exception:
+                log.exception('Exception closing socket')
+            sock = None
+    if gather_hostkeys:
+        sock = None
+        t = None
+        key_index = 0
+        # Now we gather hostkeys
+        while key_index >= 0:
+            try:
+                sock = socket.socket(family, socket.SOCK_STREAM)
+                # Do this early so that we don't trigger exceptions
+                t = paramiko.transport.Transport(sock)
+                opts = paramiko.transport.SecurityOptions(t)
+                # We use each supported key type in turn to ensure we get
+                # back that specific host key type.
+                key_types = opts.key_types
+                opts.key_types = [key_types[key_index]]
+                key_index += 1
+                if key_index >= len(key_types):
+                    key_index = -1
+
+                sock.settimeout(10)
+                sock.connect(sockaddr)
+                t.start_client(timeout=timeout)
+                key = t.get_remote_server_key()
+                if key:
+                    keys.append("%s %s" % (key.get_name(), key.get_base64()))
+                    log.debug('Added ssh host key: %s', key.get_name())
+            except paramiko.SSHException as e:
+                msg = str(e)
+                if 'no acceptable host key' not in msg:
+                    # We expect some host keys to not be valid when scanning
+                    # only log if the error isn't due to mismatched host key
+                    # types.
+                    log.exception("ssh-keyscan failure")
+            except socket.error:
+                log.exception(
+                    'Exception connecting to %s on port %s:' % (ip, port))
+            except Exception:
+                log.exception("ssh-keyscan failure")
+            finally:
+                try:
+                    if t:
+                        t.close()
+                except Exception:
+                    log.exception('Exception closing paramiko')
+                t = None
+                try:
+                    if sock:
+                        sock.close()
+                except Exception:
+                    log.exception('Exception closing socket')
+                sock = None
 
     return keys