From cb1860565f741cae1ffb62091cc756bae155a3f1 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Tue, 3 Nov 2020 13:14:48 -0800
Subject: [PATCH] Have nodepool scan as many ssh host keys as possible

We are seeing users try to enable fips on their test nodes. This
presents a problem because the ssh host key which we've been using is a
ed25519 key which fips disables. Fips forces the use of another key
which ansible doesn't trust and subsequent ssh connections fail.

Address this by trying to scan all available host keys on the server and
not just the first one that paramiko returns.

Change-Id: Ibb2a07a29681dcefd4017eb2fd6134ee33ab726c
---
 nodepool/nodeutils.py | 74 ++++++++++++++++++++++++++++++++-----------
 1 file changed, 56 insertions(+), 18 deletions(-)

diff --git a/nodepool/nodeutils.py b/nodepool/nodeutils.py
index eb0d828d0..8bff58e3a 100644
--- a/nodepool/nodeutils.py
+++ b/nodepool/nodeutils.py
@@ -73,41 +73,79 @@ def nodescan(ip, port=22, timeout=60, gather_hostkeys=True):
 
     keys = []
     key = None
+    # First we wait for the sshd to be listening
     for count in iterate_timeout(
             timeout, exceptions.ConnectionTimeoutException,
             "connection to %s on port %s" % (ip, port)):
         sock = None
-        t = None
         try:
             sock = socket.socket(family, socket.SOCK_STREAM)
             sock.settimeout(10)
             sock.connect(sockaddr)
-            if gather_hostkeys:
-                t = paramiko.transport.Transport(sock)
-                t.start_client(timeout=timeout)
-                key = t.get_remote_server_key()
             break
         except socket.error as e:
             if e.errno not in [errno.ECONNREFUSED, errno.EHOSTUNREACH, None]:
                 log.exception(
                     'Exception connecting to %s on port %s:' % (ip, port))
-        except Exception as e:
-            log.exception("ssh-keyscan failure: %s", e)
+        except Exception:
+            log.exception("ssh socket connection failure")
         finally:
-            try:
-                if t:
-                    t.close()
-            except Exception as e:
-                log.exception('Exception closing paramiko: %s', e)
             try:
                 if sock:
                     sock.close()
-            except Exception as e:
-                log.exception('Exception closing socket: %s', e)
+            except Exception:
+                log.exception('Exception closing socket')
+            sock = None
+    if gather_hostkeys:
+        sock = None
+        t = None
+        key_index = 0
+        # Now we gather hostkeys
+        while key_index >= 0:
+            try:
+                sock = socket.socket(family, socket.SOCK_STREAM)
+                # Do this early so that we don't trigger exceptions
+                t = paramiko.transport.Transport(sock)
+                opts = paramiko.transport.SecurityOptions(t)
+                # We use each supported key type in turn to ensure we get
+                # back that specific host key type.
+                key_types = opts.key_types
+                opts.key_types = [key_types[key_index]]
+                key_index += 1
+                if key_index >= len(key_types):
+                    key_index = -1
 
-    # Paramiko, at this time, seems to return only the ssh-rsa key, so
-    # only the single key is placed into the list.
-    if key:
-        keys.append("%s %s" % (key.get_name(), key.get_base64()))
+                sock.settimeout(10)
+                sock.connect(sockaddr)
+                t.start_client(timeout=timeout)
+                key = t.get_remote_server_key()
+                if key:
+                    keys.append("%s %s" % (key.get_name(), key.get_base64()))
+                    log.debug('Added ssh host key: %s', key.get_name())
+            except paramiko.SSHException as e:
+                msg = str(e)
+                if 'no acceptable host key' not in msg:
+                    # We expect some host keys to not be valid when scanning
+                    # only log if the error isn't due to mismatched host key
+                    # types.
+                    log.exception("ssh-keyscan failure")
+            except socket.error:
+                log.exception(
+                    'Exception connecting to %s on port %s:' % (ip, port))
+            except Exception:
+                log.exception("ssh-keyscan failure")
+            finally:
+                try:
+                    if t:
+                        t.close()
+                except Exception:
+                    log.exception('Exception closing paramiko')
+                t = None
+                try:
+                    if sock:
+                        sock.close()
+                except Exception:
+                    log.exception('Exception closing socket')
+                sock = None
 
     return keys