Fetch server console log if ssh connection fails

Currently, if the ssh connection fails, we are blind to what the
possible failures are.  As a result, attempt to fetch the server
console log to help debug the failure.

This is the continuation of I39ec1fe591d6602a3d494ac79ffa6d2203b5676b
but for the feature/zuulv3 branch. This was done to avoid merge
conflicts on the recent changes to nodepool.yaml layout.

Change-Id: I75ccb6d01956fb6052473f44cce8f097a56dd16a
Signed-off-by: Paul Belanger <pabelanger@redhat.com>

devstack/plugin.sh

@@ -246,26 +246,32 @@ providers:
             diskimage: centos-7
             min-ram: 1024
             flavor-name: 'nodepool'
+            console-log: True
           - name: debian-jessie
             diskimage: debian-jessie
             min-ram: 512
             flavor-name: 'nodepool'
+            console-log: True
           - name: fedora-25
             diskimage: fedora-25
             min-ram: 1024
             flavor-name: 'nodepool'
+            console-log: True
           - name: ubuntu-precise
             diskimage: ubuntu-precise
             min-ram: 512
             flavor-name: 'nodepool'
+            console-log: True
           - name: ubuntu-trusty
             diskimage: ubuntu-trusty
             min-ram: 512
             flavor-name: 'nodepool'
+            console-log: True
           - name: ubuntu-xenial
             diskimage: ubuntu-xenial
             min-ram: 512
             flavor-name: 'nodepool'
+            console-log: True
 
 diskimages:
   - name: centos-7

doc/source/configuration.rst

@@ -237,6 +237,7 @@ Example::
             - name: trusty
               min-ram: 8192
               diskimage: trusty
+              console-log: True
             - name: precise
               min-ram: 8192
               diskimage: precise
@@ -356,6 +357,7 @@ Example::
         - name: trusty
           min-ram: 8192
           diskimage: trusty
+          console-log: True
         - name: precise
           min-ram: 8192
           diskimage: precise
@@ -475,6 +477,7 @@ Example configuration::
     - name: precise
       min-ram: 8192
       flavor-name: 'something to match'
+      console-log: True
 
 **required**
 
@@ -509,3 +512,7 @@ Example configuration::
   ``key-name``
     If given, is the name of a keypair that will be used when booting each
     server.
+
+  ``console-log`` (default: False)
+    On the failure of the ssh ready check, download the server console log to
+    aid in debuging the problem.

nodepool/config.py

@@ -270,6 +270,8 @@ def loadConfig(config_path):
                 pl.min_ram = label.get('min-ram', 0)
                 pl.flavor_name = label.get('flavor-name', None)
                 pl.key_name = label.get('key-name')
+                pl.console_log = label.get('console-log', False)
+
                 top_label = newconfig.labels[pl.name]
                 top_label.pools.append(pp)
 

nodepool/launcher.py

@@ -255,6 +255,15 @@ class NodeLauncher(threading.Thread, StatsReporter):
             self._diskimage = None
         self._cloud_image = self._provider.cloud_images.get(self._label.cloud_image, None)
 
+    def logConsole(self, server_id, hostname):
+        if not self._label.console_log:
+            return
+        console = self._manager.getServerConsole(server_id)
+        if console:
+            self.log.debug('Console log from hostname %s:' % hostname)
+            for line in console.splitlines():
+                self.log.debug(line.rstrip())
+
     def _launchNode(self):
         if self._label.diskimage:
             # launch using diskimage
@@ -359,11 +368,16 @@ class NodeLauncher(threading.Thread, StatsReporter):
              self._node.public_ipv4, self._node.public_ipv6))
 
         # Get the SSH public keys for the new node and record in ZooKeeper
-        self.log.debug("Gathering host keys for node %s", self._node.id)
-        host_keys = utils.keyscan(
-            interface_ip, timeout=self._provider.boot_timeout)
-        if not host_keys:
-            raise LaunchKeyscanException("Unable to gather host keys")
+        try:
+            self.log.debug("Gathering host keys for node %s", self._node.id)
+            host_keys = utils.keyscan(
+                interface_ip, timeout=self._provider.boot_timeout)
+            if not host_keys:
+                raise LaunchKeyscanException("Unable to gather host keys")
+        except exceptions.SSHTimeoutException:
+            self.logConsole(self._node.external_id, self._node.hostname)
+            raise
+
         self._node.host_keys = host_keys
         self._zk.storeNode(self._node)
 

nodepool/provider_manager.py

@@ -238,6 +238,13 @@ class ProviderManager(object):
         with shade_inner_exceptions():
             return self._client.get_server(server_id)
 
+    def getServerConsole(self, server_id):
+        try:
+            with shade_inner_exceptions():
+                return self._client.get_server_console(server_id)
+        except shade.OpenStackCloudException:
+            return None
+
     def waitForServer(self, server, timeout=3600):
         with shade_inner_exceptions():
             return self._client.wait_for_server(