Record diagnostic info from bay nodes

In functional tests, copy logs from bay nodes to host before the
tests tear down. The bay node logs could be helpful for
trouble-shooting.

Change-Id: I102d2ce37c2253c743719e4ccf978c23a92a3a3f

magnum/tests/contrib/copy_instance_logs.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Save trace setting
+XTRACE=$(set +o | grep xtrace)
+set -o xtrace
+
+echo "Magnum's copy_instance_logs.sh was called..."
+
+SSH_IP=$1
+COE=${2-kubernetes}
+NODE_TYPE=${3-master}
+LOG_PATH=/opt/stack/logs/bay-nodes/${NODE_TYPE}-${SSH_IP}
+
+function remote_exec {
+    local ssh_user=$1
+    local cmd=$2
+    local logfile=${LOG_PATH}/$3
+    ssh -o StrictHostKeyChecking=no ${ssh_user}@${SSH_IP} "${cmd}" > ${logfile} 2>&1
+}
+
+mkdir -p $LOG_PATH
+
+if [[ "$COE" == "kubernetes" ]]; then
+    SSH_USER=minion
+    remote_exec $SSH_USER "sudo systemctl --full list-units" systemctl_list_units.log
+    remote_exec $SSH_USER "sudo journalctl -u cloud-config --no-pager" cloud-config.log
+    remote_exec $SSH_USER "sudo journalctl -u cloud-final --no-pager" cloud-final.log
+    remote_exec $SSH_USER "sudo journalctl -u cloud-init-local --no-pager" cloud-init-local.log
+    remote_exec $SSH_USER "sudo journalctl -u cloud-init --no-pager" cloud-init.log
+    remote_exec $SSH_USER "sudo journalctl -u kubelet --no-pager" kubelet.log
+    remote_exec $SSH_USER "sudo journalctl -u kube-proxy --no-pager" kube-proxy.log
+    remote_exec $SSH_USER "sudo journalctl -u etcd --no-pager" etcd.log
+    remote_exec $SSH_USER "sudo journalctl -u kube-apiserver --no-pager" kube-apiserver.log
+    remote_exec $SSH_USER "sudo journalctl -u kube-scheduler --no-pager" kube-scheduler.log
+    remote_exec $SSH_USER "sudo journalctl -u kube-controller-manager --no-pager" kube-controller-manager.log
+    remote_exec $SSH_USER "sudo journalctl -u docker --no-pager" docker.log
+    remote_exec $SSH_USER "sudo journalctl -u flanneld --no-pager" flanneld.log
+elif [[ "$COE" == "swarm" ]]; then
+    SSH_USER=fedora
+    remote_exec $SSH_USER "sudo systemctl list-units" systemctl_list_units.log
+    remote_exec $SSH_USER "sudo journalctl -u cloud-config --no-pager" cloud-config.log
+    remote_exec $SSH_USER "sudo journalctl -u cloud-final --no-pager" cloud-final.log
+    remote_exec $SSH_USER "sudo journalctl -u cloud-init-local --no-pager" cloud-init-local.log
+    remote_exec $SSH_USER "sudo journalctl -u cloud-init --no-pager" cloud-init.log
+    remote_exec $SSH_USER "sudo journalctl -u etcd --no-pager" etcd.log
+    remote_exec $SSH_USER "sudo journalctl -u swarm-manager --no-pager" swarm-manager.log
+    remote_exec $SSH_USER "sudo journalctl -u swarm-agent --no-pager" swarm-agent.log
+    remote_exec $SSH_USER "sudo journalctl -u docker --no-pager" docker.log
+    remote_exec $SSH_USER "sudo journalctl -u flanneld --no-pager" flanneld.log
+else
+    echo "ERROR: Unknown COE '${COE}'"
+    EXIT_CODE=1
+fi
+
+# Restore xtrace
+$XTRACE
+
+exit $EXIT_CODE

magnum/tests/contrib/post_test_hook.sh

@@ -80,6 +80,7 @@ image_id = $IMAGE_ID
 nic_id = $NIC_ID
 keypair_id = default
 flavor_id = m1.magnum
+copy_logs = true
 EOF
 
 # Note(eliqiao): Let's keep this only for debugging on gate.
@@ -88,7 +89,8 @@ cat $CREDS_FILE
 
 # Create a keypair for use in the functional tests.
 echo_summary "Generate a key-pair"
-nova keypair-add default
+ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa
+nova keypair-add --pub-key ~/.ssh/id_rsa.pub default
 
 # Create magnum specific flavor for use in functional tests.
 echo_summary "Create a flavor"

magnum/tests/functional/k8s/test_k8s_python_client.py

@@ -10,12 +10,10 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-
 from magnum.common.pythonk8sclient.swagger_client import api_client
 from magnum.common.pythonk8sclient.swagger_client.apis import apiv_api
 from magnum.tests.functional.python_client_base import BayAPITLSTest
 from magnum.tests.functional.python_client_base import BayTest
-from magnumclient.openstack.common.apiclient import exceptions
 
 
 class TestBayModelResource(BayTest):
@@ -64,18 +62,6 @@ extendedKeyUsage = clientAuth
                                           ca_certs=cls.ca_file)
         cls.k8s_api = apiv_api.ApivApi(k8s_client)
 
-    @classmethod
-    def tearDownClass(cls):
-        cls._delete_bay(cls.bay.uuid)
-        try:
-            cls._wait_on_status(cls.bay,
-                                ["CREATE_COMPLETE",
-                                 "DELETE_IN_PROGRESS", "CREATE_FAILED"],
-                                ["DELETE_FAILED", "DELETE_COMPLETE"])
-        except exceptions.NotFound:
-            pass
-        cls._delete_baymodel(cls.baymodel.uuid)
-
     def test_pod_apis(self):
         pod_manifest = {'apiVersion': 'v1',
                         'kind': 'Pod',

magnum/tests/functional/python_client_base.py

@@ -51,6 +51,7 @@ class BaseMagnumClient(base.TestCase):
         nic_id = cliutils.env('NIC_ID')
         flavor_id = cliutils.env('FLAVOR_ID')
         keypair_id = cliutils.env('KEYPAIR_ID')
+        copy_logs = cliutils.env('COPY_LOGS')
 
         config = ConfigParser.RawConfigParser()
         if config.read('functional_creds.conf'):
@@ -65,11 +66,16 @@ class BaseMagnumClient(base.TestCase):
             nic_id = nic_id or config.get('magnum', 'nic_id')
             flavor_id = flavor_id or config.get('magnum', 'flavor_id')
             keypair_id = keypair_id or config.get('magnum', 'keypair_id')
+            try:
+                copy_logs = copy_logs or config.get('magnum', 'copy_logs')
+            except ConfigParser.NoOptionError:
+                pass
 
         cls.image_id = image_id
         cls.nic_id = nic_id
         cls.flavor_id = flavor_id
         cls.keypair_id = keypair_id
+        cls.copy_logs = bool(copy_logs)
         cls.cs = v1client.Client(username=user,
                                  api_key=passwd,
                                  project_id=tenant_id,
@@ -134,6 +140,11 @@ class BaseMagnumClient(base.TestCase):
                                  "CREATE_COMPLETE"])
         return bay
 
+    @classmethod
+    def _show_bay(cls, name):
+        bay = cls.cs.bays.get(name)
+        return bay
+
     @classmethod
     def _delete_baymodel(cls, baymodel_uuid):
         cls.cs.baymodels.delete(baymodel_uuid)
@@ -142,6 +153,19 @@ class BaseMagnumClient(base.TestCase):
     def _delete_bay(cls, bay_uuid):
         cls.cs.bays.delete(bay_uuid)
 
+    @classmethod
+    def _copy_logs(cls):
+        if not cls.copy_logs:
+            return
+
+        cls.bay = cls._show_bay(cls.bay.uuid)
+        for node_addr in cls.bay.node_addresses:
+            subprocess.call(["magnum/tests/contrib/copy_instance_logs.sh",
+                             node_addr, cls.baymodel.coe, "worker"])
+        for node_addr in getattr(cls.bay, 'master_addresses', []):
+            subprocess.call(["magnum/tests/contrib/copy_instance_logs.sh",
+                             node_addr, cls.baymodel.coe, "master"])
+
 
 class BayTest(BaseMagnumClient):
 
@@ -207,6 +231,10 @@ class BayAPITLSTest(BaseMagnumClient):
 
     @classmethod
     def tearDownClass(cls):
+        try:
+            cls._copy_logs()
+        except Exception as e:
+            print("WARNING: Failed to copy logs. Error: " + repr(e))
 
         if cls.ca_dir:
             rmtree_without_raise(cls.ca_dir)