diff --git a/magnum/tests/contrib/copy_instance_logs.sh b/magnum/tests/contrib/copy_instance_logs.sh new file mode 100755 index 0000000000..b021b5ab83 --- /dev/null +++ b/magnum/tests/contrib/copy_instance_logs.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Save trace setting +XTRACE=$(set +o | grep xtrace) +set -o xtrace + +echo "Magnum's copy_instance_logs.sh was called..." + +SSH_IP=$1 +COE=${2-kubernetes} +NODE_TYPE=${3-master} +LOG_PATH=/opt/stack/logs/bay-nodes/${NODE_TYPE}-${SSH_IP} + +function remote_exec { + local ssh_user=$1 + local cmd=$2 + local logfile=${LOG_PATH}/$3 + ssh -o StrictHostKeyChecking=no ${ssh_user}@${SSH_IP} "${cmd}" > ${logfile} 2>&1 +} + +mkdir -p $LOG_PATH + +if [[ "$COE" == "kubernetes" ]]; then + SSH_USER=minion + remote_exec $SSH_USER "sudo systemctl --full list-units" systemctl_list_units.log + remote_exec $SSH_USER "sudo journalctl -u cloud-config --no-pager" cloud-config.log + remote_exec $SSH_USER "sudo journalctl -u cloud-final --no-pager" cloud-final.log + remote_exec $SSH_USER "sudo journalctl -u cloud-init-local --no-pager" cloud-init-local.log + remote_exec $SSH_USER "sudo journalctl -u cloud-init --no-pager" cloud-init.log + remote_exec $SSH_USER "sudo journalctl -u kubelet --no-pager" kubelet.log + remote_exec $SSH_USER "sudo journalctl -u kube-proxy --no-pager" kube-proxy.log + remote_exec $SSH_USER "sudo journalctl -u etcd --no-pager" etcd.log + remote_exec $SSH_USER "sudo journalctl -u kube-apiserver --no-pager" kube-apiserver.log + remote_exec $SSH_USER "sudo journalctl -u kube-scheduler --no-pager" kube-scheduler.log + remote_exec $SSH_USER "sudo journalctl -u kube-controller-manager --no-pager" kube-controller-manager.log + remote_exec $SSH_USER "sudo journalctl -u docker --no-pager" docker.log + remote_exec $SSH_USER "sudo journalctl -u flanneld --no-pager" flanneld.log +elif [[ "$COE" == "swarm" ]]; then + SSH_USER=fedora + remote_exec $SSH_USER "sudo systemctl list-units" systemctl_list_units.log + remote_exec $SSH_USER "sudo journalctl -u cloud-config --no-pager" cloud-config.log + remote_exec $SSH_USER "sudo journalctl -u cloud-final --no-pager" cloud-final.log + remote_exec $SSH_USER "sudo journalctl -u cloud-init-local --no-pager" cloud-init-local.log + remote_exec $SSH_USER "sudo journalctl -u cloud-init --no-pager" cloud-init.log + remote_exec $SSH_USER "sudo journalctl -u etcd --no-pager" etcd.log + remote_exec $SSH_USER "sudo journalctl -u swarm-manager --no-pager" swarm-manager.log + remote_exec $SSH_USER "sudo journalctl -u swarm-agent --no-pager" swarm-agent.log + remote_exec $SSH_USER "sudo journalctl -u docker --no-pager" docker.log + remote_exec $SSH_USER "sudo journalctl -u flanneld --no-pager" flanneld.log +else + echo "ERROR: Unknown COE '${COE}'" + EXIT_CODE=1 +fi + +# Restore xtrace +$XTRACE + +exit $EXIT_CODE diff --git a/magnum/tests/contrib/post_test_hook.sh b/magnum/tests/contrib/post_test_hook.sh index a9a8235cbd..75f8180336 100755 --- a/magnum/tests/contrib/post_test_hook.sh +++ b/magnum/tests/contrib/post_test_hook.sh @@ -80,6 +80,7 @@ image_id = $IMAGE_ID nic_id = $NIC_ID keypair_id = default flavor_id = m1.magnum +copy_logs = true EOF # Note(eliqiao): Let's keep this only for debugging on gate. @@ -88,7 +89,8 @@ cat $CREDS_FILE # Create a keypair for use in the functional tests. echo_summary "Generate a key-pair" -nova keypair-add default +ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa +nova keypair-add --pub-key ~/.ssh/id_rsa.pub default # Create magnum specific flavor for use in functional tests. echo_summary "Create a flavor" diff --git a/magnum/tests/functional/k8s/test_k8s_python_client.py b/magnum/tests/functional/k8s/test_k8s_python_client.py index 24cbb910a2..da24fa5618 100644 --- a/magnum/tests/functional/k8s/test_k8s_python_client.py +++ b/magnum/tests/functional/k8s/test_k8s_python_client.py @@ -10,12 +10,10 @@ # License for the specific language governing permissions and limitations # under the License. - from magnum.common.pythonk8sclient.swagger_client import api_client from magnum.common.pythonk8sclient.swagger_client.apis import apiv_api from magnum.tests.functional.python_client_base import BayAPITLSTest from magnum.tests.functional.python_client_base import BayTest -from magnumclient.openstack.common.apiclient import exceptions class TestBayModelResource(BayTest): @@ -64,18 +62,6 @@ extendedKeyUsage = clientAuth ca_certs=cls.ca_file) cls.k8s_api = apiv_api.ApivApi(k8s_client) - @classmethod - def tearDownClass(cls): - cls._delete_bay(cls.bay.uuid) - try: - cls._wait_on_status(cls.bay, - ["CREATE_COMPLETE", - "DELETE_IN_PROGRESS", "CREATE_FAILED"], - ["DELETE_FAILED", "DELETE_COMPLETE"]) - except exceptions.NotFound: - pass - cls._delete_baymodel(cls.baymodel.uuid) - def test_pod_apis(self): pod_manifest = {'apiVersion': 'v1', 'kind': 'Pod', diff --git a/magnum/tests/functional/python_client_base.py b/magnum/tests/functional/python_client_base.py index d1ff026a7e..1c8b845862 100644 --- a/magnum/tests/functional/python_client_base.py +++ b/magnum/tests/functional/python_client_base.py @@ -51,6 +51,7 @@ class BaseMagnumClient(base.TestCase): nic_id = cliutils.env('NIC_ID') flavor_id = cliutils.env('FLAVOR_ID') keypair_id = cliutils.env('KEYPAIR_ID') + copy_logs = cliutils.env('COPY_LOGS') config = ConfigParser.RawConfigParser() if config.read('functional_creds.conf'): @@ -65,11 +66,16 @@ class BaseMagnumClient(base.TestCase): nic_id = nic_id or config.get('magnum', 'nic_id') flavor_id = flavor_id or config.get('magnum', 'flavor_id') keypair_id = keypair_id or config.get('magnum', 'keypair_id') + try: + copy_logs = copy_logs or config.get('magnum', 'copy_logs') + except ConfigParser.NoOptionError: + pass cls.image_id = image_id cls.nic_id = nic_id cls.flavor_id = flavor_id cls.keypair_id = keypair_id + cls.copy_logs = bool(copy_logs) cls.cs = v1client.Client(username=user, api_key=passwd, project_id=tenant_id, @@ -134,6 +140,11 @@ class BaseMagnumClient(base.TestCase): "CREATE_COMPLETE"]) return bay + @classmethod + def _show_bay(cls, name): + bay = cls.cs.bays.get(name) + return bay + @classmethod def _delete_baymodel(cls, baymodel_uuid): cls.cs.baymodels.delete(baymodel_uuid) @@ -142,6 +153,19 @@ class BaseMagnumClient(base.TestCase): def _delete_bay(cls, bay_uuid): cls.cs.bays.delete(bay_uuid) + @classmethod + def _copy_logs(cls): + if not cls.copy_logs: + return + + cls.bay = cls._show_bay(cls.bay.uuid) + for node_addr in cls.bay.node_addresses: + subprocess.call(["magnum/tests/contrib/copy_instance_logs.sh", + node_addr, cls.baymodel.coe, "worker"]) + for node_addr in getattr(cls.bay, 'master_addresses', []): + subprocess.call(["magnum/tests/contrib/copy_instance_logs.sh", + node_addr, cls.baymodel.coe, "master"]) + class BayTest(BaseMagnumClient): @@ -207,6 +231,10 @@ class BayAPITLSTest(BaseMagnumClient): @classmethod def tearDownClass(cls): + try: + cls._copy_logs() + except Exception as e: + print("WARNING: Failed to copy logs. Error: " + repr(e)) if cls.ca_dir: rmtree_without_raise(cls.ca_dir)