openstack
/
magnum
Code Issues Proposed changes

Record diagnostic info from bay nodes 

In functional tests, copy logs from bay nodes to host before the
tests tear down. The bay node logs could be helpful for
trouble-shooting.

Change-Id: I102d2ce37c2253c743719e4ccf978c23a92a3a3f
This commit is contained in:
Hongbin Lu 2015-11-06 18:10:27 -05:00
parent 08c42e29e5
commit 035a07fe4d
4 changed files with 101 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
magnum/tests/contrib/copy_instance_logs.sh Executable file
View File

@ -0,0 +1,70 @@
#!/usr/bin/env bash
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Save trace setting
XTRACE=$(set +o | grep xtrace)
set -o xtrace
echo "Magnum's copy_instance_logs.sh was called..."
SSH_IP=$1
COE=${2-kubernetes}
NODE_TYPE=${3-master}
LOG_PATH=/opt/stack/logs/bay-nodes/${NODE_TYPE}-${SSH_IP}
function remote_exec {
local ssh_user=$1
local cmd=$2
local logfile=${LOG_PATH}/$3
ssh -o StrictHostKeyChecking=no ${ssh_user}@${SSH_IP} "${cmd}" > ${logfile} 2>&1
}
mkdir -p $LOG_PATH
if [[ "$COE" == "kubernetes" ]]; then
SSH_USER=minion
remote_exec $SSH_USER "sudo systemctl --full list-units" systemctl_list_units.log
remote_exec $SSH_USER "sudo journalctl -u cloud-config --no-pager" cloud-config.log
remote_exec $SSH_USER "sudo journalctl -u cloud-final --no-pager" cloud-final.log
remote_exec $SSH_USER "sudo journalctl -u cloud-init-local --no-pager" cloud-init-local.log
remote_exec $SSH_USER "sudo journalctl -u cloud-init --no-pager" cloud-init.log
remote_exec $SSH_USER "sudo journalctl -u kubelet --no-pager" kubelet.log
remote_exec $SSH_USER "sudo journalctl -u kube-proxy --no-pager" kube-proxy.log
remote_exec $SSH_USER "sudo journalctl -u etcd --no-pager" etcd.log
remote_exec $SSH_USER "sudo journalctl -u kube-apiserver --no-pager" kube-apiserver.log
remote_exec $SSH_USER "sudo journalctl -u kube-scheduler --no-pager" kube-scheduler.log
remote_exec $SSH_USER "sudo journalctl -u kube-controller-manager --no-pager" kube-controller-manager.log
remote_exec $SSH_USER "sudo journalctl -u docker --no-pager" docker.log
remote_exec $SSH_USER "sudo journalctl -u flanneld --no-pager" flanneld.log
elif [[ "$COE" == "swarm" ]]; then
SSH_USER=fedora
remote_exec $SSH_USER "sudo systemctl list-units" systemctl_list_units.log
remote_exec $SSH_USER "sudo journalctl -u cloud-config --no-pager" cloud-config.log
remote_exec $SSH_USER "sudo journalctl -u cloud-final --no-pager" cloud-final.log
remote_exec $SSH_USER "sudo journalctl -u cloud-init-local --no-pager" cloud-init-local.log
remote_exec $SSH_USER "sudo journalctl -u cloud-init --no-pager" cloud-init.log
remote_exec $SSH_USER "sudo journalctl -u etcd --no-pager" etcd.log
remote_exec $SSH_USER "sudo journalctl -u swarm-manager --no-pager" swarm-manager.log
remote_exec $SSH_USER "sudo journalctl -u swarm-agent --no-pager" swarm-agent.log
remote_exec $SSH_USER "sudo journalctl -u docker --no-pager" docker.log
remote_exec $SSH_USER "sudo journalctl -u flanneld --no-pager" flanneld.log
else
echo "ERROR: Unknown COE '${COE}'"
EXIT_CODE=1
fi
# Restore xtrace
$XTRACE
exit $EXIT_CODE

4
magnum/tests/contrib/post_test_hook.sh
View File

@ -80,6 +80,7 @@ image_id = $IMAGE_ID
nic_id = $NIC_ID
keypair_id = default
flavor_id = m1.magnum
copy_logs = true
EOF
# Note(eliqiao): Let's keep this only for debugging on gate.
@ -88,7 +89,8 @@ cat $CREDS_FILE
# Create a keypair for use in the functional tests.
echo_summary "Generate a key-pair"
nova keypair-add default
ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa
nova keypair-add --pub-key ~/.ssh/id_rsa.pub default
# Create magnum specific flavor for use in functional tests.
echo_summary "Create a flavor"

14
magnum/tests/functional/k8s/test_k8s_python_client.py
View File

@ -10,12 +10,10 @@
# License for the specific language governing permissions and limitations
# under the License.
from magnum.common.pythonk8sclient.swagger_client import api_client
from magnum.common.pythonk8sclient.swagger_client.apis import apiv_api
from magnum.tests.functional.python_client_base import BayAPITLSTest
from magnum.tests.functional.python_client_base import BayTest
from magnumclient.openstack.common.apiclient import exceptions
class TestBayModelResource(BayTest):
@ -64,18 +62,6 @@ extendedKeyUsage = clientAuth
ca_certs=cls.ca_file)
cls.k8s_api = apiv_api.ApivApi(k8s_client)
@classmethod
def tearDownClass(cls):
cls._delete_bay(cls.bay.uuid)
try:
cls._wait_on_status(cls.bay,
["CREATE_COMPLETE",
"DELETE_IN_PROGRESS", "CREATE_FAILED"],
["DELETE_FAILED", "DELETE_COMPLETE"])
except exceptions.NotFound:
pass
cls._delete_baymodel(cls.baymodel.uuid)
def test_pod_apis(self):
pod_manifest = {'apiVersion': 'v1',
'kind': 'Pod',

28
magnum/tests/functional/python_client_base.py
View File

@ -51,6 +51,7 @@ class BaseMagnumClient(base.TestCase):
nic_id = cliutils.env('NIC_ID')
flavor_id = cliutils.env('FLAVOR_ID')
keypair_id = cliutils.env('KEYPAIR_ID')
copy_logs = cliutils.env('COPY_LOGS')
config = ConfigParser.RawConfigParser()
if config.read('functional_creds.conf'):
@ -65,11 +66,16 @@ class BaseMagnumClient(base.TestCase):
nic_id = nic_id or config.get('magnum', 'nic_id')
flavor_id = flavor_id or config.get('magnum', 'flavor_id')
keypair_id = keypair_id or config.get('magnum', 'keypair_id')
try:
copy_logs = copy_logs or config.get('magnum', 'copy_logs')
except ConfigParser.NoOptionError:
pass
cls.image_id = image_id
cls.nic_id = nic_id
cls.flavor_id = flavor_id
cls.keypair_id = keypair_id
cls.copy_logs = bool(copy_logs)
cls.cs = v1client.Client(username=user,
api_key=passwd,
project_id=tenant_id,
@ -134,6 +140,11 @@ class BaseMagnumClient(base.TestCase):
"CREATE_COMPLETE"])
return bay
@classmethod
def _show_bay(cls, name):
bay = cls.cs.bays.get(name)
return bay
@classmethod
def _delete_baymodel(cls, baymodel_uuid):
cls.cs.baymodels.delete(baymodel_uuid)
@ -142,6 +153,19 @@ class BaseMagnumClient(base.TestCase):
def _delete_bay(cls, bay_uuid):
cls.cs.bays.delete(bay_uuid)
@classmethod
def _copy_logs(cls):
if not cls.copy_logs:
return
cls.bay = cls._show_bay(cls.bay.uuid)
for node_addr in cls.bay.node_addresses:
subprocess.call(["magnum/tests/contrib/copy_instance_logs.sh",
node_addr, cls.baymodel.coe, "worker"])
for node_addr in getattr(cls.bay, 'master_addresses', []):
subprocess.call(["magnum/tests/contrib/copy_instance_logs.sh",
node_addr, cls.baymodel.coe, "master"])
class BayTest(BaseMagnumClient):
@ -207,6 +231,10 @@ class BayAPITLSTest(BaseMagnumClient):
@classmethod
def tearDownClass(cls):
try:
cls._copy_logs()
except Exception as e:
print("WARNING: Failed to copy logs. Error: " + repr(e))
if cls.ca_dir:
rmtree_without_raise(cls.ca_dir)