Record diagnostic info from bay nodes
In functional tests, copy logs from bay nodes to host before the tests tear down. The bay node logs could be helpful for trouble-shooting. Change-Id: I102d2ce37c2253c743719e4ccf978c23a92a3a3f
This commit is contained in:
parent
08c42e29e5
commit
035a07fe4d
|
@ -0,0 +1,70 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Save trace setting
|
||||
XTRACE=$(set +o | grep xtrace)
|
||||
set -o xtrace
|
||||
|
||||
echo "Magnum's copy_instance_logs.sh was called..."
|
||||
|
||||
SSH_IP=$1
|
||||
COE=${2-kubernetes}
|
||||
NODE_TYPE=${3-master}
|
||||
LOG_PATH=/opt/stack/logs/bay-nodes/${NODE_TYPE}-${SSH_IP}
|
||||
|
||||
function remote_exec {
|
||||
local ssh_user=$1
|
||||
local cmd=$2
|
||||
local logfile=${LOG_PATH}/$3
|
||||
ssh -o StrictHostKeyChecking=no ${ssh_user}@${SSH_IP} "${cmd}" > ${logfile} 2>&1
|
||||
}
|
||||
|
||||
mkdir -p $LOG_PATH
|
||||
|
||||
if [[ "$COE" == "kubernetes" ]]; then
|
||||
SSH_USER=minion
|
||||
remote_exec $SSH_USER "sudo systemctl --full list-units" systemctl_list_units.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u cloud-config --no-pager" cloud-config.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u cloud-final --no-pager" cloud-final.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u cloud-init-local --no-pager" cloud-init-local.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u cloud-init --no-pager" cloud-init.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u kubelet --no-pager" kubelet.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u kube-proxy --no-pager" kube-proxy.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u etcd --no-pager" etcd.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u kube-apiserver --no-pager" kube-apiserver.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u kube-scheduler --no-pager" kube-scheduler.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u kube-controller-manager --no-pager" kube-controller-manager.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u docker --no-pager" docker.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u flanneld --no-pager" flanneld.log
|
||||
elif [[ "$COE" == "swarm" ]]; then
|
||||
SSH_USER=fedora
|
||||
remote_exec $SSH_USER "sudo systemctl list-units" systemctl_list_units.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u cloud-config --no-pager" cloud-config.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u cloud-final --no-pager" cloud-final.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u cloud-init-local --no-pager" cloud-init-local.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u cloud-init --no-pager" cloud-init.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u etcd --no-pager" etcd.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u swarm-manager --no-pager" swarm-manager.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u swarm-agent --no-pager" swarm-agent.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u docker --no-pager" docker.log
|
||||
remote_exec $SSH_USER "sudo journalctl -u flanneld --no-pager" flanneld.log
|
||||
else
|
||||
echo "ERROR: Unknown COE '${COE}'"
|
||||
EXIT_CODE=1
|
||||
fi
|
||||
|
||||
# Restore xtrace
|
||||
$XTRACE
|
||||
|
||||
exit $EXIT_CODE
|
|
@ -80,6 +80,7 @@ image_id = $IMAGE_ID
|
|||
nic_id = $NIC_ID
|
||||
keypair_id = default
|
||||
flavor_id = m1.magnum
|
||||
copy_logs = true
|
||||
EOF
|
||||
|
||||
# Note(eliqiao): Let's keep this only for debugging on gate.
|
||||
|
@ -88,7 +89,8 @@ cat $CREDS_FILE
|
|||
|
||||
# Create a keypair for use in the functional tests.
|
||||
echo_summary "Generate a key-pair"
|
||||
nova keypair-add default
|
||||
ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa
|
||||
nova keypair-add --pub-key ~/.ssh/id_rsa.pub default
|
||||
|
||||
# Create magnum specific flavor for use in functional tests.
|
||||
echo_summary "Create a flavor"
|
||||
|
|
|
@ -10,12 +10,10 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
from magnum.common.pythonk8sclient.swagger_client import api_client
|
||||
from magnum.common.pythonk8sclient.swagger_client.apis import apiv_api
|
||||
from magnum.tests.functional.python_client_base import BayAPITLSTest
|
||||
from magnum.tests.functional.python_client_base import BayTest
|
||||
from magnumclient.openstack.common.apiclient import exceptions
|
||||
|
||||
|
||||
class TestBayModelResource(BayTest):
|
||||
|
@ -64,18 +62,6 @@ extendedKeyUsage = clientAuth
|
|||
ca_certs=cls.ca_file)
|
||||
cls.k8s_api = apiv_api.ApivApi(k8s_client)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls._delete_bay(cls.bay.uuid)
|
||||
try:
|
||||
cls._wait_on_status(cls.bay,
|
||||
["CREATE_COMPLETE",
|
||||
"DELETE_IN_PROGRESS", "CREATE_FAILED"],
|
||||
["DELETE_FAILED", "DELETE_COMPLETE"])
|
||||
except exceptions.NotFound:
|
||||
pass
|
||||
cls._delete_baymodel(cls.baymodel.uuid)
|
||||
|
||||
def test_pod_apis(self):
|
||||
pod_manifest = {'apiVersion': 'v1',
|
||||
'kind': 'Pod',
|
||||
|
|
|
@ -51,6 +51,7 @@ class BaseMagnumClient(base.TestCase):
|
|||
nic_id = cliutils.env('NIC_ID')
|
||||
flavor_id = cliutils.env('FLAVOR_ID')
|
||||
keypair_id = cliutils.env('KEYPAIR_ID')
|
||||
copy_logs = cliutils.env('COPY_LOGS')
|
||||
|
||||
config = ConfigParser.RawConfigParser()
|
||||
if config.read('functional_creds.conf'):
|
||||
|
@ -65,11 +66,16 @@ class BaseMagnumClient(base.TestCase):
|
|||
nic_id = nic_id or config.get('magnum', 'nic_id')
|
||||
flavor_id = flavor_id or config.get('magnum', 'flavor_id')
|
||||
keypair_id = keypair_id or config.get('magnum', 'keypair_id')
|
||||
try:
|
||||
copy_logs = copy_logs or config.get('magnum', 'copy_logs')
|
||||
except ConfigParser.NoOptionError:
|
||||
pass
|
||||
|
||||
cls.image_id = image_id
|
||||
cls.nic_id = nic_id
|
||||
cls.flavor_id = flavor_id
|
||||
cls.keypair_id = keypair_id
|
||||
cls.copy_logs = bool(copy_logs)
|
||||
cls.cs = v1client.Client(username=user,
|
||||
api_key=passwd,
|
||||
project_id=tenant_id,
|
||||
|
@ -134,6 +140,11 @@ class BaseMagnumClient(base.TestCase):
|
|||
"CREATE_COMPLETE"])
|
||||
return bay
|
||||
|
||||
@classmethod
|
||||
def _show_bay(cls, name):
|
||||
bay = cls.cs.bays.get(name)
|
||||
return bay
|
||||
|
||||
@classmethod
|
||||
def _delete_baymodel(cls, baymodel_uuid):
|
||||
cls.cs.baymodels.delete(baymodel_uuid)
|
||||
|
@ -142,6 +153,19 @@ class BaseMagnumClient(base.TestCase):
|
|||
def _delete_bay(cls, bay_uuid):
|
||||
cls.cs.bays.delete(bay_uuid)
|
||||
|
||||
@classmethod
|
||||
def _copy_logs(cls):
|
||||
if not cls.copy_logs:
|
||||
return
|
||||
|
||||
cls.bay = cls._show_bay(cls.bay.uuid)
|
||||
for node_addr in cls.bay.node_addresses:
|
||||
subprocess.call(["magnum/tests/contrib/copy_instance_logs.sh",
|
||||
node_addr, cls.baymodel.coe, "worker"])
|
||||
for node_addr in getattr(cls.bay, 'master_addresses', []):
|
||||
subprocess.call(["magnum/tests/contrib/copy_instance_logs.sh",
|
||||
node_addr, cls.baymodel.coe, "master"])
|
||||
|
||||
|
||||
class BayTest(BaseMagnumClient):
|
||||
|
||||
|
@ -207,6 +231,10 @@ class BayAPITLSTest(BaseMagnumClient):
|
|||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
try:
|
||||
cls._copy_logs()
|
||||
except Exception as e:
|
||||
print("WARNING: Failed to copy logs. Error: " + repr(e))
|
||||
|
||||
if cls.ca_dir:
|
||||
rmtree_without_raise(cls.ca_dir)
|
||||
|
|
Loading…
Reference in New Issue