Record diagnostic info from bay nodes
In functional tests, copy logs from bay nodes to host before the tests tear down. The bay node logs could be helpful for trouble-shooting. Change-Id: I102d2ce37c2253c743719e4ccf978c23a92a3a3f
This commit is contained in:
parent
08c42e29e5
commit
035a07fe4d
|
@ -0,0 +1,70 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
# Save trace setting
|
||||||
|
XTRACE=$(set +o | grep xtrace)
|
||||||
|
set -o xtrace
|
||||||
|
|
||||||
|
echo "Magnum's copy_instance_logs.sh was called..."
|
||||||
|
|
||||||
|
SSH_IP=$1
|
||||||
|
COE=${2-kubernetes}
|
||||||
|
NODE_TYPE=${3-master}
|
||||||
|
LOG_PATH=/opt/stack/logs/bay-nodes/${NODE_TYPE}-${SSH_IP}
|
||||||
|
|
||||||
|
function remote_exec {
|
||||||
|
local ssh_user=$1
|
||||||
|
local cmd=$2
|
||||||
|
local logfile=${LOG_PATH}/$3
|
||||||
|
ssh -o StrictHostKeyChecking=no ${ssh_user}@${SSH_IP} "${cmd}" > ${logfile} 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
mkdir -p $LOG_PATH
|
||||||
|
|
||||||
|
if [[ "$COE" == "kubernetes" ]]; then
|
||||||
|
SSH_USER=minion
|
||||||
|
remote_exec $SSH_USER "sudo systemctl --full list-units" systemctl_list_units.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u cloud-config --no-pager" cloud-config.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u cloud-final --no-pager" cloud-final.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u cloud-init-local --no-pager" cloud-init-local.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u cloud-init --no-pager" cloud-init.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u kubelet --no-pager" kubelet.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u kube-proxy --no-pager" kube-proxy.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u etcd --no-pager" etcd.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u kube-apiserver --no-pager" kube-apiserver.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u kube-scheduler --no-pager" kube-scheduler.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u kube-controller-manager --no-pager" kube-controller-manager.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u docker --no-pager" docker.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u flanneld --no-pager" flanneld.log
|
||||||
|
elif [[ "$COE" == "swarm" ]]; then
|
||||||
|
SSH_USER=fedora
|
||||||
|
remote_exec $SSH_USER "sudo systemctl list-units" systemctl_list_units.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u cloud-config --no-pager" cloud-config.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u cloud-final --no-pager" cloud-final.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u cloud-init-local --no-pager" cloud-init-local.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u cloud-init --no-pager" cloud-init.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u etcd --no-pager" etcd.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u swarm-manager --no-pager" swarm-manager.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u swarm-agent --no-pager" swarm-agent.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u docker --no-pager" docker.log
|
||||||
|
remote_exec $SSH_USER "sudo journalctl -u flanneld --no-pager" flanneld.log
|
||||||
|
else
|
||||||
|
echo "ERROR: Unknown COE '${COE}'"
|
||||||
|
EXIT_CODE=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Restore xtrace
|
||||||
|
$XTRACE
|
||||||
|
|
||||||
|
exit $EXIT_CODE
|
|
@ -80,6 +80,7 @@ image_id = $IMAGE_ID
|
||||||
nic_id = $NIC_ID
|
nic_id = $NIC_ID
|
||||||
keypair_id = default
|
keypair_id = default
|
||||||
flavor_id = m1.magnum
|
flavor_id = m1.magnum
|
||||||
|
copy_logs = true
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
# Note(eliqiao): Let's keep this only for debugging on gate.
|
# Note(eliqiao): Let's keep this only for debugging on gate.
|
||||||
|
@ -88,7 +89,8 @@ cat $CREDS_FILE
|
||||||
|
|
||||||
# Create a keypair for use in the functional tests.
|
# Create a keypair for use in the functional tests.
|
||||||
echo_summary "Generate a key-pair"
|
echo_summary "Generate a key-pair"
|
||||||
nova keypair-add default
|
ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa
|
||||||
|
nova keypair-add --pub-key ~/.ssh/id_rsa.pub default
|
||||||
|
|
||||||
# Create magnum specific flavor for use in functional tests.
|
# Create magnum specific flavor for use in functional tests.
|
||||||
echo_summary "Create a flavor"
|
echo_summary "Create a flavor"
|
||||||
|
|
|
@ -10,12 +10,10 @@
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
|
|
||||||
from magnum.common.pythonk8sclient.swagger_client import api_client
|
from magnum.common.pythonk8sclient.swagger_client import api_client
|
||||||
from magnum.common.pythonk8sclient.swagger_client.apis import apiv_api
|
from magnum.common.pythonk8sclient.swagger_client.apis import apiv_api
|
||||||
from magnum.tests.functional.python_client_base import BayAPITLSTest
|
from magnum.tests.functional.python_client_base import BayAPITLSTest
|
||||||
from magnum.tests.functional.python_client_base import BayTest
|
from magnum.tests.functional.python_client_base import BayTest
|
||||||
from magnumclient.openstack.common.apiclient import exceptions
|
|
||||||
|
|
||||||
|
|
||||||
class TestBayModelResource(BayTest):
|
class TestBayModelResource(BayTest):
|
||||||
|
@ -64,18 +62,6 @@ extendedKeyUsage = clientAuth
|
||||||
ca_certs=cls.ca_file)
|
ca_certs=cls.ca_file)
|
||||||
cls.k8s_api = apiv_api.ApivApi(k8s_client)
|
cls.k8s_api = apiv_api.ApivApi(k8s_client)
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def tearDownClass(cls):
|
|
||||||
cls._delete_bay(cls.bay.uuid)
|
|
||||||
try:
|
|
||||||
cls._wait_on_status(cls.bay,
|
|
||||||
["CREATE_COMPLETE",
|
|
||||||
"DELETE_IN_PROGRESS", "CREATE_FAILED"],
|
|
||||||
["DELETE_FAILED", "DELETE_COMPLETE"])
|
|
||||||
except exceptions.NotFound:
|
|
||||||
pass
|
|
||||||
cls._delete_baymodel(cls.baymodel.uuid)
|
|
||||||
|
|
||||||
def test_pod_apis(self):
|
def test_pod_apis(self):
|
||||||
pod_manifest = {'apiVersion': 'v1',
|
pod_manifest = {'apiVersion': 'v1',
|
||||||
'kind': 'Pod',
|
'kind': 'Pod',
|
||||||
|
|
|
@ -51,6 +51,7 @@ class BaseMagnumClient(base.TestCase):
|
||||||
nic_id = cliutils.env('NIC_ID')
|
nic_id = cliutils.env('NIC_ID')
|
||||||
flavor_id = cliutils.env('FLAVOR_ID')
|
flavor_id = cliutils.env('FLAVOR_ID')
|
||||||
keypair_id = cliutils.env('KEYPAIR_ID')
|
keypair_id = cliutils.env('KEYPAIR_ID')
|
||||||
|
copy_logs = cliutils.env('COPY_LOGS')
|
||||||
|
|
||||||
config = ConfigParser.RawConfigParser()
|
config = ConfigParser.RawConfigParser()
|
||||||
if config.read('functional_creds.conf'):
|
if config.read('functional_creds.conf'):
|
||||||
|
@ -65,11 +66,16 @@ class BaseMagnumClient(base.TestCase):
|
||||||
nic_id = nic_id or config.get('magnum', 'nic_id')
|
nic_id = nic_id or config.get('magnum', 'nic_id')
|
||||||
flavor_id = flavor_id or config.get('magnum', 'flavor_id')
|
flavor_id = flavor_id or config.get('magnum', 'flavor_id')
|
||||||
keypair_id = keypair_id or config.get('magnum', 'keypair_id')
|
keypair_id = keypair_id or config.get('magnum', 'keypair_id')
|
||||||
|
try:
|
||||||
|
copy_logs = copy_logs or config.get('magnum', 'copy_logs')
|
||||||
|
except ConfigParser.NoOptionError:
|
||||||
|
pass
|
||||||
|
|
||||||
cls.image_id = image_id
|
cls.image_id = image_id
|
||||||
cls.nic_id = nic_id
|
cls.nic_id = nic_id
|
||||||
cls.flavor_id = flavor_id
|
cls.flavor_id = flavor_id
|
||||||
cls.keypair_id = keypair_id
|
cls.keypair_id = keypair_id
|
||||||
|
cls.copy_logs = bool(copy_logs)
|
||||||
cls.cs = v1client.Client(username=user,
|
cls.cs = v1client.Client(username=user,
|
||||||
api_key=passwd,
|
api_key=passwd,
|
||||||
project_id=tenant_id,
|
project_id=tenant_id,
|
||||||
|
@ -134,6 +140,11 @@ class BaseMagnumClient(base.TestCase):
|
||||||
"CREATE_COMPLETE"])
|
"CREATE_COMPLETE"])
|
||||||
return bay
|
return bay
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _show_bay(cls, name):
|
||||||
|
bay = cls.cs.bays.get(name)
|
||||||
|
return bay
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _delete_baymodel(cls, baymodel_uuid):
|
def _delete_baymodel(cls, baymodel_uuid):
|
||||||
cls.cs.baymodels.delete(baymodel_uuid)
|
cls.cs.baymodels.delete(baymodel_uuid)
|
||||||
|
@ -142,6 +153,19 @@ class BaseMagnumClient(base.TestCase):
|
||||||
def _delete_bay(cls, bay_uuid):
|
def _delete_bay(cls, bay_uuid):
|
||||||
cls.cs.bays.delete(bay_uuid)
|
cls.cs.bays.delete(bay_uuid)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _copy_logs(cls):
|
||||||
|
if not cls.copy_logs:
|
||||||
|
return
|
||||||
|
|
||||||
|
cls.bay = cls._show_bay(cls.bay.uuid)
|
||||||
|
for node_addr in cls.bay.node_addresses:
|
||||||
|
subprocess.call(["magnum/tests/contrib/copy_instance_logs.sh",
|
||||||
|
node_addr, cls.baymodel.coe, "worker"])
|
||||||
|
for node_addr in getattr(cls.bay, 'master_addresses', []):
|
||||||
|
subprocess.call(["magnum/tests/contrib/copy_instance_logs.sh",
|
||||||
|
node_addr, cls.baymodel.coe, "master"])
|
||||||
|
|
||||||
|
|
||||||
class BayTest(BaseMagnumClient):
|
class BayTest(BaseMagnumClient):
|
||||||
|
|
||||||
|
@ -207,6 +231,10 @@ class BayAPITLSTest(BaseMagnumClient):
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def tearDownClass(cls):
|
def tearDownClass(cls):
|
||||||
|
try:
|
||||||
|
cls._copy_logs()
|
||||||
|
except Exception as e:
|
||||||
|
print("WARNING: Failed to copy logs. Error: " + repr(e))
|
||||||
|
|
||||||
if cls.ca_dir:
|
if cls.ca_dir:
|
||||||
rmtree_without_raise(cls.ca_dir)
|
rmtree_without_raise(cls.ca_dir)
|
||||||
|
|
Loading…
Reference in New Issue