Merge "Copy logs on test failure"
This commit is contained in:
commit
9c9059dd07
@ -22,12 +22,21 @@ SSH_IP=$1
|
|||||||
COE=${2-kubernetes}
|
COE=${2-kubernetes}
|
||||||
NODE_TYPE=${3-master}
|
NODE_TYPE=${3-master}
|
||||||
LOG_PATH=/opt/stack/logs/bay-nodes/${NODE_TYPE}-${SSH_IP}
|
LOG_PATH=/opt/stack/logs/bay-nodes/${NODE_TYPE}-${SSH_IP}
|
||||||
|
KEYPAIR=${4-default}
|
||||||
|
|
||||||
function remote_exec {
|
function remote_exec {
|
||||||
|
local priv_key
|
||||||
|
echo "If private key is specified, save to temp and use that; else, use default"
|
||||||
|
if [[ "$KEYPAIR" == "default" ]]; then
|
||||||
|
priv_key="~/.ssh/id_rsa"
|
||||||
|
else
|
||||||
|
priv_key="$(mktemp id_rsa.$SSH_IP.XXX)"
|
||||||
|
echo -en "$KEYPAIR" > $priv_key
|
||||||
|
fi
|
||||||
local ssh_user=$1
|
local ssh_user=$1
|
||||||
local cmd=$2
|
local cmd=$2
|
||||||
local logfile=${LOG_PATH}/$3
|
local logfile=${LOG_PATH}/$3
|
||||||
ssh -o StrictHostKeyChecking=no ${ssh_user}@${SSH_IP} "${cmd}" > ${logfile} 2>&1
|
ssh -i $priv_key -o StrictHostKeyChecking=no ${ssh_user}@${SSH_IP} "${cmd}" > ${logfile} 2>&1
|
||||||
}
|
}
|
||||||
|
|
||||||
mkdir -p $LOG_PATH
|
mkdir -p $LOG_PATH
|
||||||
@ -55,7 +64,7 @@ if [[ "$COE" == "kubernetes" ]]; then
|
|||||||
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker" docker.sysconfig.env.log
|
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker" docker.sysconfig.env.log
|
||||||
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker-storage" docker-storage.sysconfig.env.log
|
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker-storage" docker-storage.sysconfig.env.log
|
||||||
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker-network" docker-network.sysconfig.env.log
|
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker-network" docker-network.sysconfig.env.log
|
||||||
remote_exec $SSH_USER "sudo docker ps --all=true --no-trunc=true" docker-containers.log
|
remote_exec $SSH_USER "sudo timeout 60s docker ps --all=true --no-trunc=true" docker-containers.log
|
||||||
remote_exec $SSH_USER "sudo tar zcvf - /var/lib/docker/containers 2>/dev/null" docker-container-configs.tar.gz
|
remote_exec $SSH_USER "sudo tar zcvf - /var/lib/docker/containers 2>/dev/null" docker-container-configs.tar.gz
|
||||||
remote_exec $SSH_USER "sudo journalctl -u flanneld --no-pager" flanneld.log
|
remote_exec $SSH_USER "sudo journalctl -u flanneld --no-pager" flanneld.log
|
||||||
remote_exec $SSH_USER "sudo ip a" ipa.log
|
remote_exec $SSH_USER "sudo ip a" ipa.log
|
||||||
@ -81,7 +90,7 @@ elif [[ "$COE" == "swarm" ]]; then
|
|||||||
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker" docker.sysconfig.env.log
|
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker" docker.sysconfig.env.log
|
||||||
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker-storage" docker-storage.sysconfig.env.log
|
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker-storage" docker-storage.sysconfig.env.log
|
||||||
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker-network" docker-network.sysconfig.env.log
|
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker-network" docker-network.sysconfig.env.log
|
||||||
remote_exec $SSH_USER "sudo docker ps --all=true --no-trunc=true" docker-containers.log
|
remote_exec $SSH_USER "sudo timeout 60s docker ps --all=true --no-trunc=true" docker-containers.log
|
||||||
remote_exec $SSH_USER "sudo tar zcvf - /var/lib/docker/containers 2>/dev/null" docker-container-configs.tar.gz
|
remote_exec $SSH_USER "sudo tar zcvf - /var/lib/docker/containers 2>/dev/null" docker-container-configs.tar.gz
|
||||||
remote_exec $SSH_USER "sudo journalctl -u flanneld --no-pager" flanneld.log
|
remote_exec $SSH_USER "sudo journalctl -u flanneld --no-pager" flanneld.log
|
||||||
remote_exec $SSH_USER "sudo ip a" ipa.log
|
remote_exec $SSH_USER "sudo ip a" ipa.log
|
||||||
|
@ -30,7 +30,8 @@ class BayTest(base.BaseMagnumTest):
|
|||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
super(BayTest, self).__init__(*args, **kwargs)
|
super(BayTest, self).__init__(*args, **kwargs)
|
||||||
self.bays = []
|
self.bays = []
|
||||||
self.credentials = None
|
self.creds = None
|
||||||
|
self.keypair = None
|
||||||
self.baymodel = None
|
self.baymodel = None
|
||||||
self.baymodel_client = None
|
self.baymodel_client = None
|
||||||
self.keypairs_client = None
|
self.keypairs_client = None
|
||||||
@ -40,18 +41,19 @@ class BayTest(base.BaseMagnumTest):
|
|||||||
def setUp(self):
|
def setUp(self):
|
||||||
try:
|
try:
|
||||||
super(BayTest, self).setUp()
|
super(BayTest, self).setUp()
|
||||||
self.credentials = self.get_credentials(type_of_creds='default')
|
(self.creds, self.keypair) = self.get_credentials_with_keypair(
|
||||||
|
type_of_creds='default')
|
||||||
(self.baymodel_client,
|
(self.baymodel_client,
|
||||||
self.keypairs_client) = self.get_clients_with_existing_creds(
|
self.keypairs_client) = self.get_clients_with_existing_creds(
|
||||||
creds=self.credentials,
|
creds=self.creds,
|
||||||
type_of_creds='default',
|
type_of_creds='default',
|
||||||
request_type='baymodel')
|
request_type='baymodel')
|
||||||
(self.bay_client, _) = self.get_clients_with_existing_creds(
|
(self.bay_client, _) = self.get_clients_with_existing_creds(
|
||||||
creds=self.credentials,
|
creds=self.creds,
|
||||||
type_of_creds='default',
|
type_of_creds='default',
|
||||||
request_type='bay')
|
request_type='bay')
|
||||||
(self.cert_client, _) = self.get_clients_with_existing_creds(
|
(self.cert_client, _) = self.get_clients_with_existing_creds(
|
||||||
creds=self.credentials,
|
creds=self.creds,
|
||||||
type_of_creds='default',
|
type_of_creds='default',
|
||||||
request_type='cert')
|
request_type='cert')
|
||||||
model = datagen.valid_swarm_baymodel()
|
model = datagen.valid_swarm_baymodel()
|
||||||
@ -91,11 +93,18 @@ class BayTest(base.BaseMagnumTest):
|
|||||||
self.LOG.debug('Response: %s' % resp)
|
self.LOG.debug('Response: %s' % resp)
|
||||||
self.assertEqual(resp.status, 201)
|
self.assertEqual(resp.status, 201)
|
||||||
self.assertIsNotNone(model.uuid)
|
self.assertIsNotNone(model.uuid)
|
||||||
|
self.bays.append(model.uuid)
|
||||||
self.assertIsNone(model.status)
|
self.assertIsNone(model.status)
|
||||||
self.assertIsNone(model.status_reason)
|
self.assertIsNone(model.status_reason)
|
||||||
self.assertEqual(model.baymodel_id, self.baymodel.uuid)
|
self.assertEqual(model.baymodel_id, self.baymodel.uuid)
|
||||||
self.bay_client.wait_for_created_bay(model.uuid)
|
self.bay_uuid = model.uuid
|
||||||
self.bays.append(model.uuid)
|
self.addOnException(self.copy_logs_handler(
|
||||||
|
lambda: list(
|
||||||
|
self._get_bay_by_id(self.bay_uuid)[1].node_addresses +
|
||||||
|
self._get_bay_by_id(self.bay_uuid)[1].master_addresses),
|
||||||
|
self.baymodel.coe,
|
||||||
|
self.keypair))
|
||||||
|
self.bay_client.wait_for_created_bay(model.uuid, delete_on_error=False)
|
||||||
return resp, model
|
return resp, model
|
||||||
|
|
||||||
def _delete_bay(self, bay_id):
|
def _delete_bay(self, bay_id):
|
||||||
@ -105,6 +114,10 @@ class BayTest(base.BaseMagnumTest):
|
|||||||
self.bay_client.wait_for_bay_to_delete(bay_id)
|
self.bay_client.wait_for_bay_to_delete(bay_id)
|
||||||
return resp, model
|
return resp, model
|
||||||
|
|
||||||
|
def _get_bay_by_id(self, bay_id):
|
||||||
|
resp, model = self.bay_client.get_bay(bay_id)
|
||||||
|
return resp, model
|
||||||
|
|
||||||
# (dimtruck) Combining all these tests in one because
|
# (dimtruck) Combining all these tests in one because
|
||||||
# they time out on the gate (2 hours not enough)
|
# they time out on the gate (2 hours not enough)
|
||||||
@testtools.testcase.attr('positive')
|
@testtools.testcase.attr('positive')
|
||||||
|
@ -11,10 +11,14 @@
|
|||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
import inspect
|
import inspect
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
|
||||||
from tempest.common import credentials_factory as common_creds
|
from tempest.common import credentials_factory as common_creds
|
||||||
from tempest_lib import base
|
from tempest_lib import base
|
||||||
|
|
||||||
|
import magnum
|
||||||
from magnum.tests.functional.common import config
|
from magnum.tests.functional.common import config
|
||||||
from magnum.tests.functional.common import manager
|
from magnum.tests.functional.common import manager
|
||||||
|
|
||||||
@ -24,6 +28,7 @@ class BaseMagnumTest(base.BaseTestCase):
|
|||||||
|
|
||||||
ic_class_list = []
|
ic_class_list = []
|
||||||
ic_method_list = []
|
ic_method_list = []
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
super(BaseMagnumTest, self).__init__(*args, **kwargs)
|
super(BaseMagnumTest, self).__init__(*args, **kwargs)
|
||||||
@ -57,6 +62,14 @@ class BaseMagnumTest(base.BaseTestCase):
|
|||||||
def get_credentials(cls, name=None,
|
def get_credentials(cls, name=None,
|
||||||
type_of_creds="default",
|
type_of_creds="default",
|
||||||
class_cleanup=False):
|
class_cleanup=False):
|
||||||
|
(creds, _) = cls.get_credentials_with_keypair(name, type_of_creds,
|
||||||
|
class_cleanup)
|
||||||
|
return creds
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_credentials_with_keypair(cls, name=None,
|
||||||
|
type_of_creds="default",
|
||||||
|
class_cleanup=False):
|
||||||
if name is None:
|
if name is None:
|
||||||
# Get name of test method
|
# Get name of test method
|
||||||
name = inspect.stack()[1][3]
|
name = inspect.stack()[1][3]
|
||||||
@ -86,12 +99,16 @@ class BaseMagnumTest(base.BaseTestCase):
|
|||||||
|
|
||||||
_, keypairs_client = cls.get_clients(
|
_, keypairs_client = cls.get_clients(
|
||||||
creds, type_of_creds, 'keypair_setup')
|
creds, type_of_creds, 'keypair_setup')
|
||||||
|
|
||||||
|
keypair = None
|
||||||
try:
|
try:
|
||||||
keypairs_client.show_keypair(config.Config.keypair_id)
|
keypairs_client.show_keypair(config.Config.keypair_id)
|
||||||
except Exception:
|
except Exception:
|
||||||
keypairs_client.create_keypair(name=config.Config.keypair_id)
|
keypair_body = keypairs_client.create_keypair(
|
||||||
|
name=config.Config.keypair_id)
|
||||||
return creds
|
cls.LOG.debug("Keypair body: %s" % keypair_body)
|
||||||
|
keypair = keypair_body['keypair']['private_key']
|
||||||
|
return (creds, keypair)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_clients(cls, creds, type_of_creds, request_type):
|
def get_clients(cls, creds, type_of_creds, request_type):
|
||||||
@ -142,3 +159,53 @@ class BaseMagnumTest(base.BaseTestCase):
|
|||||||
"""
|
"""
|
||||||
creds = cls.get_credentials(name, type_of_creds, class_cleanup)
|
creds = cls.get_credentials(name, type_of_creds, class_cleanup)
|
||||||
return cls.get_clients(creds, type_of_creds, request_type)
|
return cls.get_clients(creds, type_of_creds, request_type)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def copy_logs_handler(cls, get_nodes_fn, coe, keypair):
|
||||||
|
"""Copy logs closure.
|
||||||
|
|
||||||
|
This method will retrieve all running nodes for a specified bay
|
||||||
|
and copy addresses from there locally.
|
||||||
|
|
||||||
|
:param get_nodes_fn: function that takes no parameters and returns
|
||||||
|
a list of node IPs to get logs from
|
||||||
|
:param coe: the COE type of the nodes
|
||||||
|
"""
|
||||||
|
|
||||||
|
if not config.Config.copy_logs:
|
||||||
|
return lambda: None
|
||||||
|
|
||||||
|
def int_copy_logs(exec_info):
|
||||||
|
try:
|
||||||
|
cls.LOG.debug("Copying logs...")
|
||||||
|
fn = exec_info[2].tb_frame.f_locals['fn']
|
||||||
|
func_name = fn.im_self._get_test_method().__name__
|
||||||
|
msg = "Failed to copy logs for bay"
|
||||||
|
nodes_addresses = get_nodes_fn()
|
||||||
|
|
||||||
|
for node_address in nodes_addresses:
|
||||||
|
log_name = "node-" + func_name
|
||||||
|
try:
|
||||||
|
base_path = os.path.split(os.path.dirname(
|
||||||
|
os.path.abspath(magnum.__file__)))[0]
|
||||||
|
script = "magnum/tests/contrib/copy_instance_logs.sh"
|
||||||
|
full_location = os.path.join(base_path, script)
|
||||||
|
cls.LOG.debug("running %s" % full_location)
|
||||||
|
cls.LOG.debug("keypair: %s" % keypair)
|
||||||
|
subprocess.check_call([
|
||||||
|
full_location,
|
||||||
|
node_address,
|
||||||
|
coe,
|
||||||
|
log_name,
|
||||||
|
str(keypair)
|
||||||
|
])
|
||||||
|
except Exception:
|
||||||
|
cls.LOG.exception(msg)
|
||||||
|
cls.LOG.exception("failed to copy from %s to %s%s-%s" %
|
||||||
|
(node_address,
|
||||||
|
"/opt/stack/logs/bay-nodes/",
|
||||||
|
log_name, node_address))
|
||||||
|
except Exception:
|
||||||
|
cls.LOG.exception(msg)
|
||||||
|
|
||||||
|
return int_copy_logs
|
||||||
|
@ -103,6 +103,12 @@ class Config(object):
|
|||||||
raise Exception('config missing csr_location key')
|
raise Exception('config missing csr_location key')
|
||||||
cls.csr_location = CONF.magnum.csr_location
|
cls.csr_location = CONF.magnum.csr_location
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def set_copy_logs(cls, config):
|
||||||
|
if 'copy_logs' not in CONF.magnum:
|
||||||
|
cls.copy_logs = True
|
||||||
|
cls.copy_logs = CONF.magnum.copy_logs
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def setUp(cls):
|
def setUp(cls):
|
||||||
cls.set_admin_creds(config)
|
cls.set_admin_creds(config)
|
||||||
@ -119,3 +125,4 @@ class Config(object):
|
|||||||
cls.set_magnum_url(config)
|
cls.set_magnum_url(config)
|
||||||
cls.set_master_flavor_id(config)
|
cls.set_master_flavor_id(config)
|
||||||
cls.set_csr_location(config)
|
cls.set_csr_location(config)
|
||||||
|
cls.set_copy_logs(config)
|
||||||
|
@ -22,11 +22,10 @@ from magnum.tests.functional.common import config
|
|||||||
|
|
||||||
|
|
||||||
class Manager(clients.Manager):
|
class Manager(clients.Manager):
|
||||||
def __init__(
|
def __init__(self, credentials=None, request_type=None):
|
||||||
self,
|
if not credentials:
|
||||||
credentials=common_creds.get_configured_credentials(
|
credentials = common_creds.get_configured_credentials(
|
||||||
'identity_admin'),
|
'identity_admin')
|
||||||
request_type=None):
|
|
||||||
super(Manager, self).__init__(credentials, 'container')
|
super(Manager, self).__init__(credentials, 'container')
|
||||||
self.auth_provider.orig_base_url = self.auth_provider.base_url
|
self.auth_provider.orig_base_url = self.auth_provider.base_url
|
||||||
self.auth_provider.base_url = self.bypassed_base_url
|
self.auth_provider.base_url = self.bypassed_base_url
|
||||||
|
@ -55,4 +55,8 @@ MagnumGroup = [
|
|||||||
cfg.StrOpt("csr_location",
|
cfg.StrOpt("csr_location",
|
||||||
default="/opt/stack/new/magnum/default.csr",
|
default="/opt/stack/new/magnum/default.csr",
|
||||||
help="CSR location for certificates."),
|
help="CSR location for certificates."),
|
||||||
|
|
||||||
|
cfg.StrOpt("copy_logs",
|
||||||
|
default=True,
|
||||||
|
help="Specify whether to copy nova server logs on failure."),
|
||||||
]
|
]
|
||||||
|
Loading…
Reference in New Issue
Block a user