Copy logs on test failure
This patch will copy logs from nova servers to local environment on test failures. Don't delete bays until teardown so log gathering will work. Give functional test types a common base. Added timeout to docker ps; else, the command would time out, failing to run the rest of the tests. Closes-Bug: 1542390 Change-Id: I016ed8fe311bede407f57b7982126c4b77749bec Co-Authored-By: Corey O'Brien <coreypobrien@gmail.com>
This commit is contained in:
parent
687931fab1
commit
7d524491c8
|
@ -30,7 +30,10 @@ from magnum.tests import policy_fixture
|
|||
|
||||
|
||||
CONF = cfg.CONF
|
||||
log.register_options(CONF)
|
||||
try:
|
||||
log.register_options(CONF)
|
||||
except cfg.ArgsAlreadyParsedError:
|
||||
pass
|
||||
CONF.set_override('use_stderr', False)
|
||||
|
||||
|
||||
|
|
|
@ -16,7 +16,6 @@
|
|||
|
||||
import fixtures
|
||||
from oslo_config import cfg
|
||||
from oslo_log import log
|
||||
|
||||
from magnum.common import config
|
||||
|
||||
|
@ -30,7 +29,6 @@ class ConfFixture(fixtures.Fixture):
|
|||
"""Fixture to manage global conf settings."""
|
||||
|
||||
def _setUp(self):
|
||||
log.register_options(cfg.CONF)
|
||||
CONF.set_default('host', 'fake-mini')
|
||||
CONF.set_default('connection', "sqlite://", group='database')
|
||||
CONF.set_default('sqlite_synchronous', False, group='database')
|
||||
|
|
|
@ -22,12 +22,21 @@ SSH_IP=$1
|
|||
COE=${2-kubernetes}
|
||||
NODE_TYPE=${3-master}
|
||||
LOG_PATH=/opt/stack/logs/bay-nodes/${NODE_TYPE}-${SSH_IP}
|
||||
KEYPAIR=${4-default}
|
||||
|
||||
function remote_exec {
|
||||
local priv_key
|
||||
echo "If private key is specified, save to temp and use that; else, use default"
|
||||
if [[ "$KEYPAIR" == "default" ]]; then
|
||||
priv_key="~/.ssh/id_rsa"
|
||||
else
|
||||
priv_key="$(mktemp id_rsa.$SSH_IP.XXX)"
|
||||
echo -en "$KEYPAIR" > $priv_key
|
||||
fi
|
||||
local ssh_user=$1
|
||||
local cmd=$2
|
||||
local logfile=${LOG_PATH}/$3
|
||||
ssh -o StrictHostKeyChecking=no ${ssh_user}@${SSH_IP} "${cmd}" > ${logfile} 2>&1
|
||||
ssh -i $priv_key -o StrictHostKeyChecking=no ${ssh_user}@${SSH_IP} "${cmd}" > ${logfile} 2>&1
|
||||
}
|
||||
|
||||
mkdir -p $LOG_PATH
|
||||
|
@ -55,7 +64,7 @@ if [[ "$COE" == "kubernetes" ]]; then
|
|||
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker" docker.sysconfig.env.log
|
||||
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker-storage" docker-storage.sysconfig.env.log
|
||||
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker-network" docker-network.sysconfig.env.log
|
||||
remote_exec $SSH_USER "sudo docker ps --all=true --no-trunc=true" docker-containers.log
|
||||
remote_exec $SSH_USER "sudo timeout 60s docker ps --all=true --no-trunc=true" docker-containers.log
|
||||
remote_exec $SSH_USER "sudo tar zcvf - /var/lib/docker/containers 2>/dev/null" docker-container-configs.tar.gz
|
||||
remote_exec $SSH_USER "sudo journalctl -u flanneld --no-pager" flanneld.log
|
||||
remote_exec $SSH_USER "sudo ip a" ipa.log
|
||||
|
@ -81,7 +90,7 @@ elif [[ "$COE" == "swarm" ]]; then
|
|||
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker" docker.sysconfig.env.log
|
||||
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker-storage" docker-storage.sysconfig.env.log
|
||||
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker-network" docker-network.sysconfig.env.log
|
||||
remote_exec $SSH_USER "sudo docker ps --all=true --no-trunc=true" docker-containers.log
|
||||
remote_exec $SSH_USER "sudo timeout 60s docker ps --all=true --no-trunc=true" docker-containers.log
|
||||
remote_exec $SSH_USER "sudo tar zcvf - /var/lib/docker/containers 2>/dev/null" docker-container-configs.tar.gz
|
||||
remote_exec $SSH_USER "sudo journalctl -u flanneld --no-pager" flanneld.log
|
||||
remote_exec $SSH_USER "sudo ip a" ipa.log
|
||||
|
|
|
@ -185,6 +185,7 @@ if [[ "api" == "$coe" ]]; then
|
|||
iniset $BASE/new/tempest/etc/tempest.conf magnum flavor_id m1.magnum2
|
||||
iniset $BASE/new/tempest/etc/tempest.conf magnum master_flavor_id m1.magnum
|
||||
iniset $BASE/new/tempest/etc/tempest.conf magnum csr_location $CSR_FILE
|
||||
iniset $BASE/new/tempest/etc/tempest.conf magnum copy_logs True
|
||||
|
||||
# show tempest config with magnum
|
||||
cat etc/tempest.conf
|
||||
|
|
|
@ -30,7 +30,8 @@ class BayTest(base.BaseMagnumTest):
|
|||
def __init__(self, *args, **kwargs):
|
||||
super(BayTest, self).__init__(*args, **kwargs)
|
||||
self.bays = []
|
||||
self.credentials = None
|
||||
self.creds = None
|
||||
self.keypair = None
|
||||
self.baymodel = None
|
||||
self.baymodel_client = None
|
||||
self.keypairs_client = None
|
||||
|
@ -40,18 +41,19 @@ class BayTest(base.BaseMagnumTest):
|
|||
def setUp(self):
|
||||
try:
|
||||
super(BayTest, self).setUp()
|
||||
self.credentials = self.get_credentials(type_of_creds='default')
|
||||
(self.creds, self.keypair) = self.get_credentials_with_keypair(
|
||||
type_of_creds='default')
|
||||
(self.baymodel_client,
|
||||
self.keypairs_client) = self.get_clients_with_existing_creds(
|
||||
creds=self.credentials,
|
||||
type_of_creds='default',
|
||||
request_type='baymodel')
|
||||
self.keypairs_client) = self.get_clients_with_existing_creds(
|
||||
creds=self.creds,
|
||||
type_of_creds='default',
|
||||
request_type='baymodel')
|
||||
(self.bay_client, _) = self.get_clients_with_existing_creds(
|
||||
creds=self.credentials,
|
||||
creds=self.creds,
|
||||
type_of_creds='default',
|
||||
request_type='bay')
|
||||
(self.cert_client, _) = self.get_clients_with_existing_creds(
|
||||
creds=self.credentials,
|
||||
creds=self.creds,
|
||||
type_of_creds='default',
|
||||
request_type='cert')
|
||||
model = datagen.valid_swarm_baymodel()
|
||||
|
@ -91,11 +93,18 @@ class BayTest(base.BaseMagnumTest):
|
|||
self.LOG.debug('Response: %s' % resp)
|
||||
self.assertEqual(resp.status, 201)
|
||||
self.assertIsNotNone(model.uuid)
|
||||
self.bays.append(model.uuid)
|
||||
self.assertIsNone(model.status)
|
||||
self.assertIsNone(model.status_reason)
|
||||
self.assertEqual(model.baymodel_id, self.baymodel.uuid)
|
||||
self.bay_client.wait_for_created_bay(model.uuid)
|
||||
self.bays.append(model.uuid)
|
||||
self.bay_uuid = model.uuid
|
||||
self.addOnException(self.copy_logs_handler(
|
||||
lambda: list(
|
||||
self._get_bay_by_id(self.bay_uuid)[1].node_addresses +
|
||||
self._get_bay_by_id(self.bay_uuid)[1].master_addresses),
|
||||
self.baymodel.coe,
|
||||
self.keypair))
|
||||
self.bay_client.wait_for_created_bay(model.uuid, delete_on_error=False)
|
||||
return resp, model
|
||||
|
||||
def _delete_bay(self, bay_id):
|
||||
|
@ -105,6 +114,10 @@ class BayTest(base.BaseMagnumTest):
|
|||
self.bay_client.wait_for_bay_to_delete(bay_id)
|
||||
return resp, model
|
||||
|
||||
def _get_bay_by_id(self, bay_id):
|
||||
resp, model = self.bay_client.get_bay(bay_id)
|
||||
return resp, model
|
||||
|
||||
# (dimtruck) Combining all these tests in one because
|
||||
# they time out on the gate (2 hours not enough)
|
||||
@testtools.testcase.attr('positive')
|
||||
|
|
|
@ -11,10 +11,14 @@
|
|||
# under the License.
|
||||
|
||||
import inspect
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from tempest.common import credentials_factory as common_creds
|
||||
from tempest_lib import base
|
||||
|
||||
import magnum
|
||||
from magnum.tests.functional.common import config
|
||||
from magnum.tests.functional.common import manager
|
||||
|
||||
|
@ -24,6 +28,7 @@ class BaseMagnumTest(base.BaseTestCase):
|
|||
|
||||
ic_class_list = []
|
||||
ic_method_list = []
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(BaseMagnumTest, self).__init__(*args, **kwargs)
|
||||
|
@ -57,6 +62,14 @@ class BaseMagnumTest(base.BaseTestCase):
|
|||
def get_credentials(cls, name=None,
|
||||
type_of_creds="default",
|
||||
class_cleanup=False):
|
||||
(creds, _) = cls.get_credentials_with_keypair(name, type_of_creds,
|
||||
class_cleanup)
|
||||
return creds
|
||||
|
||||
@classmethod
|
||||
def get_credentials_with_keypair(cls, name=None,
|
||||
type_of_creds="default",
|
||||
class_cleanup=False):
|
||||
if name is None:
|
||||
# Get name of test method
|
||||
name = inspect.stack()[1][3]
|
||||
|
@ -86,12 +99,16 @@ class BaseMagnumTest(base.BaseTestCase):
|
|||
|
||||
_, keypairs_client = cls.get_clients(
|
||||
creds, type_of_creds, 'keypair_setup')
|
||||
|
||||
keypair = None
|
||||
try:
|
||||
keypairs_client.show_keypair(config.Config.keypair_id)
|
||||
except Exception:
|
||||
keypairs_client.create_keypair(name=config.Config.keypair_id)
|
||||
|
||||
return creds
|
||||
keypair_body = keypairs_client.create_keypair(
|
||||
name=config.Config.keypair_id)
|
||||
cls.LOG.debug("Keypair body: %s" % keypair_body)
|
||||
keypair = keypair_body['keypair']['private_key']
|
||||
return (creds, keypair)
|
||||
|
||||
@classmethod
|
||||
def get_clients(cls, creds, type_of_creds, request_type):
|
||||
|
@ -142,3 +159,53 @@ class BaseMagnumTest(base.BaseTestCase):
|
|||
"""
|
||||
creds = cls.get_credentials(name, type_of_creds, class_cleanup)
|
||||
return cls.get_clients(creds, type_of_creds, request_type)
|
||||
|
||||
@classmethod
|
||||
def copy_logs_handler(cls, get_nodes_fn, coe, keypair):
|
||||
"""Copy logs closure.
|
||||
|
||||
This method will retrieve all running nodes for a specified bay
|
||||
and copy addresses from there locally.
|
||||
|
||||
:param get_nodes_fn: function that takes no parameters and returns
|
||||
a list of node IPs to get logs from
|
||||
:param coe: the COE type of the nodes
|
||||
"""
|
||||
|
||||
if not config.Config.copy_logs:
|
||||
return lambda: None
|
||||
|
||||
def int_copy_logs(exec_info):
|
||||
try:
|
||||
cls.LOG.debug("Copying logs...")
|
||||
fn = exec_info[2].tb_frame.f_locals['fn']
|
||||
func_name = fn.im_self._get_test_method().__name__
|
||||
msg = "Failed to copy logs for bay"
|
||||
nodes_addresses = get_nodes_fn()
|
||||
|
||||
for node_address in nodes_addresses:
|
||||
log_name = "node-" + func_name
|
||||
try:
|
||||
base_path = os.path.split(os.path.dirname(
|
||||
os.path.abspath(magnum.__file__)))[0]
|
||||
script = "magnum/tests/contrib/copy_instance_logs.sh"
|
||||
full_location = os.path.join(base_path, script)
|
||||
cls.LOG.debug("running %s" % full_location)
|
||||
cls.LOG.debug("keypair: %s" % keypair)
|
||||
subprocess.check_call([
|
||||
full_location,
|
||||
node_address,
|
||||
coe,
|
||||
log_name,
|
||||
str(keypair)
|
||||
])
|
||||
except Exception:
|
||||
cls.LOG.exception(msg)
|
||||
cls.LOG.exception("failed to copy from %s to %s%s-%s" %
|
||||
(node_address,
|
||||
"/opt/stack/logs/bay-nodes/",
|
||||
log_name, node_address))
|
||||
except Exception:
|
||||
cls.LOG.exception(msg)
|
||||
|
||||
return int_copy_logs
|
||||
|
|
|
@ -103,6 +103,12 @@ class Config(object):
|
|||
raise Exception('config missing csr_location key')
|
||||
cls.csr_location = CONF.magnum.csr_location
|
||||
|
||||
@classmethod
|
||||
def set_copy_logs(cls, config):
|
||||
if 'copy_logs' not in CONF.magnum:
|
||||
cls.copy_logs = True
|
||||
cls.copy_logs = CONF.magnum.copy_logs
|
||||
|
||||
@classmethod
|
||||
def setUp(cls):
|
||||
cls.set_admin_creds(config)
|
||||
|
@ -119,3 +125,4 @@ class Config(object):
|
|||
cls.set_magnum_url(config)
|
||||
cls.set_master_flavor_id(config)
|
||||
cls.set_csr_location(config)
|
||||
cls.set_copy_logs(config)
|
||||
|
|
|
@ -22,11 +22,10 @@ from magnum.tests.functional.common import config
|
|||
|
||||
|
||||
class Manager(clients.Manager):
|
||||
def __init__(
|
||||
self,
|
||||
credentials=common_creds.get_configured_credentials(
|
||||
'identity_admin'),
|
||||
request_type=None):
|
||||
def __init__(self, credentials=None, request_type=None):
|
||||
if not credentials:
|
||||
credentials = common_creds.get_configured_credentials(
|
||||
'identity_admin')
|
||||
super(Manager, self).__init__(credentials, 'container')
|
||||
self.auth_provider.orig_base_url = self.auth_provider.base_url
|
||||
self.auth_provider.base_url = self.bypassed_base_url
|
||||
|
|
|
@ -24,15 +24,14 @@ class TestKubernetesAPIs(BayTest):
|
|||
"fixed_network": '192.168.0.0/24'
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
super(TestKubernetesAPIs, cls).setUpClass()
|
||||
cls.kube_api_url = cls.cs.bays.get(cls.bay.uuid).api_address
|
||||
k8s_client = api_client.ApiClient(cls.kube_api_url,
|
||||
key_file=cls.key_file,
|
||||
cert_file=cls.cert_file,
|
||||
ca_certs=cls.ca_file)
|
||||
cls.k8s_api = apiv_api.ApivApi(k8s_client)
|
||||
def setUp(self):
|
||||
super(TestKubernetesAPIs, self).setUp()
|
||||
self.kube_api_url = self.cs.bays.get(self.bay.uuid).api_address
|
||||
k8s_client = api_client.ApiClient(self.kube_api_url,
|
||||
key_file=self.key_file,
|
||||
cert_file=self.cert_file,
|
||||
ca_certs=self.ca_file)
|
||||
self.k8s_api = apiv_api.ApivApi(k8s_client)
|
||||
|
||||
def test_pod_apis(self):
|
||||
pod_manifest = {'apiVersion': 'v1',
|
||||
|
|
|
@ -26,13 +26,13 @@ import fixtures
|
|||
from six.moves import configparser
|
||||
|
||||
from magnum.common.utils import rmtree_without_raise
|
||||
from magnum.tests import base
|
||||
from magnum.tests.functional.common import base
|
||||
from magnumclient.common.apiclient import exceptions
|
||||
from magnumclient.common import cliutils
|
||||
from magnumclient.v1 import client as v1client
|
||||
|
||||
|
||||
class BaseMagnumClient(base.TestCase):
|
||||
class BaseMagnumClient(base.BaseMagnumTest):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
|
@ -40,6 +40,7 @@ class BaseMagnumClient(base.TestCase):
|
|||
#
|
||||
# Support the existence of a functional_creds.conf for
|
||||
# testing. This makes it possible to use a config file.
|
||||
super(BaseMagnumClient, cls).setUpClass()
|
||||
user = cliutils.env('OS_USERNAME')
|
||||
passwd = cliutils.env('OS_PASSWORD')
|
||||
tenant = cliutils.env('OS_TENANT_NAME')
|
||||
|
@ -133,22 +134,13 @@ class BaseMagnumClient(base.TestCase):
|
|||
return baymodel
|
||||
|
||||
@classmethod
|
||||
def _create_bay(cls, name, baymodel_uuid, wait=True):
|
||||
def _create_bay(cls, name, baymodel_uuid):
|
||||
bay = cls.cs.bays.create(
|
||||
name=name,
|
||||
baymodel_id=baymodel_uuid,
|
||||
node_count=None,
|
||||
)
|
||||
|
||||
if wait:
|
||||
cls._wait_on_status(bay,
|
||||
[None, "CREATE_IN_PROGRESS"],
|
||||
["CREATE_FAILED",
|
||||
"CREATE_COMPLETE"])
|
||||
|
||||
if cls.cs.bays.get(bay.uuid).status == 'CREATE_FAILED':
|
||||
raise Exception("bay %s created failed" % bay.uuid)
|
||||
|
||||
return bay
|
||||
|
||||
@classmethod
|
||||
|
@ -175,21 +167,16 @@ class BaseMagnumClient(base.TestCase):
|
|||
if cls._show_bay(cls.bay.uuid).status == 'DELETE_FAILED':
|
||||
raise Exception("bay %s delete failed" % cls.bay.uuid)
|
||||
|
||||
def _copy_logs(self, exec_info):
|
||||
if not self.copy_logs:
|
||||
return
|
||||
fn = exec_info[2].tb_frame.f_locals['fn']
|
||||
func_name = fn.im_self._get_test_method().__name__
|
||||
def _wait_for_bay_complete(self, bay):
|
||||
self._wait_on_status(
|
||||
bay,
|
||||
[None, "CREATE_IN_PROGRESS"],
|
||||
["CREATE_FAILED", "CREATE_COMPLETE"])
|
||||
|
||||
bay = self._show_bay(self.bay.uuid)
|
||||
for node_addr in bay.node_addresses:
|
||||
subprocess.call(["magnum/tests/contrib/copy_instance_logs.sh",
|
||||
node_addr, self.baymodel.coe,
|
||||
"worker-" + func_name])
|
||||
for node_addr in getattr(bay, 'master_addresses', []):
|
||||
subprocess.call(["magnum/tests/contrib/copy_instance_logs.sh",
|
||||
node_addr, self.baymodel.coe,
|
||||
"master-" + func_name])
|
||||
if self.cs.bays.get(bay.uuid).status == 'CREATE_FAILED':
|
||||
raise Exception("bay %s created failed" % bay.uuid)
|
||||
|
||||
return bay
|
||||
|
||||
|
||||
class BayTest(BaseMagnumClient):
|
||||
|
@ -242,6 +229,14 @@ extendedKeyUsage = clientAuth
|
|||
if test_timeout > 0:
|
||||
self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
|
||||
|
||||
self.addOnException(
|
||||
self.copy_logs_handler(
|
||||
lambda: list(self.cs.bays.get(self.bay.uuid).node_addresses +
|
||||
self.cs.bays.get(self.bay.uuid).master_addresses),
|
||||
self.baymodel.coe,
|
||||
'default'))
|
||||
self._wait_for_bay_complete(self.bay)
|
||||
|
||||
@classmethod
|
||||
def _create_tls_ca_files(cls, client_conf_contents):
|
||||
"""Creates ca files by client_conf_contents."""
|
||||
|
|
|
@ -55,4 +55,8 @@ MagnumGroup = [
|
|||
cfg.StrOpt("csr_location",
|
||||
default="/opt/stack/new/magnum/default.csr",
|
||||
help="CSR location for certificates."),
|
||||
|
||||
cfg.StrOpt("copy_logs",
|
||||
default=True,
|
||||
help="Specify whether to copy nova server logs on failure."),
|
||||
]
|
||||
|
|
Loading…
Reference in New Issue