openstack
/
magnum
Code Issues Proposed changes

Merge "Copy logs on test failure"

This commit is contained in:
Jenkins 2016-02-29 22:47:53 +00:00 committed by Gerrit Code Review
parent c64e8eb502 7d524491c8
commit 32be5b1ee6
11 changed files with 154 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
magnum/tests/base.py
View File

@ -30,7 +30,10 @@ from magnum.tests import policy_fixture
CONF = cfg.CONF
log.register_options(CONF)
try:
log.register_options(CONF)
except cfg.ArgsAlreadyParsedError:
pass
CONF.set_override('use_stderr', False)

2
magnum/tests/conf_fixture.py
View File

@ -16,7 +16,6 @@
import fixtures
from oslo_config import cfg
from oslo_log import log
from magnum.common import config
@ -30,7 +29,6 @@ class ConfFixture(fixtures.Fixture):
"""Fixture to manage global conf settings."""
def _setUp(self):
log.register_options(cfg.CONF)
CONF.set_default('host', 'fake-mini')
CONF.set_default('connection', "sqlite://", group='database')
CONF.set_default('sqlite_synchronous', False, group='database')

15
magnum/tests/contrib/copy_instance_logs.sh
View File

@ -22,12 +22,21 @@ SSH_IP=$1
COE=${2-kubernetes}
NODE_TYPE=${3-master}
LOG_PATH=/opt/stack/logs/bay-nodes/${NODE_TYPE}-${SSH_IP}
KEYPAIR=${4-default}
function remote_exec {
local priv_key
echo "If private key is specified, save to temp and use that; else, use default"
if [[ "$KEYPAIR" == "default" ]]; then
priv_key="~/.ssh/id_rsa"
else
priv_key="$(mktemp id_rsa.$SSH_IP.XXX)"
echo -en "$KEYPAIR" > $priv_key
fi
local ssh_user=$1
local cmd=$2
local logfile=${LOG_PATH}/$3
ssh -o StrictHostKeyChecking=no ${ssh_user}@${SSH_IP} "${cmd}" > ${logfile} 2>&1
ssh -i $priv_key -o StrictHostKeyChecking=no ${ssh_user}@${SSH_IP} "${cmd}" > ${logfile} 2>&1
}
mkdir -p $LOG_PATH
@ -55,7 +64,7 @@ if [[ "$COE" == "kubernetes" ]]; then
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker" docker.sysconfig.env.log
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker-storage" docker-storage.sysconfig.env.log
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker-network" docker-network.sysconfig.env.log
remote_exec $SSH_USER "sudo docker ps --all=true --no-trunc=true" docker-containers.log
remote_exec $SSH_USER "sudo timeout 60s docker ps --all=true --no-trunc=true" docker-containers.log
remote_exec $SSH_USER "sudo tar zcvf - /var/lib/docker/containers 2>/dev/null" docker-container-configs.tar.gz
remote_exec $SSH_USER "sudo journalctl -u flanneld --no-pager" flanneld.log
remote_exec $SSH_USER "sudo ip a" ipa.log
@ -81,7 +90,7 @@ elif [[ "$COE" == "swarm" ]]; then
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker" docker.sysconfig.env.log
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker-storage" docker-storage.sysconfig.env.log
remote_exec $SSH_USER "sudo cat /etc/sysconfig/docker-network" docker-network.sysconfig.env.log
remote_exec $SSH_USER "sudo docker ps --all=true --no-trunc=true" docker-containers.log
remote_exec $SSH_USER "sudo timeout 60s docker ps --all=true --no-trunc=true" docker-containers.log
remote_exec $SSH_USER "sudo tar zcvf - /var/lib/docker/containers 2>/dev/null" docker-container-configs.tar.gz
remote_exec $SSH_USER "sudo journalctl -u flanneld --no-pager" flanneld.log
remote_exec $SSH_USER "sudo ip a" ipa.log

1
magnum/tests/contrib/post_test_hook.sh
View File

@ -185,6 +185,7 @@ if [[ "api" == "$coe" ]]; then
iniset $BASE/new/tempest/etc/tempest.conf magnum flavor_id m1.magnum2
iniset $BASE/new/tempest/etc/tempest.conf magnum master_flavor_id m1.magnum
iniset $BASE/new/tempest/etc/tempest.conf magnum csr_location $CSR_FILE
iniset $BASE/new/tempest/etc/tempest.conf magnum copy_logs True
# show tempest config with magnum
cat etc/tempest.conf

33
magnum/tests/functional/api/v1/test_bay.py
View File

@ -30,7 +30,8 @@ class BayTest(base.BaseMagnumTest):
def __init__(self, *args, **kwargs):
super(BayTest, self).__init__(*args, **kwargs)
self.bays = []
self.credentials = None
self.creds = None
self.keypair = None
self.baymodel = None
self.baymodel_client = None
self.keypairs_client = None
@ -40,18 +41,19 @@ class BayTest(base.BaseMagnumTest):
def setUp(self):
try:
super(BayTest, self).setUp()
self.credentials = self.get_credentials(type_of_creds='default')
(self.creds, self.keypair) = self.get_credentials_with_keypair(
type_of_creds='default')
(self.baymodel_client,
self.keypairs_client) = self.get_clients_with_existing_creds(
creds=self.credentials,
type_of_creds='default',
request_type='baymodel')
self.keypairs_client) = self.get_clients_with_existing_creds(
creds=self.creds,
type_of_creds='default',
request_type='baymodel')
(self.bay_client, _) = self.get_clients_with_existing_creds(
creds=self.credentials,
creds=self.creds,
type_of_creds='default',
request_type='bay')
(self.cert_client, _) = self.get_clients_with_existing_creds(
creds=self.credentials,
creds=self.creds,
type_of_creds='default',
request_type='cert')
model = datagen.valid_swarm_baymodel()
@ -91,11 +93,18 @@ class BayTest(base.BaseMagnumTest):
self.LOG.debug('Response: %s' % resp)
self.assertEqual(resp.status, 201)
self.assertIsNotNone(model.uuid)
self.bays.append(model.uuid)
self.assertIsNone(model.status)
self.assertIsNone(model.status_reason)
self.assertEqual(model.baymodel_id, self.baymodel.uuid)
self.bay_client.wait_for_created_bay(model.uuid)
self.bays.append(model.uuid)
self.bay_uuid = model.uuid
self.addOnException(self.copy_logs_handler(
lambda: list(
self._get_bay_by_id(self.bay_uuid)[1].node_addresses +
self._get_bay_by_id(self.bay_uuid)[1].master_addresses),
self.baymodel.coe,
self.keypair))
self.bay_client.wait_for_created_bay(model.uuid, delete_on_error=False)
return resp, model
def _delete_bay(self, bay_id):
@ -105,6 +114,10 @@ class BayTest(base.BaseMagnumTest):
self.bay_client.wait_for_bay_to_delete(bay_id)
return resp, model
def _get_bay_by_id(self, bay_id):
resp, model = self.bay_client.get_bay(bay_id)
return resp, model
# (dimtruck) Combining all these tests in one because
# they time out on the gate (2 hours not enough)
@testtools.testcase.attr('positive')

73
magnum/tests/functional/common/base.py
View File

@ -11,10 +11,14 @@
# under the License.
import inspect
import logging
import os
import subprocess
from tempest.common import credentials_factory as common_creds
from tempest_lib import base
import magnum
from magnum.tests.functional.common import config
from magnum.tests.functional.common import manager
@ -24,6 +28,7 @@ class BaseMagnumTest(base.BaseTestCase):
ic_class_list = []
ic_method_list = []
LOG = logging.getLogger(__name__)
def __init__(self, *args, **kwargs):
super(BaseMagnumTest, self).__init__(*args, **kwargs)
@ -57,6 +62,14 @@ class BaseMagnumTest(base.BaseTestCase):
def get_credentials(cls, name=None,
type_of_creds="default",
class_cleanup=False):
(creds, _) = cls.get_credentials_with_keypair(name, type_of_creds,
class_cleanup)
return creds
@classmethod
def get_credentials_with_keypair(cls, name=None,
type_of_creds="default",
class_cleanup=False):
if name is None:
# Get name of test method
name = inspect.stack()[1][3]
@ -86,12 +99,16 @@ class BaseMagnumTest(base.BaseTestCase):
_, keypairs_client = cls.get_clients(
creds, type_of_creds, 'keypair_setup')
keypair = None
try:
keypairs_client.show_keypair(config.Config.keypair_id)
except Exception:
keypairs_client.create_keypair(name=config.Config.keypair_id)
return creds
keypair_body = keypairs_client.create_keypair(
name=config.Config.keypair_id)
cls.LOG.debug("Keypair body: %s" % keypair_body)
keypair = keypair_body['keypair']['private_key']
return (creds, keypair)
@classmethod
def get_clients(cls, creds, type_of_creds, request_type):
@ -142,3 +159,53 @@ class BaseMagnumTest(base.BaseTestCase):
"""
creds = cls.get_credentials(name, type_of_creds, class_cleanup)
return cls.get_clients(creds, type_of_creds, request_type)
@classmethod
def copy_logs_handler(cls, get_nodes_fn, coe, keypair):
"""Copy logs closure.
This method will retrieve all running nodes for a specified bay
and copy addresses from there locally.
:param get_nodes_fn: function that takes no parameters and returns
a list of node IPs to get logs from
:param coe: the COE type of the nodes
"""
if not config.Config.copy_logs:
return lambda: None
def int_copy_logs(exec_info):
try:
cls.LOG.debug("Copying logs...")
fn = exec_info[2].tb_frame.f_locals['fn']
func_name = fn.im_self._get_test_method().__name__
msg = "Failed to copy logs for bay"
nodes_addresses = get_nodes_fn()
for node_address in nodes_addresses:
log_name = "node-" + func_name
try:
base_path = os.path.split(os.path.dirname(
os.path.abspath(magnum.__file__)))[0]
script = "magnum/tests/contrib/copy_instance_logs.sh"
full_location = os.path.join(base_path, script)
cls.LOG.debug("running %s" % full_location)
cls.LOG.debug("keypair: %s" % keypair)
subprocess.check_call([
full_location,
node_address,
coe,
log_name,
str(keypair)
])
except Exception:
cls.LOG.exception(msg)
cls.LOG.exception("failed to copy from %s to %s%s-%s" %
(node_address,
"/opt/stack/logs/bay-nodes/",
log_name, node_address))
except Exception:
cls.LOG.exception(msg)
return int_copy_logs

7
magnum/tests/functional/common/config.py
View File

@ -103,6 +103,12 @@ class Config(object):
raise Exception('config missing csr_location key')
cls.csr_location = CONF.magnum.csr_location
@classmethod
def set_copy_logs(cls, config):
if 'copy_logs' not in CONF.magnum:
cls.copy_logs = True
cls.copy_logs = CONF.magnum.copy_logs
@classmethod
def setUp(cls):
cls.set_admin_creds(config)
@ -119,3 +125,4 @@ class Config(object):
cls.set_magnum_url(config)
cls.set_master_flavor_id(config)
cls.set_csr_location(config)
cls.set_copy_logs(config)

9
magnum/tests/functional/common/manager.py
View File

@ -22,11 +22,10 @@ from magnum.tests.functional.common import config
class Manager(clients.Manager):
def __init__(
self,
credentials=common_creds.get_configured_credentials(
'identity_admin'),
request_type=None):
def __init__(self, credentials=None, request_type=None):
if not credentials:
credentials = common_creds.get_configured_credentials(
'identity_admin')
super(Manager, self).__init__(credentials, 'container')
self.auth_provider.orig_base_url = self.auth_provider.base_url
self.auth_provider.base_url = self.bypassed_base_url

17
magnum/tests/functional/k8s/test_k8s_python_client.py
View File

@ -24,15 +24,14 @@ class TestKubernetesAPIs(BayTest):
"fixed_network": '192.168.0.0/24'
}
@classmethod
def setUpClass(cls):
super(TestKubernetesAPIs, cls).setUpClass()
cls.kube_api_url = cls.cs.bays.get(cls.bay.uuid).api_address
k8s_client = api_client.ApiClient(cls.kube_api_url,
key_file=cls.key_file,
cert_file=cls.cert_file,
ca_certs=cls.ca_file)
cls.k8s_api = apiv_api.ApivApi(k8s_client)
def setUp(self):
super(TestKubernetesAPIs, self).setUp()
self.kube_api_url = self.cs.bays.get(self.bay.uuid).api_address
k8s_client = api_client.ApiClient(self.kube_api_url,
key_file=self.key_file,
cert_file=self.cert_file,
ca_certs=self.ca_file)
self.k8s_api = apiv_api.ApivApi(k8s_client)
def test_pod_apis(self):
pod_manifest = {'apiVersion': 'v1',

47
magnum/tests/functional/python_client_base.py
View File

@ -26,13 +26,13 @@ import fixtures
from six.moves import configparser
from magnum.common.utils import rmtree_without_raise
from magnum.tests import base
from magnum.tests.functional.common import base
from magnumclient.common.apiclient import exceptions
from magnumclient.common import cliutils
from magnumclient.v1 import client as v1client
class BaseMagnumClient(base.TestCase):
class BaseMagnumClient(base.BaseMagnumTest):
@classmethod
def setUpClass(cls):
@ -40,6 +40,7 @@ class BaseMagnumClient(base.TestCase):
#
# Support the existence of a functional_creds.conf for
# testing. This makes it possible to use a config file.
super(BaseMagnumClient, cls).setUpClass()
user = cliutils.env('OS_USERNAME')
passwd = cliutils.env('OS_PASSWORD')
tenant = cliutils.env('OS_TENANT_NAME')
@ -133,22 +134,13 @@ class BaseMagnumClient(base.TestCase):
return baymodel
@classmethod
def _create_bay(cls, name, baymodel_uuid, wait=True):
def _create_bay(cls, name, baymodel_uuid):
bay = cls.cs.bays.create(
name=name,
baymodel_id=baymodel_uuid,
node_count=None,
)
if wait:
cls._wait_on_status(bay,
[None, "CREATE_IN_PROGRESS"],
["CREATE_FAILED",
"CREATE_COMPLETE"])
if cls.cs.bays.get(bay.uuid).status == 'CREATE_FAILED':
raise Exception("bay %s created failed" % bay.uuid)
return bay
@classmethod
@ -175,21 +167,16 @@ class BaseMagnumClient(base.TestCase):
if cls._show_bay(cls.bay.uuid).status == 'DELETE_FAILED':
raise Exception("bay %s delete failed" % cls.bay.uuid)
def _copy_logs(self, exec_info):
if not self.copy_logs:
return
fn = exec_info[2].tb_frame.f_locals['fn']
func_name = fn.im_self._get_test_method().__name__
def _wait_for_bay_complete(self, bay):
self._wait_on_status(
bay,
[None, "CREATE_IN_PROGRESS"],
["CREATE_FAILED", "CREATE_COMPLETE"])
bay = self._show_bay(self.bay.uuid)
for node_addr in bay.node_addresses:
subprocess.call(["magnum/tests/contrib/copy_instance_logs.sh",
node_addr, self.baymodel.coe,
"worker-" + func_name])
for node_addr in getattr(bay, 'master_addresses', []):
subprocess.call(["magnum/tests/contrib/copy_instance_logs.sh",
node_addr, self.baymodel.coe,
"master-" + func_name])
if self.cs.bays.get(bay.uuid).status == 'CREATE_FAILED':
raise Exception("bay %s created failed" % bay.uuid)
return bay
class BayTest(BaseMagnumClient):
@ -242,6 +229,14 @@ extendedKeyUsage = clientAuth
if test_timeout > 0:
self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
self.addOnException(
self.copy_logs_handler(
lambda: list(self.cs.bays.get(self.bay.uuid).node_addresses +
self.cs.bays.get(self.bay.uuid).master_addresses),
self.baymodel.coe,
'default'))
self._wait_for_bay_complete(self.bay)
@classmethod
def _create_tls_ca_files(cls, client_conf_contents):
"""Creates ca files by client_conf_contents."""

4
magnum/tests/functional/tempest_tests/config.py
View File

@ -55,4 +55,8 @@ MagnumGroup = [
cfg.StrOpt("csr_location",
default="/opt/stack/new/magnum/default.csr",
help="CSR location for certificates."),
cfg.StrOpt("copy_logs",
default=True,
help="Specify whether to copy nova server logs on failure."),
]