x/ci-cd-pipeline-app-murano
Code Issues Proposed changes

Collect murano-agent logs after test execution

Browse Source 
* Added creation of keypair in every test, remove usage of
  alredy prepared and uploaded to CI
* Creation of new keypair gave an ability to implement method
  that collect murano-agent logs from VMs with apps after test
  and publish them in job artifacts directory
* Small refactoring of setupUp method was made

Change-Id: Icb6833e3f57f37670315bd21bcdd6d4309ca1d43
This commit is contained in:
Anastasia Kuznetsova 2016-07-13 13:16:12 +03:00
parent 9348c45ac8
commit 68c3a85d5c
4 changed files with 126 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
test-requirements.txt
View File

@ -1,6 +1,8 @@
testrepository
testtools
requests
paramiko
python-muranoclient
python-heatclient
python-novaclient

151
tests/base.py
View File

@ -20,6 +20,7 @@ import socket
import time
import uuid
import paramiko
import requests
import testtools
import yaml
@ -47,9 +48,10 @@ LOG.addHandler(ch)
# Sometimes need to pass some boolean from bash env. Since each bash
# variable is string, we need such simply hack
_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
'0': False, 'no': False, 'false': False, 'off': False}
_boolean_states = {
'1': True, 'yes': True, 'true': True, 'on': True,
'0': False, 'no': False, 'false': False, 'off': False
}
def str2bool(name, default):
value = os.environ.get(name, '')
@ -60,53 +62,64 @@ class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
def setUp(self):
super(MuranoTestsBase, self).setUp()
# counter, for murano deployment logger
self.latest_report = 0
self.flavor = os.environ.get('OS_FLAVOR', 'm1.medium')
self.image = os.environ.get('OS_IMAGE')
self.keyname = os.environ.get('OS_KEYNAME', None)
self.availability_zone = os.environ.get('OS_ZONE', 'nova')
self.deploy_timeout = (60 * 60) * 2
# Since its really useful to debug deployment after it fail...lets
# add such possibility
self.os_cleanup_before = str2bool('OS_CLEANUP_BEFORE', False)
self.os_cleanup_after = str2bool('OS_CLEANUP_AFTER', True)
#
self.os_username = os.environ.get('OS_USERNAME')
self.os_password = os.environ.get('OS_PASSWORD')
self.os_tenant_name = os.environ.get('OS_TENANT_NAME')
self.os_auth_uri = os.environ.get('OS_AUTH_URL')
# Data for Nodepool app
self.os_np_username = os.environ.get('OS_NP_USERNAME', self.os_username)
self.os_np_password = os.environ.get('OS_NP_PASSWORD', self.os_password)
self.os_np_tenant_name = os.environ.get('OS_NP_TENANT_NAME',
self.os_tenant_name)
self.os_np_auth_uri = os.environ.get('OS_NP_AUTH_URL', self.os_auth_uri)
self.os_np_cleanup_before = str2bool('OS_NP_CLEANUP_BEFORE', False)
self.keystone = self.initialize_keystone_client()
self.heat = self.initialize_heat_client(self.keystone)
self.murano = self.initialize_murano_client(self.keystone)
self.headers = {
'X-Auth-Token': self.murano.http_client.auth_token,
'content-type': 'application/json'
}
self.envs = []
self.nova = self.initialize_nova_client(self.keystone)
# Since its really useful to debug deployment after it fail lets
# add such possibility
self.os_cleanup_before = str2bool('OS_CLEANUP_BEFORE', False)
self.os_cleanup_after = str2bool('OS_CLEANUP_AFTER', True)
# Data for Nodepool app
self.os_np_username = os.environ.get('OS_NP_USERNAME', self.os_username)
self.os_np_password = os.environ.get('OS_NP_PASSWORD', self.os_password)
self.os_np_tenant_name = os.environ.get(
'OS_NP_TENANT_NAME',
self.os_tenant_name
)
self.os_np_auth_uri = os.environ.get('OS_NP_AUTH_URL', self.os_auth_uri)
self.os_np_cleanup_before = str2bool('OS_NP_CLEANUP_BEFORE', False)
if self.os_cleanup_before:
self.cleanup_up_tenant()
if self.os_np_cleanup_before:
self.cleanup_up_np_tenant()
# Counter for murano deployment logger
self.latest_report = 0
# Application instance parameters
self.flavor = os.environ.get('OS_FLAVOR', 'm1.medium')
self.image = os.environ.get('OS_IMAGE')
self.files = []
self.keyname, self.key_file = self._create_keypair()
self.availability_zone = os.environ.get('OS_ZONE', 'nova')
self.envs = []
LOG.info('Running test: {0}'.format(self._testMethodName))
def tearDown(self):
for env in self.envs:
self._collect_murano_agent_logs(env)
if self.os_cleanup_after:
for env in self.envs:
try:
self.delete_env(env)
except Exception:
self.delete_stack(env)
self.nova.keypairs.delete(self.keyname)
for file in self.files:
os.remove(file)
super(MuranoTestsBase, self).tearDown()
@staticmethod
@ -117,23 +130,22 @@ class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
def generate_id():
return uuid.uuid4()
def _get_stack(self, environment_id):
for stack in self.heat.stacks.list():
if environment_id in stack.description:
return stack
def create_file(self, name, context):
with open(name, 'w') as f:
f.write(context)
path_to_file = os.path.join(os.getcwd(), name)
self.files.append(path_to_file)
return path_to_file
def cleanup_up_tenant(self):
LOG.debug('Removing EVERYTHING in tenant: {0}'.format(
self.keystone.tenant_name))
for env in self.murano.environments.list():
self.delete_env(env)
for stack in self.heat.stacks.list():
try:
self.heat.stacks.delete(stack.id)
except Exception as e:
LOG.warning("Unable delete stack:{}".format(stack))
LOG.exception(e)
pass
self.delete_stack(env)
for key in self.nova.keypairs.list():
if key.name.startswith("murano_ci_keypair"):
self.nova.keypairs.delete(key)
return
def cleanup_up_np_tenant(self):
@ -141,12 +153,31 @@ class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
LOG.warning('NodePool cleanup not implemented yet!')
return
def _create_keypair(self):
kp_name = self.rand_name('murano_ci_keypair_')
keypair = self.nova.keypairs.create(kp_name)
pr_key_file = self.create_file(
'id_{}'.format(kp_name), keypair.private_key
)
self.create_file('id_{}.pub'.format(kp_name), keypair.public_key)
return kp_name, pr_key_file
def _get_stack(self, environment_id):
for stack in self.heat.stacks.list():
if environment_id in stack.description:
return stack
def delete_stack(self, environment):
stack = self._get_stack(environment.id)
if not stack:
return
else:
self.heat.stacks.delete(stack.id)
try:
self.heat.stacks.delete(stack.id)
except Exception as e:
LOG.warning("Unable delete stack:{}".format(stack))
LOG.exception(e)
pass
def create_env(self):
name = self.rand_name()
@ -225,7 +256,29 @@ class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
self.latest_report = tmp
return history
def wait_for_environment_deploy(self, env):
def _collect_murano_agent_logs(self, environment):
fips = self.get_services_fips(environment)
logs_dir = "{0}/{1}".format(ARTIFACTS_DIR, environment.name)
os.makedirs(logs_dir)
self.files.append(logs_dir)
for service, fip in fips.iteritems():
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(fip, username='ubuntu', key_filename=self.key_file)
ftp = ssh.open_sftp()
ftp.get(
'/var/log/murano-agent.log',
os.path.join(logs_dir, '{0}-agent.log'.format(service))
)
ftp.close()
except Exception as e:
LOG.warning(
"Couldn't collect murano-agent "
"logs of {0} (IP: {1}): {2}".format(service, fip, e)
)
def wait_for_environment_deploy(self, env, timeout=7200):
start_time = time.time()
status = self.get_env(env).manager.get(env.id).status
@ -234,11 +287,11 @@ class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
LOG.debug('Deployment status:{}...nothing new..'.format(status))
self._log_latest(env)
if time.time() - start_time > self.deploy_timeout:
if time.time() - start_time > timeout:
time.sleep(60)
self.fail(
'Environment deployment wasn\'t'
'finished in {} seconds'.format(self.deploy_timeout)
'finished in {} seconds'.format(self.timeout)
)
elif status == 'deploy failure':
self._log_report(env)
@ -286,6 +339,13 @@ class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
return result
def get_services_fips(self, environment):
fips = {}
for service in environment.services:
fips.update(self.guess_fip(service))
return fips
def check_ports_open(self, ip, ports):
for port in ports:
result = 1
@ -329,10 +389,7 @@ class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
'Deployment status is "{0}"'.format(deployment.state)
)
fips = {}
for service in environment.services:
fips.update(self.guess_fip(service))
fips = self.get_services_fips(environment)
for service in services_map:
LOG.debug(

21
tests/clients.py
View File

@ -18,6 +18,7 @@ import os
from heatclient import client as heatclient
from keystoneclient.v2_0 import client as keystoneclient
from muranoclient import client as muranoclient
from novaclient import client as novaclient
class ClientsBase(object):
@ -27,13 +28,13 @@ class ClientsBase(object):
username = os.environ.get('OS_USERNAME')
password = os.environ.get('OS_PASSWORD')
tenant_name = os.environ.get('OS_TENANT_NAME')
os_auth_uri = os.environ.get('OS_AUTH_URL')
auth_url = os.environ.get('OS_AUTH_URL')
keystone = keystoneclient.Client(
username=username,
password=password,
tenant_name=tenant_name,
auth_url=os_auth_uri
auth_url=auth_url
)
return keystone
@ -82,3 +83,19 @@ class ClientsBase(object):
return heat
@classmethod
def initialize_nova_client(cls, auth_client=None):
ks_client = (auth_client if auth_client
else cls.initialize_keystone_client())
nova = novaclient.Client(
'2',
username=None,
service_type='compute',
endpoint_type='publicURL',
auth_token=ks_client.auth_token,
auth_url=ks_client.auth_url
)
nova.client.management_url = cls.get_endpoint('compute', 'publicURL')
return nova

2
tests/test_cicd_apps.py
View File

@ -88,4 +88,4 @@ class MuranoCiCdTest(base.MuranoTestsBase):
base.LOG.debug("Run second deployment, w\o any changes..")
session = self.create_session(environment)
self.deploy_env(environment, session)
self.deployment_success_check(environment, check_services)
self.deployment_success_check(environment, check_services)