stackforge/fuel-ostf-plugin
Code Issues Proposed changes

ut refactor, defunct process handling

Browse Source
This commit is contained in:
Dmitry Shulyak
2013-07-18 12:22:51 +03:00
parent 1872b7ff27
commit d1981d56a6
8 changed files with 100 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
functional/manual/post_testrun.py
View File

@@ -18,4 +18,4 @@ def make_requests(claster_id, test_set):
if __name__ == '__main__':
make_requests(27, 'plugin_general')
make_requests('307', 'plugin_general')

2
functional/manual/put_test_run_with_tests.py
View File

@@ -17,4 +17,4 @@ def make_requests(claster_id, test_set):
if __name__ == '__main__':
make_requests(364, 'plugin_general')
make_requests(370, 'plugin_general')

2
functional/manual/update_testrun.py
View File

@@ -13,4 +13,4 @@ def make_requests(claster_id, test_set):
pprint.pprint(data)
if __name__ == '__main__':
make_requests(356, 'plugin_stopped')
make_requests(378, 'plugin_stopped')

3
ostf_adapter/api.py
View File

@@ -110,8 +110,7 @@ class API(object):
command, transport = self._find_command(test_run.type)
cleanup = command.get('cleanup')
killed = transport.obj.kill(
test_run.id, test_run.external_id, test_run.type,
test_path=command.get('test_path'), cleanup=cleanup)
test_run.id, test_run.external_id, cleanup=cleanup)
if killed:
if cleanup:
status = 'cleanup'

127
ostf_adapter/transport/nose_adapter.py
View File

@@ -1,11 +1,10 @@
import multiprocessing
from nose import main
from nose import core
import os
from ostf_adapter.storage import get_storage
from ostf_adapter import storage
from ostf_adapter.transport import nose_utils
from ostf_adapter.transport import nose_storage_plugin
import logging
from ostf_adapter.api import parse_json_file
from ostf_adapter import exceptions as exc
from pecan import conf
@@ -20,61 +19,65 @@ class NoseDriver(object):
def __init__(self):
log.info('NoseDriver initialized')
self.storage = get_storage()
self.storage = storage.get_storage()
self._named_threads = {}
self._configs = parse_json_file('config_templates.json')
def clean_process(self):
items = self._named_threads.items()
for uid, proc in items:
if not proc.is_alive():
proc.terminate()
self._named_threads.pop(uid)
def check_current_running(self, unique_id):
return unique_id in self._named_threads
def run(self, test_run_id, external_id,
conf, test_set, tests=None, test_path=None, argv=None):
if conf:
test_conf_path = self.prepare_config(
conf, test_path, external_id, test_set)
else:
test_conf_path = ''
self.clean_process()
argv_add = argv or []
tests = tests or []
if tests:
log.info('TESTS RECEIVED %s' % tests)
argv_add += map(nose_utils.modify_test_name_for_nose, tests)
else:
argv_add.append(test_path)
log.info('Additional args: %s' % argv_add)
proc = multiprocessing.Process(
target=self._run_tests,
args=(test_run_id, external_id, argv_add, test_conf_path))
args=(test_run_id, external_id, argv_add))
proc.daemon = True
proc.start()
self._named_threads[test_run_id] = proc
self._named_threads[int(test_run_id)] = proc
log.info('NAMED PROCESS %s' % self._named_threads)
def tests_discovery(self, test_set, test_path, argv_add):
try:
log.info('Started test discovery %s' % test_set)
main(defaultTest=test_path,
addplugins=[nose_storage_plugin.StoragePlugin(
test_set, '', discovery=True)],
exit=False,
argv=['tests', '--collect-only'] + argv_add)
except Exception, e:
log.info('Finished tests discovery %s' % test_set)
log.info('Started test discovery %s' % test_set)
def _run_tests(self, test_run_id, external_id,
argv_add, test_conf_path=''):
try:
log.info('Nose Driver spawn process for TEST RUN: %s\n'
core.TestProgram(
defaultTest=test_path,
addplugins=[nose_storage_plugin.StoragePlugin(
test_set, '', discovery=True)],
exit=False,
argv=['tests', '--collect-only'] + argv_add)
def _run_tests(self, test_run_id, external_id, argv_add):
log.info('Nose Driver spawn process for TEST RUN: %s\n'
'ARGS: %s' % (test_run_id, argv_add))
main(addplugins=[nose_storage_plugin.StoragePlugin(
test_run_id, str(external_id), test_conf_path=test_conf_path)],
try:
core.TestProgram(addplugins=[nose_storage_plugin.StoragePlugin(
test_run_id, str(external_id))],
exit=False,
argv=['tests']+argv_add)
log.info('Test run %s finished successfully' % test_run_id)
if test_run_id in self._named_threads:
del self._named_threads[test_run_id]
self.storage.update_test_run(test_run_id, status='finished')
#To close thread we need to catch any exception
self._named_threads.pop(int(test_run_id), None)
raise SystemExit
except Exception, e:
log.info('Close process TEST_RUN: %s\n'
'Thread closed with exception: %s' % (test_run_id,
@@ -82,71 +85,55 @@ class NoseDriver(object):
self.storage.update_test_run(test_run_id, status='error')
self.storage.update_running_tests(test_run_id,
status='error')
if test_run_id in self._named_threads:
del self._named_threads[test_run_id]
def kill(self, test_run_id, external_id, test_set,
test_path=None, cleanup=False):
def kill(self, test_run_id, external_id, cleanup=None):
log.info('Trying to stop process %s\n'
'%s' % (test_run_id, self._named_threads))
self.clean_process()
if test_run_id in self._named_threads:
log.info('Terminating process: %s' % test_run_id)
self._named_threads[test_run_id].terminate()
del self._named_threads[test_run_id]
self._named_threads[int(test_run_id)].terminate()
self._named_threads.pop(int(test_run_id), None)
if cleanup:
proc = multiprocessing.Process(
target=self._clean_up,
args=(test_run_id, external_id, test_set,
test_path, cleanup))
args=(test_run_id, external_id, cleanup))
proc.daemon = True
proc.start()
self._named_threads[int(test_run_id)] = proc
else:
self.storage.update_test_run(test_run_id, status='stopped')
return True
return False
def _clean_up(self,
test_run_id, external_id, test_set, test_path, cleanup):
stor = get_storage(conf.dbpath)
test_run_id, external_id, cleanup,
storage=storage.get_storage):
#Had problems with mocking storage.get_storage
storage = storage()
try:
log.info("TRYING TO CLEAN")
module_obj = __import__(cleanup, -1)
os.environ['OSTF_CONF_PATH'] = nose_utils.config_name_generator(
test_path, test_set, external_id)
os.environ['NAILGUN_HOST'] = str(conf.nailgun.host)
os.environ['NAILGUN_PORT'] = str(conf.nailgun.port)
os.environ['CLUSTER_ID'] = str(external_id)
log.info('STARTING CLEANUP FUNCTION')
module_obj.cleanup.cleanup()
log.info('CLEANUP IS SUCCESSFULL')
stor.update_test_run(test_run_id, status='stopped')
except BaseException, e:
log.error('EXCITED WITH EXCEPTIOBN %s' % e)
stor.update_test_run(test_run_id, status='error_on_cleanup')
def prepare_config(self, config, test_path, external_id, test_set):
template = []
for group_name, group_items in self._configs.iteritems():
template_group = []
for group_item in group_items:
if group_item in config:
if not template_group:
template_group.append('[{0}]'.format(group_name))
template_group.append('{0} = {1}'.format(
group_item, config[group_item]))
with_group = '{0}_{1}'.format(group_name, group_item)
if with_group in config:
if not template_group:
template_group.append('[{0}]'.format(group_name))
template_group.append('{0} = {1}'.format(
group_item, config[with_group]))
template.extend(template_group)
if template:
conf_path = nose_utils.config_name_generator(
test_path, test_set, external_id)
with open(conf_path, 'w') as f:
f.write(u'\n'.join(template))
return conf_path
storage.update_test_run(test_run_id, status='stopped')
raise SystemExit
except Exception, e:
log.error('EXCITED WITH EXCEPTION %s' % e)
storage.update_test_run(test_run_id, status='error_on_cleanup')

13
ostf_adapter/transport/nose_storage_plugin.py
View File

@@ -21,21 +21,18 @@ class StoragePlugin(Plugin):
score = 15000
def __init__(
self, test_run_id, cluster_id, discovery=False,
test_conf_path=''):
self, test_run_id, cluster_id, discovery=False):
self._capture = []
self.test_run_id = test_run_id
self.cluster_id = cluster_id
self.storage = get_storage()
self.discovery = discovery
self.test_conf_path = test_conf_path
super(StoragePlugin, self).__init__()
log.info('Storage Plugin initialized')
self._start_time = None
self._started = False
def options(self, parser, env=os.environ):
env['CUSTOM_FUEL_CONFIG'] = self.test_conf_path
log.info('NAILGUN HOST %s '
'AND PORT %s' % (conf.nailgun.host, conf.nailgun.port))
env['NAILGUN_HOST'] = str(conf.nailgun.host)
@@ -52,13 +49,15 @@ class StoragePlugin(Plugin):
if not self._started:
self.storage.update_test_run(self.test_run_id, status='running')
self._started = True
data = {}
data = dict()
data['name'], data['description'], data['duration'] =\
nose_utils.get_description(test)
if err:
exc_type, exc_value, exc_traceback = err
log.info('Error %s' % exc_value)
data['message'] = nose_utils.get_exc_message(exc_value)
data['message'] = u''
if not status == 'error':
data['message'] = nose_utils.get_exc_message(exc_value)
data['traceback'] = nose_utils.format_exception(err)
else:
data['message'] = None
@@ -76,7 +75,7 @@ class StoragePlugin(Plugin):
def addSuccess(self, test, capt=None):
log.info('SUCCESS for %s' % test)
if self.discovery:
data = {}
data = dict()
data['name'], data['description'], data['duration'] =\
nose_utils.get_description(test)
data['message'] = None

6
tests/test_api.py
View File

@@ -66,5 +66,7 @@ class TestApi(unittest.TestCase):
test_run = {'id': 1}
status = 'stopped'
with patch.object(self.api, '_find_command') as command_mock:
command_mock.return_value = (self.command, self.transport)
self.api.kill(test_run)
with patch.object(self.api, '_prepare_test_run') as test_run_mock:
command_mock.return_value = (self.command, self.transport)
self.api.kill(test_run)
self.assertEqual(test_run_mock.call_count, 1)

49
tests/test_nose_adapter.py
View File

@@ -24,26 +24,28 @@ class NoExitStriongIO(io.StringIO):
class TestNoseAdapters(unittest.TestCase):
@patch('ostf_adapter.transport.nose_adapter.get_storage')
def setUp(self, get_storage_mock):
@patch('ostf_adapter.transport.nose_adapter.storage')
def setUp(self, storage_mock):
self.thread = MagicMock()
self.storage = MagicMock()
self.module_mock = MagicMock()
self.config_out = NoExitStriongIO()
get_storage_mock.return_value = self.storage
storage_mock.get_storage.return_value = self.storage
self.driver = nose_adapter.NoseDriver()
@patch('__builtin__.open')
def test_prepare_config_conf(self, open_mock):
open_mock.return_value = self.config_out
conf = {'network_catalog_type': 'TEST_TYPE',
'url': 'http://localhost:8989/v1/'}
test_path = 'fuel_health.tests'
external_id = 12
test_set = 'fuel_health'
self.driver.prepare_config(
conf, test_path, external_id, test_set)
self.assertEqual(self.config_out.getvalue(),
u'[network]\ncatalog_type = TEST_TYPE\n[identity]\nurl = http://localhost:8989/v1/')
def test_kill(self):
pass
@patch('ostf_adapter.transport.nose_adapter.conf')
@patch('__builtin__.__import__')
def test_cleanup_call(self, import_mock, conf_mock):
import_mock.return_value = self.module_mock
conf_mock.nailgun.host = 'NAILGUN_HOST'
conf_mock.nailgun.port = 'NAILGUN_PORT'
self.driver._named_threads[1] = self.thread
self.driver._clean_up(1, '101', 'cleanup', storage=MagicMock)
class TestNoseUtils(unittest.TestCase):
@@ -66,8 +68,10 @@ class TestNoseUtils(unittest.TestCase):
self.assertEqual(test_path.split('/')[-1],
'test_plugin_general_12.conf')
class TestNoseStoragePlugin(unittest.TestCase):
@patch('ostf_adapter.transport.nose_storage_plugin.get_storage')
def setUp(self, get_storage_mock):
self.storage = MagicMock()
@@ -77,16 +81,21 @@ class TestNoseStoragePlugin(unittest.TestCase):
self.test_parent_id = 12
self.cluster_id = '14'
self.plugin = nose_storage_plugin.StoragePlugin(
self.test_parent_id, self.cluster_id, discovery=False,
test_conf_path='/etc/config.conf')
self.test_parent_id, self.cluster_id, discovery=False)
def test_options_interface_defined(self):
@patch('ostf_adapter.transport.nose_storage_plugin.conf')
def test_options_interface_defined(self, conf_mock):
conf_mock.nailgun.host = 'NAILGUN_HOST'
conf_mock.nailgun.port = 'NAILGUN_PORT'
self.plugin.options({})
self.assertEqual(os.environ['CUSTOM_FUEL_CONFIG'],
self.plugin.test_conf_path)
self.assertEqual(os.environ['CLUSTER_ID'],
self.cluster_id)
self.assertEqual(os.environ['NAILGUN_HOST'],
'NAILGUN_HOST')
self.assertEqual(os.environ['NAILGUN_PORT'],
'NAILGUN_PORT')
def test_add_success_discover_false(self):
with patch.object(self.plugin, '_add_message') as add_mock: