From d1981d56a66ad6ce6ba5f0c3235fb30f36fc2495 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 18 Jul 2013 12:22:51 +0300 Subject: [PATCH] ut refactor, defunct process handling --- functional/manual/post_testrun.py | 2 +- functional/manual/put_test_run_with_tests.py | 2 +- functional/manual/update_testrun.py | 2 +- ostf_adapter/api.py | 3 +- ostf_adapter/transport/nose_adapter.py | 127 ++++++++---------- ostf_adapter/transport/nose_storage_plugin.py | 13 +- tests/test_api.py | 6 +- tests/test_nose_adapter.py | 49 ++++--- 8 files changed, 100 insertions(+), 104 deletions(-) diff --git a/functional/manual/post_testrun.py b/functional/manual/post_testrun.py index c6b315f..af04aba 100644 --- a/functional/manual/post_testrun.py +++ b/functional/manual/post_testrun.py @@ -18,4 +18,4 @@ def make_requests(claster_id, test_set): if __name__ == '__main__': - make_requests(27, 'plugin_general') \ No newline at end of file + make_requests('307', 'plugin_general') \ No newline at end of file diff --git a/functional/manual/put_test_run_with_tests.py b/functional/manual/put_test_run_with_tests.py index d1a7558..fd67ad0 100644 --- a/functional/manual/put_test_run_with_tests.py +++ b/functional/manual/put_test_run_with_tests.py @@ -17,4 +17,4 @@ def make_requests(claster_id, test_set): if __name__ == '__main__': - make_requests(364, 'plugin_general') \ No newline at end of file + make_requests(370, 'plugin_general') \ No newline at end of file diff --git a/functional/manual/update_testrun.py b/functional/manual/update_testrun.py index fe40928..b880495 100644 --- a/functional/manual/update_testrun.py +++ b/functional/manual/update_testrun.py @@ -13,4 +13,4 @@ def make_requests(claster_id, test_set): pprint.pprint(data) if __name__ == '__main__': - make_requests(356, 'plugin_stopped') \ No newline at end of file + make_requests(378, 'plugin_stopped') \ No newline at end of file diff --git a/ostf_adapter/api.py b/ostf_adapter/api.py index 4655066..6e7672f 100644 --- a/ostf_adapter/api.py +++ b/ostf_adapter/api.py @@ -110,8 +110,7 @@ class API(object): command, transport = self._find_command(test_run.type) cleanup = command.get('cleanup') killed = transport.obj.kill( - test_run.id, test_run.external_id, test_run.type, - test_path=command.get('test_path'), cleanup=cleanup) + test_run.id, test_run.external_id, cleanup=cleanup) if killed: if cleanup: status = 'cleanup' diff --git a/ostf_adapter/transport/nose_adapter.py b/ostf_adapter/transport/nose_adapter.py index 96923b4..b43f9ac 100644 --- a/ostf_adapter/transport/nose_adapter.py +++ b/ostf_adapter/transport/nose_adapter.py @@ -1,11 +1,10 @@ import multiprocessing -from nose import main +from nose import core import os -from ostf_adapter.storage import get_storage +from ostf_adapter import storage from ostf_adapter.transport import nose_utils from ostf_adapter.transport import nose_storage_plugin import logging -from ostf_adapter.api import parse_json_file from ostf_adapter import exceptions as exc from pecan import conf @@ -20,61 +19,65 @@ class NoseDriver(object): def __init__(self): log.info('NoseDriver initialized') - self.storage = get_storage() + self.storage = storage.get_storage() self._named_threads = {} - self._configs = parse_json_file('config_templates.json') + + def clean_process(self): + items = self._named_threads.items() + for uid, proc in items: + if not proc.is_alive(): + proc.terminate() + self._named_threads.pop(uid) def check_current_running(self, unique_id): return unique_id in self._named_threads def run(self, test_run_id, external_id, conf, test_set, tests=None, test_path=None, argv=None): - if conf: - test_conf_path = self.prepare_config( - conf, test_path, external_id, test_set) - else: - test_conf_path = '' + self.clean_process() argv_add = argv or [] tests = tests or [] + if tests: log.info('TESTS RECEIVED %s' % tests) argv_add += map(nose_utils.modify_test_name_for_nose, tests) else: argv_add.append(test_path) log.info('Additional args: %s' % argv_add) + proc = multiprocessing.Process( target=self._run_tests, - args=(test_run_id, external_id, argv_add, test_conf_path)) + args=(test_run_id, external_id, argv_add)) proc.daemon = True proc.start() - self._named_threads[test_run_id] = proc + + self._named_threads[int(test_run_id)] = proc log.info('NAMED PROCESS %s' % self._named_threads) def tests_discovery(self, test_set, test_path, argv_add): - try: - log.info('Started test discovery %s' % test_set) - main(defaultTest=test_path, - addplugins=[nose_storage_plugin.StoragePlugin( - test_set, '', discovery=True)], - exit=False, - argv=['tests', '--collect-only'] + argv_add) - except Exception, e: - log.info('Finished tests discovery %s' % test_set) + log.info('Started test discovery %s' % test_set) - def _run_tests(self, test_run_id, external_id, - argv_add, test_conf_path=''): - try: - log.info('Nose Driver spawn process for TEST RUN: %s\n' + core.TestProgram( + defaultTest=test_path, + addplugins=[nose_storage_plugin.StoragePlugin( + test_set, '', discovery=True)], + exit=False, + argv=['tests', '--collect-only'] + argv_add) + + def _run_tests(self, test_run_id, external_id, argv_add): + log.info('Nose Driver spawn process for TEST RUN: %s\n' 'ARGS: %s' % (test_run_id, argv_add)) - main(addplugins=[nose_storage_plugin.StoragePlugin( - test_run_id, str(external_id), test_conf_path=test_conf_path)], + + try: + core.TestProgram(addplugins=[nose_storage_plugin.StoragePlugin( + test_run_id, str(external_id))], exit=False, argv=['tests']+argv_add) + log.info('Test run %s finished successfully' % test_run_id) - if test_run_id in self._named_threads: - del self._named_threads[test_run_id] self.storage.update_test_run(test_run_id, status='finished') - #To close thread we need to catch any exception + self._named_threads.pop(int(test_run_id), None) + raise SystemExit except Exception, e: log.info('Close process TEST_RUN: %s\n' 'Thread closed with exception: %s' % (test_run_id, @@ -82,71 +85,55 @@ class NoseDriver(object): self.storage.update_test_run(test_run_id, status='error') self.storage.update_running_tests(test_run_id, status='error') - if test_run_id in self._named_threads: - del self._named_threads[test_run_id] - def kill(self, test_run_id, external_id, test_set, - test_path=None, cleanup=False): + def kill(self, test_run_id, external_id, cleanup=None): log.info('Trying to stop process %s\n' '%s' % (test_run_id, self._named_threads)) + self.clean_process() if test_run_id in self._named_threads: + log.info('Terminating process: %s' % test_run_id) - self._named_threads[test_run_id].terminate() - del self._named_threads[test_run_id] + + self._named_threads[int(test_run_id)].terminate() + self._named_threads.pop(int(test_run_id), None) + if cleanup: proc = multiprocessing.Process( target=self._clean_up, - args=(test_run_id, external_id, test_set, - test_path, cleanup)) + args=(test_run_id, external_id, cleanup)) proc.daemon = True proc.start() + self._named_threads[int(test_run_id)] = proc else: self.storage.update_test_run(test_run_id, status='stopped') + return True return False def _clean_up(self, - test_run_id, external_id, test_set, test_path, cleanup): - stor = get_storage(conf.dbpath) + test_run_id, external_id, cleanup, + storage=storage.get_storage): + #Had problems with mocking storage.get_storage + storage = storage() try: + log.info("TRYING TO CLEAN") module_obj = __import__(cleanup, -1) - os.environ['OSTF_CONF_PATH'] = nose_utils.config_name_generator( - test_path, test_set, external_id) + os.environ['NAILGUN_HOST'] = str(conf.nailgun.host) + os.environ['NAILGUN_PORT'] = str(conf.nailgun.port) + os.environ['CLUSTER_ID'] = str(external_id) + log.info('STARTING CLEANUP FUNCTION') module_obj.cleanup.cleanup() log.info('CLEANUP IS SUCCESSFULL') - stor.update_test_run(test_run_id, status='stopped') - except BaseException, e: - log.error('EXCITED WITH EXCEPTIOBN %s' % e) - stor.update_test_run(test_run_id, status='error_on_cleanup') - def prepare_config(self, config, test_path, external_id, test_set): - template = [] - for group_name, group_items in self._configs.iteritems(): - template_group = [] - for group_item in group_items: - if group_item in config: - if not template_group: - template_group.append('[{0}]'.format(group_name)) - template_group.append('{0} = {1}'.format( - group_item, config[group_item])) - with_group = '{0}_{1}'.format(group_name, group_item) - if with_group in config: - if not template_group: - template_group.append('[{0}]'.format(group_name)) - template_group.append('{0} = {1}'.format( - group_item, config[with_group])) - template.extend(template_group) - - if template: - conf_path = nose_utils.config_name_generator( - test_path, test_set, external_id) - with open(conf_path, 'w') as f: - f.write(u'\n'.join(template)) - return conf_path + storage.update_test_run(test_run_id, status='stopped') + raise SystemExit + except Exception, e: + log.error('EXCITED WITH EXCEPTION %s' % e) + storage.update_test_run(test_run_id, status='error_on_cleanup') diff --git a/ostf_adapter/transport/nose_storage_plugin.py b/ostf_adapter/transport/nose_storage_plugin.py index 29e3423..2345824 100644 --- a/ostf_adapter/transport/nose_storage_plugin.py +++ b/ostf_adapter/transport/nose_storage_plugin.py @@ -21,21 +21,18 @@ class StoragePlugin(Plugin): score = 15000 def __init__( - self, test_run_id, cluster_id, discovery=False, - test_conf_path=''): + self, test_run_id, cluster_id, discovery=False): self._capture = [] self.test_run_id = test_run_id self.cluster_id = cluster_id self.storage = get_storage() self.discovery = discovery - self.test_conf_path = test_conf_path super(StoragePlugin, self).__init__() log.info('Storage Plugin initialized') self._start_time = None self._started = False def options(self, parser, env=os.environ): - env['CUSTOM_FUEL_CONFIG'] = self.test_conf_path log.info('NAILGUN HOST %s ' 'AND PORT %s' % (conf.nailgun.host, conf.nailgun.port)) env['NAILGUN_HOST'] = str(conf.nailgun.host) @@ -52,13 +49,15 @@ class StoragePlugin(Plugin): if not self._started: self.storage.update_test_run(self.test_run_id, status='running') self._started = True - data = {} + data = dict() data['name'], data['description'], data['duration'] =\ nose_utils.get_description(test) if err: exc_type, exc_value, exc_traceback = err log.info('Error %s' % exc_value) - data['message'] = nose_utils.get_exc_message(exc_value) + data['message'] = u'' + if not status == 'error': + data['message'] = nose_utils.get_exc_message(exc_value) data['traceback'] = nose_utils.format_exception(err) else: data['message'] = None @@ -76,7 +75,7 @@ class StoragePlugin(Plugin): def addSuccess(self, test, capt=None): log.info('SUCCESS for %s' % test) if self.discovery: - data = {} + data = dict() data['name'], data['description'], data['duration'] =\ nose_utils.get_description(test) data['message'] = None diff --git a/tests/test_api.py b/tests/test_api.py index d1de44d..4c7b0df 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -66,5 +66,7 @@ class TestApi(unittest.TestCase): test_run = {'id': 1} status = 'stopped' with patch.object(self.api, '_find_command') as command_mock: - command_mock.return_value = (self.command, self.transport) - self.api.kill(test_run) \ No newline at end of file + with patch.object(self.api, '_prepare_test_run') as test_run_mock: + command_mock.return_value = (self.command, self.transport) + self.api.kill(test_run) + self.assertEqual(test_run_mock.call_count, 1) \ No newline at end of file diff --git a/tests/test_nose_adapter.py b/tests/test_nose_adapter.py index 7c4e22c..dd62a84 100644 --- a/tests/test_nose_adapter.py +++ b/tests/test_nose_adapter.py @@ -24,26 +24,28 @@ class NoExitStriongIO(io.StringIO): class TestNoseAdapters(unittest.TestCase): - @patch('ostf_adapter.transport.nose_adapter.get_storage') - def setUp(self, get_storage_mock): + @patch('ostf_adapter.transport.nose_adapter.storage') + def setUp(self, storage_mock): self.thread = MagicMock() self.storage = MagicMock() + self.module_mock = MagicMock() self.config_out = NoExitStriongIO() - get_storage_mock.return_value = self.storage + storage_mock.get_storage.return_value = self.storage self.driver = nose_adapter.NoseDriver() - @patch('__builtin__.open') - def test_prepare_config_conf(self, open_mock): - open_mock.return_value = self.config_out - conf = {'network_catalog_type': 'TEST_TYPE', - 'url': 'http://localhost:8989/v1/'} - test_path = 'fuel_health.tests' - external_id = 12 - test_set = 'fuel_health' - self.driver.prepare_config( - conf, test_path, external_id, test_set) - self.assertEqual(self.config_out.getvalue(), - u'[network]\ncatalog_type = TEST_TYPE\n[identity]\nurl = http://localhost:8989/v1/') + def test_kill(self): + pass + + @patch('ostf_adapter.transport.nose_adapter.conf') + @patch('__builtin__.__import__') + def test_cleanup_call(self, import_mock, conf_mock): + import_mock.return_value = self.module_mock + + conf_mock.nailgun.host = 'NAILGUN_HOST' + conf_mock.nailgun.port = 'NAILGUN_PORT' + self.driver._named_threads[1] = self.thread + + self.driver._clean_up(1, '101', 'cleanup', storage=MagicMock) class TestNoseUtils(unittest.TestCase): @@ -66,8 +68,10 @@ class TestNoseUtils(unittest.TestCase): self.assertEqual(test_path.split('/')[-1], 'test_plugin_general_12.conf') + class TestNoseStoragePlugin(unittest.TestCase): + @patch('ostf_adapter.transport.nose_storage_plugin.get_storage') def setUp(self, get_storage_mock): self.storage = MagicMock() @@ -77,16 +81,21 @@ class TestNoseStoragePlugin(unittest.TestCase): self.test_parent_id = 12 self.cluster_id = '14' self.plugin = nose_storage_plugin.StoragePlugin( - self.test_parent_id, self.cluster_id, discovery=False, - test_conf_path='/etc/config.conf') + self.test_parent_id, self.cluster_id, discovery=False) - def test_options_interface_defined(self): + @patch('ostf_adapter.transport.nose_storage_plugin.conf') + def test_options_interface_defined(self, conf_mock): + conf_mock.nailgun.host = 'NAILGUN_HOST' + conf_mock.nailgun.port = 'NAILGUN_PORT' self.plugin.options({}) - self.assertEqual(os.environ['CUSTOM_FUEL_CONFIG'], - self.plugin.test_conf_path) self.assertEqual(os.environ['CLUSTER_ID'], self.cluster_id) + self.assertEqual(os.environ['NAILGUN_HOST'], + 'NAILGUN_HOST') + self.assertEqual(os.environ['NAILGUN_PORT'], + 'NAILGUN_PORT') + def test_add_success_discover_false(self): with patch.object(self.plugin, '_add_message') as add_mock: