ut refactor, defunct process handling

functional/manual/post_testrun.py

@@ -18,4 +18,4 @@ def make_requests(claster_id, test_set):
 
 
 if __name__ == '__main__':
-    make_requests(27, 'plugin_general')
+    make_requests('307', 'plugin_general')

functional/manual/put_test_run_with_tests.py

@@ -17,4 +17,4 @@ def make_requests(claster_id, test_set):
 
 
 if __name__ == '__main__':
-    make_requests(364, 'plugin_general')
+    make_requests(370, 'plugin_general')

functional/manual/update_testrun.py

@@ -13,4 +13,4 @@ def make_requests(claster_id, test_set):
     pprint.pprint(data)
 
 if __name__ == '__main__':
-    make_requests(356, 'plugin_stopped')
+    make_requests(378, 'plugin_stopped')

ostf_adapter/api.py

@@ -110,8 +110,7 @@ class API(object):
         command, transport = self._find_command(test_run.type)
         cleanup = command.get('cleanup')
         killed = transport.obj.kill(
-            test_run.id, test_run.external_id, test_run.type,
+            test_run.id, test_run.external_id, cleanup=cleanup)
-            test_path=command.get('test_path'), cleanup=cleanup)
         if killed:
             if cleanup:
                 status = 'cleanup'

ostf_adapter/transport/nose_adapter.py

@@ -1,11 +1,10 @@
 import multiprocessing
-from nose import main
+from nose import core
 import os
-from ostf_adapter.storage import get_storage
+from ostf_adapter import storage
 from ostf_adapter.transport import nose_utils
 from ostf_adapter.transport import nose_storage_plugin
 import logging
-from ostf_adapter.api import parse_json_file
 from ostf_adapter import exceptions as exc
 from pecan import conf
 
@@ -20,61 +19,65 @@ class NoseDriver(object):
 
     def __init__(self):
         log.info('NoseDriver initialized')
-        self.storage = get_storage()
+        self.storage = storage.get_storage()
         self._named_threads = {}
-        self._configs = parse_json_file('config_templates.json')
+
+    def clean_process(self):
+        items = self._named_threads.items()
+        for uid, proc in items:
+            if not proc.is_alive():
+                proc.terminate()
+                self._named_threads.pop(uid)
 
     def check_current_running(self, unique_id):
         return unique_id in self._named_threads
 
     def run(self, test_run_id, external_id,
             conf, test_set, tests=None, test_path=None, argv=None):
-        if conf:
+        self.clean_process()
-            test_conf_path = self.prepare_config(
-                conf, test_path, external_id, test_set)
-        else:
-            test_conf_path = ''
         argv_add = argv or []
         tests = tests or []
+
         if tests:
             log.info('TESTS RECEIVED %s' % tests)
             argv_add += map(nose_utils.modify_test_name_for_nose, tests)
         else:
             argv_add.append(test_path)
         log.info('Additional args: %s' % argv_add)
+
         proc = multiprocessing.Process(
             target=self._run_tests,
-            args=(test_run_id, external_id, argv_add, test_conf_path))
+            args=(test_run_id, external_id, argv_add))
         proc.daemon = True
         proc.start()
-        self._named_threads[test_run_id] = proc
+
+        self._named_threads[int(test_run_id)] = proc
         log.info('NAMED PROCESS %s' % self._named_threads)
 
     def tests_discovery(self, test_set, test_path, argv_add):
-        try:
         log.info('Started test discovery %s' % test_set)
-            main(defaultTest=test_path,
+
+        core.TestProgram(
+            defaultTest=test_path,
             addplugins=[nose_storage_plugin.StoragePlugin(
                 test_set, '', discovery=True)],
             exit=False,
             argv=['tests', '--collect-only'] + argv_add)
-        except Exception, e:
-            log.info('Finished tests discovery %s' % test_set)
 
-    def _run_tests(self, test_run_id, external_id,
+    def _run_tests(self, test_run_id, external_id, argv_add):
-                   argv_add, test_conf_path=''):
-        try:
         log.info('Nose Driver spawn process for TEST RUN: %s\n'
                      'ARGS: %s' % (test_run_id, argv_add))
-            main(addplugins=[nose_storage_plugin.StoragePlugin(
+
-                test_run_id, str(external_id), test_conf_path=test_conf_path)],
+        try:
+            core.TestProgram(addplugins=[nose_storage_plugin.StoragePlugin(
+                test_run_id, str(external_id))],
                 exit=False,
                 argv=['tests']+argv_add)
+
             log.info('Test run %s finished successfully' % test_run_id)
-            if test_run_id in self._named_threads:
-                del self._named_threads[test_run_id]
             self.storage.update_test_run(test_run_id, status='finished')
-        #To close thread we need to catch any exception
+            self._named_threads.pop(int(test_run_id), None)
+            raise SystemExit
         except Exception, e:
             log.info('Close process TEST_RUN: %s\n'
                      'Thread closed with exception: %s' % (test_run_id,
@@ -82,71 +85,55 @@ class NoseDriver(object):
             self.storage.update_test_run(test_run_id, status='error')
             self.storage.update_running_tests(test_run_id,
                                               status='error')
-            if test_run_id in self._named_threads:
-                del self._named_threads[test_run_id]
 
-    def kill(self, test_run_id, external_id, test_set,
+    def kill(self, test_run_id, external_id, cleanup=None):
-             test_path=None, cleanup=False):
         log.info('Trying to stop process %s\n'
                  '%s' % (test_run_id, self._named_threads))
+        self.clean_process()
         if test_run_id in self._named_threads:
+
             log.info('Terminating process: %s' % test_run_id)
-            self._named_threads[test_run_id].terminate()
+
-            del self._named_threads[test_run_id]
+            self._named_threads[int(test_run_id)].terminate()
+            self._named_threads.pop(int(test_run_id), None)
+
             if cleanup:
                 proc = multiprocessing.Process(
                     target=self._clean_up,
-                    args=(test_run_id, external_id, test_set,
+                    args=(test_run_id, external_id, cleanup))
-                          test_path, cleanup))
                 proc.daemon = True
                 proc.start()
+                self._named_threads[int(test_run_id)] = proc
             else:
                 self.storage.update_test_run(test_run_id, status='stopped')
+
             return True
         return False
 
     def _clean_up(self,
-                  test_run_id, external_id, test_set, test_path, cleanup):
+                  test_run_id, external_id, cleanup,
-        stor = get_storage(conf.dbpath)
+                  storage=storage.get_storage):
+        #Had problems with mocking storage.get_storage
+        storage = storage()
         try:
+
             log.info("TRYING TO CLEAN")
             module_obj = __import__(cleanup, -1)
 
-            os.environ['OSTF_CONF_PATH'] = nose_utils.config_name_generator(
+            os.environ['NAILGUN_HOST'] = str(conf.nailgun.host)
-                test_path, test_set, external_id)
+            os.environ['NAILGUN_PORT'] = str(conf.nailgun.port)
+            os.environ['CLUSTER_ID'] = str(external_id)
+
             log.info('STARTING CLEANUP FUNCTION')
             module_obj.cleanup.cleanup()
             log.info('CLEANUP IS SUCCESSFULL')
-            stor.update_test_run(test_run_id, status='stopped')
-        except BaseException, e:
-            log.error('EXCITED WITH EXCEPTIOBN %s' % e)
-            stor.update_test_run(test_run_id, status='error_on_cleanup')
 
-    def prepare_config(self, config, test_path, external_id, test_set):
+            storage.update_test_run(test_run_id, status='stopped')
-        template = []
+            raise SystemExit
-        for group_name, group_items in self._configs.iteritems():
+        except Exception, e:
-            template_group = []
-            for group_item in group_items:
-                if group_item in config:
-                    if not template_group:
-                        template_group.append('[{0}]'.format(group_name))
-                    template_group.append('{0} = {1}'.format(
-                        group_item, config[group_item]))
-                with_group = '{0}_{1}'.format(group_name, group_item)
-                if with_group in config:
-                    if not template_group:
-                        template_group.append('[{0}]'.format(group_name))
-                    template_group.append('{0} = {1}'.format(
-                        group_item, config[with_group]))
-            template.extend(template_group)
-
-        if template:
-            conf_path = nose_utils.config_name_generator(
-                test_path, test_set, external_id)
-            with open(conf_path, 'w') as f:
-                f.write(u'\n'.join(template))
-            return conf_path
 
+            log.error('EXCITED WITH EXCEPTION %s' % e)
+            storage.update_test_run(test_run_id, status='error_on_cleanup')
 
 
 

ostf_adapter/transport/nose_storage_plugin.py

@@ -21,21 +21,18 @@ class StoragePlugin(Plugin):
     score = 15000
 
     def __init__(
-            self, test_run_id, cluster_id, discovery=False,
+            self, test_run_id, cluster_id, discovery=False):
-            test_conf_path=''):
         self._capture = []
         self.test_run_id = test_run_id
         self.cluster_id = cluster_id
         self.storage = get_storage()
         self.discovery = discovery
-        self.test_conf_path = test_conf_path
         super(StoragePlugin, self).__init__()
         log.info('Storage Plugin initialized')
         self._start_time = None
         self._started = False
 
     def options(self, parser, env=os.environ):
-        env['CUSTOM_FUEL_CONFIG'] = self.test_conf_path
         log.info('NAILGUN HOST %s '
                  'AND PORT %s' % (conf.nailgun.host, conf.nailgun.port))
         env['NAILGUN_HOST'] = str(conf.nailgun.host)
@@ -52,12 +49,14 @@ class StoragePlugin(Plugin):
         if not self._started:
             self.storage.update_test_run(self.test_run_id, status='running')
         self._started = True
-        data = {}
+        data = dict()
         data['name'], data['description'], data['duration'] =\
             nose_utils.get_description(test)
         if err:
             exc_type, exc_value, exc_traceback = err
             log.info('Error %s' % exc_value)
+            data['message'] = u''
+            if not status == 'error':
                 data['message'] = nose_utils.get_exc_message(exc_value)
             data['traceback'] = nose_utils.format_exception(err)
         else:
@@ -76,7 +75,7 @@ class StoragePlugin(Plugin):
     def addSuccess(self, test, capt=None):
         log.info('SUCCESS for %s' % test)
         if self.discovery:
-            data = {}
+            data = dict()
             data['name'], data['description'], data['duration'] =\
                 nose_utils.get_description(test)
             data['message'] = None

tests/test_api.py

@@ -66,5 +66,7 @@ class TestApi(unittest.TestCase):
         test_run = {'id': 1}
         status = 'stopped'
         with patch.object(self.api, '_find_command') as command_mock:
+            with patch.object(self.api, '_prepare_test_run') as test_run_mock:
                 command_mock.return_value = (self.command, self.transport)
                 self.api.kill(test_run)
+        self.assertEqual(test_run_mock.call_count, 1)

tests/test_nose_adapter.py

@@ -24,26 +24,28 @@ class NoExitStriongIO(io.StringIO):
 
 class TestNoseAdapters(unittest.TestCase):
 
-    @patch('ostf_adapter.transport.nose_adapter.get_storage')
+    @patch('ostf_adapter.transport.nose_adapter.storage')
-    def setUp(self, get_storage_mock):
+    def setUp(self, storage_mock):
         self.thread = MagicMock()
         self.storage = MagicMock()
+        self.module_mock = MagicMock()
         self.config_out = NoExitStriongIO()
-        get_storage_mock.return_value = self.storage
+        storage_mock.get_storage.return_value = self.storage
         self.driver = nose_adapter.NoseDriver()
 
-    @patch('__builtin__.open')
+    def test_kill(self):
-    def test_prepare_config_conf(self, open_mock):
+        pass
-        open_mock.return_value = self.config_out
+
-        conf = {'network_catalog_type': 'TEST_TYPE',
+    @patch('ostf_adapter.transport.nose_adapter.conf')
-                'url': 'http://localhost:8989/v1/'}
+    @patch('__builtin__.__import__')
-        test_path = 'fuel_health.tests'
+    def test_cleanup_call(self, import_mock, conf_mock):
-        external_id = 12
+        import_mock.return_value = self.module_mock
-        test_set = 'fuel_health'
+
-        self.driver.prepare_config(
+        conf_mock.nailgun.host = 'NAILGUN_HOST'
-            conf, test_path, external_id, test_set)
+        conf_mock.nailgun.port = 'NAILGUN_PORT'
-        self.assertEqual(self.config_out.getvalue(),
+        self.driver._named_threads[1] = self.thread
-                         u'[network]\ncatalog_type = TEST_TYPE\n[identity]\nurl = http://localhost:8989/v1/')
+
+        self.driver._clean_up(1, '101', 'cleanup', storage=MagicMock)
 
 
 class TestNoseUtils(unittest.TestCase):
@@ -66,8 +68,10 @@ class TestNoseUtils(unittest.TestCase):
         self.assertEqual(test_path.split('/')[-1],
                          'test_plugin_general_12.conf')
 
+
 class TestNoseStoragePlugin(unittest.TestCase):
 
+
     @patch('ostf_adapter.transport.nose_storage_plugin.get_storage')
     def setUp(self, get_storage_mock):
         self.storage = MagicMock()
@@ -77,16 +81,21 @@ class TestNoseStoragePlugin(unittest.TestCase):
         self.test_parent_id = 12
         self.cluster_id = '14'
         self.plugin = nose_storage_plugin.StoragePlugin(
-            self.test_parent_id, self.cluster_id, discovery=False,
+            self.test_parent_id, self.cluster_id, discovery=False)
-            test_conf_path='/etc/config.conf')
 
-    def test_options_interface_defined(self):
+    @patch('ostf_adapter.transport.nose_storage_plugin.conf')
+    def test_options_interface_defined(self, conf_mock):
+        conf_mock.nailgun.host = 'NAILGUN_HOST'
+        conf_mock.nailgun.port = 'NAILGUN_PORT'
         self.plugin.options({})
 
-        self.assertEqual(os.environ['CUSTOM_FUEL_CONFIG'],
-                         self.plugin.test_conf_path)
         self.assertEqual(os.environ['CLUSTER_ID'],
                          self.cluster_id)
+        self.assertEqual(os.environ['NAILGUN_HOST'],
+                         'NAILGUN_HOST')
+        self.assertEqual(os.environ['NAILGUN_PORT'],
+                         'NAILGUN_PORT')
+
 
     def test_add_success_discover_false(self):
         with patch.object(self.plugin, '_add_message') as add_mock: