Add persister.py unit tests

Add unit tests for persister.py in root directory.
Additionally:
* switched to testr as it seems to be used in many
openstack projects
* enabled coverage

Change-Id: I429ef6a900808c192ad5613c13808583d33fde24

.gitignore

@@ -7,3 +7,8 @@ logs/
 .idea/
 **/*.pyc
 virtenv/*
+.coverage
+cover/
+.tox/
+*egg-info/
+.testrepository/

.testr.conf

@@ -0,0 +1,9 @@
+[DEFAULT]
+test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
+             OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
+             OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-160} \
+             ${PYTHON:-python} -m subunit.run discover -t ./ ${OS_TEST_PATH:-./monasca_persister/tests} $LISTOPT $IDOPTION
+
+test_id_option=--load-list $IDFILE
+test_list_option=--list
+group_regex=monasca_persister\.tests(?:\.|_)([^_]+)

monasca_persister/tests/test_persister_main.py

@@ -0,0 +1,245 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from mock import patch
+from mock import call
+from mock import Mock
+import signal
+
+from oslo_config import cfg
+from oslotest import base
+
+CONF = cfg.CONF
+
+NUMBER_OF_METRICS_PROCESSES = 2
+NUMBER_OF_ALARM_HIST_PROCESSES = 3
+
+
+class FakeException(Exception):
+    pass
+
+
+class TestPersister(base.BaseTestCase):
+
+    def setUp(self):
+        super(TestPersister, self).setUp()
+
+        from monasca_persister import persister
+        self.persister = persister
+
+        self._set_patchers()
+        self._set_mocks()
+
+    def _set_patchers(self):
+        self.sys_exit_patcher = patch.object(self.persister.sys, 'exit')
+        self.log_patcher = patch.object(self.persister, 'log')
+        self.simport_patcher = patch.object(self.persister, 'simport')
+        self.cfg_patcher = patch.object(self.persister, 'cfg')
+        self.sleep_patcher = patch.object(self.persister.time, 'sleep')
+        self.process_patcher = patch.object(self.persister.multiprocessing, 'Process')
+
+    def _set_mocks(self):
+        self.mock_sys_exit = self.sys_exit_patcher.start()
+        self.mock_log = self.log_patcher.start()
+        self.mock_simport = self.simport_patcher.start()
+        self.mock_cfg = self.cfg_patcher.start()
+        self.mock_sleep = self.sleep_patcher.start()
+        self.mock_process_class = self.process_patcher.start()
+
+        self.mock_cfg.CONF.kafka_metrics.num_processors = NUMBER_OF_METRICS_PROCESSES
+        self.mock_cfg.CONF.kafka_alarm_history.num_processors = NUMBER_OF_ALARM_HIST_PROCESSES
+        self.mock_cfg.CONF.zookeeper = 'zookeeper'
+
+        self.mock_sleep.side_effect = [FakeException, None]
+
+        self.process_pool = _get_process_list(
+            NUMBER_OF_METRICS_PROCESSES + NUMBER_OF_ALARM_HIST_PROCESSES + 1)
+        self.mock_process_class.side_effect = self.process_pool
+
+    def tearDown(self):
+        super(TestPersister, self).tearDown()
+
+        self.mock_sleep.side_effect = None
+
+        self.mock_sys_exit.side_effect = None
+        self.mock_sys_exit.reset_mock()
+
+        self.sys_exit_patcher.stop()
+        self.log_patcher.stop()
+        self.simport_patcher.stop()
+        self.cfg_patcher.stop()
+        self.sleep_patcher.stop()
+        self.process_patcher.stop()
+
+        self.persister.processors = []
+        self.persister.exiting = False
+
+    def _run_persister(self):
+        self.persister.main()
+        self.assertTrue(self.mock_process_class.called)
+
+    def _assert_clean_exit_handler_terminates_with_expected_signal(
+            self, signum, expected):
+        # in order to really exit, an exception is thrown
+
+        self.mock_sys_exit.side_effect = FakeException
+        self.assertRaises(FakeException, self.persister.clean_exit, signum)
+
+        self.mock_sys_exit.assert_called_once_with(expected)
+
+    def _assert_correct_number_of_processes_created(self):
+
+        for p in self.process_pool[:-1]:
+            self.assertTrue(p.start.called)
+
+        self.assertFalse(self.process_pool[-1].called)
+
+    def _assert_process_alive_status_called(self):
+
+        for p in self.process_pool[:-1]:
+            self.assertTrue(p.is_alive.called)
+
+    def _assert_process_terminate_called(self):
+
+        for p in self.process_pool[:-1]:
+            self.assertTrue(p.terminate.called)
+
+    def test_active_children_are_killed_during_exit(self):
+
+        with patch.object(self.persister.multiprocessing, 'active_children') as active_children,\
+             patch.object(self.persister.os, 'kill') as mock_kill:
+
+            active_children.return_value = [Mock(name='child-1', pid=1),
+                                            Mock(name='child-2', pid=2)]
+
+            self.persister.clean_exit(0)
+
+            mock_kill.assert_has_calls([call(1, signal.SIGKILL), call(2, signal.SIGKILL)])
+
+    def test_active_children_kill_exception_is_ignored(self):
+
+        with patch.object(self.persister.multiprocessing,
+                          'active_children') as active_children, \
+                patch.object(self.persister.os, 'kill') as mock_kill:
+
+            active_children.return_value = [Mock()]
+            mock_kill.side_effect = FakeException
+
+            self.persister.clean_exit(0)
+
+            self.assertTrue(mock_kill.called)
+
+    def test_clean_exit_does_nothing_when_exiting_is_true(self):
+        self.persister.exiting = True
+
+        self.assertEqual(None, self.persister.clean_exit(0))
+
+        self.assertFalse(self.mock_sys_exit.called)
+
+    def test_exception_during_process_termination_is_ignored(self):
+        dead_process = _get_process('raises_when_terminated')
+        dead_process.terminate.side_effect = FakeException
+        self.process_pool[0] = dead_process
+
+        self._run_persister()
+
+        self.assertTrue(dead_process.terminate.called)
+
+    def test_if_not_sigterm_then_clean_exit_handler_terminates_with_signal(self):
+
+        self._assert_clean_exit_handler_terminates_with_expected_signal(
+            signal.SIGINT, signal.SIGINT)
+
+    def test_if_sigterm_then_clean_exit_handler_terminates_with_zero(self):
+
+        self._assert_clean_exit_handler_terminates_with_expected_signal(
+            signal.SIGTERM, 0)
+
+    def test_non_running_process_not_terminated(self):
+        dead_process = _get_process('dead_process', is_alive=False)
+        self.process_pool[0] = dead_process
+
+        self._run_persister()
+
+        self.assertTrue(dead_process.is_alive.called)
+        self.assertFalse(dead_process.terminate.called)
+
+    def test_running_processes_are_created_and_terminated(self):
+
+        self._run_persister()
+
+        self._assert_correct_number_of_processes_created()
+        self._assert_process_alive_status_called()
+        self._assert_process_terminate_called()
+
+    def test_start_process_handler_creates_and_runs_persister(self):
+        fake_kafka_config = Mock()
+        fake_repository = Mock()
+
+        with patch('monasca_persister.repositories.persister.Persister') as mock_persister_class:
+            self.persister.start_process(fake_repository, fake_kafka_config)
+
+            mock_persister_class.assert_called_once_with(
+                fake_kafka_config, 'zookeeper', fake_repository)
+
+
+class TestPersisterConfig(base.BaseTestCase):
+
+    def setUp(self):
+        super(TestPersisterConfig, self).setUp()
+
+        from monasca_persister import persister
+
+        self.assertIsNotNone(persister)
+
+    def _test_zookeeper_options(self):
+        str_opts = ['uri']
+        int_opts = ['partition_interval_recheck_seconds']
+
+        self._assert_cfg_registered('zookeeper', str_opts, int_opts)
+
+    def _assert_cfg_registered(self, group_name, str_opts, int_opts):
+        options = {group_name: str_opts + int_opts}
+
+        for key, values in options.items():
+            for v in values:
+                self.assertIsNone(CONF[key][v])
+
+    def _test_kafka_options(self):
+        str_opts = ['uri', 'group_id', 'topic', 'consumer_id', 'client_id', 'zookeeper_path']
+        int_opts = ['database_batch_size', 'max_wait_time_seconds', 'fetch_size_bytes',
+                    'buffer_size', 'max_buffer_size', 'num_processors']
+
+        self._assert_cfg_registered('kafka_metrics', str_opts, int_opts)
+        self._assert_cfg_registered('kafka_alarm_history', str_opts, int_opts)
+
+    def _test_repositories_options(self):
+        str_opts = ['metrics_driver', 'alarm_state_history_driver']
+        int_opts = []
+
+        self._assert_cfg_registered('repositories', str_opts, int_opts)
+
+    def test_correct_config_options_are_registered(self):
+        self._test_zookeeper_options()
+        self._test_kafka_options()
+        self._test_repositories_options()
+
+
+def _get_process(name, is_alive=True):
+    return Mock(name=name, is_alive=Mock(return_value=is_alive))
+
+
+def _get_process_list(number_of_processes):
+    processes = []
+    for i in range(number_of_processes):
+        processes.append(_get_process(name='process_{}'.format(i)))
+    return processes

setup.cfg

@@ -12,6 +12,10 @@ description-file = README.md
 home-page = https://github.com/openstack/monasca-persister
 license = Apache
 
+[global]
+setup-hooks =
+    pbr.hooks.setup_hook
+
 [entry_points]
 console_scripts =
     monasca-persister = monasca_persister.persister:main
@@ -19,15 +23,9 @@ console_scripts =
 [files]
 packages = monasca_persister
 
-[flake8]
-max-line-length = 120
-
 [pbr]
 autodoc_index_modules = True
 
-[pep8]
-max-line-length = 120
-
 [wheel]
 universal = 1
 

test-requirements.txt

@@ -1,6 +1,10 @@
 # The order of packages is significant, because pip processes them in the order
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
-hacking<0.12,>=0.11.0 # Apache-2.0
+flake8>=2.5.4,<2.6.0  # MIT
+hacking>=0.12.0,!=0.13.0,<0.14  # Apache-2.0
+coverage>=4.0 # Apache-2.0
 nose # LGPL
 mock>=2.0 # BSD
+oslotest>=1.10.0 # Apache-2.0
+os-testr>=0.8.0 # Apache-2.0

tox.ini

@@ -1,19 +1,53 @@
 [tox]
-envlist = py27,pypy,pep8
-minversion = 2.0
+envlist = py27,py35,pep8
+minversion = 2.1
 skipsdist = True
 
 [testenv]
-setenv = VIRTUAL_ENV={envdir}
+setenv =
+    VIRTUAL_ENV={envdir}
+    DISCOVER_DIRECTORY=tests
+    CLIENT_NAME=monasca-persister
+passenv = http_proxy
+          HTTP_PROXY
+          https_proxy
+          HTTPS_PROXY
+          no_proxy
+          NO_PROXY
 usedevelop = True
+whitelist_externals = bash
+                      find
+                      rm
 install_command =
     {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
-whitelist_externals = find
 commands =
-  find . -type f -name "*.pyc" -delete
-  nosetests
+  find {toxinidir} -type f -name "*.py[c|o]" -delete
+
+[testenv:py27]
+basepython = python2.7
+commands =
+  {[testenv]commands}
+  ostestr {posargs}
+
+[testenv:py35]
+basepython = python3.5
+commands =
+  {[testenv]commands}
+  ostestr {posargs}
+
+[testenv:cover]
+commands =
+  {[testenv]commands}
+  coverage erase
+  python setup.py test --coverage --testr-args='{posargs}' --coverage-package-name=monasca_persister
+  coverage report
+
+[testenv:debug]
+commands =
+  {[testenv]commands}
+  oslo_debug_helper -t monasca_persister/tests {posargs}
 
 [testenv:pep8]
 commands = flake8
@@ -23,7 +57,7 @@ commands = {posargs}
 
 [flake8]
 max-line-length = 120
-# TODO: ignored checks should be enabled in the future 
+# TODO: ignored checks should be enabled in the future
 # H405  multi line docstring summary not separated with an empty line
 # H904  Wrap long lines in parentheses instead of a backslash
 ignore = F821,H405,H904,E126,E125,H306,E302,E122