From 7aef952a4153c810ac290813d6830b7faec7221c Mon Sep 17 00:00:00 2001 From: Kengo Takahara Date: Fri, 17 Mar 2017 17:52:23 +0900 Subject: [PATCH] Add testcases of ha and processmonitor This patch added missing testcases for ha and processmonitor directory. And also improved existing testcases for them. Change-Id: Ifd9681b53b71122aa9ca738e6e6bf49f282b1f61 --- masakarimonitors/tests/unit/ha/__init__.py | 0 .../tests/unit/ha/test_masakari.py | 178 ++++++++++++ .../process_handler/test_handle_process.py | 266 +++++++++++++++++- .../tests/unit/processmonitor/test_process.py | 98 ++++++- 4 files changed, 534 insertions(+), 8 deletions(-) create mode 100644 masakarimonitors/tests/unit/ha/__init__.py create mode 100644 masakarimonitors/tests/unit/ha/test_masakari.py diff --git a/masakarimonitors/tests/unit/ha/__init__.py b/masakarimonitors/tests/unit/ha/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/masakarimonitors/tests/unit/ha/test_masakari.py b/masakarimonitors/tests/unit/ha/test_masakari.py new file mode 100644 index 0000000..af8d830 --- /dev/null +++ b/masakarimonitors/tests/unit/ha/test_masakari.py @@ -0,0 +1,178 @@ +# Copyright(c) 2017 Nippon Telegraph and Telephone Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import testtools + +import eventlet +from openstack import connection +from openstack import exceptions +from openstack import profile +from oslo_utils import timeutils + +from masakariclient.sdk.ha import ha_service +from masakarimonitors.ha import masakari +from masakarimonitors.objects import event_constants as ec + +PROFILE_TYPE = "ha" +PROFILE_NAME = "masakari" + + +class TestSendNotification(testtools.TestCase): + + def setUp(self): + super(TestSendNotification, self).setUp() + self.api_retry_max = 3 + self.api_retry_interval = 1 + self.event = { + 'notification': { + 'type': ec.EventConstants.TYPE_COMPUTE_HOST, + 'hostname': 'compute-node1', + 'generated_time': timeutils.utcnow(), + 'payload': { + 'event': ec.EventConstants.EVENT_STOPPED, + 'cluster_status': 'OFFLINE', + 'host_status': ec.EventConstants.HOST_STATUS_NORMAL + } + } + } + + @mock.patch.object(connection, 'Connection') + @mock.patch.object(profile, 'Profile') + def test_send_notification(self, + mock_Profile, + mock_Connection): + + mock_prof = mock.Mock() + mock_Profile.return_value = mock_prof + mock_conn = mock.Mock() + mock_Connection.return_value = mock_conn + + notifier = masakari.SendNotification() + notifier.send_notification( + self.api_retry_max, self.api_retry_interval, self.event) + + mock_prof._add_service.assert_called_once_with( + ha_service.HAService(version='v1')) + mock_prof.set_name.assert_called_once_with( + PROFILE_TYPE, PROFILE_NAME) + mock_prof.set_region.assert_called_once_with( + PROFILE_TYPE, 'RegionOne') + mock_prof.set_version.assert_called_once_with( + PROFILE_TYPE, 'v1') + mock_prof.set_interface.assert_called_once_with( + PROFILE_TYPE, 'public') + + mock_Connection.assert_called_once_with( + auth_url=None, + project_name=None, + username=None, + password=None, + project_domain_id=None, + user_domain_id=None, + profile=mock_prof) + mock_conn.ha.create_notification.assert_called_once_with( + type=self.event['notification']['type'], + hostname=self.event['notification']['hostname'], + generated_time=self.event['notification']['generated_time'], + payload=self.event['notification']['payload']) + + @mock.patch.object(connection, 'Connection') + @mock.patch.object(profile, 'Profile') + def test_send_notification_409_error(self, + mock_Profile, + mock_Connection): + + mock_prof = mock.Mock() + mock_Profile.return_value = mock_prof + mock_conn = mock.Mock() + mock_Connection.return_value = mock_conn + mock_conn.ha.create_notification.side_effect = \ + exceptions.HttpException(http_status=409) + + notifier = masakari.SendNotification() + notifier.send_notification( + self.api_retry_max, self.api_retry_interval, self.event) + + mock_prof._add_service.assert_called_once_with( + ha_service.HAService(version='v1')) + mock_prof.set_name.assert_called_once_with( + PROFILE_TYPE, PROFILE_NAME) + mock_prof.set_region.assert_called_once_with( + PROFILE_TYPE, 'RegionOne') + mock_prof.set_version.assert_called_once_with( + PROFILE_TYPE, 'v1') + mock_prof.set_interface.assert_called_once_with( + PROFILE_TYPE, 'public') + + mock_Connection.assert_called_once_with( + auth_url=None, + project_name=None, + username=None, + password=None, + project_domain_id=None, + user_domain_id=None, + profile=mock_prof) + mock_conn.ha.create_notification.assert_called_once_with( + type=self.event['notification']['type'], + hostname=self.event['notification']['hostname'], + generated_time=self.event['notification']['generated_time'], + payload=self.event['notification']['payload']) + + @mock.patch.object(eventlet.greenthread, 'sleep') + @mock.patch.object(connection, 'Connection') + @mock.patch.object(profile, 'Profile') + def test_send_notification_500_error(self, + mock_Profile, + mock_Connection, + mock_sleep): + + mock_prof = mock.Mock() + mock_Profile.return_value = mock_prof + mock_conn = mock.Mock() + mock_Connection.return_value = mock_conn + mock_conn.ha.create_notification.side_effect = \ + exceptions.HttpException(http_status=500) + mock_sleep.return_value = None + + notifier = masakari.SendNotification() + notifier.send_notification( + self.api_retry_max, self.api_retry_interval, self.event) + + mock_prof._add_service.assert_called_once_with( + ha_service.HAService(version='v1')) + mock_prof.set_name.assert_called_once_with( + PROFILE_TYPE, PROFILE_NAME) + mock_prof.set_region.assert_called_once_with( + PROFILE_TYPE, 'RegionOne') + mock_prof.set_version.assert_called_once_with( + PROFILE_TYPE, 'v1') + mock_prof.set_interface.assert_called_once_with( + PROFILE_TYPE, 'public') + + mock_Connection.assert_called_once_with( + auth_url=None, + project_name=None, + username=None, + password=None, + project_domain_id=None, + user_domain_id=None, + profile=mock_prof) + mock_conn.ha.create_notification.assert_called_with( + type=self.event['notification']['type'], + hostname=self.event['notification']['hostname'], + generated_time=self.event['notification']['generated_time'], + payload=self.event['notification']['payload']) + self.assertEqual(self.api_retry_max + 1, + mock_conn.ha.create_notification.call_count) diff --git a/masakarimonitors/tests/unit/processmonitor/process_handler/test_handle_process.py b/masakarimonitors/tests/unit/processmonitor/process_handler/test_handle_process.py index e8f1aaf..912bd05 100644 --- a/masakarimonitors/tests/unit/processmonitor/process_handler/test_handle_process.py +++ b/masakarimonitors/tests/unit/processmonitor/process_handler/test_handle_process.py @@ -13,13 +13,19 @@ # limitations under the License. import mock +import socket import testtools import eventlet +from oslo_utils import timeutils +import masakarimonitors.conf +from masakarimonitors.ha import masakari +from masakarimonitors.objects import event_constants as ec from masakarimonitors.processmonitor.process_handler import handle_process from masakarimonitors import utils +CONF = masakarimonitors.conf.CONF eventlet.monkey_patch(os=False) MOCK_PROCESS_LIST = [ @@ -65,6 +71,8 @@ class TestHandleProcess(testtools.TestCase): obj = handle_process.HandleProcess() obj.set_process_list(process_list) + self.assertEqual(process_list, obj.process_list) + @mock.patch.object(utils, 'execute') def test_start_processes(self, mock_execute): @@ -88,6 +96,20 @@ class TestHandleProcess(testtools.TestCase): MOCK_PROCESS_LIST[0].get('post_start_command'), run_as_root=MOCK_PROCESS_LIST[0].get('run_as_root')) + @mock.patch.object(utils, 'execute') + def test_start_processes_pre_cmd_fail(self, mock_execute): + process_list = MOCK_PROCESS_LIST + obj = handle_process.HandleProcess() + obj.set_process_list(process_list) + + mock_execute.return_value = ('test_stdout', 'test_stderr') + + obj.start_processes() + + mock_execute.assert_called_once_with( + MOCK_PROCESS_LIST[0].get('pre_start_command'), + run_as_root=MOCK_PROCESS_LIST[0].get('run_as_root')) + @mock.patch.object(utils, 'execute') def test_monitor_processes(self, mock_execute): @@ -99,6 +121,32 @@ class TestHandleProcess(testtools.TestCase): down_process_list = obj.monitor_processes() self.assertEqual([], down_process_list) + mock_execute.assert_called_once_with( + 'ps', '-ef', run_as_root=False) + + @mock.patch.object(utils, 'execute') + def test_monitor_processes_not_found(self, mock_execute): + process_list = MOCK_PROCESS_LIST + obj = handle_process.HandleProcess() + obj.set_process_list(process_list) + + mock_execute.return_value = ('', '') + + down_process_list = obj.monitor_processes() + self.assertEqual(MOCK_PROCESS_LIST, down_process_list) + mock_execute.assert_called_once_with( + 'ps', '-ef', run_as_root=False) + + @mock.patch.object(utils, 'execute') + def test_monitor_processes_exception(self, mock_execute): + process_list = MOCK_PROCESS_LIST + obj = handle_process.HandleProcess() + obj.set_process_list(process_list) + + mock_execute.side_effect = Exception("Test exception.") + + down_process_list = obj.monitor_processes() + self.assertEqual([], down_process_list) @mock.patch.object(utils, 'execute') def test_restart_processes(self, @@ -115,11 +163,217 @@ class TestHandleProcess(testtools.TestCase): obj.restart_processes(down_process_list) mock_execute.assert_any_call( - MOCK_DOWN_PROCESS_LIST[0].get('pre_restart_command'), - run_as_root=MOCK_DOWN_PROCESS_LIST[0].get('run_as_root')) + down_process_list[0].get('pre_restart_command'), + run_as_root=down_process_list[0].get('run_as_root')) mock_execute.assert_any_call( - MOCK_DOWN_PROCESS_LIST[0].get('restart_command'), - run_as_root=MOCK_DOWN_PROCESS_LIST[0].get('run_as_root')) + down_process_list[0].get('restart_command'), + run_as_root=down_process_list[0].get('run_as_root')) mock_execute.assert_any_call( - MOCK_DOWN_PROCESS_LIST[0].get('post_restart_command'), - run_as_root=MOCK_DOWN_PROCESS_LIST[0].get('run_as_root')) + down_process_list[0].get('post_restart_command'), + run_as_root=down_process_list[0].get('run_as_root')) + self.assertEqual([], obj.restart_failure_list) + + @mock.patch.object(utils, 'execute') + def test_restart_processes_failed_to_restart_previously( + self, mock_execute): + process_list = MOCK_PROCESS_LIST + obj = handle_process.HandleProcess() + obj.set_process_list(process_list) + restart_failure_list = [MOCK_DOWN_PROCESS_LIST[0].get('process_name')] + obj.restart_failure_list = restart_failure_list + down_process_list = MOCK_DOWN_PROCESS_LIST + + obj.restart_processes(down_process_list) + + self.assertEqual(restart_failure_list, obj.restart_failure_list) + mock_execute.assert_not_called() + + @mock.patch.object(masakari.SendNotification, 'send_notification') + @mock.patch.object(timeutils, 'utcnow') + @mock.patch.object(eventlet.greenthread, 'sleep') + @mock.patch.object(utils, 'execute') + def test_restart_processes_pre_restart_command_retry_over( + self, mock_execute, mock_sleep, mock_utcnow, mock_send_notification): + process_list = MOCK_PROCESS_LIST + obj = handle_process.HandleProcess() + obj.set_process_list(process_list) + down_process_list = MOCK_DOWN_PROCESS_LIST + + mock_execute.side_effect = [('test_stdout', 'test_stderr'), + ('test_stdout', 'test_stderr'), + ('test_stdout', 'test_stderr'), + ('test_stdout', 'test_stderr')] + mock_sleep.return_value = None + current_time = timeutils.utcnow() + mock_utcnow.return_value = current_time + mock_send_notification.return_value = None + + obj.restart_processes(down_process_list) + + pre_execute_count = CONF.process.restart_retries + 1 + self.assertEqual(pre_execute_count, mock_execute.call_count) + + for var in range(0, mock_execute.call_count): + args, kwargs = mock_execute.call_args_list[var] + self.assertEqual( + (down_process_list[0].get('pre_restart_command'),), + args) + self.assertEqual({'run_as_root': True}, kwargs) + + event = { + 'notification': { + 'type': ec.EventConstants.TYPE_PROCESS, + 'hostname': socket.gethostname(), + 'generated_time': current_time, + 'payload': { + 'event': ec.EventConstants.EVENT_STOPPED, + 'process_name': down_process_list[0].get('process_name') + } + } + } + mock_send_notification.assert_called_once_with( + CONF.process.api_retry_max, + CONF.process.api_retry_interval, + event) + + self.assertEqual( + [down_process_list[0].get('process_name')], + obj.restart_failure_list) + + @mock.patch.object(masakari.SendNotification, 'send_notification') + @mock.patch.object(timeutils, 'utcnow') + @mock.patch.object(eventlet.greenthread, 'sleep') + @mock.patch.object(utils, 'execute') + def test_restart_processes_restart_command_retry_over( + self, mock_execute, mock_sleep, mock_utcnow, mock_send_notification): + process_list = MOCK_PROCESS_LIST + obj = handle_process.HandleProcess() + obj.set_process_list(process_list) + down_process_list = MOCK_DOWN_PROCESS_LIST + + mock_execute.side_effect = [('test_stdout', ''), + ('test_stdout', 'test_stderr'), + ('test_stdout', ''), + ('test_stdout', 'test_stderr'), + ('test_stdout', ''), + ('test_stdout', 'test_stderr'), + ('test_stdout', ''), + ('test_stdout', 'test_stderr')] + mock_sleep.return_value = None + current_time = timeutils.utcnow() + mock_utcnow.return_value = current_time + mock_send_notification.return_value = None + + obj.restart_processes(down_process_list) + + pre_execute_count = CONF.process.restart_retries + 1 + execute_count = CONF.process.restart_retries + 1 + total_execute_count = pre_execute_count + execute_count + self.assertEqual(total_execute_count, mock_execute.call_count) + + for var in range(0, mock_execute.call_count): + # Execute order of restart_command is the second. + execute_order = 2 + + if (var + 1) % execute_order == 0: + args, kwargs = mock_execute.call_args_list[var] + self.assertEqual( + (down_process_list[0].get('restart_command'),), + args) + self.assertEqual({'run_as_root': True}, kwargs) + + event = { + 'notification': { + 'type': ec.EventConstants.TYPE_PROCESS, + 'hostname': socket.gethostname(), + 'generated_time': current_time, + 'payload': { + 'event': ec.EventConstants.EVENT_STOPPED, + 'process_name': down_process_list[0].get('process_name') + } + } + } + mock_send_notification.assert_called_once_with( + CONF.process.api_retry_max, + CONF.process.api_retry_interval, + event) + self.assertEqual( + [down_process_list[0].get('process_name')], + obj.restart_failure_list) + + @mock.patch.object(masakari.SendNotification, 'send_notification') + @mock.patch.object(timeutils, 'utcnow') + @mock.patch.object(eventlet.greenthread, 'sleep') + @mock.patch.object(utils, 'execute') + def test_restart_processes_post_restart_command_retry_over( + self, mock_execute, mock_sleep, mock_utcnow, mock_send_notification): + process_list = MOCK_PROCESS_LIST + obj = handle_process.HandleProcess() + obj.set_process_list(process_list) + down_process_list = MOCK_DOWN_PROCESS_LIST + + mock_execute.side_effect = [('test_stdout', ''), + ('test_stdout', ''), + ('test_stdout', 'test_stderr'), + ('test_stdout', ''), + ('test_stdout', ''), + ('test_stdout', 'test_stderr'), + ('test_stdout', ''), + ('test_stdout', ''), + ('test_stdout', 'test_stderr'), + ('test_stdout', ''), + ('test_stdout', ''), + ('test_stdout', 'test_stderr')] + mock_sleep.return_value = None + current_time = timeutils.utcnow() + mock_utcnow.return_value = current_time + mock_send_notification.return_value = None + + obj.restart_processes(down_process_list) + + pre_execute_count = CONF.process.restart_retries + 1 + execute_count = CONF.process.restart_retries + 1 + post_execute_count = CONF.process.restart_retries + 1 + total_execute_count = \ + pre_execute_count + execute_count + post_execute_count + self.assertEqual(total_execute_count, mock_execute.call_count) + + for var in range(0, mock_execute.call_count): + # Execute order of restart_command is the third. + execute_order = 3 + + if (var + 1) % execute_order == 0: + args, kwargs = mock_execute.call_args_list[var] + self.assertEqual( + (down_process_list[0].get('post_restart_command'),), + args) + self.assertEqual({'run_as_root': True}, kwargs) + + event = { + 'notification': { + 'type': ec.EventConstants.TYPE_PROCESS, + 'hostname': socket.gethostname(), + 'generated_time': current_time, + 'payload': { + 'event': ec.EventConstants.EVENT_STOPPED, + 'process_name': down_process_list[0].get('process_name') + } + } + } + mock_send_notification.assert_called_once_with( + CONF.process.api_retry_max, + CONF.process.api_retry_interval, + event) + self.assertEqual( + [down_process_list[0].get('process_name')], + obj.restart_failure_list) + + @mock.patch.object(utils, 'execute') + def test_execute_cmd_exception(self, mock_execute): + mock_execute.side_effect = Exception("Test exception.") + + obj = handle_process.HandleProcess() + ret = obj._execute_cmd(MOCK_PROCESS_LIST[0].get('start_command'), + MOCK_PROCESS_LIST[0].get('run_as_root')) + + self.assertEqual(ret, 1) diff --git a/masakarimonitors/tests/unit/processmonitor/test_process.py b/masakarimonitors/tests/unit/processmonitor/test_process.py index fea8b12..5f93b44 100644 --- a/masakarimonitors/tests/unit/processmonitor/test_process.py +++ b/masakarimonitors/tests/unit/processmonitor/test_process.py @@ -47,6 +47,19 @@ MOCK_PROCESS_LIST = [ }, ] +MOCK_DOWN_PROCESS_LIST = [ + { + 'process_name': 'mock_process_name_A', + 'start_command': 'mock_start_command', + 'pre_start_command': 'mock_pre_start_command', + 'post_start_command': 'mock_post_start_command', + 'restart_command': 'mock_restart_command', + 'pre_restart_command': 'mock_pre_restart_command', + 'post_restart_command': 'mock_post_restart_command', + 'run_as_root': True + }, +] + class TestProcessmonitorManager(testtools.TestCase): @@ -59,21 +72,102 @@ class TestProcessmonitorManager(testtools.TestCase): else: return + @mock.patch.object(eventlet.greenthread, 'sleep') + @mock.patch.object(handle_process.HandleProcess, 'restart_processes') @mock.patch.object(handle_process.HandleProcess, 'monitor_processes') @mock.patch.object(handle_process.HandleProcess, 'start_processes') @mock.patch.object(handle_process.HandleProcess, 'set_process_list') @mock.patch.object(yaml, 'load') + @mock.patch('__builtin__.file') def test_main(self, + mock_file, mock_load, mock_set_process_list, mock_start_processes, - mock_monitor_processes): + mock_monitor_processes, + mock_restart_processes, + mock_sleep): + mock_file.return_value = None mock_load.side_effect = [self._get_mock_process_list(0), + self._get_mock_process_list(0), self._get_mock_process_list(1)] mock_set_process_list.return_value = None mock_start_processes.return_value = None - mock_monitor_processes.return_value = [] + mock_monitor_processes.side_effect = [MOCK_DOWN_PROCESS_LIST, []] + mock_restart_processes.return_value = None + mock_sleep.return_value = None obj = processmonitor_manager.ProcessmonitorManager() obj.main() + + mock_set_process_list.assert_called_with(MOCK_PROCESS_LIST) + mock_start_processes.assert_called_once_with() + self.assertEqual(2, mock_monitor_processes.call_count) + mock_restart_processes.assert_called_once_with(MOCK_DOWN_PROCESS_LIST) + + @mock.patch.object(handle_process.HandleProcess, 'restart_processes') + @mock.patch.object(handle_process.HandleProcess, 'monitor_processes') + @mock.patch.object(handle_process.HandleProcess, 'start_processes') + @mock.patch.object(handle_process.HandleProcess, 'set_process_list') + @mock.patch.object(yaml, 'load') + @mock.patch('__builtin__.file') + def test_main_exception(self, + mock_file, + mock_load, + mock_set_process_list, + mock_start_processes, + mock_monitor_processes, + mock_restart_processes): + + mock_file.return_value = None + mock_load.return_value = self._get_mock_process_list(0) + mock_set_process_list.return_value = None + mock_start_processes.side_effect = Exception("Test exception.") + + obj = processmonitor_manager.ProcessmonitorManager() + obj.main() + + mock_set_process_list.assert_called_once_with(MOCK_PROCESS_LIST) + mock_start_processes.assert_called_once_with() + mock_monitor_processes.assert_not_called() + mock_restart_processes.assert_not_called() + + @mock.patch.object(handle_process.HandleProcess, 'set_process_list') + @mock.patch.object(yaml, 'load') + @mock.patch('__builtin__.file') + def test_load_process_list_yaml_error(self, + mock_file, + mock_load, + mock_set_process_list): + + mock_file.return_value = None + mock_load.side_effect = yaml.YAMLError + + obj = processmonitor_manager.ProcessmonitorManager() + obj.main() + + mock_set_process_list.assert_not_called() + + @mock.patch.object(handle_process.HandleProcess, 'set_process_list') + @mock.patch.object(yaml, 'load') + @mock.patch('__builtin__.file') + def test_load_process_list_exception(self, + mock_file, + mock_load, + mock_set_process_list): + + mock_file.return_value = None + mock_load.side_effect = Exception("Test exception.") + + obj = processmonitor_manager.ProcessmonitorManager() + obj.main() + + mock_set_process_list.assert_not_called() + + def test_stop(self): + + obj = processmonitor_manager.ProcessmonitorManager() + obj.stop() + + self.assertFalse(obj.running)