Marcin Wilk
5dff73cb3a
Previously, the charm configured masakari-monitors with the following
defaults for host monitoring:
- monitoring_interval: 60 seconds
- monitoring_samples: 1
The values were hardcoded in the configuration file template.
With only a single sample, any transient OFFLINE state captured
during a masakari control plane shutdown or restart could trigger
a notification — potentially placing healthy compute nodes into
maintenance. This can occur during ownership transitions between
control nodes, when pacemaker-remote nodes may briefly appear offline.
This commit mitigates the issue by exposing the aforementioned
configurations knobs as Juju config options. Now, a user has
possibility to adjust the interval and samples on a running cloud,
within the predefined boundaries:
- monitoring-interval option: between 10 and 300 seconds
- monitoring-samples option: between 1 and 5
On the pacemaker-remote nodes, ('restrict_to_remotes==True'),
masakari-monitors use 'crm_mon -X' command to check the cluster status.
The pacemaker-remote node queries a cluster node running the Pacemaker,
which continuously maintains the state (via subscribtion to the Corosync
events) in the Cluster Information Base (CIB). When responding to the
crm_mon request, Pacemaker doesn't query Corosync or collect that
information on demand. Instead, it returns the data it already has in the CIB.
Therefore the performance impact on the cluster, of increasing the
frequency of the crm_mon one-shot calls, is negligible, even if the
change affects all the pacemaker-remote nodes running masakari-monitors.
Change-Id: I138b8a932aebaed172c9ed0f369009ace491aea8
Signed-off-by: Marcin Wilk <marcin.wilk@canonical.com>
102 lines
4.0 KiB
Python
102 lines
4.0 KiB
Python
# Copyright 2019 Canonical Ltd
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from unittest import mock
|
|
|
|
import charmhelpers
|
|
|
|
import charm.openstack.masakari_monitors as masakari_monitors
|
|
|
|
import charms_openstack.test_utils as test_utils
|
|
|
|
|
|
class Helper(test_utils.PatchHelper):
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
self.patch_release(masakari_monitors.MasakariMonitorsCharm.release)
|
|
|
|
|
|
class TestMasakariMonitorsCharm(Helper):
|
|
|
|
def _patch_config_and_charm(self, config):
|
|
self.patch_object(charmhelpers.core.hookenv, 'config')
|
|
|
|
def cf(key=None):
|
|
if key is not None:
|
|
return config[key]
|
|
return config
|
|
|
|
self.config.side_effect = cf
|
|
c = masakari_monitors.MasakariMonitorsCharm()
|
|
return c
|
|
|
|
def test_request_credentials(self):
|
|
keystone_relation = mock.MagicMock()
|
|
self.patch('charms.reactive.relations.endpoint_from_flag',
|
|
name='endpoint_from_flag',
|
|
return_value=keystone_relation)
|
|
c = self._patch_config_and_charm({})
|
|
c.request_credentials()
|
|
keystone_relation.request_credentials.assert_called_once_with(
|
|
'masakari-monitors', project='services')
|
|
|
|
def test_validated_monitoring_interval_valid(self):
|
|
self.patch_object(charmhelpers.core.hookenv, 'config')
|
|
self.patch_object(charmhelpers.core.hookenv, 'log')
|
|
self.config.return_value = 60
|
|
value = masakari_monitors.validated_monitoring_interval(None)
|
|
self.assertEqual(value, 60)
|
|
self.log.assert_not_called()
|
|
|
|
def test_validated_monitoring_interval_too_low(self):
|
|
self.patch_object(charmhelpers.core.hookenv, 'config')
|
|
self.patch_object(charmhelpers.core.hookenv, 'log')
|
|
self.config.return_value = 5
|
|
value = masakari_monitors.validated_monitoring_interval(None)
|
|
self.assertEqual(value, masakari_monitors.MONITORING_INTERVAL_DEFAULT)
|
|
self.log.assert_called_once()
|
|
|
|
def test_validated_monitoring_interval_too_high(self):
|
|
self.patch_object(charmhelpers.core.hookenv, 'config')
|
|
self.patch_object(charmhelpers.core.hookenv, 'log')
|
|
self.config.return_value = 301
|
|
value = masakari_monitors.validated_monitoring_interval(None)
|
|
self.assertEqual(value, masakari_monitors.MONITORING_INTERVAL_DEFAULT)
|
|
self.log.assert_called_once()
|
|
|
|
def test_validated_monitoring_samples_valid(self):
|
|
self.patch_object(charmhelpers.core.hookenv, 'config')
|
|
self.patch_object(charmhelpers.core.hookenv, 'log')
|
|
self.config.return_value = 1
|
|
value = masakari_monitors.validated_monitoring_samples(None)
|
|
self.assertEqual(value, 1)
|
|
self.log.assert_not_called()
|
|
|
|
def test_validated_monitoring_samples_too_low(self):
|
|
self.patch_object(charmhelpers.core.hookenv, 'config')
|
|
self.patch_object(charmhelpers.core.hookenv, 'log')
|
|
self.config.return_value = 0
|
|
value = masakari_monitors.validated_monitoring_samples(None)
|
|
self.assertEqual(value, masakari_monitors.MONITORING_SAMPLES_DEFAULT)
|
|
self.log.assert_called_once()
|
|
|
|
def test_validated_monitoring_samples_too_high(self):
|
|
self.patch_object(charmhelpers.core.hookenv, 'config')
|
|
self.patch_object(charmhelpers.core.hookenv, 'log')
|
|
self.config.return_value = 6
|
|
value = masakari_monitors.validated_monitoring_samples(None)
|
|
self.assertEqual(value, masakari_monitors.MONITORING_SAMPLES_DEFAULT)
|
|
self.log.assert_called_once()
|