Add host monitoring interval and samples config options

Previously, the charm configured masakari-monitors with the following
defaults for host monitoring:

   - monitoring_interval: 60 seconds
   - monitoring_samples: 1

The values were hardcoded in the configuration file template.
With only a single sample, any transient OFFLINE state captured
during a masakari control plane shutdown or restart could trigger
a notification — potentially placing healthy compute nodes into
maintenance. This can occur during ownership transitions between
control nodes, when pacemaker-remote nodes may briefly appear offline.

This commit mitigates the issue by exposing the aforementioned
configurations knobs as Juju config options. Now, a user has
possibility to adjust the interval and samples on a running cloud,
within the predefined boundaries:
  - monitoring-interval option: between 10 and 300 seconds
  - monitoring-samples option: between 1 and 5

On the pacemaker-remote nodes, ('restrict_to_remotes==True'),
masakari-monitors use 'crm_mon -X' command to check the cluster status.
The pacemaker-remote node queries a cluster node running the Pacemaker,
which continuously maintains the state (via subscribtion to the Corosync
events) in the Cluster Information Base (CIB). When responding to the
crm_mon request, Pacemaker doesn't query Corosync or collect that
information on demand. Instead, it returns the data it already has in the CIB.
Therefore the performance impact on the cluster, of increasing the
frequency of the crm_mon one-shot calls, is negligible, even if the
change affects all the pacemaker-remote nodes running masakari-monitors.

Change-Id: I138b8a932aebaed172c9ed0f369009ace491aea8
Signed-off-by: Marcin Wilk <marcin.wilk@canonical.com>

src/config.yaml

@@ -9,3 +9,15 @@ options:
     description: |
       Comma-separated list of key=value config flags. These values will be
       placed in the masakarimonitors.conf [DEFAULT] section.
+  monitoring-interval:
+    type: int
+    default: 60
+    description: |
+      Interval, in seconds, between host monitoring checks. Must be
+      between 10 and 300 seconds. Default is 60 seconds.
+  monitoring-samples:
+    type: int
+    default: 1
+    description: |
+      Number of consecutive samples that must confirm a host state change
+      before masakari-monitors acts on it. Must be between 1 and 5. Default is 1.

src/lib/charm/openstack/masakari_monitors.py

@@ -4,15 +4,52 @@ import socket
 import charms_openstack.adapters
 import charms_openstack.charm
 import charms.reactive.relations as relations
+import charmhelpers.core.hookenv as hookenv
 
 charms_openstack.charm.use_defaults('charm.default-select-release')
 
+MONITORING_INTERVAL_MIN = 10
+MONITORING_INTERVAL_MAX = 300
+MONITORING_INTERVAL_DEFAULT = 60
+MONITORING_SAMPLES_MIN = 1
+MONITORING_SAMPLES_MAX = 5
+MONITORING_SAMPLES_DEFAULT = 1
+
 
 @charms_openstack.adapters.config_property
 def hostname(config):
     return socket.getfqdn()
 
 
+@charms_openstack.adapters.config_property
+def validated_monitoring_interval(cls):
+    value = hookenv.config('monitoring-interval')
+    if not (MONITORING_INTERVAL_MIN <= value <= MONITORING_INTERVAL_MAX):
+        hookenv.log(
+            "monitoring-interval={} is outside of the supported range "
+            "({}-{} seconds); falling back to the default of {} "
+            "seconds".format(
+                value, MONITORING_INTERVAL_MIN, MONITORING_INTERVAL_MAX,
+                MONITORING_INTERVAL_DEFAULT),
+            level=hookenv.WARNING)
+        return MONITORING_INTERVAL_DEFAULT
+    return value
+
+
+@charms_openstack.adapters.config_property
+def validated_monitoring_samples(cls):
+    value = hookenv.config('monitoring-samples')
+    if not (MONITORING_SAMPLES_MIN <= value <= MONITORING_SAMPLES_MAX):
+        hookenv.log(
+            "monitoring-samples={} is outside of the supported range "
+            "({}-{}); falling back to the default of {}".format(
+                value, MONITORING_SAMPLES_MIN, MONITORING_SAMPLES_MAX,
+                MONITORING_SAMPLES_DEFAULT),
+            level=hookenv.WARNING)
+        return MONITORING_SAMPLES_DEFAULT
+    return value
+
+
 class MasakariMonitorsCharm(charms_openstack.charm.OpenStackCharm):
 
     # Internal name of charm

src/templates/ussuri/masakarimonitors.conf

@@ -25,6 +25,8 @@ region = {{ options.region }}
 [host]
 corosync_multicast_interfaces = ens3
 corosync_multicast_ports = 5405
+monitoring_interval = {{ options.validated_monitoring_interval }}
+monitoring_samples = {{ options.validated_monitoring_samples }}
 # XXX LY restrict_to_remotes is a WIP
 restrict_to_remotes = True
 disable_ipmi_check = True

unit_tests/test_lib_charm_openstack_masakari_monitors.py

@@ -51,3 +51,51 @@ class TestMasakariMonitorsCharm(Helper):
         c.request_credentials()
         keystone_relation.request_credentials.assert_called_once_with(
             'masakari-monitors', project='services')
+
+    def test_validated_monitoring_interval_valid(self):
+        self.patch_object(charmhelpers.core.hookenv, 'config')
+        self.patch_object(charmhelpers.core.hookenv, 'log')
+        self.config.return_value = 60
+        value = masakari_monitors.validated_monitoring_interval(None)
+        self.assertEqual(value, 60)
+        self.log.assert_not_called()
+
+    def test_validated_monitoring_interval_too_low(self):
+        self.patch_object(charmhelpers.core.hookenv, 'config')
+        self.patch_object(charmhelpers.core.hookenv, 'log')
+        self.config.return_value = 5
+        value = masakari_monitors.validated_monitoring_interval(None)
+        self.assertEqual(value, masakari_monitors.MONITORING_INTERVAL_DEFAULT)
+        self.log.assert_called_once()
+
+    def test_validated_monitoring_interval_too_high(self):
+        self.patch_object(charmhelpers.core.hookenv, 'config')
+        self.patch_object(charmhelpers.core.hookenv, 'log')
+        self.config.return_value = 301
+        value = masakari_monitors.validated_monitoring_interval(None)
+        self.assertEqual(value, masakari_monitors.MONITORING_INTERVAL_DEFAULT)
+        self.log.assert_called_once()
+
+    def test_validated_monitoring_samples_valid(self):
+        self.patch_object(charmhelpers.core.hookenv, 'config')
+        self.patch_object(charmhelpers.core.hookenv, 'log')
+        self.config.return_value = 1
+        value = masakari_monitors.validated_monitoring_samples(None)
+        self.assertEqual(value, 1)
+        self.log.assert_not_called()
+
+    def test_validated_monitoring_samples_too_low(self):
+        self.patch_object(charmhelpers.core.hookenv, 'config')
+        self.patch_object(charmhelpers.core.hookenv, 'log')
+        self.config.return_value = 0
+        value = masakari_monitors.validated_monitoring_samples(None)
+        self.assertEqual(value, masakari_monitors.MONITORING_SAMPLES_DEFAULT)
+        self.log.assert_called_once()
+
+    def test_validated_monitoring_samples_too_high(self):
+        self.patch_object(charmhelpers.core.hookenv, 'config')
+        self.patch_object(charmhelpers.core.hookenv, 'log')
+        self.config.return_value = 6
+        value = masakari_monitors.validated_monitoring_samples(None)
+        self.assertEqual(value, masakari_monitors.MONITORING_SAMPLES_DEFAULT)
+        self.log.assert_called_once()