Merge "Add NRPE checks for services"

src/config.yaml

@@ -7,3 +7,17 @@ options:
       override YAML files in the service's policy.d directory.  The resource
       file should be a ZIP file containing at least one yaml file with a .yaml
       or .yml extension.  If False then remove the overrides.
+  nagios_context:
+    default: "juju"
+    type: string
+    description: |
+      A string that will be prepended to instance name to set the host name
+      in nagios. So for instance the hostname would be something like:
+          juju-myservice-0
+      If you're running multiple environments with the same services in them
+      this allows you to differentiate between them.
+  nagios_servicegroups:
+    default: ""
+    type: string
+    description: |
+      Comma separated list of nagios servicegroups for the service checks.

src/layer.yaml

@@ -1,4 +1,4 @@
-includes: ['layer:openstack-api', 'interface:mongodb']
+includes: ['layer:openstack-api', 'interface:mongodb', 'interface:nrpe-external-master']
 options:
   basic:
     use_venv: True

src/lib/charm/openstack/aodh.py

@@ -17,6 +17,7 @@ import os
 import subprocess
 
 import charmhelpers.core.host as ch_host
+import charmhelpers.contrib.charmsupport.nrpe as nrpe
 
 import charms_openstack.charm
 import charms_openstack.adapters
@@ -132,6 +133,15 @@ class AodhCharm(charms_openstack.plugins.PolicydOverridePlugin,
             subprocess.check_call(['systemctl', 'daemon-reload'])
             ch_host.service_restart('aodh-api')
 
+    def render_nrpe_checks(self):
+        """Configure Nagios NRPE checks."""
+        hostname = nrpe.get_nagios_hostname()
+        current_unit = nrpe.get_nagios_unit_name()
+        charm_nrpe = nrpe.NRPE(hostname=hostname)
+        nrpe.add_init_service_checks(
+            charm_nrpe, self.services, current_unit)
+        charm_nrpe.write()
+
 
 class AodhCharmNewton(AodhCharm):
     """Newton uses the aodh-api standalone systemd. If the systemd definition
@@ -291,3 +301,8 @@ def reload_and_restart():
     """Reload systemd and restart aodh API when override file changes
     """
     AodhCharm.singleton.reload_and_restart()
+
+
+def render_nrpe():
+    """Render NRPE service monitors."""
+    AodhCharm.singleton.render_nrpe_checks()

src/metadata.yaml

@@ -23,6 +23,10 @@ subordinate: false
 requires:
   mongodb:
     interface: mongodb
+provides:
+  nrpe-external-master:
+    interface: nrpe-external-master
+    scope: container
 resources:
   policyd-override:
     type: file

src/reactive/aodh_handlers.py

@@ -104,3 +104,14 @@ def run_db_migration():
 @reactive.when('ha.connected')
 def cluster_connected(hacluster):
     aodh.configure_ha_resources(hacluster)
+
+
+@reactive.when_none('charm.paused', 'is-update-status-hook')
+@reactive.when('config.complete')
+@reactive.when_any('config.changed.nagios_context',
+                   'config.changed.nagios_servicegroups',
+                   'endpoint.nrpe-external-master.changed',
+                   'nrpe-external-master.available')
+def configure_nrpe():
+    """Handle config-changed for NRPE options."""
+    aodh.render_nrpe()

src/tests/bundles/bionic-queens.yaml

@@ -92,6 +92,8 @@ applications:
     num_units: 1
     to:
       - '12'
+  nrpe:
+    charm: cs:nrpe
 relations:
   - - 'keystone:shared-db'
     - 'percona-cluster:shared-db'
@@ -119,3 +121,5 @@ relations:
     - 'ceph-mon:client'
   - - 'gnocchi:coordinator-memcached'
     - 'memcached:cache'
+  - - 'aodh:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'

src/tests/bundles/bionic-rocky.yaml

@@ -92,6 +92,8 @@ applications:
     num_units: 1
     to:
       - '12'
+  nrpe:
+    charm: cs:nrpe
 relations:
   - - 'keystone:shared-db'
     - 'percona-cluster:shared-db'
@@ -119,3 +121,5 @@ relations:
     - 'ceph-mon:client'
   - - 'gnocchi:coordinator-memcached'
     - 'memcached:cache'
+  - - 'aodh:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'

src/tests/bundles/bionic-stein.yaml

@@ -92,6 +92,8 @@ applications:
     num_units: 1
     to:
       - '12'
+  nrpe:
+    charm: cs:nrpe
 relations:
   - - 'keystone:shared-db'
     - 'percona-cluster:shared-db'
@@ -119,3 +121,5 @@ relations:
     - 'ceph-mon:client'
   - - 'gnocchi:coordinator-memcached'
     - 'memcached:cache'
+  - - 'aodh:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'

src/tests/bundles/bionic-train.yaml

@@ -92,6 +92,8 @@ applications:
     num_units: 1
     to:
       - '12'
+  nrpe:
+    charm: cs:nrpe
 relations:
   - - 'keystone:shared-db'
     - 'percona-cluster:shared-db'
@@ -119,3 +121,5 @@ relations:
     - 'ceph-mon:client'
   - - 'gnocchi:coordinator-memcached'
     - 'memcached:cache'
+  - - 'aodh:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'

src/tests/bundles/bionic-ussuri.yaml

@@ -92,6 +92,8 @@ applications:
     num_units: 1
     to:
       - '12'
+  nrpe:
+    charm: cs:nrpe
 relations:
   - - 'keystone:shared-db'
     - 'percona-cluster:shared-db'
@@ -119,3 +121,5 @@ relations:
     - 'ceph-mon:client'
   - - 'gnocchi:coordinator-memcached'
     - 'memcached:cache'
+  - - 'aodh:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'

src/tests/bundles/focal-ussuri.yaml

@@ -115,6 +115,9 @@ applications:
     to:
       - '14'
 
+  nrpe:
+    charm: cs:nrpe
+
 relations:
 
   - - 'keystone:shared-db'
@@ -162,3 +165,6 @@ relations:
 
   - - 'gnocchi:coordinator-memcached'
     - 'memcached:cache'
+
+  - - 'aodh:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'

src/tests/bundles/focal-victoria.yaml

@@ -115,6 +115,9 @@ applications:
     to:
       - '14'
 
+  nrpe:
+    charm: cs:nrpe
+
 relations:
 
   - - 'keystone:shared-db'
@@ -162,3 +165,6 @@ relations:
 
   - - 'gnocchi:coordinator-memcached'
     - 'memcached:cache'
+
+  - - 'aodh:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'

src/tests/bundles/groovy-victoria.yaml

@@ -116,6 +116,9 @@ applications:
     to:
       - '14'
 
+  nrpe:
+    charm: cs:nrpe
+
 relations:
 
   - - 'keystone:shared-db'
@@ -163,3 +166,6 @@ relations:
 
   - - 'gnocchi:coordinator-memcached'
     - 'memcached:cache'
+
+  - - 'aodh:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'

src/tests/bundles/trusty-mitaka.yaml

@@ -56,6 +56,8 @@ applications:
     num_units: 1
     to:
       - '5'
+  nrpe:
+    charm: cs:nrpe
 relations:
   - - 'keystone:shared-db'
     - 'percona-cluster:shared-db'
@@ -75,3 +77,5 @@ relations:
     - 'mongodb:database'
   - - 'ceilometer:identity-service'
     - 'keystone:identity-service'
+  - - 'aodh:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'

src/tests/bundles/xenial-mitaka.yaml

@@ -56,6 +56,8 @@ applications:
     num_units: 1
     to:
       - '5'
+  nrpe:
+    charm: cs:nrpe
 relations:
   - - 'keystone:shared-db'
     - 'percona-cluster:shared-db'
@@ -75,3 +77,5 @@ relations:
     - 'mongodb:database'
   - - 'ceilometer:identity-service'
     - 'keystone:identity-service'
+  - - 'aodh:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'

src/tests/bundles/xenial-ocata.yaml

@@ -98,6 +98,8 @@ applications:
     num_units: 1
     to:
       - '13'
+  nrpe:
+    charm: cs:nrpe
 relations:
   - - 'keystone:shared-db'
     - 'percona-cluster:shared-db'
@@ -129,3 +131,5 @@ relations:
     - 'ceph-mon:client'
   - - 'gnocchi:coordinator-memcached'
     - 'memcached:cache'
+  - - 'aodh:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'

src/tests/bundles/xenial-pike.yaml

@@ -98,6 +98,8 @@ applications:
     num_units: 1
     to:
       - '13'
+  nrpe:
+    charm: cs:nrpe
 relations:
   - - 'keystone:shared-db'
     - 'percona-cluster:shared-db'
@@ -129,3 +131,5 @@ relations:
     - 'ceph-mon:client'
   - - 'gnocchi:coordinator-memcached'
     - 'memcached:cache'
+  - - 'aodh:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'

src/tests/bundles/xenial-queens.yaml

@@ -92,6 +92,8 @@ applications:
     num_units: 1
     to:
       - '12'
+  nrpe:
+    charm: cs:nrpe
 relations:
   - - 'keystone:shared-db'
     - 'percona-cluster:shared-db'
@@ -119,3 +121,5 @@ relations:
     - 'ceph-mon:client'
   - - 'gnocchi:coordinator-memcached'
     - 'memcached:cache'
+  - - 'aodh:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'

src/tests/tests.yaml

@@ -29,6 +29,9 @@ target_deploy_status:
   mongodb:
     workload-status: unknown
     workload-status-message: ''
+  nrpe:
+    workload-status: blocked
+    workload-status-message: "Nagios server not configured or related"
 tests_options:
   force_deploy:
     - trusty-mitaka

unit_tests/__init__.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import mock
 import sys
 
 sys.path.append('src')
@@ -20,3 +21,4 @@ sys.path.append('src/lib')
 # Mock out charmhelpers so that we can test without it.
 import charms_openstack.test_mocks  # noqa
 charms_openstack.test_mocks.mock_charmhelpers()
+sys.modules['charmhelpers.contrib.charmsupport.nrpe'] = mock.MagicMock()

unit_tests/test_aodh_handlers.py

@@ -15,41 +15,63 @@
 import unittest
 
 import mock
-
+import charms_openstack.test_utils as test_utils
 import reactive.aodh_handlers as handlers
 
 
-_when_args = {}
+class TestRegisteredHooks(test_utils.TestRegisteredHooks):
-_when_not_args = {}
+    """Run tests to ensure hooks are registered."""
 
-
+    def test_registered_hooks(self):
-def mock_hook_factory(d):
+        """Test that the correct hooks are registered."""
-
+        defaults = [
-    def mock_hook(*args, **kwargs):
+            'charm.installed',
-
+            'config.changed',
-        def inner(f):
+            'charm.default-select-release',
-            # remember what we were passed.  Note that we can't actually
+            'update-status',
-            # determine the class we're attached to, as the decorator only gets
+            'upgrade-charm',
-            # the function.
+        ]
-            try:
+        hook_set = {
-                d[f.__name__].append(dict(args=args, kwargs=kwargs))
+            'when': {
-            except KeyError:
+                'setup_amqp_req': ('amqp.connected', ),
-                d[f.__name__] = [dict(args=args, kwargs=kwargs)]
+                'setup_database': ('shared-db.connected', ),
-            return f
+                'setup_endpoint': ('identity-service.connected', ),
-        return inner
+                'render_unclustered': ('charm.installed',
-    return mock_hook
+                                       'shared-db.available',
+                                       'identity-service.available',
+                                       'amqp.available',),
+                'render_clustered': ('charm.installed',
+                                     'shared-db.available',
+                                     'identity-service.available',
+                                     'amqp.available',
+                                     'cluster.available',),
+                'run_db_migration': ('charm.installed',
+                                     'config.complete', ),
+                'cluster_connected': ('ha.connected', ),
+                'configure_nrpe': ('config.complete', ),
+            },
+            'when_not': {
+                'install_packages': ('charm.installed', ),
+                'render_unclustered': ('cluster.available', ),
+                'run_db_migration': ('db.synced', ),
+            },
+            'when_none': {
+                'configure_nrpe': ('charm.paused', 'is-update-status-hook', ),
+            },
+            'when_any': {
+                'configure_nrpe': ('config.changed.nagios_context',
+                                   'config.changed.nagios_servicegroups',
+                                   'endpoint.nrpe-external-master.changed',
+                                   'nrpe-external-master.available', ),
+            },
+        }
+        self.registered_hooks_test_helper(handlers, hook_set, defaults)
 
 
 class TestAodhHandlers(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        cls._patched_when = mock.patch('charms.reactive.when',
-                                       mock_hook_factory(_when_args))
-        cls._patched_when_started = cls._patched_when.start()
-        cls._patched_when_not = mock.patch('charms.reactive.when_not',
-                                           mock_hook_factory(_when_not_args))
-        cls._patched_when_not_started = cls._patched_when_not.start()
         # force requires to rerun the mock_hook decorator:
         # try except is Python2/Python3 compatibility as Python3 has moved
         # reload to importlib.
@@ -61,12 +83,6 @@ class TestAodhHandlers(unittest.TestCase):
 
     @classmethod
     def tearDownClass(cls):
-        cls._patched_when.stop()
-        cls._patched_when_started = None
-        cls._patched_when = None
-        cls._patched_when_not.stop()
-        cls._patched_when_not_started = None
-        cls._patched_when_not = None
         # and fix any breakage we did to the module
         try:
             reload(handlers)
@@ -94,46 +110,6 @@ class TestAodhHandlers(unittest.TestCase):
         self._patches_start[attr] = started
         setattr(self, attr, started)
 
-    def test_registered_hooks(self):
-        # test that the hooks actually registered the relation expressions that
-        # are meaningful for this interface: this is to handle regressions.
-        # The keys are the function names that the hook attaches to.
-        when_patterns = {
-            'setup_amqp_req': ('amqp.connected', ),
-            'setup_database': ('shared-db.connected', ),
-            'setup_endpoint': ('identity-service.connected', ),
-            'render_unclustered': ('charm.installed',
-                                   'shared-db.available',
-                                   'identity-service.available',
-                                   'amqp.available',),
-            'render_clustered': ('charm.installed',
-                                 'shared-db.available',
-                                 'identity-service.available',
-                                 'amqp.available',
-                                 'cluster.available',),
-            'run_db_migration': ('charm.installed',
-                                 'config.complete', ),
-            'cluster_connected': ('ha.connected', ),
-        }
-        when_not_patterns = {
-            'install_packages': ('charm.installed', ),
-            'render_unclustered': ('cluster.available', ),
-            'run_db_migration': ('db.synced', ),
-        }
-        # check the when hooks are attached to the expected functions
-        for t, p in [(_when_args, when_patterns),
-                     (_when_not_args, when_not_patterns)]:
-            for f, args in t.items():
-                # check that function is in patterns
-                self.assertTrue(f in p.keys(),
-                                "{} not found".format(f))
-                # check that the lists are equal
-                items = []
-                for a in args:
-                    items += a['args'][:]
-                self.assertEqual(sorted(items), sorted(p[f]),
-                                 "{}: incorrect state registration".format(f))
-
     def test_install_packages(self):
         self.patch(handlers.aodh, 'install')
         self.patch(handlers.reactive, 'set_state')

unit_tests/test_lib_charm_openstack_aodh.py

@@ -123,6 +123,29 @@ class TestAodhCharm(Helper):
         self.check_call.assert_called_once_with(['systemctl', 'daemon-reload'])
         self.service_restart.assert_called_once_with('aodh-api')
 
+    def test_render_nrpe(self):
+        """Test NRPE renders correctly pre Ocata."""
+        self.patch_object(aodh.nrpe, 'NRPE')
+        self.patch_object(aodh.nrpe, 'add_init_service_checks')
+        services = ['aodh-api',
+                    'aodh-evaluator',
+                    'aodh-notifier',
+                    'aodh-listener',
+                    ]
+        target = aodh.AodhCharmNewton()
+        target.render_nrpe_checks()
+        # Note that this list is valid for Ussuri
+        self.add_init_service_checks.assert_has_calls([
+            mock.call().add_init_service_checks(
+                mock.ANY,
+                services,
+                mock.ANY
+            ),
+        ])
+        self.NRPE.assert_has_calls([
+            mock.call().write(),
+        ])
+
 
 class TestAodhCharmOcata(Helper):
 
@@ -136,3 +159,23 @@ class TestAodhCharmOcata(Helper):
         self.init_is_systemd.return_value = True
         aodh.AodhCharmOcata.reload_and_restart()
         self.check_call.assert_called_once_with(['systemctl', 'daemon-reload'])
+
+    def test_render_nrpe(self):
+        """Test NRPE renders correctly in Ocata."""
+        self.patch_object(aodh.nrpe, 'NRPE')
+        self.patch_object(aodh.nrpe, 'add_init_service_checks')
+        services = ['aodh-evaluator', 'aodh-notifier',
+                    'aodh-listener', 'apache2']
+        target = aodh.AodhCharmOcata()
+        target.render_nrpe_checks()
+        # Note that this list is valid for Ussuri
+        self.add_init_service_checks.assert_has_calls([
+            mock.call().add_init_service_checks(
+                mock.ANY,
+                services,
+                mock.ANY
+            ),
+        ])
+        self.NRPE.assert_has_calls([
+            mock.call().write(),
+        ])