Add NRPE checks for services

Adds the nrpe-external-master layer and checks for services managed by
this charm.

func-test-pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/436

Change-Id: Ice5eada838626059f8cb2c427f7b9b0884317660
Closes-Bug: #1897809
Related-Bug: #1933643
Co-authored-by: Aurelien Lourot <aurelien.lourot@canonical.com>

src/config.yaml

@@ -128,3 +128,18 @@ options:
     description: |
       Default for how old deleted zones should be (deleted_at) to be
       purged, in seconds.
+  nagios_context:
+    default: "juju"
+    type: string
+    description: |
+      A string that will be prepended to instance name to set the host name
+      in nagios. So for instance the hostname would be something like:
+          juju-myservice-0
+      If you're running multiple environments with the same services in them
+      this allows you to differentiate between them.
+  nagios_servicegroups:
+    default: ""
+    type: string
+    description: |
+      A comma-separated list of nagios servicegroups. If left empty, the
+      nagios_context will be used as the servicegroup.

src/layer.yaml

@@ -1,4 +1,4 @@
-includes: ['layer:openstack-api', 'layer:leadership', 'interface:bind-rndc', 'interface:hacluster', 'interface:openstack-ha', 'interface:memcache', 'interface:designate']
+includes: ['layer:openstack-api', 'layer:leadership', 'interface:bind-rndc', 'interface:hacluster', 'interface:openstack-ha', 'interface:memcache', 'interface:designate', 'interface:nrpe-external-master']
 options:
   basic:
     use_venv: True

src/lib/charm/openstack/designate.py

@@ -19,6 +19,7 @@ import subprocess
 import uuid
 
 import charmhelpers.contrib.openstack.utils as ch_utils
+import charmhelpers.contrib.charmsupport.nrpe as nrpe
 import charms_openstack.adapters as openstack_adapters
 import charms_openstack.charm as openstack_charm
 import charms_openstack.ip as os_ip
@@ -613,6 +614,15 @@ class DesignateCharm(ch_plugins.PolicydOverridePlugin,
             hookenv.leader_set({'pool-manager-cache-sync-done': True})
             self.restart_all()
 
+    def render_nrpe(self):
+        """Configure Nagios NRPE checks."""
+        hostname = nrpe.get_nagios_hostname()
+        current_unit = nrpe.get_nagios_unit_name()
+        charm_nrpe = nrpe.NRPE(hostname=hostname)
+        nrpe.add_init_service_checks(
+            charm_nrpe, self.services, current_unit)
+        charm_nrpe.write()
+
 
 class DesignateCharmQueens(DesignateCharm):
 

src/metadata.yaml

@@ -24,6 +24,9 @@ provides:
     interface: designate
   nova-designate:
     interface: nova-designate
+  nrpe-external-master:
+    interface: nrpe-external-master
+    scope: container
 requires:
   dns-backend:
     interface: bind-rndc

src/reactive/designate_handlers.py

@@ -254,3 +254,15 @@ def reset_shared_db():
     """
     reactive.remove_state('shared-db.setup')
     reactive.remove_state('db.synched')
+
+
+@reactive.when_not('is-update-status-hook')
+@reactive.when('base-config.rendered')
+@reactive.when_any('config.changed.nagios_context',
+                   'config.changed.nagios_servicegroups',
+                   'endpoint.nrpe-external-master.changed',
+                   'nrpe-external-master.available')
+def configure_nrpe():
+    """Handle config-changed for NRPE options."""
+    with charm.provide_charm_instance() as charm_instance:
+        charm_instance.render_nrpe()

src/tests/bundles/bionic-queens.yaml

@@ -48,6 +48,8 @@ applications:
       nameservers: 'ns1.amuletexample.com. ns2.amuletexample.com.'
     to:
     - '5'
+  nrpe:
+    charm: cs:nrpe
 
 relations:
 - - 'designate:shared-db'
@@ -70,3 +72,5 @@ relations:
   - 'percona-cluster:shared-db'
 - - 'neutron-api:amqp'
   - 'rabbitmq-server:amqp'
+- - 'designate:nrpe-external-master'
+  - 'nrpe:nrpe-external-master'

src/tests/bundles/bionic-rocky.yaml

@@ -53,6 +53,8 @@ applications:
       openstack-origin: cloud:bionic-rocky
     to:
     - '5'
+  nrpe:
+    charm: cs:nrpe
 
 relations:
 - - 'designate:shared-db'
@@ -75,3 +77,5 @@ relations:
   - 'percona-cluster:shared-db'
 - - 'neutron-api:amqp'
   - 'rabbitmq-server:amqp'
+- - 'designate:nrpe-external-master'
+  - 'nrpe:nrpe-external-master'

src/tests/bundles/bionic-stein.yaml

@@ -53,6 +53,8 @@ applications:
       openstack-origin: cloud:bionic-stein
     to:
     - '5'
+  nrpe:
+    charm: cs:nrpe
 
 relations:
 - - 'designate:shared-db'
@@ -75,3 +77,5 @@ relations:
   - 'percona-cluster:shared-db'
 - - 'neutron-api:amqp'
   - 'rabbitmq-server:amqp'
+- - 'designate:nrpe-external-master'
+  - 'nrpe:nrpe-external-master'

src/tests/bundles/bionic-train.yaml

@@ -53,6 +53,8 @@ applications:
       openstack-origin: cloud:bionic-train
     to:
     - '5'
+  nrpe:
+    charm: cs:nrpe
 
 relations:
 - - 'designate:shared-db'
@@ -75,3 +77,5 @@ relations:
   - 'percona-cluster:shared-db'
 - - 'neutron-api:amqp'
   - 'rabbitmq-server:amqp'
+- - 'designate:nrpe-external-master'
+  - 'nrpe:nrpe-external-master'

src/tests/bundles/bionic-ussuri.yaml

@@ -54,6 +54,8 @@ applications:
       openstack-origin: cloud:bionic-ussuri
     to:
     - '5'
+  nrpe:
+    charm: cs:nrpe
 
 relations:
 - - 'designate:shared-db'
@@ -76,3 +78,5 @@ relations:
   - 'percona-cluster:shared-db'
 - - 'neutron-api:amqp'
   - 'rabbitmq-server:amqp'
+- - 'designate:nrpe-external-master'
+  - 'nrpe:nrpe-external-master'

src/tests/bundles/focal-ussuri.yaml

@@ -74,7 +74,7 @@ applications:
   memcached:
     charm: cs:~memcached-team/memcached
     num_units: 1
-    # holding charm at bionic as it isn't available for focal yet
+    # holding charm at bionic as it isn't available for focal+ yet
     series: bionic
     to:
     - '7'
@@ -88,6 +88,9 @@ applications:
     to:
     - '8'
 
+  nrpe:
+    charm: cs:nrpe
+
 relations:
   - - 'keystone:shared-db'
     - 'keystone-mysql-router:shared-db'
@@ -124,3 +127,6 @@ relations:
 
   - - 'neutron-api:amqp'
     - 'rabbitmq-server:amqp'
+
+  - - 'designate:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'

src/tests/bundles/focal-victoria.yaml

@@ -74,7 +74,7 @@ applications:
   memcached:
     charm: cs:~memcached-team/memcached
     num_units: 1
-    # holding charm at bionic as it isn't available for focal yet
+    # holding charm at bionic as it isn't available for focal+ yet
     series: bionic
     to:
     - '7'
@@ -88,6 +88,9 @@ applications:
     to:
     - '8'
 
+  nrpe:
+    charm: cs:nrpe
+
 relations:
   - - 'keystone:shared-db'
     - 'keystone-mysql-router:shared-db'
@@ -124,3 +127,6 @@ relations:
 
   - - 'neutron-api:amqp'
     - 'rabbitmq-server:amqp'
+
+  - - 'designate:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'

src/tests/bundles/focal-wallaby.yaml

@@ -74,7 +74,7 @@ applications:
   memcached:
     charm: cs:~memcached-team/memcached
     num_units: 1
-    # holding charm at bionic as it isn't available for focal yet
+    # holding charm at bionic as it isn't available for focal+ yet
     series: bionic
     to:
     - '7'
@@ -88,6 +88,9 @@ applications:
     to:
     - '8'
 
+  nrpe:
+    charm: cs:nrpe
+
 relations:
   - - 'keystone:shared-db'
     - 'keystone-mysql-router:shared-db'
@@ -124,3 +127,6 @@ relations:
 
   - - 'neutron-api:amqp'
     - 'rabbitmq-server:amqp'
+
+  - - 'designate:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'

src/tests/bundles/groovy-victoria.yaml

@@ -74,7 +74,7 @@ applications:
   memcached:
     charm: cs:~memcached-team/memcached
     num_units: 1
-    # holding charm at bionic as it isn't available for focal/groovy yet
+    # holding charm at bionic as it isn't available for focal+ yet
     series: bionic
     to:
     - '7'
@@ -88,6 +88,9 @@ applications:
     to:
     - '8'
 
+  nrpe:
+    charm: cs:nrpe
+
 relations:
   - - 'keystone:shared-db'
     - 'keystone-mysql-router:shared-db'
@@ -124,3 +127,6 @@ relations:
 
   - - 'neutron-api:amqp'
     - 'rabbitmq-server:amqp'
+
+  - - 'designate:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'

src/tests/bundles/hirsute-wallaby.yaml

@@ -74,7 +74,7 @@ applications:
   memcached:
     charm: cs:~memcached-team/memcached
     num_units: 1
-    # holding charm at bionic as it isn't available for focal/groovy yet
+    # holding charm at bionic as it isn't available for focal+ yet
     series: bionic
     to:
     - '7'
@@ -88,6 +88,9 @@ applications:
     to:
     - '8'
 
+  nrpe:
+    charm: cs:nrpe
+
 relations:
   - - 'keystone:shared-db'
     - 'keystone-mysql-router:shared-db'
@@ -124,3 +127,6 @@ relations:
 
   - - 'neutron-api:amqp'
     - 'rabbitmq-server:amqp'
+
+  - - 'designate:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'

src/tests/bundles/xenial-mitaka.yaml

@@ -48,6 +48,8 @@ applications:
       nameservers: 'ns1.amuletexample.com. ns2.amuletexample.com.'
     to:
     - '5'
+  nrpe:
+    charm: cs:nrpe
 
 relations:
 - - 'designate:shared-db'
@@ -70,3 +72,5 @@ relations:
   - 'percona-cluster:shared-db'
 - - 'neutron-api:amqp'
   - 'rabbitmq-server:amqp'
+- - 'designate:nrpe-external-master'
+  - 'nrpe:nrpe-external-master'

src/tests/bundles/xenial-ocata.yaml

@@ -53,6 +53,8 @@ applications:
       openstack-origin: cloud:xenial-ocata
     to:
     - '5'
+  nrpe:
+    charm: cs:nrpe
 
 relations:
 - - 'designate:shared-db'
@@ -75,3 +77,5 @@ relations:
   - 'percona-cluster:shared-db'
 - - 'neutron-api:amqp'
   - 'rabbitmq-server:amqp'
+- - 'designate:nrpe-external-master'
+  - 'nrpe:nrpe-external-master'

src/tests/bundles/xenial-pike.yaml

@@ -53,6 +53,8 @@ applications:
       openstack-origin: cloud:xenial-pike
     to:
     - '5'
+  nrpe:
+    charm: cs:nrpe
 
 relations:
 - - 'designate:shared-db'
@@ -75,3 +77,5 @@ relations:
   - 'percona-cluster:shared-db'
 - - 'neutron-api:amqp'
   - 'rabbitmq-server:amqp'
+- - 'designate:nrpe-external-master'
+  - 'nrpe:nrpe-external-master'

src/tests/bundles/xenial-queens.yaml

@@ -53,6 +53,8 @@ applications:
       openstack-origin: cloud:xenial-queens
     to:
     - '5'
+  nrpe:
+    charm: cs:nrpe
 
 relations:
 - - 'designate:shared-db'
@@ -75,3 +77,5 @@ relations:
   - 'percona-cluster:shared-db'
 - - 'neutron-api:amqp'
   - 'rabbitmq-server:amqp'
+- - 'designate:nrpe-external-master'
+  - 'nrpe:nrpe-external-master'

src/tests/tests.yaml

@@ -25,6 +25,14 @@ dev_bundles:
 tests:
   - zaza.openstack.charm_tests.designate.tests.DesignateTests
 
+target_deploy_status:
+  nrpe:
+    workload-status: blocked
+    workload-status-message: "Nagios server not configured or related"
+
 tests_options:
   force_deploy:
+    # NOTE(lourot): this is needed because the NRPE charm isn't available on
+    # non-LTS Ubuntu series. See lp:1933643
+    - groovy-victoria
     - hirsute-wallaby

unit_tests/__init__.py

@@ -23,6 +23,7 @@ import charms_openstack.test_mocks  # noqa
 charms_openstack.test_mocks.mock_charmhelpers()
 sys.modules['charmhelpers.core.decorators'] = (
     charms_openstack.test_mocks.charmhelpers.core.decorators)
+sys.modules['charmhelpers.contrib.charmsupport.nrpe'] = mock.MagicMock()
 
 
 def _fake_retry(num_retries, base_delay=0, exc_type=Exception):

unit_tests/test_designate_handlers.py

@@ -40,6 +40,7 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks):
                 'remote_pools_updated': (
                     'leadership.changed.pool-yaml-hash', ),
                 'reset_shared_db': ('shared-db.setup', ),
+                'configure_nrpe': ('base-config.rendered', ),
             },
             'when_not': {
                 'setup_amqp_req': ('amqp.requested-access', ),
@@ -52,10 +53,17 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks):
                 'run_assess_status_on_every_hook': (
                     'dont-set-assess-status', ),
                 'reset_shared_db': ('shared-db.connected', ),
+                'configure_nrpe': ('is-update-status-hook', ),
             },
             'when_any': {
                 'set_dns_config_available': (
                     'dns-slaves-config-valid', 'dns-backend.available', ),
+                'configure_nrpe': (
+                    'config.changed.nagios_context',
+                    'config.changed.nagios_servicegroups',
+                    'endpoint.nrpe-external-master.changed',
+                    'nrpe-external-master.available',
+                ),
             },
             'when_none': {
                 'clear_dns_config_available': (

unit_tests/test_lib_charm_openstack_designate.py

@@ -363,6 +363,18 @@ class TestDesignateCharm(Helper):
             pass
         self.assertFalse(self.leader_set.called)
 
+    def test_render_nrpe(self):
+        self.patch_object(designate.nrpe, 'add_init_service_checks')
+        charm_instance = designate.DesignateCharm(release='queens')
+        charm_instance.render_nrpe()
+        self.add_init_service_checks.assert_has_calls([
+            mock.call().add_init_service_checks(
+                mock.ANY,
+                charm_instance.services,
+                mock.ANY
+            ),
+        ])
+
 
 class TestDesignateQueensCharm(Helper):