Merge "Add NRPE checks for services"

This commit is contained in:
Zuul 2021-06-08 06:27:25 +00:00 committed by Gerrit Code Review
commit 20fe970bf7
22 changed files with 198 additions and 72 deletions

View File

@ -7,3 +7,17 @@ options:
override YAML files in the service's policy.d directory. The resource override YAML files in the service's policy.d directory. The resource
file should be a ZIP file containing at least one yaml file with a .yaml file should be a ZIP file containing at least one yaml file with a .yaml
or .yml extension. If False then remove the overrides. or .yml extension. If False then remove the overrides.
nagios_context:
default: "juju"
type: string
description: |
A string that will be prepended to instance name to set the host name
in nagios. So for instance the hostname would be something like:
juju-myservice-0
If you're running multiple environments with the same services in them
this allows you to differentiate between them.
nagios_servicegroups:
default: ""
type: string
description: |
Comma separated list of nagios servicegroups for the service checks.

View File

@ -1,4 +1,4 @@
includes: ['layer:openstack-api', 'interface:mongodb'] includes: ['layer:openstack-api', 'interface:mongodb', 'interface:nrpe-external-master']
options: options:
basic: basic:
use_venv: True use_venv: True

View File

@ -17,6 +17,7 @@ import os
import subprocess import subprocess
import charmhelpers.core.host as ch_host import charmhelpers.core.host as ch_host
import charmhelpers.contrib.charmsupport.nrpe as nrpe
import charms_openstack.charm import charms_openstack.charm
import charms_openstack.adapters import charms_openstack.adapters
@ -132,6 +133,15 @@ class AodhCharm(charms_openstack.plugins.PolicydOverridePlugin,
subprocess.check_call(['systemctl', 'daemon-reload']) subprocess.check_call(['systemctl', 'daemon-reload'])
ch_host.service_restart('aodh-api') ch_host.service_restart('aodh-api')
def render_nrpe_checks(self):
"""Configure Nagios NRPE checks."""
hostname = nrpe.get_nagios_hostname()
current_unit = nrpe.get_nagios_unit_name()
charm_nrpe = nrpe.NRPE(hostname=hostname)
nrpe.add_init_service_checks(
charm_nrpe, self.services, current_unit)
charm_nrpe.write()
class AodhCharmNewton(AodhCharm): class AodhCharmNewton(AodhCharm):
"""Newton uses the aodh-api standalone systemd. If the systemd definition """Newton uses the aodh-api standalone systemd. If the systemd definition
@ -291,3 +301,8 @@ def reload_and_restart():
"""Reload systemd and restart aodh API when override file changes """Reload systemd and restart aodh API when override file changes
""" """
AodhCharm.singleton.reload_and_restart() AodhCharm.singleton.reload_and_restart()
def render_nrpe():
"""Render NRPE service monitors."""
AodhCharm.singleton.render_nrpe_checks()

View File

@ -23,6 +23,10 @@ subordinate: false
requires: requires:
mongodb: mongodb:
interface: mongodb interface: mongodb
provides:
nrpe-external-master:
interface: nrpe-external-master
scope: container
resources: resources:
policyd-override: policyd-override:
type: file type: file

View File

@ -104,3 +104,14 @@ def run_db_migration():
@reactive.when('ha.connected') @reactive.when('ha.connected')
def cluster_connected(hacluster): def cluster_connected(hacluster):
aodh.configure_ha_resources(hacluster) aodh.configure_ha_resources(hacluster)
@reactive.when_none('charm.paused', 'is-update-status-hook')
@reactive.when('config.complete')
@reactive.when_any('config.changed.nagios_context',
'config.changed.nagios_servicegroups',
'endpoint.nrpe-external-master.changed',
'nrpe-external-master.available')
def configure_nrpe():
"""Handle config-changed for NRPE options."""
aodh.render_nrpe()

View File

@ -92,6 +92,8 @@ applications:
num_units: 1 num_units: 1
to: to:
- '12' - '12'
nrpe:
charm: cs:nrpe
relations: relations:
- - 'keystone:shared-db' - - 'keystone:shared-db'
- 'percona-cluster:shared-db' - 'percona-cluster:shared-db'
@ -119,3 +121,5 @@ relations:
- 'ceph-mon:client' - 'ceph-mon:client'
- - 'gnocchi:coordinator-memcached' - - 'gnocchi:coordinator-memcached'
- 'memcached:cache' - 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

View File

@ -92,6 +92,8 @@ applications:
num_units: 1 num_units: 1
to: to:
- '12' - '12'
nrpe:
charm: cs:nrpe
relations: relations:
- - 'keystone:shared-db' - - 'keystone:shared-db'
- 'percona-cluster:shared-db' - 'percona-cluster:shared-db'
@ -119,3 +121,5 @@ relations:
- 'ceph-mon:client' - 'ceph-mon:client'
- - 'gnocchi:coordinator-memcached' - - 'gnocchi:coordinator-memcached'
- 'memcached:cache' - 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

View File

@ -92,6 +92,8 @@ applications:
num_units: 1 num_units: 1
to: to:
- '12' - '12'
nrpe:
charm: cs:nrpe
relations: relations:
- - 'keystone:shared-db' - - 'keystone:shared-db'
- 'percona-cluster:shared-db' - 'percona-cluster:shared-db'
@ -119,3 +121,5 @@ relations:
- 'ceph-mon:client' - 'ceph-mon:client'
- - 'gnocchi:coordinator-memcached' - - 'gnocchi:coordinator-memcached'
- 'memcached:cache' - 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

View File

@ -92,6 +92,8 @@ applications:
num_units: 1 num_units: 1
to: to:
- '12' - '12'
nrpe:
charm: cs:nrpe
relations: relations:
- - 'keystone:shared-db' - - 'keystone:shared-db'
- 'percona-cluster:shared-db' - 'percona-cluster:shared-db'
@ -119,3 +121,5 @@ relations:
- 'ceph-mon:client' - 'ceph-mon:client'
- - 'gnocchi:coordinator-memcached' - - 'gnocchi:coordinator-memcached'
- 'memcached:cache' - 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

View File

@ -92,6 +92,8 @@ applications:
num_units: 1 num_units: 1
to: to:
- '12' - '12'
nrpe:
charm: cs:nrpe
relations: relations:
- - 'keystone:shared-db' - - 'keystone:shared-db'
- 'percona-cluster:shared-db' - 'percona-cluster:shared-db'
@ -119,3 +121,5 @@ relations:
- 'ceph-mon:client' - 'ceph-mon:client'
- - 'gnocchi:coordinator-memcached' - - 'gnocchi:coordinator-memcached'
- 'memcached:cache' - 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

View File

@ -115,6 +115,9 @@ applications:
to: to:
- '14' - '14'
nrpe:
charm: cs:nrpe
relations: relations:
- - 'keystone:shared-db' - - 'keystone:shared-db'
@ -162,3 +165,6 @@ relations:
- - 'gnocchi:coordinator-memcached' - - 'gnocchi:coordinator-memcached'
- 'memcached:cache' - 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

View File

@ -115,6 +115,9 @@ applications:
to: to:
- '14' - '14'
nrpe:
charm: cs:nrpe
relations: relations:
- - 'keystone:shared-db' - - 'keystone:shared-db'
@ -162,3 +165,6 @@ relations:
- - 'gnocchi:coordinator-memcached' - - 'gnocchi:coordinator-memcached'
- 'memcached:cache' - 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

View File

@ -116,6 +116,9 @@ applications:
to: to:
- '14' - '14'
nrpe:
charm: cs:nrpe
relations: relations:
- - 'keystone:shared-db' - - 'keystone:shared-db'
@ -163,3 +166,6 @@ relations:
- - 'gnocchi:coordinator-memcached' - - 'gnocchi:coordinator-memcached'
- 'memcached:cache' - 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

View File

@ -56,6 +56,8 @@ applications:
num_units: 1 num_units: 1
to: to:
- '5' - '5'
nrpe:
charm: cs:nrpe
relations: relations:
- - 'keystone:shared-db' - - 'keystone:shared-db'
- 'percona-cluster:shared-db' - 'percona-cluster:shared-db'
@ -75,3 +77,5 @@ relations:
- 'mongodb:database' - 'mongodb:database'
- - 'ceilometer:identity-service' - - 'ceilometer:identity-service'
- 'keystone:identity-service' - 'keystone:identity-service'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

View File

@ -56,6 +56,8 @@ applications:
num_units: 1 num_units: 1
to: to:
- '5' - '5'
nrpe:
charm: cs:nrpe
relations: relations:
- - 'keystone:shared-db' - - 'keystone:shared-db'
- 'percona-cluster:shared-db' - 'percona-cluster:shared-db'
@ -75,3 +77,5 @@ relations:
- 'mongodb:database' - 'mongodb:database'
- - 'ceilometer:identity-service' - - 'ceilometer:identity-service'
- 'keystone:identity-service' - 'keystone:identity-service'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

View File

@ -98,6 +98,8 @@ applications:
num_units: 1 num_units: 1
to: to:
- '13' - '13'
nrpe:
charm: cs:nrpe
relations: relations:
- - 'keystone:shared-db' - - 'keystone:shared-db'
- 'percona-cluster:shared-db' - 'percona-cluster:shared-db'
@ -129,3 +131,5 @@ relations:
- 'ceph-mon:client' - 'ceph-mon:client'
- - 'gnocchi:coordinator-memcached' - - 'gnocchi:coordinator-memcached'
- 'memcached:cache' - 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

View File

@ -98,6 +98,8 @@ applications:
num_units: 1 num_units: 1
to: to:
- '13' - '13'
nrpe:
charm: cs:nrpe
relations: relations:
- - 'keystone:shared-db' - - 'keystone:shared-db'
- 'percona-cluster:shared-db' - 'percona-cluster:shared-db'
@ -129,3 +131,5 @@ relations:
- 'ceph-mon:client' - 'ceph-mon:client'
- - 'gnocchi:coordinator-memcached' - - 'gnocchi:coordinator-memcached'
- 'memcached:cache' - 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

View File

@ -92,6 +92,8 @@ applications:
num_units: 1 num_units: 1
to: to:
- '12' - '12'
nrpe:
charm: cs:nrpe
relations: relations:
- - 'keystone:shared-db' - - 'keystone:shared-db'
- 'percona-cluster:shared-db' - 'percona-cluster:shared-db'
@ -119,3 +121,5 @@ relations:
- 'ceph-mon:client' - 'ceph-mon:client'
- - 'gnocchi:coordinator-memcached' - - 'gnocchi:coordinator-memcached'
- 'memcached:cache' - 'memcached:cache'
- - 'aodh:nrpe-external-master'
- 'nrpe:nrpe-external-master'

View File

@ -29,6 +29,9 @@ target_deploy_status:
mongodb: mongodb:
workload-status: unknown workload-status: unknown
workload-status-message: '' workload-status-message: ''
nrpe:
workload-status: blocked
workload-status-message: "Nagios server not configured or related"
tests_options: tests_options:
force_deploy: force_deploy:
- trusty-mitaka - trusty-mitaka

View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import mock
import sys import sys
sys.path.append('src') sys.path.append('src')
@ -20,3 +21,4 @@ sys.path.append('src/lib')
# Mock out charmhelpers so that we can test without it. # Mock out charmhelpers so that we can test without it.
import charms_openstack.test_mocks # noqa import charms_openstack.test_mocks # noqa
charms_openstack.test_mocks.mock_charmhelpers() charms_openstack.test_mocks.mock_charmhelpers()
sys.modules['charmhelpers.contrib.charmsupport.nrpe'] = mock.MagicMock()

View File

@ -15,41 +15,63 @@
import unittest import unittest
import mock import mock
import charms_openstack.test_utils as test_utils
import reactive.aodh_handlers as handlers import reactive.aodh_handlers as handlers
_when_args = {} class TestRegisteredHooks(test_utils.TestRegisteredHooks):
_when_not_args = {} """Run tests to ensure hooks are registered."""
def test_registered_hooks(self):
def mock_hook_factory(d): """Test that the correct hooks are registered."""
defaults = [
def mock_hook(*args, **kwargs): 'charm.installed',
'config.changed',
def inner(f): 'charm.default-select-release',
# remember what we were passed. Note that we can't actually 'update-status',
# determine the class we're attached to, as the decorator only gets 'upgrade-charm',
# the function. ]
try: hook_set = {
d[f.__name__].append(dict(args=args, kwargs=kwargs)) 'when': {
except KeyError: 'setup_amqp_req': ('amqp.connected', ),
d[f.__name__] = [dict(args=args, kwargs=kwargs)] 'setup_database': ('shared-db.connected', ),
return f 'setup_endpoint': ('identity-service.connected', ),
return inner 'render_unclustered': ('charm.installed',
return mock_hook 'shared-db.available',
'identity-service.available',
'amqp.available',),
'render_clustered': ('charm.installed',
'shared-db.available',
'identity-service.available',
'amqp.available',
'cluster.available',),
'run_db_migration': ('charm.installed',
'config.complete', ),
'cluster_connected': ('ha.connected', ),
'configure_nrpe': ('config.complete', ),
},
'when_not': {
'install_packages': ('charm.installed', ),
'render_unclustered': ('cluster.available', ),
'run_db_migration': ('db.synced', ),
},
'when_none': {
'configure_nrpe': ('charm.paused', 'is-update-status-hook', ),
},
'when_any': {
'configure_nrpe': ('config.changed.nagios_context',
'config.changed.nagios_servicegroups',
'endpoint.nrpe-external-master.changed',
'nrpe-external-master.available', ),
},
}
self.registered_hooks_test_helper(handlers, hook_set, defaults)
class TestAodhHandlers(unittest.TestCase): class TestAodhHandlers(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls._patched_when = mock.patch('charms.reactive.when',
mock_hook_factory(_when_args))
cls._patched_when_started = cls._patched_when.start()
cls._patched_when_not = mock.patch('charms.reactive.when_not',
mock_hook_factory(_when_not_args))
cls._patched_when_not_started = cls._patched_when_not.start()
# force requires to rerun the mock_hook decorator: # force requires to rerun the mock_hook decorator:
# try except is Python2/Python3 compatibility as Python3 has moved # try except is Python2/Python3 compatibility as Python3 has moved
# reload to importlib. # reload to importlib.
@ -61,12 +83,6 @@ class TestAodhHandlers(unittest.TestCase):
@classmethod @classmethod
def tearDownClass(cls): def tearDownClass(cls):
cls._patched_when.stop()
cls._patched_when_started = None
cls._patched_when = None
cls._patched_when_not.stop()
cls._patched_when_not_started = None
cls._patched_when_not = None
# and fix any breakage we did to the module # and fix any breakage we did to the module
try: try:
reload(handlers) reload(handlers)
@ -94,46 +110,6 @@ class TestAodhHandlers(unittest.TestCase):
self._patches_start[attr] = started self._patches_start[attr] = started
setattr(self, attr, started) setattr(self, attr, started)
def test_registered_hooks(self):
# test that the hooks actually registered the relation expressions that
# are meaningful for this interface: this is to handle regressions.
# The keys are the function names that the hook attaches to.
when_patterns = {
'setup_amqp_req': ('amqp.connected', ),
'setup_database': ('shared-db.connected', ),
'setup_endpoint': ('identity-service.connected', ),
'render_unclustered': ('charm.installed',
'shared-db.available',
'identity-service.available',
'amqp.available',),
'render_clustered': ('charm.installed',
'shared-db.available',
'identity-service.available',
'amqp.available',
'cluster.available',),
'run_db_migration': ('charm.installed',
'config.complete', ),
'cluster_connected': ('ha.connected', ),
}
when_not_patterns = {
'install_packages': ('charm.installed', ),
'render_unclustered': ('cluster.available', ),
'run_db_migration': ('db.synced', ),
}
# check the when hooks are attached to the expected functions
for t, p in [(_when_args, when_patterns),
(_when_not_args, when_not_patterns)]:
for f, args in t.items():
# check that function is in patterns
self.assertTrue(f in p.keys(),
"{} not found".format(f))
# check that the lists are equal
items = []
for a in args:
items += a['args'][:]
self.assertEqual(sorted(items), sorted(p[f]),
"{}: incorrect state registration".format(f))
def test_install_packages(self): def test_install_packages(self):
self.patch(handlers.aodh, 'install') self.patch(handlers.aodh, 'install')
self.patch(handlers.reactive, 'set_state') self.patch(handlers.reactive, 'set_state')

View File

@ -123,6 +123,29 @@ class TestAodhCharm(Helper):
self.check_call.assert_called_once_with(['systemctl', 'daemon-reload']) self.check_call.assert_called_once_with(['systemctl', 'daemon-reload'])
self.service_restart.assert_called_once_with('aodh-api') self.service_restart.assert_called_once_with('aodh-api')
def test_render_nrpe(self):
"""Test NRPE renders correctly pre Ocata."""
self.patch_object(aodh.nrpe, 'NRPE')
self.patch_object(aodh.nrpe, 'add_init_service_checks')
services = ['aodh-api',
'aodh-evaluator',
'aodh-notifier',
'aodh-listener',
]
target = aodh.AodhCharmNewton()
target.render_nrpe_checks()
# Note that this list is valid for Ussuri
self.add_init_service_checks.assert_has_calls([
mock.call().add_init_service_checks(
mock.ANY,
services,
mock.ANY
),
])
self.NRPE.assert_has_calls([
mock.call().write(),
])
class TestAodhCharmOcata(Helper): class TestAodhCharmOcata(Helper):
@ -136,3 +159,23 @@ class TestAodhCharmOcata(Helper):
self.init_is_systemd.return_value = True self.init_is_systemd.return_value = True
aodh.AodhCharmOcata.reload_and_restart() aodh.AodhCharmOcata.reload_and_restart()
self.check_call.assert_called_once_with(['systemctl', 'daemon-reload']) self.check_call.assert_called_once_with(['systemctl', 'daemon-reload'])
def test_render_nrpe(self):
"""Test NRPE renders correctly in Ocata."""
self.patch_object(aodh.nrpe, 'NRPE')
self.patch_object(aodh.nrpe, 'add_init_service_checks')
services = ['aodh-evaluator', 'aodh-notifier',
'aodh-listener', 'apache2']
target = aodh.AodhCharmOcata()
target.render_nrpe_checks()
# Note that this list is valid for Ussuri
self.add_init_service_checks.assert_has_calls([
mock.call().add_init_service_checks(
mock.ANY,
services,
mock.ANY
),
])
self.NRPE.assert_has_calls([
mock.call().write(),
])