Merge "Add support for Ceph's prometheus monitoring"

README.md

@@ -73,6 +73,17 @@ implications of segregating Ceph network traffic.
   or `ceph-cluster-network` options will continue to honour them. Furthermore,
   these options override any space bindings, if set.
 
+## Monitoring
+
+The charm supports Ceph metric monitoring with Prometheus. Add relations to the
+[prometheus][prometheus-charm] application in this way:
+
+    juju deploy cs:prometheus2
+    juju add-relation ceph-mon prometheus2
+
+> **Note**: Prometheus support is available starting with Ceph Luminous
+  (xenial-queens UCA pocket).
+
 ## Actions
 
 This section lists Juju [actions][juju-docs-actions] supported by the charm.
@@ -203,3 +214,4 @@ For general charm questions refer to the OpenStack [Charm Guide][cg].
 [ceph-docs-monitors]: https://docs.ceph.com/docs/master/dev/mon-bootstrap
 [lp-bugs-charm-ceph-mon]: https://bugs.launchpad.net/charm-ceph-mon/+filebug
 [cdg-install-openstack]: https://docs.openstack.org/project-deploy-guide/charm-deployment-guide/latest/install-openstack.html
+[prometheus-charm]: https://jaas.ai/prometheus2

hooks/ceph_hooks.py

@@ -33,6 +33,7 @@ from charmhelpers.core.hookenv import (
     DEBUG,
     ERROR,
     INFO,
+    WARNING,
     config,
     relation_ids,
     related_units,
@@ -91,7 +92,9 @@ from utils import (
     has_rbd_mirrors,
     get_ceph_osd_releases,
     execute_post_osd_upgrade_steps,
+    mgr_disable_module,
     mgr_enable_module,
+    is_mgr_module_enabled,
 )
 
 from charmhelpers.contrib.charmsupport import nrpe
@@ -416,6 +419,39 @@ def bootstrap_source_relation_changed():
         mon_relation()
 
 
+@hooks.hook('prometheus-relation-joined',
+            'prometheus-relation-changed')
+def prometheus_relation(relid=None, unit=None, prometheus_permitted=None,
+                        module_enabled=None):
+    if not ceph.is_bootstrapped():
+        return
+    if prometheus_permitted is None:
+        prometheus_permitted = cmp_pkgrevno('ceph', '12.2.0') >= 0
+    if module_enabled is None:
+        module_enabled = (is_mgr_module_enabled('prometheus') or
+                          mgr_enable_module('prometheus'))
+    log("checking if prometheus module is enabled")
+    if prometheus_permitted and module_enabled:
+        log("Updating prometheus")
+        addr = get_public_addr()
+        data = {
+            'hostname': format_ipv6_addr(addr) or addr,
+            'port': 9283,
+        }
+        relation_set(relation_id=relid,
+                     relation_settings=data)
+    else:
+        log("Couldn't enable prometheus, but are related. "
+            "Prometheus is available in Ceph version: {} ; "
+            "Prometheus Module is enabled: {}".format(
+                prometheus_permitted, module_enabled), level=WARNING)
+
+
+@hooks.hook('prometheus-relation-departed')
+def prometheus_left():
+    mgr_disable_module('prometheus')
+
+
 @hooks.hook('mon-relation-departed',
             'mon-relation-changed',
             'leader-settings-changed',
@@ -499,11 +535,24 @@ def mon_relation():
             notify_radosgws()
             notify_client()
             notify_rbd_mirrors()
+            notify_prometheus()
     else:
         log('Not enough mons ({}), punting.'
             .format(len(get_mon_hosts())))
 
 
+def notify_prometheus():
+    if relation_ids('prometheus') and ceph.is_bootstrapped():
+        prometheus_permitted = cmp_pkgrevno('ceph', '12.2.0') >= 0
+        module_enabled = (is_mgr_module_enabled('prometheus') or
+                          mgr_enable_module('prometheus'))
+    for relid in relation_ids('prometheus'):
+        for unit in related_units(relid):
+            prometheus_relation(relid=relid, unit=unit,
+                                prometheus_permitted=prometheus_permitted,
+                                module_enabled=module_enabled)
+
+
 def notify_osds():
     for relid in relation_ids('osd'):
         for unit in related_units(relid):
@@ -859,6 +908,7 @@ def upgrade_charm():
     notify_client()
     notify_radosgws()
     notify_rbd_mirrors()
+    notify_prometheus()
 
 
 @hooks.hook('nrpe-external-master-relation-joined')

hooks/prometheus-relation-changed

@@ -0,0 +1 @@
+ceph_hooks.py

hooks/prometheus-relation-departed

@@ -0,0 +1 @@
+ceph_hooks.py

hooks/prometheus-relation-joined

@@ -0,0 +1 @@
+ceph_hooks.py

hooks/utils.py

@@ -70,6 +70,17 @@ def enable_pocket(pocket):
                 sources.write(line)
 
 
+def is_mgr_module_enabled(module):
+    """Is a given manager module enabled.
+
+    :param module:
+    :type module: str
+    :returns: Whether the named module is enabled
+    :rtype: bool
+    """
+    return module in ceph.enabled_manager_modules()
+
+
 def mgr_enable_module(module):
     """Enable a Ceph Manager Module.
 
@@ -78,12 +89,26 @@ def mgr_enable_module(module):
 
     :raises: subprocess.CalledProcessError
     """
-    if module not in ceph.enabled_manager_modules():
+    if not is_mgr_module_enabled(module):
         subprocess.check_call(['ceph', 'mgr', 'module', 'enable', module])
         return True
     return False
 
 
+def mgr_disable_module(module):
+    """Enable a Ceph Manager Module.
+
+    :param module: The module name to enable
+    :type module: str
+
+    :raises: subprocess.CalledProcessError
+    """
+    if is_mgr_module_enabled(module):
+        subprocess.check_call(['ceph', 'mgr', 'module', 'disable', module])
+        return True
+    return False
+
+
 @cached
 def get_unit_hostname():
     return socket.gethostname()

metadata.yaml

@@ -39,6 +39,8 @@ provides:
   nrpe-external-master:
     interface: nrpe-external-master
     scope: container
+  prometheus:
+    interface: http
 requires:
   bootstrap-source:
     interface: ceph-bootstrap

tests/bundles/bionic-queens.yaml

@@ -47,6 +47,9 @@ applications:
     expose: True
     charm: cs:~openstack-charmers-next/nova-cloud-controller
     num_units: 1
+  prometheus2:
+    charm: cs:prometheus2
+    num_units: 1
 relations:
 - - nova-compute:amqp
   - rabbitmq-server:amqp
@@ -88,3 +91,5 @@ relations:
   - nova-compute:cloud-compute
 - - nova-cloud-controller:image-service
   - glance:image-service
+- - ceph-mon:prometheus
+  - prometheus2:target

tests/bundles/bionic-rocky.yaml

@@ -61,6 +61,9 @@ applications:
     num_units: 1
     options:
       openstack-origin: cloud:bionic-rocky
+  prometheus2:
+    charm: cs:prometheus2
+    num_units: 1
 relations:
 - - nova-compute:amqp
   - rabbitmq-server:amqp
@@ -102,3 +105,5 @@ relations:
   - nova-compute:cloud-compute
 - - nova-cloud-controller:image-service
   - glance:image-service
+- - ceph-mon:prometheus
+  - prometheus2:target

tests/bundles/bionic-stein.yaml

@@ -61,6 +61,9 @@ applications:
     num_units: 1
     options:
       openstack-origin: cloud:bionic-stein
+  prometheus2:
+    charm: cs:prometheus2
+    num_units: 1
 relations:
 - - nova-compute:amqp
   - rabbitmq-server:amqp
@@ -102,3 +105,5 @@ relations:
   - nova-compute:cloud-compute
 - - nova-cloud-controller:image-service
   - glance:image-service
+- - ceph-mon:prometheus
+  - prometheus2:target

tests/bundles/bionic-train.yaml

@@ -65,6 +65,9 @@ applications:
     num_units: 1
     options:
       openstack-origin: cloud:bionic-train
+  prometheus2:
+    charm: cs:prometheus2
+    num_units: 1
 relations:
 - - nova-compute:amqp
   - rabbitmq-server:amqp
@@ -112,3 +115,5 @@ relations:
   - keystone
 - - placement
   - nova-cloud-controller
+- - ceph-mon:prometheus
+  - prometheus2:target

tests/bundles/xenial-queens.yaml

@@ -61,6 +61,9 @@ applications:
     num_units: 1
     options:
       openstack-origin: cloud:xenial-queens
+  prometheus2:
+    charm: cs:prometheus2
+    num_units: 1
 relations:
 - - nova-compute:amqp
   - rabbitmq-server:amqp
@@ -102,3 +105,5 @@ relations:
   - nova-compute:cloud-compute
 - - nova-cloud-controller:image-service
   - glance:image-service
+- - ceph-mon:prometheus
+  - prometheus2:target

tests/tests.yaml

@@ -20,3 +20,4 @@ tests:
   - zaza.openstack.charm_tests.ceph.tests.CephRelationTest
   - zaza.openstack.charm_tests.ceph.tests.CephTest
   - zaza.openstack.charm_tests.ceph.osd.tests.SecurityTest
+  - zaza.openstack.charm_tests.ceph.tests.CephPrometheusTest

unit_tests/test_ceph_hooks.py

@@ -198,6 +198,7 @@ class CephHooksTestCase(test_utils.CharmTestCase):
         mocks["apt_install"].assert_called_once_with(
             ["python-dbus", "lockfile-progs"])
 
+    @patch.object(ceph_hooks, 'notify_prometheus')
     @patch.object(ceph_hooks, 'notify_rbd_mirrors')
     @patch.object(ceph_hooks, 'service_pause')
     @patch.object(ceph_hooks, 'notify_radosgws')
@@ -211,7 +212,8 @@ class CephHooksTestCase(test_utils.CharmTestCase):
             mock_ceph,
             mock_notify_radosgws,
             mock_service_pause,
-            mock_notify_rbd_mirrors):
+            mock_notify_rbd_mirrors,
+            mock_notify_prometheus):
         config = copy.deepcopy(CHARM_CONFIG)
         mock_config.side_effect = lambda key: config[key]
         with patch.multiple(
@@ -232,6 +234,7 @@ class CephHooksTestCase(test_utils.CharmTestCase):
         mock_notify_client.assert_called_once_with()
         mock_notify_radosgws.assert_called_once_with()
         mock_ceph.update_monfs.assert_called_once_with()
+        mock_notify_prometheus.assert_called_once_with()
         mock_service_pause.assert_called_with('ceph-create-keys')
 
     @patch.object(ceph_hooks, 'mds_relation_joined')

unit_tests/test_ceph_utils.py

@@ -39,6 +39,24 @@ class CephUtilsTestCase(test_utils.CharmTestCase):
         _relation_ids.assert_called_once_with('rbd-mirror')
         _related_units.assert_called_once_with('arelid')
 
+    @mock.patch.object(utils.ceph, 'enabled_manager_modules')
+    def test_mgr_module_enabled(self, _enabled_modules):
+        _enabled_modules.return_value = []
+        self.assertFalse(utils.is_mgr_module_enabled('test-module'))
+
+    @mock.patch.object(utils.ceph, 'enabled_manager_modules')
+    def test_mgr_module__is_enabled(self, _enabled_modules):
+        _enabled_modules.return_value = ['test-module']
+        self.assertTrue(utils.is_mgr_module_enabled('test-module'))
+
+    @mock.patch.object(utils.ceph, 'enabled_manager_modules')
+    @mock.patch.object(utils.subprocess, 'check_call')
+    def test_mgr_disable_module(self, _call, _enabled_modules):
+        _enabled_modules.return_value = ['test-module']
+        utils.mgr_disable_module('test-module')
+        _call.assert_called_once_with(
+            ['ceph', 'mgr', 'module', 'disable', 'test-module'])
+
     @mock.patch.object(utils.ceph, 'enabled_manager_modules')
     @mock.patch.object(utils.subprocess, 'check_call')
     def test_mgr_enable_module(self, _call, _enabled_modules):