Add support for snapshots from ceph

The data-mover service requires direct access to the ceph cluster
in order to snapshot volumes etc.

Add support for the ceph-client interface; configuration and keyring
are writen to a charm specific location to avoid interference with
any other ceph services that may be deployed in the same unit.

TrilioVault requires some slight specialised ceph keyring and conf
placement - see code for notes.

Change-Id: If6998d668fc3e7bd164069ba01a157e136133706

src/layer.yaml

@@ -1 +1,5 @@
-includes: ['layer:openstack', 'interface:rabbitmq']
+includes: [
+   'layer:openstack',
+   'interface:rabbitmq',
+   'interface:ceph-client',
+ ]

src/lib/charm/openstack/trilio_dm.py

@@ -20,16 +20,24 @@ import charmhelpers.core.host as host
 import charmhelpers.contrib.openstack.utils as os_utils
 import charmhelpers.fetch as fetch
 
+import charms.reactive as reactive
+
 import charms_openstack.charm
+import charms_openstack.plugins
 import charms_openstack.adapters as os_adapters
 
 # select the default release function
-charms_openstack.charm.use_defaults('charm.default-select-release')
+charms_openstack.charm.use_defaults("charm.default-select-release")
 
 VALID_BACKUP_TARGETS = ["nfs"]
 TV_MOUNTS = "/var/triliovault-mounts"
 
 
+@os_adapters.config_property
+def ceph_dir(ceph):
+    return os.path.join("/var/lib/charm", hookenv.service_name())
+
+
 class NFSShareNotMountedException(Exception):
     """Signal that the trilio nfs share is not mount"""
 
@@ -42,13 +50,27 @@ class GhostShareAlreadyMountedException(Exception):
     pass
 
 
-class TrilioDataMoverBaseCharm(charms_openstack.charm.OpenStackCharm):
+class DataMoverRelationAdapaters(os_adapters.OpenStackAPIRelationAdapters):
+    """
+    Adapters collection for TrilioVault data mover
+    """
+
+    relation_adapters = {
+        "ceph": charms_openstack.plugins.CephRelationAdapter,
+        "amqp": os_adapters.RabbitMQRelationAdapter,
+    }
+
+
+class TrilioDataMoverBaseCharm(
+    charms_openstack.charm.OpenStackCharm,
+    charms_openstack.plugins.BaseOpenStackCephCharm,
+):
 
     release = "queens"
 
     service_name = name = "trilio-data-mover"
 
-    adapters_class = os_adapters.OpenStackAPIRelationAdapters
+    adapters_class = DataMoverRelationAdapaters
 
     data_mover_conf = "/etc/tvault-contego/tvault-contego.conf"
     logrotate_conf = "/etc/logrotate.d/tvault-contego"
@@ -66,12 +88,50 @@ class TrilioDataMoverBaseCharm(charms_openstack.charm.OpenStackCharm):
 
     # Setting an empty source_config_key activates special handling of release
     # selection suitable for subordinate charms
-    source_config_key = ''
+    source_config_key = ""
 
     # Use nova-common package to drive OpenStack Release versioning.
     release_pkg = "nova-common"
     package_codenames = os_utils.PACKAGE_CODENAMES
 
+    # Set ceph keyring prefix to charm specific location
+    @property
+    def ceph_keyring_path_prefix(self):
+        return os.path.join("/var/lib/charm", hookenv.service_name())
+
+    @property
+    def ceph_conf(self):
+        return os.path.join(self.ceph_keyring_path, "ceph.conf")
+
+    def configure_ceph_keyring(self, key, cluster_name=None):
+        """Creates or updates a Ceph keyring file.
+
+        :param key: Key data
+        :type key: str
+        :param cluster_name: (Optional) Name of Ceph cluster to operate on.
+                             Defaults to value of ``self.ceph_cluster_name``.
+        :type cluster_name: str
+        :returns: Absolute path to keyring file
+        :rtype: str
+        :raises: subprocess.CalledProcessError, OSError
+        """
+        keyring_absolute_path = super().configure_ceph_keyring(
+            key, cluster_name
+        )
+        # TODO: add support for custom permissions into charms.openstack
+        if os.path.exists(keyring_absolute_path):
+            # NOTE: triliovault access the keyring as the nova user, so
+            #       set permissions so it can do this.
+            os.chmod(keyring_absolute_path, 0o640)
+            ceph_keyring = os.path.join(
+                "/etc/ceph", os.path.basename(keyring_absolute_path)
+            )
+            # NOTE: triliovault needs a keyring in /etc/ceph as well as in the
+            #       charm specific location for qemu commands to work
+            if not os.path.exists(ceph_keyring):
+                os.symlink(keyring_absolute_path, ceph_keyring)
+        return keyring_absolute_path
+
     def get_amqp_credentials(self):
         return ("datamover", "openstack")
 
@@ -90,7 +150,14 @@ class TrilioDataMoverBaseCharm(charms_openstack.charm.OpenStackCharm):
 
     @property
     def restart_map(self):
-        return {self.data_mover_conf: self.services}
+        if reactive.flags.is_flag_set("ceph.available"):
+            return {
+                self.data_mover_conf: self.services,
+                self.ceph_conf: self.services,
+            }
+        return {
+            self.data_mover_conf: self.services,
+        }
 
     def install(self):
         self.configure_source()
@@ -141,6 +208,19 @@ class TrilioDataMoverBaseCharm(charms_openstack.charm.OpenStackCharm):
             return "blocked", "nfs-shares configuration not set"
         return None, None
 
+    def request_access_to_groups(self, ceph):
+        """Request access to pool types needed for dm operation.
+
+        :param ceph: ceph interface
+        :type ceph: ceph-interface:CephClientRequires
+        """
+        for ceph_group in ("volumes", "images", "vms"):
+            ceph.request_access_to_group(
+                name=ceph_group,
+                object_prefix_permissions={"class-read": ["rbd_children"]},
+                permission="rwx",
+            )
+
 
 class TrilioDataMoverRockyCharm(TrilioDataMoverBaseCharm):
 

src/metadata.yaml

@@ -23,6 +23,8 @@ requires:
   juju-info:
     interface: juju-info
     scope: container
+  ceph:
+    interface: ceph-client
 provides:
   data-mover:
     interface: data-mover

src/reactive/data_mover_handlers.py

@@ -18,9 +18,7 @@ import charms.reactive as reactive
 import charm.openstack.trilio_dm as trilio_dm  # noqa
 
 charm.use_defaults(
-    "charm.installed",
-    "config.changed",
-    "update-status",
+    "charm.installed", "config.changed", "update-status",
 )
 
 
@@ -29,9 +27,12 @@ def render_config(*args):
     """Render the configuration for charm when all the interfaces are
     available.
     """
-    with charm.provide_charm_instance() as charm_class:
-        charm_class.render_with_interfaces(args)
-        charm_class.assess_status()
+    ceph = reactive.endpoint_from_flag("ceph.available")
+    if ceph:
+        args = (ceph,) + args
+    with charm.provide_charm_instance() as charm_instance:
+        charm_instance.render_with_interfaces(args)
+        charm_instance.assess_status()
     reactive.set_state("config.rendered")
 
 
@@ -43,13 +44,28 @@ def default_amqp_connection(amqp):
     This requires that the charm implements get_amqp_credentials() to
     provide a tuple of the (user, vhost) for the amqp server
     """
-    with charm.provide_charm_instance() as instance:
-        user, vhost = instance.get_amqp_credentials()
+    with charm.provide_charm_instance() as charm_instance:
+        user, vhost = charm_instance.get_amqp_credentials()
         amqp.request_access(username=user, vhost=vhost)
-        instance.assess_status()
+        charm_instance.assess_status()
 
 
 @reactive.when("config.changed.triliovault-pkg-source")
 def install_source_changed():
     """Trigger re-install of charm if source configuration options change"""
     reactive.clear_flag("charm.installed")
+
+
+@reactive.when_not("ceph.access.req.sent")
+@reactive.when("ceph.connected")
+def ceph_connected(ceph):
+    with charm.provide_charm_instance() as charm_instance:
+        charm_instance.request_access_to_groups(ceph)
+        reactive.set_flag("ceph.access.req.sent")
+
+
+@reactive.when("ceph.available")
+def configure_ceph(ceph):
+    with charm.provide_charm_instance() as charm_instance:
+        charm_instance.configure_ceph_keyring(ceph.key())
+        charm_instance.assess_status()

src/templates/ceph.conf

@@ -0,0 +1,10 @@
+###############################################################################
+# [ WARNING ]
+# configuration file maintained by Juju
+# local changes will be overwritten.
+###############################################################################
+[global]
+{% if ceph.auth -%}
+auth_supported = {{ ceph.auth }}
+mon host = {{ ceph.monitors }}
+{% endif -%}

src/templates/tvault-contego.conf

@@ -44,4 +44,7 @@ qemu_agent_ping_timeout = {{ options.tv_datamover_qemu_agent_ping_timeout }}
 helper_command = sudo /usr/bin/privsep-helper
 
 [conductor]
-use_local = True
+use_local = True
+
+[ceph]
+ceph_dir = {{ options.ceph_dir }}

unit_tests/test_data_mover.py

@@ -85,9 +85,19 @@ class Test(test_utils.PatchHelper):
             "default_amqp_connection": ("amqp.connected",),
             "install_source_changed": (
                 "config.changed.triliovault-pkg-source",
-            )
+            ),
+            "configure_ceph": (
+                "ceph.available",
+            ),
+            "ceph_connected": (
+                "ceph.connected",
+            ),
+        }
+        when_not_patterns = {
+            "ceph_connected": (
+                "ceph.access.req.sent",
+            ),
         }
-        when_not_patterns = {}
         # check the when hooks are attached to the expected functions
         for t, p in [
             (_when_args, when_patterns),