Use client cert for keystone-identity

When manila-ganesha is related to vault it needs a client cert to
configure the keystone-auth section of manila.conf to communicate with
keystone. This patch sets that up and removes the broken server cert
auto configuration which ended up masking the manila-share service.

func-test-pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/1253
Change-Id: I55e9aa09b88684517d4052dc56eed0cab05a0262
Closes-Bug: #2064487

osci.yaml

@@ -2,8 +2,17 @@
     templates:
       - charm-unit-jobs-py310
       - charm-functional-jobs
+    check:
+      jobs:
+        - noble-caracal-vault_manila-ganesha
     vars:
       needs_charm_build: true
       charm_build_name: manila-ganesha
       build_type: charmcraft
       charmcraft_channel: 3.x/stable
+
+- job:
+    name: noble-caracal-vault_manila-ganesha
+    parent: func-target
+    vars:
+      tox_extra_args: '-- vault:noble-caracal-vault'

src/lib/charm/openstack/manila_ganesha.py

@@ -14,22 +14,30 @@
 
 import collections
 import errno
+import os
 import socket
 import subprocess
 
 import charms_openstack.charm
 import charms_openstack.adapters
 import charms_openstack.plugins
+import charms_openstack.charm.utils
 import charmhelpers.contrib.network.ip as ch_net_ip
+import charms.reactive.relations as relations
 from charmhelpers.core.host import (
     cmp_pkgrevno,
     service_pause,
+    mkdir,
+    path_hash,
+    write_file,
 )
 from charmhelpers.core.hookenv import (
+    ERROR,
     config,
     goal_state,
     local_unit,
     log,
+    network_get,
 )
 from charmhelpers.contrib.hahelpers.cluster import (
     is_clustered,
@@ -49,6 +57,10 @@ from lib.nfs_ganesha_nrpe import (
 
 MANILA_DIR = '/etc/manila/'
 MANILA_CONF = MANILA_DIR + "manila.conf"
+MANILA_SSL_DIR = MANILA_DIR + "ssl/"
+MANILA_CLIENT_CERT_FILE = MANILA_SSL_DIR + "cert.crt"
+MANILA_CLIENT_KEY_FILE = MANILA_SSL_DIR + "cert.key"
+MANILA_CLIENT_CA_FILE = MANILA_SSL_DIR + "ca.crt"
 MANILA_LOGGING_CONF = MANILA_DIR + "logging.conf"
 MANILA_API_PASTE_CONF = MANILA_DIR + "api-paste.ini"
 CEPH_CONF = '/etc/ceph/ceph.conf'
@@ -152,6 +164,28 @@ class KeystoneCredentialAdapter(
         return self.credentials_username
 
 
+class TlsCertificatesAdapter(
+        charms_openstack.adapters.OpenStackRelationAdapter):
+    """Modifies the keystone-credentials interface to act like keystone."""
+
+    def _resolve_file_name(self, path):
+        if os.path.exists(path):
+            return path
+        return None
+
+    @property
+    def certfile(self):
+        return self._resolve_file_name(MANILA_CLIENT_CERT_FILE)
+
+    @property
+    def keyfile(self):
+        return self._resolve_file_name(MANILA_CLIENT_KEY_FILE)
+
+    @property
+    def cafile(self):
+        return self._resolve_file_name(MANILA_CLIENT_CA_FILE)
+
+
 class GaneshaCharmRelationAdapters(
         charms_openstack.adapters.OpenStackRelationAdapters):
     relation_adapters = {
@@ -160,6 +194,7 @@ class GaneshaCharmRelationAdapters(
         'manila-ganesha': charms_openstack.adapters.OpenStackRelationAdapter,
         'identity-service': KeystoneCredentialAdapter,
         'shared_db': charms_openstack.adapters.DatabaseRelationAdapter,
+        'certificates': TlsCertificatesAdapter,
     }
 
 
@@ -222,6 +257,10 @@ class ManilaGaneshaCharm(charms_openstack.charm.HAOpenStackCharm,
             ('12', 'wallaby'),
             ('13', 'xena'),
             ('14', 'yoga'),
+            ('15', 'zed'),
+            ('16', 'antelope'),
+            ('17', 'bobcat'),
+            ('18', 'caracal'),
         ]),
     }
 
@@ -411,6 +450,78 @@ class ManilaGaneshaCharm(charms_openstack.charm.HAOpenStackCharm,
                    'client': ch_core.hookenv.application_name()})
         ceph.send_request_if_needed(rq)
 
+    def get_client_cert_cn_sans(self):
+        """Get the tuple (cn, [sans]) for a client certificiate.
+
+        This is for the keystone endpoint/interface, so generate the client
+        cert data for that.
+        """
+        try:
+            ingress = network_get('identity-service')['ingress-addresses']
+        except Exception as e:
+            # if it didn't work, log it as an error, and return (None, None)
+            log(f"Getting ingress for identity-service failed: {str(e)}",
+                level=ERROR)
+            return (None, None)
+        return (ingress[0], ingress[1:])
+
+    def handle_changed_client_cert_files(self, ca, cert, key):
+        """Handle changes to client cert, key or ca.
+
+        If the client certs have changed on disk, rerender and restart manila.
+
+        The cert and key need to be written to:
+
+        - /etc/manila/ssl/cert.crt - MANILA_CLIENT_CERT_FILE
+        - /etc/manila/ssl/cert.key - MANILA_CLIENT_KEY_FILE
+        - /etc/manila/ssl/ca.cert  - MANILA_CLIENT_CA_FILE
+        """
+        # lives ensrure that the cert dir exists
+        mkdir(MANILA_SSL_DIR)
+        paths = {
+            MANILA_CLIENT_CA_FILE: ca,
+            MANILA_CLIENT_CERT_FILE: cert,
+            MANILA_CLIENT_KEY_FILE: key,
+        }
+        checksums = {path: path_hash(path) for path in paths.keys()}
+        # write or remove the files.
+        for path, contents in paths.items():
+            if contents is None:
+                # delete the file
+                realpath = os.path.abspath(path)
+                path_exists = os.path.exists(realpath)
+                if path_exists:
+                    try:
+                        os.remove(path)
+                    except OSError as e:
+                        log("Path {} couldn't be deleted: {}"
+                            .format(path, str(e)), level=ERROR)
+            else:
+                write_file(path,
+                           contents.encode(),
+                           owner=self.user,
+                           group=self.group,
+                           perms=0o640)
+        new_checksums = {path: path_hash(path) for path in paths.keys()}
+        if new_checksums != checksums:
+            interfaces = (
+                'ceph.available',
+                'amqp.available',
+                'manila-plugin.available',
+                'shared-db.available',
+                'identity-service.available',
+                'certificates.available',
+            )
+            # check all the interfaces are available
+            endpoints = []
+            for interface in interfaces:
+                endpoint = relations.endpoint_from_flag(interface)
+                if not endpoint:
+                    # if not available don't attempt to render
+                    return
+                endpoints.append(endpoint)
+            self.render_with_interfaces(endpoints)
+
     def install_nrpe_checks(self, enable_cron=True):
         return install_nrpe_checks(enable_cron=enable_cron)
 

src/reactive/manila_ganesha.py

@@ -23,7 +23,10 @@ charm.use_defaults(
     'config.changed',
     'update-status',
     'upgrade-charm',
-    'certificates.available',
+    # TODO: remove follwoing commented out code.
+    # remove certificates.available as we want to wire in the call ourselves
+    # directly.
+    # 'certificates.available',
 )
 
 
@@ -78,7 +81,14 @@ def render_things(*args):
                             level=ch_core.hookenv.INFO)
         charm_instance.configure_ceph_keyring(ceph_relation.key)
 
-        charm_instance.render_with_interfaces(args)
+        # add in optional certificates.available relation for https to keystone
+        certificates = relations.endpoint_from_flag('certificates.available')
+        if certificates:
+            interfaces = list(args) + [certificates]
+        else:
+            interfaces = list(args)
+
+        charm_instance.render_with_interfaces(interfaces)
 
         reactive.set_flag('config.rendered')
         charm_instance.assess_status()
@@ -156,6 +166,68 @@ def disable_services():
     reactive.set_flag('services-disabled')
 
 
+@reactive.when('certificates.ca.available')
+def install_root_ca_cert():
+    print("running install_root_ca_cert")
+    cert_provider = relations.endpoint_from_flag('certificates.ca.available')
+    if cert_provider:
+        print("cert_provider lives")
+        update_client_certs_and_ca(cert_provider)
+
+
+@reactive.when('certificates.available')
+def set_client_cert_request():
+    """Set up the client certificate request.
+
+    If the charm is related to vault then it will send a client cert request
+    (set it on the relation) so that the keystone auth can be configured with a
+    client cert, key and CA to authenticate with keystone (HTTP).
+    """
+    print("running set_client_cert_request")
+    cert_provider = relations.endpoint_from_flag('certificates.available')
+    if cert_provider:
+        print("cert_provider lives")
+        with charm.provide_charm_instance() as the_charm:
+            client_cn, client_sans = the_charm.get_client_cert_cn_sans()
+            print(f"client_cn: {client_cn}, client_sans: {client_sans}")
+            if client_cn:
+                cert_provider.request_client_cert(client_cn, client_sans)
+
+
+@reactive.when('certificates.certs.available')
+def update_client_cert():
+    print("running update_client_cert")
+    cert_provider = relations.endpoint_from_flag('certificates.available')
+    if cert_provider:
+        print("cert_provider lives")
+        update_client_certs_and_ca(cert_provider)
+
+
+def update_client_certs_and_ca(cert_provider):
+    """Get the CA, and client cert, key and then update the config."""
+    ca = cert_provider.root_ca_cert
+    chain = cert_provider.root_ca_chain
+    if ca and chain:
+        if ca not in chain:
+            ca = chain + ca
+        else:
+            ca = chain
+    cert = key = None
+    try:
+        client_cert = cert_provider.client_certs[0]  # only requested one cert
+        cert = client_cert.cert
+        key = client_cert.key
+    except IndexError:
+        pass
+    with charm.provide_charm_instance() as the_charm:
+        print(f"updating: {ca}\n{cert}\n{key}")
+        if ca:
+            the_charm.configure_ca(ca)
+        if chain:
+            the_charm.configure_ca(chain, postfix="chain")
+        the_charm.handle_changed_client_cert_files(ca, cert, key)
+
+
 @reactive.when('nrpe-external-master.available')
 def configure_nrpe():
     """Config and install NRPE plugins."""

src/templates/rocky/manila.conf

@@ -29,6 +29,20 @@ lock_path = /var/lib/manila/tmp
 # parts/section-keystone-authtoken includes the [keystone_authtoken] section
 # identifier
 {% include "parts/section-keystone-authtoken" %}
+{% if certificates -%}
+# Certificates for https connections to keystone when tls-certificates is
+# available
+{# NOTE(ajkavanagh) 'certificates' is an optional relation and so we have to -#}
+{# check for its existence before access the .parts. -#}
+{% if certificates.certfile -%}
+certfile = {{ certificates.certfile }}
+keyfile = {{ certificates.keyfile }}
+{% endif -%}
+{% if certificates.cafile -%}
+cafile = {{ certificates.cafile }}
+insecure = false
+{% endif -%}
+{% endif %}
 
 [oslo_messaging_amqp]
 

src/tests/bundles/noble-caracal-vault.yaml

@@ -0,0 +1,368 @@
+variables:
+  openstack-origin: &openstack-origin distro
+
+local_overlay_enabled: True
+
+series: noble
+
+comment:
+- 'machines section to decide order of deployment. database sooner = faster'
+machines:
+  '0':
+    constraints: mem=3072M
+  '1':
+    constraints: mem=3072M
+  '2':
+    constraints: mem=3072M
+  '3':
+  '4':
+  '5':
+  '6':
+  '7':
+  '8':
+  '9':
+  '10':
+  '11':
+  '12':
+  '13':
+  '14':
+  '15':
+  '16':
+  '17':
+    constraints: mem=8G
+  '18':
+    constraints: mem=8G
+  '19':
+  '20':
+  '21':
+  '22':
+  '23':
+  '24':
+
+applications:
+
+  manila-mysql-router:
+    charm: ch:mysql-router
+    channel: latest/edge
+  manila-ganesha-mysql-router:
+    charm: ch:mysql-router
+    channel: latest/edge
+  keystone-mysql-router:
+    charm: ch:mysql-router
+    channel: latest/edge
+  neutron-api-mysql-router:
+    charm: ch:mysql-router
+    channel: latest/edge
+  nova-cloud-controller-mysql-router:
+    charm: ch:mysql-router
+    channel: latest/edge
+  glance-mysql-router:
+    charm: ch:mysql-router
+    channel: latest/edge
+  placement-mysql-router:
+    charm: ch:mysql-router
+    channel: latest/edge
+
+  mysql-innodb-cluster:
+    charm: ch:mysql-innodb-cluster
+    num_units: 3
+    options:
+      source: *openstack-origin
+    to:
+      - '0'
+      - '1'
+      - '2'
+    channel: latest/edge
+
+  manila-ganesha-az1:
+    num_units: 3
+    charm: ../../../manila-ganesha.charm
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '3'
+      - '4'
+      - '5'
+
+  ceph-mon:
+    charm: ch:ceph-mon
+    num_units: 3
+    options:
+      source: *openstack-origin
+    to:
+      - '6'
+      - '7'
+      - '8'
+    channel: latest/edge
+
+  ceph-osd:
+    charm: ch:ceph-osd
+    num_units: 3
+    options:
+      source: *openstack-origin
+    storage:
+      osd-devices: 'cinder,10G'
+    to:
+      - '9'
+      - '10'
+      - '11'
+    channel: latest/edge
+
+  ceph-fs:
+    charm: ch:ceph-fs
+    num_units: 2
+    options:
+      source: *openstack-origin
+    to:
+      - '12'
+      - '13'
+    channel: latest/edge
+
+  manila:
+    charm: ch:manila
+    num_units: 1
+    options:
+      default-share-backend: cephfsnfs1
+      share-protocols: NFS
+      openstack-origin: *openstack-origin
+    to:
+      - '14'
+    channel: latest/edge
+
+  nova-cloud-controller:
+    charm: ch:nova-cloud-controller
+    num_units: 1
+    options:
+      network-manager: Neutron
+      openstack-origin: *openstack-origin
+    to:
+      - '15'
+    channel: latest/edge
+
+  placement:
+    charm: ch:placement
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '16'
+    channel: latest/edge
+
+  nova-compute:
+    charm: ch:nova-compute
+    num_units: 2
+    options:
+      config-flags: default_ephemeral_format=ext4
+      enable-live-migration: true
+      enable-resize: true
+      migration-auth-type: ssh
+      openstack-origin: *openstack-origin
+    to:
+      - '17'
+      - '18'
+    channel: latest/edge
+
+  glance:
+    charm: ch:glance
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '19'
+    channel: latest/edge
+
+  neutron-api:
+    charm: ch:neutron-api
+    num_units: 1
+    options:
+      manage-neutron-plugin-legacy-mode: true
+      neutron-plugin: ovs
+      flat-network-providers: physnet1
+      neutron-security-groups: true
+      openstack-origin: *openstack-origin
+    to:
+      - '20'
+    channel: latest/edge
+
+  neutron-openvswitch:
+    charm: ch:neutron-openvswitch
+    channel: latest/edge
+
+  neutron-gateway:
+    charm: ch:neutron-gateway
+    num_units: 1
+    options:
+      bridge-mappings: physnet1:br-ex
+      openstack-origin: *openstack-origin
+    to:
+      - '21'
+    channel: latest/edge
+
+  rabbitmq-server:
+    charm: ch:rabbitmq-server
+    num_units: 1
+    to:
+      - '22'
+    channel: latest/edge
+
+  keystone:
+    charm: ch:keystone
+    num_units: 1
+    options:
+      openstack-origin: *openstack-origin
+    to:
+      - '23'
+    channel: latest/edge
+
+  nrpe:
+    charm: ch:nrpe
+    channel: latest/edge
+
+  vault:
+    charm: vault
+    channel: 1.8/edge
+    num_units: 1
+    to:
+      - '24'
+
+relations:
+
+  - - 'ceph-mon'
+    - 'ceph-osd'
+
+  - - 'ceph-mon'
+    - 'ceph-fs'
+
+  - - 'ceph-mon'
+    - 'manila-ganesha-az1'
+
+  - - 'manila:shared-db'
+    - 'manila-mysql-router:shared-db'
+  - - 'manila-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'manila-ganesha-az1'
+    - 'rabbitmq-server'
+
+  - - 'manila-ganesha-az1'
+    - 'keystone'
+
+  - - 'manila'
+    - 'manila-ganesha-az1'
+
+  - - 'manila-ganesha-az1:shared-db'
+    - 'manila-ganesha-mysql-router:shared-db'
+  - - 'manila-ganesha-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'manila'
+    - 'rabbitmq-server'
+
+  - - 'manila'
+    - 'keystone'
+
+  - - 'keystone:shared-db'
+    - 'keystone-mysql-router:shared-db'
+  - - 'keystone-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'neutron-api:shared-db'
+    - 'neutron-api-mysql-router:shared-db'
+  - - 'neutron-api-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'neutron-api:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'neutron-api:neutron-api'
+    - 'nova-cloud-controller:neutron-api'
+
+  - - 'placement:placement'
+    - 'nova-cloud-controller:placement'
+
+  - - 'placement:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'placement:shared-db'
+    - 'placement-mysql-router:shared-db'
+  - - 'placement-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'placement:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'neutron-api:neutron-plugin-api'
+    - 'neutron-gateway:neutron-plugin-api'
+
+  - - 'neutron-api:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'nova-compute:neutron-plugin'
+    - 'neutron-openvswitch:neutron-plugin'
+
+  - - 'nova-cloud-controller:shared-db'
+    - 'nova-cloud-controller-mysql-router:shared-db'
+  - - 'nova-cloud-controller-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'neutron-gateway:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'nova-cloud-controller:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'nova-compute:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'neutron-openvswitch:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'nova-cloud-controller:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'nova-cloud-controller:cloud-compute'
+    - 'nova-compute:cloud-compute'
+
+  - - 'glance:identity-service'
+    - 'keystone:identity-service'
+
+  - - 'glance:shared-db'
+    - 'glance-mysql-router:shared-db'
+  - - 'glance-mysql-router:db-router'
+    - 'mysql-innodb-cluster:db-router'
+
+  - - 'glance:amqp'
+    - 'rabbitmq-server:amqp'
+
+  - - 'nova-compute:image-service'
+    - 'glance:image-service'
+
+  - - 'nova-cloud-controller:image-service'
+    - 'glance:image-service'
+
+  - - 'nova-cloud-controller:quantum-network-service'
+    - 'neutron-gateway:quantum-network-service'
+
+  - - 'manila-ganesha-az1:nrpe-external-master'
+    - 'nrpe:nrpe-external-master'
+
+  - - 'keystone'
+    - 'vault'
+
+  - - 'glance'
+    - 'vault'
+
+  - - 'manila'
+    - 'vault'
+
+  - - 'manila-ganesha-az1'
+    - 'vault'
+
+  - - 'neutron-api'
+    - 'vault'
+
+  - - 'nova-cloud-controller'
+    - 'vault'
+
+  - - 'placement'
+    - 'vault'

src/tests/tests.yaml

@@ -4,12 +4,18 @@ gate_bundles:
   - noble-caracal
 dev_bundles:
   - noble-caracal
+  - noble-caracal-vault
 smoke_bundles:
   - noble-caracal
-target_deploy_status: {}
+target_deploy_status:
+  vault:
+    workload-status: blocked
+    workload-status-message-prefix: Vault needs to be initialized
 
 tests:
   - zaza.openstack.charm_tests.manila_ganesha.tests.ManilaGaneshaTests
+  - vault:
+    - zaza.openstack.charm_tests.manila_ganesha.tests.ManilaGaneshaTests
 
 configure:
   - zaza.openstack.charm_tests.glance.setup.add_lts_image
@@ -18,6 +24,14 @@ configure:
   - zaza.openstack.charm_tests.nova.setup.manage_ssh_key
   - zaza.openstack.charm_tests.keystone.setup.add_demo_user
   - zaza.openstack.charm_tests.manila_ganesha.setup.setup_ganesha_share_type
+  - vault:
+    - zaza.openstack.charm_tests.vault.setup.auto_initialize
+    - zaza.openstack.charm_tests.glance.setup.add_lts_image
+    - zaza.openstack.charm_tests.neutron.setup.basic_overcloud_network
+    - zaza.openstack.charm_tests.nova.setup.create_flavors
+    - zaza.openstack.charm_tests.nova.setup.manage_ssh_key
+    - zaza.openstack.charm_tests.keystone.setup.add_demo_user
+    - zaza.openstack.charm_tests.manila_ganesha.setup.setup_ganesha_share_type
 
 tests_options:
   force_deploy:

unit_tests/test_lib_charm_openstack_manila_ganesha.py

@@ -26,6 +26,51 @@ class Helper(test_utils.PatchHelper):
         self.patch_release(manila_ganesha.ManilaGaneshaCharm.release)
 
 
+class TestTlsCertificatesAdapter(Helper):
+
+    def test__resolve_file_name(self):
+        self.patch('os.path.exists', name='os_path_exists')
+        self.os_path_exists.return_value = False
+        relation = mock.MagicMock()
+        a = manila_ganesha.TlsCertificatesAdapter(relation)
+        self.assertEqual(a._resolve_file_name('some-path'), None)
+        self.os_path_exists.return_value = True
+        self.assertEqual(a._resolve_file_name('some-path'), 'some-path')
+
+    def test_certfile_property(self):
+        relation = mock.MagicMock()
+        a = manila_ganesha.TlsCertificatesAdapter(relation)
+        self.patch_object(a,
+                          '_resolve_file_name',
+                          name='mock_resolve_file_name')
+        self.mock_resolve_file_name.return_value = None
+        self.assertEqual(a.certfile, None)
+        self.mock_resolve_file_name.return_value = 'the-certfile'
+        self.assertEqual(a.certfile, 'the-certfile')
+
+    def test_keyfile_property(self):
+        relation = mock.MagicMock()
+        a = manila_ganesha.TlsCertificatesAdapter(relation)
+        self.patch_object(a,
+                          '_resolve_file_name',
+                          name='mock_resolve_file_name')
+        self.mock_resolve_file_name.return_value = None
+        self.assertEqual(a.keyfile, None)
+        self.mock_resolve_file_name.return_value = 'the-keyfile'
+        self.assertEqual(a.certfile, 'the-keyfile')
+
+    def test_cafile_property(self):
+        relation = mock.MagicMock()
+        a = manila_ganesha.TlsCertificatesAdapter(relation)
+        self.patch_object(a,
+                          '_resolve_file_name',
+                          name='mock_resolve_file_name')
+        self.mock_resolve_file_name.return_value = None
+        self.assertEqual(a.cafile, None)
+        self.mock_resolve_file_name.return_value = 'the-cafile'
+        self.assertEqual(a.certfile, 'the-cafile')
+
+
 class TestManilaGaneshaCharm(Helper):
 
     def test_request_ceph_permissions(self):
@@ -63,3 +108,176 @@ class TestManilaGaneshaCharm(Helper):
         self.get_relation_ip.assert_called_once_with('tenant-storage')
         self.is_address_in_network.assert_called_once_with(
             '10.0.0.0/24', '10.0.0.10')
+
+    def test_get_client_cert_cn_sans(self):
+        c = manila_ganesha.ManilaGaneshaCharm()
+        self.patch_object(manila_ganesha, 'network_get')
+        self.network_get.return_value = {
+            'ingress-addresses': ['ip1', 'ip2', 'ip3'],
+        }
+        self.assertEqual(c.get_client_cert_cn_sans(), ('ip1', ['ip2', 'ip3']))
+        self.network_get.assert_called_once_with('identity-service')
+
+        def raises(*args, **kwargs):
+            raise Exception('bang!')
+
+        self.network_get.side_effect = raises
+        self.patch_object(manila_ganesha, 'log')
+        self.assertEqual(c.get_client_cert_cn_sans(), (None, None))
+        self.log.assert_called_once()
+        self.assertRegex(self.log.call_args.args[0],
+                         r"^Getting ingress.*failed")
+
+    def test_handle_changed_client_cert_files__none(self):
+        # test that calling with None on all values ensures not files
+        self.patch_object(manila_ganesha, 'mkdir', name='mock_mkdir')
+        self.patch_object(manila_ganesha, 'path_hash', name='mock_path_hash')
+        self.patch('os.path.exists', name='mock_os_path_exists')
+        self.patch_object(manila_ganesha, 'log', name='mock_log')
+        self.patch('os.remove', name='mock_os_remove')
+        self.patch_object(manila_ganesha, 'write_file', name='mock_write_file')
+        self.patch_object(manila_ganesha.relations, 'endpoint_from_flag',
+                          name='mock_endpoint_from_flag')
+        c = manila_ganesha.ManilaGaneshaCharm()
+        self.patch_object(c,
+                          'render_with_interfaces',
+                          name='mock_render_with_interfaces')
+
+        # Set up test conditions.
+        self.mock_path_hash.return_value = None  # no file changes at all
+        self.mock_os_path_exists.side_effect = [False, True, False]
+
+        # call with all None.
+        c.handle_changed_client_cert_files(None, None, None)
+
+        # validate that things got called
+        self.mock_os_remove.assert_called_once_with(
+            manila_ganesha.MANILA_CLIENT_CERT_FILE)
+        self.mock_write_file.assert_not_called()
+        self.mock_endpoint_from_flag.assert_not_called()
+        self.mock_render_with_interfaces.assert_not_called()
+
+    def test_handle_changed_client_cert_files__none_os_remove_error(self):
+        # test that calling with None on all values ensures not files
+        self.patch_object(manila_ganesha, 'mkdir', name='mock_mkdir')
+        self.patch_object(manila_ganesha, 'path_hash', name='mock_path_hash')
+        self.patch('os.path.exists', name='mock_os_path_exists')
+        self.patch_object(manila_ganesha, 'log', name='mock_log')
+        self.patch('os.remove', name='mock_os_remove')
+        self.patch_object(manila_ganesha, 'write_file', name='mock_write_file')
+        self.patch_object(manila_ganesha.relations, 'endpoint_from_flag',
+                          name='mock_endpoint_from_flag')
+        c = manila_ganesha.ManilaGaneshaCharm()
+        self.patch_object(c,
+                          'render_with_interfaces',
+                          name='mock_render_with_interfaces')
+
+        # Set up test conditions.
+        def raises(_path):
+            if _path == manila_ganesha.MANILA_CLIENT_CERT_FILE:
+                raise OSError('bang!')
+
+        self.mock_path_hash.return_value = None  # no file changes at all
+        self.mock_os_path_exists.side_effect = [True, True, False]
+        self.mock_os_remove.side_effect = raises
+
+        # call with all None.
+        c.handle_changed_client_cert_files(None, None, None)
+
+        # validate that things got called
+        self.mock_os_remove.assert_has_calls([
+            mock.call(manila_ganesha.MANILA_CLIENT_CA_FILE),
+            mock.call(manila_ganesha.MANILA_CLIENT_CERT_FILE),
+        ])
+        self.assertRegex(self.mock_log.call_args.args[0],
+                         r"^Path " + manila_ganesha.MANILA_CLIENT_CERT_FILE +
+                         r".*deleted")
+        self.mock_write_file.assert_not_called()
+        self.mock_endpoint_from_flag.assert_not_called()
+        self.mock_render_with_interfaces.assert_not_called()
+
+    def test_handle_changed_client_cert_files__all(self):
+        # test that calling with None on all values ensures not files
+        self.patch_object(manila_ganesha, 'mkdir', name='mock_mkdir')
+        self.patch_object(manila_ganesha, 'path_hash', name='mock_path_hash')
+        self.patch('os.path.exists', name='mock_os_path_exists')
+        self.patch_object(manila_ganesha, 'log', name='mock_log')
+        self.patch('os.remove', name='mock_os_remove')
+        self.patch_object(manila_ganesha, 'write_file', name='mock_write_file')
+        self.patch_object(manila_ganesha.relations, 'endpoint_from_flag',
+                          name='mock_endpoint_from_flag')
+        c = manila_ganesha.ManilaGaneshaCharm()
+        self.patch_object(c,
+                          'render_with_interfaces',
+                          name='mock_render_with_interfaces')
+
+        # Set up test conditions.
+        self.mock_path_hash.side_effect = [None, None, None, 'h1', 'h2', 'h3']
+        self.mock_endpoint_from_flag.side_effect = [
+            'e1', 'e2', 'e3', 'e4', 'e5', 'e6']
+
+        # call with all None.
+        c.handle_changed_client_cert_files('ca', 'cert', 'key')
+
+        # validate that things got called
+        self.mock_os_remove.assert_not_called()
+        self.mock_write_file.assert_has_calls([
+            mock.call(manila_ganesha.MANILA_CLIENT_CA_FILE, b"ca",
+                      owner=c.user, group=c.group, perms=0o640),
+            mock.call(manila_ganesha.MANILA_CLIENT_CERT_FILE, b"cert",
+                      owner=c.user, group=c.group, perms=0o640),
+            mock.call(manila_ganesha.MANILA_CLIENT_KEY_FILE, b"key",
+                      owner=c.user, group=c.group, perms=0o640),
+        ])
+        self.mock_endpoint_from_flag.assert_has_calls([
+            mock.call('ceph.available'),
+            mock.call('amqp.available'),
+            mock.call('manila-plugin.available'),
+            mock.call('shared-db.available'),
+            mock.call('identity-service.available'),
+            mock.call('certificates.available'),
+        ])
+        self.mock_render_with_interfaces.assert_called_once_with(
+            ['e1', 'e2', 'e3', 'e4', 'e5', 'e6'])
+
+    def test_handle_changed_client_cert_files__all_not_all_endpoints(self):
+        # test that calling with None on all values ensures not files
+        self.patch_object(manila_ganesha, 'mkdir', name='mock_mkdir')
+        self.patch_object(manila_ganesha, 'path_hash', name='mock_path_hash')
+        self.patch('os.path.exists', name='mock_os_path_exists')
+        self.patch_object(manila_ganesha, 'log', name='mock_log')
+        self.patch('os.remove', name='mock_os_remove')
+        self.patch_object(manila_ganesha, 'write_file', name='mock_write_file')
+        self.patch_object(manila_ganesha.relations, 'endpoint_from_flag',
+                          name='mock_endpoint_from_flag')
+        c = manila_ganesha.ManilaGaneshaCharm()
+        self.patch_object(c,
+                          'render_with_interfaces',
+                          name='mock_render_with_interfaces')
+
+        # Set up test conditions.
+        self.mock_path_hash.side_effect = [None, None, None, 'h1', 'h2', 'h3']
+        self.mock_endpoint_from_flag.side_effect = [
+            'e1', 'e2', 'e3', 'e4', None, 'e6']
+
+        # call with all None.
+        c.handle_changed_client_cert_files('ca', 'cert', 'key')
+
+        # validate that things got called
+        self.mock_os_remove.assert_not_called()
+        self.mock_write_file.assert_has_calls([
+            mock.call(manila_ganesha.MANILA_CLIENT_CA_FILE, b"ca",
+                      owner=c.user, group=c.group, perms=0o640),
+            mock.call(manila_ganesha.MANILA_CLIENT_CERT_FILE, b"cert",
+                      owner=c.user, group=c.group, perms=0o640),
+            mock.call(manila_ganesha.MANILA_CLIENT_KEY_FILE, b"key",
+                      owner=c.user, group=c.group, perms=0o640),
+        ])
+        self.mock_endpoint_from_flag.assert_has_calls([
+            mock.call('ceph.available'),
+            mock.call('amqp.available'),
+            mock.call('manila-plugin.available'),
+            mock.call('shared-db.available'),
+            mock.call('identity-service.available'),
+        ])
+        self.mock_render_with_interfaces.assert_not_called()

unit_tests/test_manila_ganesha_handlers.py

@@ -52,6 +52,9 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks):
                 'configure_nrpe': ('nrpe-external-master.available',),
                 'update_ident_username': ('config.changed.service-user',
                                           'identity-service.connected',),
+                'install_root_ca_cert': ('certificates.ca.available',),
+                'set_client_cert_request': ('certificates.available',),
+                'update_client_cert': ('certificates.certs.available',),
             },
             'when_not': {
                 'ceph_connected': ('ganesha-pool-configured',),