Implement key rotation on Quincy

This patchset implements key rotation for Quincy. This needs to be
done before merging to master as this action requires coordination
between ceph-mon and ceph-osd, and these use quincy/edge to run the
test bundles.

Change-Id: I2a49b89c7438626f55347b4201803496557b28ec
func-test-pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/1201

.gitreview

@@ -2,3 +2,5 @@
 host=review.opendev.org
 port=29418
 project=openstack/charm-ceph-osd.git
+
+defaultbranch=stable/quincy.2

hooks/ceph_hooks.py

@@ -83,6 +83,7 @@ from utils import (
     import_osd_upgrade_key,
     import_osd_removal_key,
     import_client_crash_key,
+    import_pending_key,
     get_host_ip,
     get_networks,
     assert_charm_supports_ipv6,
@@ -719,8 +720,23 @@ def get_bdev_enable_discard():
                           "bdev-enable-discard: %s") % bdev_enable_discard)
 
 
+def handle_pending_key(pending_key):
+    key_map = json.loads(pending_key)
+    for osd_id, key in key_map.items():
+        if not os.path.exists('/var/lib/ceph/osd/ceph-%s' % osd_id):
+            continue
+
+        import_pending_key(key, osd_id)
+        service_restart('ceph-osd@%s' % osd_id)
+
+
 @hooks.hook('mon-relation-changed')
 def mon_relation():
+    pending_key = relation_get('pending_key')
+    if pending_key:
+        handle_pending_key(pending_key)
+        return
+
     bootstrap_key = relation_get('osd_bootstrap_key')
     upgrade_key = relation_get('osd_upgrade_key')
     removal_key = relation_get('osd_disk_removal_key')

hooks/utils.py

@@ -85,15 +85,17 @@ def is_osd_bootstrap_ready():
     return os.path.exists(_bootstrap_keyring)
 
 
-def _import_key(key, path, name):
-    if not os.path.exists(path):
+def _import_key(key, path, name, override=False):
+    exists = os.path.exists(path)
+    if not exists or override:
+        create = ['--create-keyring'] if not exists else []
         cmd = [
             'sudo',
             '-u',
             ceph.ceph_user(),
             'ceph-authtool',
-            path,
-            '--create-keyring',
+            path
+        ] + create + [
             '--name={}'.format(name),
             '--add-key={}'.format(key)
         ]
@@ -140,6 +142,19 @@ def import_client_crash_key(key):
     _import_key(key, _client_crash_keyring, 'client.crash')
 
 
+def import_pending_key(key, osd_id):
+    """
+    Import a pending key, used for key rotation.
+
+    :param key: The pending cephx key that will replace the current one.
+    :type key: str
+    :param osd_id: The OSD id whose key will be replaced.
+    :type osd_id: str
+    :raises: subprocess.CalledProcessError"""
+    _import_key(key, '/var/lib/ceph/osd/ceph-%s/keyring' % osd_id,
+                'osd.%s' % osd_id, override=True)
+
+
 def render_template(template_name, context, template_dir=TEMPLATES_DIR):
     """Render Jinja2 template.
 

tests/bundles/focal-xena.yaml

@@ -1,238 +0,0 @@
-variables:
-  openstack-origin: &openstack-origin cloud:focal-xena
-
-series: focal
-
-comment:
-- 'machines section to decide order of deployment. database sooner = faster'
-machines:
-  '0':
-    constraints: mem=3072M
-  '1':
-    constraints: mem=3072M
-  '2':
-    constraints: mem=3072M
-  '3':
-  '4':
-  '5':
-  '6':
-  '7':
-  '8':
-  '9':
-  '10':
-  '11':
-  '12':
-  '13':
-  '14':
-  '15':
-
-applications:
-
-  keystone-mysql-router:
-    charm: ch:mysql-router
-    channel: latest/edge
-  glance-mysql-router:
-    charm: ch:mysql-router
-    channel: latest/edge
-  cinder-mysql-router:
-    charm: ch:mysql-router
-    channel: latest/edge
-  nova-cloud-controller-mysql-router:
-    charm: ch:mysql-router
-    channel: latest/edge
-  placement-mysql-router:
-    charm: ch:mysql-router
-    channel: latest/edge
-
-  mysql-innodb-cluster:
-    charm: ch:mysql-innodb-cluster
-    num_units: 3
-    options:
-      source: *openstack-origin
-    to:
-      - '0'
-      - '1'
-      - '2'
-    channel: latest/edge
-
-  ceph-osd:
-    charm: ../../ceph-osd.charm
-    num_units: 3
-    storage:
-      osd-devices: 'cinder,10G'
-    options:
-      osd-devices: '/dev/test-non-existent'
-      source: *openstack-origin
-      aa-profile-mode: enforce
-    to:
-      - '3'
-      - '4'
-      - '5'
-
-  ceph-mon:
-    charm: ch:ceph-mon
-    num_units: 3
-    options:
-      monitor-count: '3'
-      source: *openstack-origin
-    to:
-      - '6'
-      - '7'
-      - '8'
-    channel: quincy/edge
-
-  rabbitmq-server:
-    charm: ch:rabbitmq-server
-    num_units: 1
-    options:
-      source: *openstack-origin
-    to:
-      - '9'
-    channel: latest/edge
-
-  keystone:
-    expose: True
-    charm: ch:keystone
-    num_units: 1
-    options:
-      openstack-origin: *openstack-origin
-    to:
-      - '10'
-    channel: yoga/edge
-
-  nova-compute:
-    charm: ch:nova-compute
-    num_units: 1
-    options:
-      openstack-origin: *openstack-origin
-    to:
-      - '11'
-    channel: yoga/edge
-
-  glance:
-    expose: True
-    charm: ch:glance
-    num_units: 1
-    options:
-      openstack-origin: *openstack-origin
-    to:
-      - '12'
-    channel: yoga/edge
-
-  cinder:
-    expose: True
-    charm: ch:cinder
-    num_units: 1
-    options:
-      openstack-origin: *openstack-origin
-      block-device: 'None'
-      glance-api-version: '2'
-    to:
-      - '13'
-    channel: yoga/edge
-
-  cinder-ceph:
-    charm: ch:cinder-ceph
-    channel: yoga/edge
-
-  nova-cloud-controller:
-    expose: True
-    charm: ch:nova-cloud-controller
-    num_units: 1
-    options:
-      openstack-origin: *openstack-origin
-    to:
-      - '14'
-    channel: yoga/edge
-
-  placement:
-    charm: ch:placement
-    num_units: 1
-    options:
-      openstack-origin: *openstack-origin
-    to:
-      - '15'
-    channel: yoga/edge
-
-relations:
-  - - 'nova-compute:amqp'
-    - 'rabbitmq-server:amqp'
-
-  - - 'nova-compute:image-service'
-    - 'glance:image-service'
-
-  - - 'nova-compute:ceph'
-    - 'ceph-mon:client'
-
-  - - 'keystone:shared-db'
-    - 'keystone-mysql-router:shared-db'
-  - - 'keystone-mysql-router:db-router'
-    - 'mysql-innodb-cluster:db-router'
-
-  - - 'glance:shared-db'
-    - 'glance-mysql-router:shared-db'
-  - - 'glance-mysql-router:db-router'
-    - 'mysql-innodb-cluster:db-router'
-
-  - - 'glance:identity-service'
-    - 'keystone:identity-service'
-
-  - - 'glance:amqp'
-    - 'rabbitmq-server:amqp'
-
-  - - 'glance:ceph'
-    - 'ceph-mon:client'
-
-  - - 'cinder:shared-db'
-    - 'cinder-mysql-router:shared-db'
-  - - 'cinder-mysql-router:db-router'
-    - 'mysql-innodb-cluster:db-router'
-
-  - - 'cinder:identity-service'
-    - 'keystone:identity-service'
-
-  - - 'cinder:amqp'
-    - 'rabbitmq-server:amqp'
-
-  - - 'cinder:image-service'
-    - 'glance:image-service'
-
-  - - 'cinder-ceph:storage-backend'
-    - 'cinder:storage-backend'
-
-  - - 'cinder-ceph:ceph'
-    - 'ceph-mon:client'
-
-  - - 'ceph-osd:mon'
-    - 'ceph-mon:osd'
-
-  - - 'nova-cloud-controller:shared-db'
-    - 'nova-cloud-controller-mysql-router:shared-db'
-  - - 'nova-cloud-controller-mysql-router:db-router'
-    - 'mysql-innodb-cluster:db-router'
-
-  - - 'nova-cloud-controller:identity-service'
-    - 'keystone:identity-service'
-
-  - - 'nova-cloud-controller:amqp'
-    - 'rabbitmq-server:amqp'
-
-  - - 'nova-cloud-controller:cloud-compute'
-    - 'nova-compute:cloud-compute'
-
-  - - 'nova-cloud-controller:image-service'
-    - 'glance:image-service'
-
-  - - 'placement:shared-db'
-    - 'placement-mysql-router:shared-db'
-  - - 'placement-mysql-router:db-router'
-    - 'mysql-innodb-cluster:db-router'
-
-  - - 'placement'
-    - 'keystone'
-
-  - - 'placement'
-    - 'nova-cloud-controller'
-
-  - - 'cinder-ceph:ceph-access'
-    - 'nova-compute:ceph-access'

tests/tests.yaml

@@ -1,13 +1,9 @@
 charm_name: ceph-osd
 
 gate_bundles:
-  - focal-xena
   - focal-yoga
   - jammy-yoga
 
-smoke_bundles:
-  - focal-xena
-
 configure:
   - zaza.openstack.charm_tests.glance.setup.add_lts_image
 
@@ -16,3 +12,6 @@ tests:
   - zaza.openstack.charm_tests.ceph.tests.CephTest
   - zaza.openstack.charm_tests.ceph.osd.tests.SecurityTest
   - zaza.openstack.charm_tests.ceph.osd.tests.ServiceTest
+  # Upgrade charm, then test key rotation.
+  - zaza.charm_tests.lifecycle.tests.UpgradeCharmsToPath;ceph-osd
+  - zaza.openstack.charm_tests.ceph.tests.CephMonKeyRotationTests

unit_tests/test_ceph_hooks.py

@@ -819,6 +819,18 @@ class CephHooksTestCase(unittest.TestCase):
             level=ceph_hooks.ERROR,
         )
 
+    @patch.object(ceph_hooks, 'service_restart')
+    @patch.object(ceph_hooks, 'import_pending_key')
+    @patch.object(ceph_hooks.os.path, 'exists')
+    def test_handle_pending_key(self, exists, import_pending_key,
+                                service_restart):
+        exists.return_value = True
+        pending_key = '{"0":"some-key"}'
+        ceph_hooks.handle_pending_key(pending_key)
+        exists.assert_called_with('/var/lib/ceph/osd/ceph-0')
+        import_pending_key.assert_called_with('some-key', '0')
+        service_restart.assert_called_with('ceph-osd@0')
+
 
 @patch.object(ceph_hooks, 'local_unit')
 @patch.object(ceph_hooks, 'relation_get')
@@ -903,3 +915,17 @@ class VaultLockerTestCase(unittest.TestCase):
         _cmp_pkgrevno.return_value = -1
         self.assertRaises(ValueError,
                           ceph_hooks.use_vaultlocker)
+
+
+class KeyRotationTestCase(unittest.TestCase):
+
+    @patch.object(ceph_hooks, 'import_pending_key')
+    @patch.object(ceph_hooks, 'service_restart')
+    @patch.object(ceph_hooks.os.path, 'exists')
+    def test_pending_key(self, exists, service_restart, import_pending_key):
+        pending_key = '{"1":"key1","2":"key2"}'
+        exists.side_effect = lambda x: x == '/var/lib/ceph/osd/ceph-2'
+
+        ceph_hooks.handle_pending_key(pending_key)
+        import_pending_key.assert_called_with("key2", "2")
+        service_restart.assert_called_with("ceph-osd@2")