From 189a1096da2b0ad6b51fd5943a385a89f56a18c4 Mon Sep 17 00:00:00 2001 From: Alan Bishop Date: Tue, 28 Nov 2017 08:04:12 -0500 Subject: [PATCH] Migrate fixed_key encryption keys to Barbican Assist users who are switching from the legacy ConfKeyManager to Barbican by automatically migrating any existing keys. Key migration is executed in its own thread spawned on cinder-volume startup. Two factors are used to determine whether existing keys need to be migrated. 1) The ConfKeyManager's fixed_key config value is set (not None). This indicates volumes may exist that were encrypted using the ConfKeyManager. 2) Barbican is the current key manager. When the both conditions are met, each instance of the cinder-volume service scans its volumes in the database, looking for volumes using the ConfKeyManager's all-zeros encryption key ID. If a volume has an all-zeros key ID, the same secret (derived from the fixed_key) is stored in Barbican, and all database references to that volume's key ID are replaced with the new Barbican key ID. Implements: blueprint migrate-fixed-key-to-barbican Change-Id: Ic70f45762cf4e426c222415e49b947a328282ca0 --- cinder/keymgr/migration.py | 162 +++++++++++++ cinder/tests/unit/keymgr/test_migration.py | 217 ++++++++++++++++++ cinder/tests/unit/volume/test_init_host.py | 12 + cinder/volume/manager.py | 5 + cinder/volume/utils.py | 20 ++ ...ixed-key-to-barbican-91dfcb829efd4bb6.yaml | 13 ++ 6 files changed, 429 insertions(+) create mode 100644 cinder/keymgr/migration.py create mode 100644 cinder/tests/unit/keymgr/test_migration.py create mode 100644 releasenotes/notes/migrate-fixed-key-to-barbican-91dfcb829efd4bb6.yaml diff --git a/cinder/keymgr/migration.py b/cinder/keymgr/migration.py new file mode 100644 index 00000000000..876a1476f70 --- /dev/null +++ b/cinder/keymgr/migration.py @@ -0,0 +1,162 @@ +# Copyright 2017 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import binascii + +from oslo_config import cfg +from oslo_log import log as logging + +from barbicanclient import client as barbican_client +from castellan import options as castellan_options +from keystoneauth1 import loading as ks_loading +from keystoneauth1 import session as ks_session + +from cinder import context +from cinder import coordination +from cinder import objects + +LOG = logging.getLogger(__name__) + +CONF = cfg.CONF + +MAX_KEY_MIGRATION_ERRORS = 3 + + +class KeyMigrator(object): + def __init__(self, conf): + self.conf = conf + self.admin_context = context.get_admin_context() + self.fixed_key_id = '00000000-0000-0000-0000-000000000000' + self.fixed_key_bytes = None + self.fixed_key_length = None + + def handle_key_migration(self, volumes): + castellan_options.set_defaults(self.conf) + self.conf.import_opt(name='fixed_key', + module_str='cinder.keymgr.conf_key_mgr', + group='key_manager') + fixed_key = self.conf.key_manager.fixed_key + backend = self.conf.key_manager.backend or '' + + backend = backend.split('.')[-1] + + if backend == 'ConfKeyManager': + LOG.info("Not migrating encryption keys because the " + "ConfKeyManager is still in use.") + elif not fixed_key: + LOG.info("Not migrating encryption keys because the " + "ConfKeyManager's fixed_key is not in use.") + elif backend != 'barbican' and backend != 'BarbicanKeyManager': + # Note: There are two ways of specifying the Barbican backend. + # The long-hand method contains the "BarbicanKeyManager" class + # name, and the short-hand method is just "barbican" with no + # module path prefix. + LOG.warning("Not migrating encryption keys because migration to " + "the '%s' key_manager backend is not supported.", + backend) + self._log_migration_status() + elif not volumes: + LOG.info("Not migrating encryption keys because there are no " + "volumes associated with this host.") + self._log_migration_status() + else: + self.fixed_key_bytes = bytes(binascii.unhexlify(fixed_key)) + self.fixed_key_length = len(self.fixed_key_bytes) * 8 + self._migrate_keys(volumes) + self._log_migration_status() + + def _migrate_keys(self, volumes): + LOG.info("Starting migration of ConfKeyManager keys.") + + # Establish a Barbican client session that will be used for the entire + # key migration process. Use cinder's own service credentials. + try: + ks_loading.register_auth_conf_options(self.conf, + 'keystone_authtoken') + auth = ks_loading.load_auth_from_conf_options(self.conf, + 'keystone_authtoken') + sess = ks_session.Session(auth=auth) + self.barbican = barbican_client.Client(session=sess) + except Exception as e: + LOG.error("Aborting encryption key migration due to " + "error creating Barbican client: %s", e) + return + + errors = 0 + for volume in volumes: + try: + self._migrate_volume_key(volume) + except Exception as e: + LOG.error("Error migrating encryption key: %s", e) + # NOTE(abishop): There really shouldn't be any soft errors, so + # if an error occurs migrating one key then chances are they + # will all fail. This avoids filling the log with the same + # error in situations where there are many keys to migrate. + errors += 1 + if errors > MAX_KEY_MIGRATION_ERRORS: + LOG.error("Aborting encryption key migration " + "(too many errors).") + break + + @coordination.synchronized('{volume.id}-{f_name}') + def _migrate_volume_key(self, volume): + if volume.encryption_key_id == self.fixed_key_id: + self._update_encryption_key_id(volume) + + def _update_encryption_key_id(self, volume): + LOG.info("Migrating volume %s encryption key to Barbican", volume.id) + + # Create a Barbican secret using the same fixed_key algorithm. + secret = self.barbican.secrets.create(algorithm='AES', + bit_length=self.fixed_key_length, + secret_type='symmetric', + mode=None, + payload=self.fixed_key_bytes) + secret_ref = secret.store() + + # Create a Barbican ACL so the volume's user can access the secret. + acl = self.barbican.acls.create(entity_ref=secret_ref, + users=[volume.user_id]) + acl.submit() + + _, _, encryption_key_id = secret_ref.rpartition('/') + volume.encryption_key_id = encryption_key_id + volume.save() + + # TODO(abishop): need to determine if any snapshot creations are + # in-flight that might be added to the db with the volume's old + # fixed key ID. + snapshots = objects.snapshot.SnapshotList.get_all_for_volume( + self.admin_context, + volume.id) + for snapshot in snapshots: + snapshot.encryption_key_id = encryption_key_id + snapshot.save() + + def _log_migration_status(self): + num_to_migrate = len(objects.volume.VolumeList.get_all( + context=self.admin_context, + filters={'encryption_key_id': self.fixed_key_id})) + if num_to_migrate == 0: + LOG.info("No volumes are using the ConfKeyManager's " + "encryption_key_id.") + else: + LOG.warning("There are still %d volume(s) using the " + "ConfKeyManager's all-zeros encryption key ID.", + num_to_migrate) + + +def migrate_fixed_key(volumes, conf=CONF): + KeyMigrator(conf).handle_key_migration(volumes) diff --git a/cinder/tests/unit/keymgr/test_migration.py b/cinder/tests/unit/keymgr/test_migration.py new file mode 100644 index 00000000000..28e52a983e4 --- /dev/null +++ b/cinder/tests/unit/keymgr/test_migration.py @@ -0,0 +1,217 @@ +# Copyright 2017 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Tests for encryption key migration.""" + +import mock +from oslo_config import cfg + +from cinder import db +from cinder.keymgr import migration +from cinder import objects +from cinder.tests.unit import fake_constants as fake +from cinder.tests.unit import utils as tests_utils +from cinder.tests.unit import volume as base + +CONF = cfg.CONF + +FIXED_KEY_ID = '00000000-0000-0000-0000-000000000000' + + +class KeyMigrationTestCase(base.BaseVolumeTestCase): + def setUp(self): + super(KeyMigrationTestCase, self).setUp() + self.conf = CONF + self.fixed_key = '1' * 64 + self.conf.import_opt(name='fixed_key', + module_str='cinder.keymgr.conf_key_mgr', + group='key_manager') + self.conf.set_override('fixed_key', + self.fixed_key, + group='key_manager') + self.conf.set_override('backend', + 'barbican', + group='key_manager') + self.my_vols = [] + + def tearDown(self): + for vol in objects.VolumeList.get_all(self.context): + self.volume.delete_volume(self.context, vol) + super(KeyMigrationTestCase, self).tearDown() + + def create_volume(self, key_id=FIXED_KEY_ID): + vol = tests_utils.create_volume(self.context, host=self.conf.host) + vol_id = self.volume.create_volume(self.context, vol) + if key_id: + db.volume_update(self.context, + vol_id, + {'encryption_key_id': key_id}) + self.my_vols = objects.VolumeList.get_all_by_host(self.context, + self.conf.host) + # Return a fully baked Volume object (not the partially baked 'vol' + # and not the DB object). + return next((v for v in self.my_vols if v.id == vol_id)) + + @mock.patch('cinder.keymgr.migration.KeyMigrator._migrate_keys') + @mock.patch('cinder.keymgr.migration.KeyMigrator._log_migration_status') + def test_no_fixed_key(self, + mock_log_migration_status, + mock_migrate_keys): + self.create_volume() + self.conf.set_override('fixed_key', None, group='key_manager') + migration.migrate_fixed_key(self.my_vols, conf=self.conf) + mock_migrate_keys.assert_not_called() + mock_log_migration_status.assert_not_called() + + @mock.patch('cinder.keymgr.migration.KeyMigrator._migrate_keys') + @mock.patch('cinder.keymgr.migration.KeyMigrator._log_migration_status') + def test_using_conf_key_manager(self, + mock_log_migration_status, + mock_migrate_keys): + self.create_volume() + self.conf.set_override('backend', + 'some.ConfKeyManager', + group='key_manager') + migration.migrate_fixed_key(self.my_vols, conf=self.conf) + mock_migrate_keys.assert_not_called() + mock_log_migration_status.assert_not_called() + + @mock.patch('cinder.keymgr.migration.KeyMigrator._migrate_keys') + @mock.patch('cinder.keymgr.migration.KeyMigrator._log_migration_status') + def test_using_barbican_module_path(self, + mock_log_migration_status, + mock_migrate_keys): + # Verify the long-hand method of specifying the Barbican backend + # is properly parsed. + self.create_volume() + self.conf.set_override( + 'backend', + 'castellan.key_manager.barbican_key_manager.BarbicanKeyManager', + group='key_manager') + migration.migrate_fixed_key(self.my_vols, conf=self.conf) + mock_migrate_keys.assert_called_once_with(self.my_vols) + mock_log_migration_status.assert_called_once_with() + + @mock.patch('cinder.keymgr.migration.KeyMigrator._migrate_keys') + @mock.patch('cinder.keymgr.migration.KeyMigrator._log_migration_status') + def test_using_unsupported_key_manager(self, + mock_log_migration_status, + mock_migrate_keys): + self.create_volume() + self.conf.set_override('backend', + 'some.OtherKeyManager', + group='key_manager') + migration.migrate_fixed_key(self.my_vols, conf=self.conf) + mock_migrate_keys.assert_not_called() + mock_log_migration_status.assert_called_once_with() + + @mock.patch('cinder.keymgr.migration.KeyMigrator._migrate_keys') + @mock.patch('cinder.keymgr.migration.KeyMigrator._log_migration_status') + def test_no_volumes(self, + mock_log_migration_status, + mock_migrate_keys): + migration.migrate_fixed_key(self.my_vols, conf=self.conf) + mock_migrate_keys.assert_not_called() + mock_log_migration_status.assert_called_once_with() + + @mock.patch('cinder.keymgr.migration.KeyMigrator._migrate_volume_key') + @mock.patch('barbicanclient.client.Client') + def test_fail_no_barbican_client(self, + mock_barbican_client, + mock_migrate_volume_key): + self.create_volume() + mock_barbican_client.side_effect = Exception + migration.migrate_fixed_key(self.my_vols, conf=self.conf) + mock_migrate_volume_key.assert_not_called() + + @mock.patch('cinder.keymgr.migration.KeyMigrator._migrate_volume_key') + @mock.patch('barbicanclient.client.Client') + def test_fail_too_many_errors(self, + mock_barbican_client, + mock_migrate_volume_key): + for n in range(0, (migration.MAX_KEY_MIGRATION_ERRORS + 3)): + self.create_volume() + mock_migrate_volume_key.side_effect = Exception + migration.migrate_fixed_key(self.my_vols, conf=self.conf) + self.assertEqual(mock_migrate_volume_key.call_count, + (migration.MAX_KEY_MIGRATION_ERRORS + 1)) + + @mock.patch('cinder.keymgr.migration.KeyMigrator._migrate_keys') + def test_migration_status_more_to_migrate(self, + mock_migrate_keys): + mock_log = self.mock_object(migration, 'LOG') + self.create_volume() + migration.migrate_fixed_key(self.my_vols, conf=self.conf) + + # Look for one warning (more to migrate) and no info log messages. + mock_log.info.assert_not_called() + self.assertEqual(mock_log.warning.call_count, 1) + + @mock.patch('cinder.keymgr.migration.KeyMigrator._migrate_keys') + def test_migration_status_all_done(self, + mock_migrate_keys): + mock_log = self.mock_object(migration, 'LOG') + self.create_volume(key_id=fake.ENCRYPTION_KEY_ID) + migration.migrate_fixed_key(self.my_vols, conf=self.conf) + + # Look for one info (all done) and no warning log messages. + mock_log.warning.assert_not_called() + self.assertEqual(mock_log.info.call_count, 1) + + @mock.patch( + 'cinder.keymgr.migration.KeyMigrator._update_encryption_key_id') + @mock.patch('barbicanclient.client.Client') + def test_migrate_fixed_key_volumes(self, + mock_barbican_client, + mock_update_encryption_key_id): + # Create two volumes with fixed key ID that needs to be migrated, and + # a couple of volumes with key IDs that don't need to be migrated, + # or no key ID. + vol_1 = self.create_volume() + self.create_volume(key_id=fake.UUID1) + self.create_volume(key_id=None) + vol_2 = self.create_volume() + self.create_volume(key_id=fake.UUID2) + + migration.migrate_fixed_key(self.my_vols, conf=self.conf) + calls = [mock.call(vol_1), mock.call(vol_2)] + mock_update_encryption_key_id.assert_has_calls(calls, any_order=True) + self.assertEqual(mock_update_encryption_key_id.call_count, len(calls)) + + @mock.patch('barbicanclient.client.Client') + def test_update_encryption_key_id(self, + mock_barbican_client): + vol = self.create_volume() + + snap_ids = [fake.SNAPSHOT_ID, fake.SNAPSHOT2_ID, fake.SNAPSHOT3_ID] + for snap_id in snap_ids: + tests_utils.create_snapshot(self.context, vol.id, id=snap_id) + + # Barbican's secret.store() returns a URI that contains the + # secret's key ID at the end. + secret_ref = 'http://some/path/' + fake.ENCRYPTION_KEY_ID + mock_secret = mock.MagicMock() + mock_secret.store.return_value = secret_ref + + mock_barbican_client.return_value.secrets.create.return_value \ + = mock_secret + + migration.migrate_fixed_key(self.my_vols, conf=self.conf) + vol_db = db.volume_get(self.context, vol.id) + self.assertEqual(fake.ENCRYPTION_KEY_ID, vol_db['encryption_key_id']) + + for snap_id in snap_ids: + snap_db = db.snapshot_get(self.context, snap_id) + self.assertEqual(fake.ENCRYPTION_KEY_ID, + snap_db['encryption_key_id']) diff --git a/cinder/tests/unit/volume/test_init_host.py b/cinder/tests/unit/volume/test_init_host.py index c8353337cc6..d6c9c242040 100644 --- a/cinder/tests/unit/volume/test_init_host.py +++ b/cinder/tests/unit/volume/test_init_host.py @@ -268,3 +268,15 @@ class VolumeInitHostTestCase(base.BaseVolumeTestCase): mock.ANY, filters={'cluster_name': cluster}) snap_get_all_mock.assert_called_once_with( mock.ANY, filters={'cluster_name': cluster}) + + @mock.patch('cinder.keymgr.migration.migrate_fixed_key') + @mock.patch('cinder.volume.manager.VolumeManager._get_my_volumes') + @mock.patch('cinder.manager.ThreadPoolManager._add_to_threadpool') + def test_init_host_key_migration(self, + mock_add_threadpool, + mock_get_my_volumes, + mock_migrate_fixed_key): + + self.volume.init_host(service_id=self.service_id) + mock_add_threadpool.assert_called_once_with(mock_migrate_fixed_key, + mock_get_my_volumes()) diff --git a/cinder/volume/manager.py b/cinder/volume/manager.py index a1efefdae45..b17eae82fe8 100644 --- a/cinder/volume/manager.py +++ b/cinder/volume/manager.py @@ -65,6 +65,7 @@ from cinder.i18n import _ from cinder.image import cache as image_cache from cinder.image import glance from cinder.image import image_utils +from cinder.keymgr import migration as key_migration from cinder import manager from cinder.message import api as message_api from cinder.message import message_field @@ -495,6 +496,10 @@ class VolumeManager(manager.CleanableManager, backend_name = vol_utils.extract_host(self.service_topic_queue) image_utils.cleanup_temporary_file(backend_name) + # Migrate any ConfKeyManager keys based on fixed_key to the currently + # configured key manager. + self._add_to_threadpool(key_migration.migrate_fixed_key, volumes) + # collect and publish service capabilities self.publish_service_capabilities(ctxt) LOG.info("Driver initialization completed successfully.", diff --git a/cinder/volume/utils.py b/cinder/volume/utils.py index 8fe83068c9a..7ba26bc279d 100644 --- a/cinder/volume/utils.py +++ b/cinder/volume/utils.py @@ -25,9 +25,12 @@ import re import time import uuid +from castellan.common.credentials import keystone_password from castellan.common import exception as castellan_exception +from castellan import key_manager as castellan_key_manager import eventlet from eventlet import tpool +from keystoneauth1 import loading as ks_loading from oslo_concurrency import processutils from oslo_config import cfg from oslo_log import log as logging @@ -913,6 +916,23 @@ def delete_encryption_key(context, key_manager, encryption_key_id): key_manager.delete(context, encryption_key_id) except castellan_exception.ManagedObjectNotFoundError: pass + except castellan_exception.KeyManagerError: + LOG.info("First attempt to delete key id %s failed, retrying with " + "cinder's service context.", encryption_key_id) + conf = CONF + ks_loading.register_auth_conf_options(conf, 'keystone_authtoken') + service_context = keystone_password.KeystonePassword( + password=conf.keystone_authtoken.password, + auth_url=conf.keystone_authtoken.auth_url, + username=conf.keystone_authtoken.username, + user_domain_name=conf.keystone_authtoken.user_domain_name, + project_name=conf.keystone_authtoken.project_name, + project_domain_name=conf.keystone_authtoken.project_domain_name) + try: + castellan_key_manager.API(conf).delete(service_context, + encryption_key_id) + except castellan_exception.ManagedObjectNotFoundError: + pass def clone_encryption_key(context, key_manager, encryption_key_id): diff --git a/releasenotes/notes/migrate-fixed-key-to-barbican-91dfcb829efd4bb6.yaml b/releasenotes/notes/migrate-fixed-key-to-barbican-91dfcb829efd4bb6.yaml new file mode 100644 index 00000000000..ed34f57161c --- /dev/null +++ b/releasenotes/notes/migrate-fixed-key-to-barbican-91dfcb829efd4bb6.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + When Barbican is the encryption key_manager backend, any encryption keys + associated with the legacy ConfKeyManager will be automatically migrated + to Barbican. All database references to the ConfKeyManager's all-zeros key + ID will be updated with a Barbican key ID. The encryption keys do not + change. Only the encryption key ID changes. + + Key migration is initiated on service startup, and entries in the + cinder-volume log will indicate the migration status. Log entries will + indicate when a volume's encryption key ID has been migrated to Barbican, + and a summary log message will indicate when key migration has finished.