From 0edf1fe46c066a09f4a251e2505f5d6a18525bf3 Mon Sep 17 00:00:00 2001 From: Lance Bragstad Date: Mon, 15 Aug 2016 19:36:17 +0000 Subject: [PATCH] Implement encryption of credentials at rest This commit implements credential encryption through the following changes: - additive schema change to store key hashes for credentials - database migration to encrypt all pre-existing credentials - contractive schema change to remove unencrypted credential column - added code to the credential Manager to handle credential encryption All credentials will be encrypted by default. There will not be a way to store unencrypted credentials in keystone from this point forward. Note that this implementation uses database triggers in the migration process. If operators use the traditional offline migration method, it would be more reliable if we didn't try to setup and tear down triggers, as they'll never be used anyway. This makes it so that expand and contract migrations can skip anything related to triggers. Co-Authored-By: Werner Mendizabal bp credential-encryption Depends-On: I433da9a257daa21ec3b5996b2bca571211f1fbba Depends-On: Id3e8922adc154cfec5f7a36613e22eb0b49eeffe Change-Id: I31b7539db436ad270462cfaa3b14213e0ed1fc04 --- keystone/cmd/cli.py | 129 ++++++++++++++++++ ...unencrypted_blob_column_from_credential.py | 60 ++++++++ .../003_migrate_unencrypted_credentials.py | 39 ++++++ ...y_hash_and_encrypted_blob_to_credential.py | 129 ++++++++++++++++++ keystone/common/sql/migration_helpers.py | 11 ++ keystone/credential/backends/sql.py | 7 +- keystone/credential/core.py | 81 ++++++++++- keystone/credential/providers/fernet/core.py | 53 ++++--- keystone/server/backends.py | 1 + .../unit/credential/test_fernet_provider.py | 24 +--- keystone/tests/unit/test_backend_sql.py | 57 ++++++++ keystone/tests/unit/test_credential.py | 16 +++ .../tests/unit/test_sql_banned_operations.py | 7 +- keystone/tests/unit/test_sql_upgrade.py | 119 +++++++++++++++- keystone/tests/unit/test_v3.py | 2 + keystone/tests/unit/test_v3_auth.py | 8 ++ keystone/tests/unit/test_v3_credential.py | 20 +++ keystone/tests/unit/test_v3_identity.py | 9 ++ keystone/tests/unit/test_v3_protection.py | 8 ++ keystone/tests/unit/test_v3_resource.py | 12 ++ setup.cfg | 3 + 21 files changed, 748 insertions(+), 47 deletions(-) create mode 100644 keystone/common/sql/contract_repo/versions/003_remove_unencrypted_blob_column_from_credential.py create mode 100644 keystone/common/sql/data_migration_repo/versions/003_migrate_unencrypted_credentials.py create mode 100644 keystone/common/sql/expand_repo/versions/003_add_key_hash_and_encrypted_blob_to_credential.py diff --git a/keystone/cmd/cli.py b/keystone/cmd/cli.py index e2b8e31fc6..83f4da4dac 100644 --- a/keystone/cmd/cli.py +++ b/keystone/cmd/cli.py @@ -626,6 +626,133 @@ class CredentialSetup(BasePermissionsSetup): ) +class CredentialRotate(BasePermissionsSetup): + """Rotate Fernet encryption keys for credential encryption. + + This assumes you have already run `keystone-manage credential_setup`. + + A new primary key is placed into rotation only if all credentials are + encrypted with the current primary key. If any credentials are encrypted + with a secondary key the rotation will abort. This protects against + removing a key that is still required to decrypt credentials. Once a key is + removed from the repository, it is impossible to recover the original data + without restoring from a backup external to keystone (more on backups + below). To make sure all credentials are encrypted with the latest primary + key, please see the `keystone-manage credential_migrate` command. Since the + maximum number of keys in the credential repository is 3, once all + credentials are encrypted with the latest primary key we can safely + introduce a new primary key. All credentials will still be decryptable + since they are all encrypted with the only secondary key in the repository. + + It is imperitive to understand the importance of backing up keys used to + encrypt credentials. In the event keys are overrotated, applying a key + repository from backup can help recover otherwise useless credentials. + Persisting snapshots of the key repository in secure and encrypted source + control, or a dedicated key management system are good examples of + encryption key backups. + + The `keystone-manage credential_rotate` and `keystone-manage + credential_migrate` commands are intended to be done in sequence. After + performing a rotation, a migration must be done before performing another + rotation. This ensures we don't over-rotate encryption keys. + + """ + + name = 'credential_rotate' + + def __init__(self): + drivers = backends.load_backends() + self.credential_provider_api = drivers['credential_provider_api'] + self.credential_api = drivers['credential_api'] + + def validate_primary_key(self): + crypto, keys = credential_fernet.get_multi_fernet_keys() + primary_key_hash = credential_fernet.primary_key_hash(keys) + + credentials = self.credential_api.driver.list_credentials( + driver_hints.Hints() + ) + for credential in credentials: + if credential['key_hash'] != primary_key_hash: + msg = _('Unable to rotate credential keys because not all ' + 'credentials are encrypted with the primary key. ' + 'Please make sure all credentials have been encrypted ' + 'with the primary key using `keystone-manage ' + 'credential_migrate`.') + raise SystemExit(msg) + + @classmethod + def main(cls): + from keystone.common import fernet_utils as utils + fernet_utils = utils.FernetUtils( + CONF.credential.key_repository, + credential_fernet.MAX_ACTIVE_KEYS + ) + + keystone_user_id, keystone_group_id = cls.get_user_group() + if fernet_utils.validate_key_repository(requires_write=True): + klass = cls() + klass.validate_primary_key() + fernet_utils.rotate_keys(keystone_user_id, keystone_group_id) + + +class CredentialMigrate(BasePermissionsSetup): + """Provides the ability to encrypt credentials using a new primary key. + + This assumes that there is already a credential key repository in place and + that the database backend has been upgraded to at least the Newton schema. + If the credential repository doesn't exist yet, you can use + ``keystone-manage credential_setup`` to create one. + + """ + + name = 'credential_migrate' + + def __init__(self): + drivers = backends.load_backends() + self.credential_provider_api = drivers['credential_provider_api'] + self.credential_api = drivers['credential_api'] + + def migrate_credentials(self): + crypto, keys = credential_fernet.get_multi_fernet_keys() + primary_key_hash = credential_fernet.primary_key_hash(keys) + + # FIXME(lbragstad): We *should* be able to use Hints() to ask only for + # credentials that have a key_hash equal to a secondary key hash or + # None, but Hints() doesn't seem to honor None values. See + # https://bugs.launchpad.net/keystone/+bug/1614154. As a workaround - + # we have to ask for *all* credentials and filter them ourselves. + credentials = self.credential_api.driver.list_credentials( + driver_hints.Hints() + ) + for credential in credentials: + if credential['key_hash'] != primary_key_hash: + # If the key_hash isn't None but doesn't match the + # primary_key_hash, then we know the credential was encrypted + # with a secondary key. Let's decrypt it, and send it through + # the update path to re-encrypt it with the new primary key. + decrypted_blob = self.credential_provider_api.decrypt( + credential['encrypted_blob'] + ) + cred = {'blob': decrypted_blob} + self.credential_api.update_credential( + credential['id'], + cred + ) + + @classmethod + def main(cls): + # Check to make sure we have a repository that works... + from keystone.common import fernet_utils as utils + fernet_utils = utils.FernetUtils( + CONF.credential.key_repository, + credential_fernet.MAX_ACTIVE_KEYS + ) + fernet_utils.validate_key_repository(requires_write=True) + klass = cls() + klass.migrate_credentials() + + class TokenFlush(BaseApp): """Flush expired tokens from the backend.""" @@ -1080,6 +1207,8 @@ class MappingPopulate(BaseApp): CMDS = [ BootStrap, + CredentialMigrate, + CredentialRotate, CredentialSetup, DbSync, DbVersion, diff --git a/keystone/common/sql/contract_repo/versions/003_remove_unencrypted_blob_column_from_credential.py b/keystone/common/sql/contract_repo/versions/003_remove_unencrypted_blob_column_from_credential.py new file mode 100644 index 0000000000..b247e4fd78 --- /dev/null +++ b/keystone/common/sql/contract_repo/versions/003_remove_unencrypted_blob_column_from_credential.py @@ -0,0 +1,60 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from keystone.common.sql import migration_helpers + +import sqlalchemy as sql + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + credential_table = sql.Table('credential', meta, autoload=True) + credential_table.c.blob.drop() + + if migration_helpers.USE_TRIGGERS: + if migrate_engine.name == 'postgresql': + drop_credential_update_trigger = ( + 'DROP TRIGGER credential_update_read_only on credential;' + ) + drop_credential_insert_trigger = ( + 'DROP TRIGGER credential_insert_read_only on credential;' + ) + elif migrate_engine.name == 'mysql': + drop_credential_update_trigger = ( + 'DROP TRIGGER credential_update_read_only;' + ) + drop_credential_insert_trigger = ( + 'DROP TRIGGER credential_insert_read_only;' + ) + else: + # NOTE(lbragstad, henry-nash): Apparently sqlalchemy and sqlite + # behave weird when using triggers, which is why we use the `IF + # EXISTS` conditional here. I think what is happening is that the + # credential_table.c.blob.drop() causes sqlalchemy to create a new + # credential table - but it doesn't copy the triggers over, which + # causes the DROP TRIGGER statement to fail without `IF EXISTS` + # because the trigger doesn't exist in the new table(?!). + drop_credential_update_trigger = ( + 'DROP TRIGGER IF EXISTS credential_update_read_only;' + ) + drop_credential_insert_trigger = ( + 'DROP TRIGGER IF EXISTS credential_insert_read_only;' + ) + migrate_engine.execute(drop_credential_update_trigger) + migrate_engine.execute(drop_credential_insert_trigger) + + # NOTE(lbragstad): We close these so that they are not nullable because + # Newton code (and anything after) would always populate these values. + credential_table.c.encrypted_blob.alter(nullable=False) + credential_table.c.key_hash.alter(nullable=False) diff --git a/keystone/common/sql/data_migration_repo/versions/003_migrate_unencrypted_credentials.py b/keystone/common/sql/data_migration_repo/versions/003_migrate_unencrypted_credentials.py new file mode 100644 index 0000000000..7f51b75e11 --- /dev/null +++ b/keystone/common/sql/data_migration_repo/versions/003_migrate_unencrypted_credentials.py @@ -0,0 +1,39 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + +from keystone.credential.providers import fernet as credential_fernet + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + session = sql.orm.sessionmaker(bind=migrate_engine)() + + credential_table = sql.Table('credential', meta, autoload=True) + credentials = list(credential_table.select().execute()) + + for credential in credentials: + crypto, keys = credential_fernet.get_multi_fernet_keys() + primary_key_hash = credential_fernet.primary_key_hash(keys) + encrypted_blob = crypto.encrypt(credential['blob'].encode('utf-8')) + values = { + 'encrypted_blob': encrypted_blob, + 'key_hash': primary_key_hash + } + update = credential_table.update().where( + credential_table.c.id == credential.id + ).values(values) + session.execute(update) + session.commit() + session.close() diff --git a/keystone/common/sql/expand_repo/versions/003_add_key_hash_and_encrypted_blob_to_credential.py b/keystone/common/sql/expand_repo/versions/003_add_key_hash_and_encrypted_blob_to_credential.py new file mode 100644 index 0000000000..a245c677f8 --- /dev/null +++ b/keystone/common/sql/expand_repo/versions/003_add_key_hash_and_encrypted_blob_to_credential.py @@ -0,0 +1,129 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + +from keystone.common import sql as ks_sql +from keystone.common.sql import migration_helpers + + +# NOTE(lbragstad): MySQL error state of 45000 is a generic unhandled exception. +# Keystone will return a 500 in this case. +MYSQL_INSERT_TRIGGER = """ +CREATE TRIGGER credential_insert_read_only BEFORE INSERT ON credential +FOR EACH ROW +BEGIN + SIGNAL SQLSTATE '45000' + SET MESSAGE_TEXT = '%s'; +END; +""" + +MYSQL_UPDATE_TRIGGER = """ +CREATE TRIGGER credential_update_read_only BEFORE UPDATE ON credential +FOR EACH ROW +BEGIN + IF NEW.encrypted_blob IS NULL THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s'; + END IF; + IF NEW.encrypted_blob IS NOT NULL AND OLD.blob IS NULL THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s'; + END IF; +END; +""" + +SQLITE_INSERT_TRIGGER = """ +CREATE TRIGGER credential_insert_read_only BEFORE INSERT ON credential +BEGIN + SELECT RAISE (ABORT, '%s'); +END; +""" + +SQLITE_UPDATE_TRIGGER = """ +CREATE TRIGGER credential_update_read_only BEFORE UPDATE ON credential +WHEN NEW.encrypted_blob IS NULL +BEGIN + SELECT RAISE (ABORT, '%s'); +END; +""" + +POSTGRESQL_INSERT_TRIGGER = """ +CREATE OR REPLACE FUNCTION keystone_read_only_insert() + RETURNS trigger AS +$BODY$ +BEGIN + RAISE EXCEPTION '%s'; +END +$BODY$ LANGUAGE plpgsql; + +CREATE TRIGGER credential_insert_read_only BEFORE INSERT ON credential +FOR EACH ROW +EXECUTE PROCEDURE keystone_read_only_insert(); +""" + +POSTGRESQL_UPDATE_TRIGGER = """ +CREATE OR REPLACE FUNCTION keystone_read_only_update() + RETURNS trigger AS +$BODY$ +BEGIN + IF NEW.encrypted_blob IS NULL THEN + RAISE EXCEPTION '%s'; + END IF; + IF NEW.encrypted_blob IS NOT NULL AND OLD.blob IS NULL THEN + RAISE EXCEPTION '%s'; + END IF; + RETURN NEW; +END +$BODY$ LANGUAGE plpgsql; + +CREATE TRIGGER credential_update_read_only BEFORE UPDATE ON credential +FOR EACH ROW +EXECUTE PROCEDURE keystone_read_only_update(); +""" + + +def upgrade(migrate_engine): + meta = sql.MetaData() + meta.bind = migrate_engine + + key_hash = sql.Column('key_hash', sql.String(64), nullable=True) + encrypted_blob = sql.Column( + 'encrypted_blob', + ks_sql.Text, + nullable=True + ) + credential_table = sql.Table('credential', meta, autoload=True) + credential_table.create_column(key_hash) + credential_table.create_column(encrypted_blob) + credential_table.c.blob.alter(nullable=True) + + if not migration_helpers.USE_TRIGGERS: + # Skip managing triggers if we're doing an offline upgrade. + return + + error_message = ('Credential migration in progress. Cannot perform ' + 'writes to credential table.') + if migrate_engine.name == 'postgresql': + credential_insert_trigger = POSTGRESQL_INSERT_TRIGGER % error_message + credential_update_trigger = POSTGRESQL_UPDATE_TRIGGER % ( + error_message, error_message + ) + elif migrate_engine.name == 'sqlite': + credential_insert_trigger = SQLITE_INSERT_TRIGGER % error_message + credential_update_trigger = SQLITE_UPDATE_TRIGGER % error_message + else: + credential_insert_trigger = MYSQL_INSERT_TRIGGER % error_message + credential_update_trigger = MYSQL_UPDATE_TRIGGER % ( + error_message, error_message + ) + + migrate_engine.execute(credential_insert_trigger) + migrate_engine.execute(credential_update_trigger) diff --git a/keystone/common/sql/migration_helpers.py b/keystone/common/sql/migration_helpers.py index 98a428f19b..c327790a17 100644 --- a/keystone/common/sql/migration_helpers.py +++ b/keystone/common/sql/migration_helpers.py @@ -30,6 +30,8 @@ from keystone.i18n import _ CONF = keystone.conf.CONF +USE_TRIGGERS = True + # Different RDBMSs use different schemes for naming the Foreign Key # Constraints. SQLAlchemy does not yet attempt to determine the name @@ -190,6 +192,15 @@ def offline_sync_database_to_version(version=None): contract phases will NOT be run. """ + global USE_TRIGGERS + + # This flags let's us bypass trigger setup & teardown for non-rolling + # upgrades. We set this as a global variable immediately before handing off + # to sqlalchemy-migrate, because we can't pass arguments directly to + # migrations that depend on it. We could also register this as a CONF + # option, but the idea here is that we aren't exposing a new API. + USE_TRIGGERS = False + if version: _sync_common_repo(version) else: diff --git a/keystone/credential/backends/sql.py b/keystone/credential/backends/sql.py index 0c08730f41..350787a95e 100644 --- a/keystone/credential/backends/sql.py +++ b/keystone/credential/backends/sql.py @@ -20,13 +20,16 @@ from keystone import exception class CredentialModel(sql.ModelBase, sql.DictBase): __tablename__ = 'credential' - attributes = ['id', 'user_id', 'project_id', 'blob', 'type'] + attributes = [ + 'id', 'user_id', 'project_id', 'encrypted_blob', 'type', 'key_hash' + ] id = sql.Column(sql.String(64), primary_key=True) user_id = sql.Column(sql.String(64), nullable=False) project_id = sql.Column(sql.String(64)) - blob = sql.Column(sql.JsonBlob(), nullable=False) + encrypted_blob = sql.Column(sql.Text(), nullable=True) type = sql.Column(sql.String(255), nullable=False) + key_hash = sql.Column(sql.String(64), nullable=True) extra = sql.Column(sql.JsonBlob()) diff --git a/keystone/credential/core.py b/keystone/credential/core.py index a35b6de426..17e9c5bb54 100644 --- a/keystone/credential/core.py +++ b/keystone/credential/core.py @@ -14,6 +14,8 @@ """Main entry point into the Credential service.""" +import json + from oslo_log import versionutils from keystone.common import dependency @@ -28,6 +30,7 @@ CONF = keystone.conf.CONF @dependency.provider('credential_api') +@dependency.requires('credential_provider_api') class Manager(manager.Manager): """Default pivot point for the Credential backend. @@ -41,17 +44,71 @@ class Manager(manager.Manager): def __init__(self): super(Manager, self).__init__(CONF.credential.driver) + def _decrypt_credential(self, credential): + """Return a decrypted credential reference.""" + if credential['type'] == 'ec2': + decrypted_blob = json.loads( + self.credential_provider_api.decrypt( + credential['encrypted_blob'], + ) + ) + else: + decrypted_blob = self.credential_provider_api.decrypt( + credential['encrypted_blob'] + ) + credential['blob'] = decrypted_blob + credential.pop('key_hash', None) + credential.pop('encrypted_blob', None) + return credential + + def _encrypt_credential(self, credential): + """Return an encrypted credential reference.""" + credential_copy = credential.copy() + if credential.get('type', None) == 'ec2': + # NOTE(lbragstad): When dealing with ec2 credentials, it's possible + # for the `blob` to be a dictionary. Let's make sure we are + # encrypting a string otherwise encryption will fail. + encrypted_blob, key_hash = self.credential_provider_api.encrypt( + json.dumps(credential['blob']) + ) + else: + encrypted_blob, key_hash = self.credential_provider_api.encrypt( + credential['blob'] + ) + credential_copy['encrypted_blob'] = encrypted_blob + credential_copy['key_hash'] = key_hash + credential_copy.pop('blob', None) + return credential_copy + @manager.response_truncated def list_credentials(self, hints=None): - return self.driver.list_credentials(hints or driver_hints.Hints()) + credentials = self.driver.list_credentials( + hints or driver_hints.Hints() + ) + for credential in credentials: + credential = self._decrypt_credential(credential) + return credentials + + def list_credentials_for_user(self, user_id, type=None): + """List credentials for a specific user.""" + credentials = self.driver.list_credentials_for_user(user_id, type=type) + for credential in credentials: + credential = self._decrypt_credential(credential) + return credentials def get_credential(self, credential_id): """Return a credential reference.""" - return self.driver.get_credential(credential_id) + credential = self.driver.get_credential(credential_id) + return self._decrypt_credential(credential) def create_credential(self, credential_id, credential): """Create a credential.""" - return self.driver.create_credential(credential_id, credential) + credential_copy = self._encrypt_credential(credential) + ref = self.driver.create_credential(credential_id, credential_copy) + ref.pop('key_hash', None) + ref.pop('encrypted_blob', None) + ref['blob'] = credential['blob'] + return ref def _validate_credential_update(self, credential_id, credential): # ec2 credentials require a "project_id" to be functional. Before we @@ -68,7 +125,23 @@ class Manager(manager.Manager): def update_credential(self, credential_id, credential): """Update an existing credential.""" self._validate_credential_update(credential_id, credential) - return self.driver.update_credential(credential_id, credential) + if 'blob' in credential: + credential_copy = self._encrypt_credential(credential) + else: + credential_copy = credential.copy() + existing_credential = self.get_credential(credential_id) + existing_blob = existing_credential['blob'] + ref = self.driver.update_credential(credential_id, credential_copy) + ref.pop('key_hash', None) + ref.pop('encrypted_blob', None) + # If the update request contains a `blob` attribute - we should return + # that in the update response. If not, then we should return the + # existing `blob` attribute since it wasn't updated. + if credential.get('blob'): + ref['blob'] = credential['blob'] + else: + ref['blob'] = existing_blob + return ref @versionutils.deprecated( diff --git a/keystone/credential/providers/fernet/core.py b/keystone/credential/providers/fernet/core.py index 5825fa3632..ed69f06e55 100644 --- a/keystone/credential/providers/fernet/core.py +++ b/keystone/credential/providers/fernet/core.py @@ -10,8 +10,11 @@ # License for the specific language governing permissions and limitations # under the License. +import hashlib + from cryptography import fernet from oslo_log import log +import six from keystone.common import fernet_utils import keystone.conf @@ -38,30 +41,38 @@ LOG = log.getLogger(__name__) MAX_ACTIVE_KEYS = 3 +def get_multi_fernet_keys(): + key_utils = fernet_utils.FernetUtils( + CONF.credential.key_repository, MAX_ACTIVE_KEYS) + keys = key_utils.load_keys() + fernet_keys = [fernet.Fernet(key) for key in keys] + crypto = fernet.MultiFernet(fernet_keys) + + return crypto, keys + + +def primary_key_hash(keys): + """Calculate a hash of the primary key used for encryption.""" + if isinstance(keys[0], six.text_type): + keys[0] = keys[0].encode('utf-8') + return hashlib.sha1(keys[0]).hexdigest() + + class Provider(core.Provider): - - @property - def crypto(self): - keys = [fernet.Fernet(key) for key in self._get_encryption_keys()] - return fernet.MultiFernet(keys) - - def _get_encryption_keys(self): - self.key_utils = fernet_utils.FernetUtils( - CONF.credential.key_repository, MAX_ACTIVE_KEYS - ) - return self.key_utils.load_keys() - def encrypt(self, credential): """Attempt to encrypt a plaintext credential. :param credential: a plaintext representation of a credential :returns: an encrypted credential """ + crypto, keys = get_multi_fernet_keys() + try: - return self.crypto.encrypt(credential.encode('utf-8')) - except (TypeError, ValueError): - msg = _('Credential could not be encrypted. Please contact the' - ' administrator') + return ( + crypto.encrypt(credential.encode('utf-8')), + primary_key_hash(keys)) + except (TypeError, ValueError) as e: + msg = 'Credential could not be encrypted: %s' % str(e) LOG.error(msg) raise exception.CredentialEncryptionError(msg) @@ -71,8 +82,16 @@ class Provider(core.Provider): :param credential: an encrypted credential string :returns: a decrypted credential """ + key_utils = fernet_utils.FernetUtils( + CONF.credential.key_repository, MAX_ACTIVE_KEYS) + keys = key_utils.load_keys() + fernet_keys = [fernet.Fernet(key) for key in keys] + crypto = fernet.MultiFernet(fernet_keys) + try: - return self.crypto.decrypt(bytes(credential)).decode('utf-8') + if isinstance(credential, six.text_type): + credential = credential.encode('utf-8') + return crypto.decrypt(credential).decode('utf-8') except (fernet.InvalidToken, TypeError, ValueError): msg = _('Credential could not be decrypted. Please contact the' ' administrator') diff --git a/keystone/server/backends.py b/keystone/server/backends.py index 68f7b5dbf8..92929572ca 100644 --- a/keystone/server/backends.py +++ b/keystone/server/backends.py @@ -51,6 +51,7 @@ def load_backends(): assignment_api=_ASSIGNMENT_API, catalog_api=catalog.Manager(), credential_api=credential.Manager(), + credential_provider_api=credential.provider.Manager(), domain_config_api=resource.DomainConfigManager(), endpoint_policy_api=endpoint_policy.Manager(), federation_api=federation.Manager(), diff --git a/keystone/tests/unit/credential/test_fernet_provider.py b/keystone/tests/unit/credential/test_fernet_provider.py index 639a6d1033..3cf8ff5d2c 100644 --- a/keystone/tests/unit/credential/test_fernet_provider.py +++ b/keystone/tests/unit/credential/test_fernet_provider.py @@ -10,12 +10,10 @@ # License for the specific language governing permissions and limitations # under the License. -import shutil import uuid import keystone.conf from keystone.credential.providers import fernet -from keystone import exception from keystone.tests import unit from keystone.tests.unit import ksfixtures from keystone.tests.unit.ksfixtures import database @@ -41,27 +39,9 @@ class TestFernetCredentialProvider(unit.TestCase): def test_valid_data_encryption(self): blob = uuid.uuid4().hex - encrypted_blob = self.provider.encrypt(blob) + encrypted_blob, primary_key_hash = self.provider.encrypt(blob) decrypted_blob = self.provider.decrypt(encrypted_blob) self.assertNotEqual(blob, encrypted_blob) self.assertEqual(blob, decrypted_blob) - - def test_encrypt_with_invalid_key_raises_exception(self): - shutil.rmtree(CONF.credential.key_repository) - blob = uuid.uuid4().hex - self.assertRaises( - exception.CredentialEncryptionError, - self.provider.encrypt, - blob - ) - - def test_decrypt_with_invalid_key_raises_exception(self): - blob = uuid.uuid4().hex - encrypted_blob = self.provider.encrypt(blob) - shutil.rmtree(CONF.credential.key_repository) - self.assertRaises( - exception.CredentialEncryptionError, - self.provider.decrypt, - encrypted_blob - ) + self.assertIsNotNone(primary_key_hash) diff --git a/keystone/tests/unit/test_backend_sql.py b/keystone/tests/unit/test_backend_sql.py index b75ad2d670..c4583f4aab 100644 --- a/keystone/tests/unit/test_backend_sql.py +++ b/keystone/tests/unit/test_backend_sql.py @@ -27,6 +27,7 @@ from testtools import matchers from keystone.common import driver_hints from keystone.common import sql import keystone.conf +from keystone.credential.providers import fernet as credential_provider from keystone import exception from keystone.identity.backends import sql_model as identity_sql from keystone.resource.backends import base as resource @@ -35,6 +36,7 @@ from keystone.tests.unit.assignment import test_backends as assignment_tests from keystone.tests.unit.catalog import test_backends as catalog_tests from keystone.tests.unit import default_fixtures from keystone.tests.unit.identity import test_backends as identity_tests +from keystone.tests.unit import ksfixtures from keystone.tests.unit.ksfixtures import database from keystone.tests.unit.policy import test_backends as policy_tests from keystone.tests.unit.resource import test_backends as resource_tests @@ -1024,7 +1026,16 @@ class SqlCredential(SqlTests): self.assertIn(cred['id'], retrived_ids) def setUp(self): + self.useFixture(database.Database()) super(SqlCredential, self).setUp() + self.useFixture( + ksfixtures.KeyRepository( + self.config_fixture, + 'credential', + credential_provider.MAX_ACTIVE_KEYS + ) + ) + self.credentials = [] for _ in range(3): self.credentials.append( @@ -1054,3 +1065,49 @@ class SqlCredential(SqlTests): credentials = self.credential_api.list_credentials_for_user( self.user_foo['id'], type=cred['type']) self._validateCredentialList(credentials, [cred]) + + def test_create_credential_is_encrypted_when_stored(self): + credential = unit.new_credential_ref(user_id=uuid.uuid4().hex) + credential_id = credential['id'] + returned_credential = self.credential_api.create_credential( + credential_id, + credential + ) + + # Make sure the `blob` is *not* encrypted when returned from the + # credential API. + self.assertEqual(returned_credential['blob'], credential['blob']) + + credential_from_backend = self.credential_api.driver.get_credential( + credential_id + ) + + # Pull the credential directly from the backend, the `blob` should be + # encrypted. + self.assertNotEqual( + credential_from_backend['encrypted_blob'], + credential['blob'] + ) + + def test_list_credentials_is_decrypted(self): + credential = unit.new_credential_ref(user_id=uuid.uuid4().hex) + credential_id = credential['id'] + + created_credential = self.credential_api.create_credential( + credential_id, + credential + ) + + # Pull the credential directly from the backend, the `blob` should be + # encrypted. + credential_from_backend = self.credential_api.driver.get_credential( + credential_id + ) + self.assertNotEqual( + credential_from_backend['encrypted_blob'], + credential['blob'] + ) + + # Make sure the `blob` values listed from the API are not encrypted. + listed_credentials = self.credential_api.list_credentials() + self.assertIn(created_credential, listed_credentials) diff --git a/keystone/tests/unit/test_credential.py b/keystone/tests/unit/test_credential.py index 466fdd481c..cec139c247 100644 --- a/keystone/tests/unit/test_credential.py +++ b/keystone/tests/unit/test_credential.py @@ -21,9 +21,11 @@ from keystone.common import context from keystone.common import request from keystone.common import utils from keystone.contrib.ec2 import controllers +from keystone.credential.providers import fernet as credential_fernet from keystone import exception from keystone.tests import unit from keystone.tests.unit import default_fixtures +from keystone.tests.unit import ksfixtures from keystone.tests.unit.ksfixtures import database from keystone.tests.unit import rest @@ -35,6 +37,13 @@ class V2CredentialEc2TestCase(rest.RestfulTestCase): super(V2CredentialEc2TestCase, self).setUp() self.user_id = self.user_foo['id'] self.project_id = self.tenant_bar['id'] + self.useFixture( + ksfixtures.KeyRepository( + self.config_fixture, + 'credential', + credential_fernet.MAX_ACTIVE_KEYS + ) + ) def _get_token_id(self, r): return r.result['access']['token']['id'] @@ -104,6 +113,13 @@ class V2CredentialEc2Controller(unit.TestCase): def setUp(self): super(V2CredentialEc2Controller, self).setUp() self.useFixture(database.Database()) + self.useFixture( + ksfixtures.KeyRepository( + self.config_fixture, + 'credential', + credential_fernet.MAX_ACTIVE_KEYS + ) + ) self.load_backends() self.load_fixtures(default_fixtures) self.user_id = self.user_foo['id'] diff --git a/keystone/tests/unit/test_sql_banned_operations.py b/keystone/tests/unit/test_sql_banned_operations.py index 3481f8a7c9..244eabd1dd 100644 --- a/keystone/tests/unit/test_sql_banned_operations.py +++ b/keystone/tests/unit/test_sql_banned_operations.py @@ -231,7 +231,12 @@ class TestKeystoneExpandSchemaMigrations( # Migration 002 changes the column type, from datetime to timestamp in # the contract phase. Adding exception here to pass expand banned # tests, otherwise fails. - 2 + 2, + # NOTE(lbragstad): The expand 003 migration alters the credential table + # to make `blob` nullable. This allows the triggers added in 003 to + # catch writes when the `blob` attribute isn't populated. We do this so + # that the triggers aren't aware of the encryption implementation. + 3 ] def setUp(self): diff --git a/keystone/tests/unit/test_sql_upgrade.py b/keystone/tests/unit/test_sql_upgrade.py index 65eae3a754..7e29518514 100644 --- a/keystone/tests/unit/test_sql_upgrade.py +++ b/keystone/tests/unit/test_sql_upgrade.py @@ -48,8 +48,10 @@ from testtools import matchers from keystone.common import sql from keystone.common.sql import migration_helpers import keystone.conf +from keystone.credential.providers import fernet as credential_fernet from keystone.tests import unit from keystone.tests.unit import default_fixtures +from keystone.tests.unit import ksfixtures from keystone.tests.unit.ksfixtures import database @@ -1560,13 +1562,20 @@ class PostgreSQLOpportunisticDataMigrationUpgradeTestCase( FIXTURE = test_base.PostgreSQLOpportunisticFixture -class SqlContractSchemaUpgradeTests(SqlMigrateBase): +class SqlContractSchemaUpgradeTests(SqlMigrateBase, unit.TestCase): def setUp(self): # Make sure the legacy, expand and data migration repos are fully # upgraded, since the contract phase is only run after these are # upgraded. super(SqlContractSchemaUpgradeTests, self).setUp() + self.useFixture( + ksfixtures.KeyRepository( + self.config_fixture, + 'credential', + credential_fernet.MAX_ACTIVE_KEYS + ) + ) self.upgrade() self.expand() self.migrate() @@ -1640,6 +1649,7 @@ class FullMigration(SqlMigrateBase, unit.TestCase): self.expand(1) self.migrate(1) self.contract(1) + password = sqlalchemy.Table('password', self.metadata, autoload=True) self.assertTrue(password.c.created_at.nullable) # upgrade each repository to 002 @@ -1650,6 +1660,113 @@ class FullMigration(SqlMigrateBase, unit.TestCase): if self.engine.name != 'sqlite': self.assertFalse(password.c.created_at.nullable) + def test_migration_003_migrate_unencrypted_credentials(self): + self.useFixture( + ksfixtures.KeyRepository( + self.config_fixture, + 'credential', + credential_fernet.MAX_ACTIVE_KEYS + ) + ) + + session = self.sessionmaker() + credential_table_name = 'credential' + + # upgrade each repository to 002 + self.expand(2) + self.migrate(2) + self.contract(2) + + # populate the credential table with some sample credentials + credentials = list() + for i in range(5): + credential = {'id': uuid.uuid4().hex, + 'blob': uuid.uuid4().hex, + 'user_id': uuid.uuid4().hex, + 'type': 'cert'} + credentials.append(credential) + self.insert_dict(session, credential_table_name, credential) + + # verify the current schema + self.assertTableColumns( + credential_table_name, + ['id', 'user_id', 'project_id', 'type', 'blob', 'extra'] + ) + + # upgrade expand repo to 003 to add new columns + self.expand(3) + + # verify encrypted_blob and key_hash columns have been added and verify + # the original blob column is still there + self.assertTableColumns( + credential_table_name, + ['id', 'user_id', 'project_id', 'type', 'blob', 'extra', + 'key_hash', 'encrypted_blob'] + ) + + # verify triggers by making sure we can't write to the credential table + credential = {'id': uuid.uuid4().hex, + 'blob': uuid.uuid4().hex, + 'user_id': uuid.uuid4().hex, + 'type': 'cert'} + self.assertRaises(db_exception.DBError, + self.insert_dict, + session, + credential_table_name, + credential) + + # upgrade migrate repo to 003 to migrate existing credentials + self.migrate(3) + + # make sure we've actually updated the credential with the + # encrypted blob and the corresponding key hash + credential_table = sqlalchemy.Table( + credential_table_name, + self.metadata, + autoload=True + ) + for credential in credentials: + filter = credential_table.c.id == credential['id'] + cols = [credential_table.c.key_hash, credential_table.c.blob, + credential_table.c.encrypted_blob] + q = sqlalchemy.select(cols).where(filter) + result = session.execute(q).fetchone() + + self.assertIsNotNone(result.encrypted_blob) + self.assertIsNotNone(result.key_hash) + # verify the original blob column is still populated + self.assertEqual(result.blob, credential['blob']) + + # verify we can't make any writes to the credential table + credential = {'id': uuid.uuid4().hex, + 'blob': uuid.uuid4().hex, + 'user_id': uuid.uuid4().hex, + 'key_hash': uuid.uuid4().hex, + 'type': 'cert'} + self.assertRaises(db_exception.DBError, + self.insert_dict, + session, + credential_table_name, + credential) + + # upgrade contract repo to 003 to remove triggers and blob column + self.contract(3) + + # verify the new schema doesn't have a blob column anymore + self.assertTableColumns( + credential_table_name, + ['id', 'user_id', 'project_id', 'type', 'extra', 'key_hash', + 'encrypted_blob'] + ) + + # verify that the triggers are gone by writing to the database + credential = {'id': uuid.uuid4().hex, + 'encrypted_blob': uuid.uuid4().hex, + 'key_hash': uuid.uuid4().hex, + 'user_id': uuid.uuid4().hex, + 'type': 'cert'} + self.insert_dict(session, credential_table_name, credential) + class MySQLOpportunisticFullMigration(FullMigration): FIXTURE = test_base.MySQLOpportunisticFixture diff --git a/keystone/tests/unit/test_v3.py b/keystone/tests/unit/test_v3.py index 98bc647b31..5db6730129 100644 --- a/keystone/tests/unit/test_v3.py +++ b/keystone/tests/unit/test_v3.py @@ -1042,6 +1042,8 @@ class RestfulTestCase(unit.SQLDriverOverrides, rest.RestfulTestCase, self.assertIsNotNone(entity.get('user_id')) self.assertIsNotNone(entity.get('blob')) self.assertIsNotNone(entity.get('type')) + self.assertNotIn('key_hash', entity) + self.assertNotIn('encrypted_blob', entity) if ref: self.assertEqual(ref['user_id'], entity['user_id']) self.assertEqual(ref['blob'], entity['blob']) diff --git a/keystone/tests/unit/test_v3_auth.py b/keystone/tests/unit/test_v3_auth.py index 2b3de453c4..a39ac6c798 100644 --- a/keystone/tests/unit/test_v3_auth.py +++ b/keystone/tests/unit/test_v3_auth.py @@ -36,6 +36,7 @@ from keystone.auth.plugins import totp from keystone.common import utils import keystone.conf from keystone.contrib.revoke import routers +from keystone.credential.providers import fernet as credential_fernet from keystone import exception from keystone.policy.backends import rules from keystone.tests.common import auth as common_auth @@ -4926,6 +4927,13 @@ class TestAuthTOTP(test_v3.RestfulTestCase): def setUp(self): super(TestAuthTOTP, self).setUp() + self.useFixture( + ksfixtures.KeyRepository( + self.config_fixture, + 'credential', + credential_fernet.MAX_ACTIVE_KEYS + ) + ) ref = unit.new_totp_credential( user_id=self.default_domain_user['id'], diff --git a/keystone/tests/unit/test_v3_credential.py b/keystone/tests/unit/test_v3_credential.py index 181430d113..76d92a94b7 100644 --- a/keystone/tests/unit/test_v3_credential.py +++ b/keystone/tests/unit/test_v3_credential.py @@ -23,8 +23,10 @@ from testtools import matchers from keystone.common import utils import keystone.conf from keystone.contrib.ec2 import controllers +from keystone.credential.providers import fernet as credential_fernet from keystone import exception from keystone.tests import unit +from keystone.tests.unit import ksfixtures from keystone.tests.unit import test_v3 @@ -33,6 +35,17 @@ CRED_TYPE_EC2 = controllers.CRED_TYPE_EC2 class CredentialBaseTestCase(test_v3.RestfulTestCase): + + def setUp(self): + super(CredentialBaseTestCase, self).setUp() + self.useFixture( + ksfixtures.KeyRepository( + self.config_fixture, + 'credential', + credential_fernet.MAX_ACTIVE_KEYS + ) + ) + def _create_dict_blob_credential(self): blob, credential = unit.new_ec2_credential(user_id=self.user['id'], project_id=self.project_id) @@ -341,6 +354,13 @@ class TestCredentialTrustScoped(test_v3.RestfulTestCase): self.trustee_user = self.identity_api.create_user(self.trustee_user) self.trustee_user['password'] = password self.trustee_user_id = self.trustee_user['id'] + self.useFixture( + ksfixtures.KeyRepository( + self.config_fixture, + 'credential', + credential_fernet.MAX_ACTIVE_KEYS + ) + ) def config_overrides(self): super(TestCredentialTrustScoped, self).config_overrides() diff --git a/keystone/tests/unit/test_v3_identity.py b/keystone/tests/unit/test_v3_identity.py index f940ceeb0f..dbd600c556 100644 --- a/keystone/tests/unit/test_v3_identity.py +++ b/keystone/tests/unit/test_v3_identity.py @@ -22,8 +22,10 @@ from testtools import matchers from keystone.common import controller import keystone.conf +from keystone.credential.providers import fernet as credential_fernet from keystone import exception from keystone.tests import unit +from keystone.tests.unit import ksfixtures from keystone.tests.unit import test_v3 @@ -70,6 +72,13 @@ class IdentityTestCase(test_v3.RestfulTestCase): def setUp(self): super(IdentityTestCase, self).setUp() + self.useFixture( + ksfixtures.KeyRepository( + self.config_fixture, + 'credential', + credential_fernet.MAX_ACTIVE_KEYS + ) + ) self.group = unit.new_group_ref(domain_id=self.domain_id) self.group = self.identity_api.create_group(self.group) diff --git a/keystone/tests/unit/test_v3_protection.py b/keystone/tests/unit/test_v3_protection.py index 60e1ae7b9b..0298ddf5f1 100644 --- a/keystone/tests/unit/test_v3_protection.py +++ b/keystone/tests/unit/test_v3_protection.py @@ -19,6 +19,7 @@ from oslo_serialization import jsonutils from six.moves import http_client import keystone.conf +from keystone.credential.providers import fernet as credential_fernet from keystone import exception from keystone.tests import unit from keystone.tests.unit import ksfixtures @@ -594,6 +595,13 @@ class IdentityTestv3CloudPolicySample(test_v3.RestfulTestCase, self.config_fixture.config( group='resource', admin_project_domain_name=self.admin_domain['name']) + self.useFixture( + ksfixtures.KeyRepository( + self.config_fixture, + 'credential', + credential_fernet.MAX_ACTIVE_KEYS + ) + ) def load_sample_data(self): # Start by creating a couple of domains diff --git a/keystone/tests/unit/test_v3_resource.py b/keystone/tests/unit/test_v3_resource.py index 8523ddb1d5..bffe546424 100644 --- a/keystone/tests/unit/test_v3_resource.py +++ b/keystone/tests/unit/test_v3_resource.py @@ -18,8 +18,10 @@ from testtools import matchers from keystone.common import controller import keystone.conf +from keystone.credential.providers import fernet as credential_fernet from keystone import exception from keystone.tests import unit +from keystone.tests.unit import ksfixtures from keystone.tests.unit import test_v3 from keystone.tests.unit import utils as test_utils @@ -31,6 +33,16 @@ class ResourceTestCase(test_v3.RestfulTestCase, test_v3.AssignmentTestMixin): """Test domains and projects.""" + def setUp(self): + super(ResourceTestCase, self).setUp() + self.useFixture( + ksfixtures.KeyRepository( + self.config_fixture, + 'credential', + credential_fernet.MAX_ACTIVE_KEYS + ) + ) + # Domain CRUD tests def test_create_domain(self): diff --git a/setup.cfg b/setup.cfg index 183efe196f..2d060f2186 100644 --- a/setup.cfg +++ b/setup.cfg @@ -125,6 +125,9 @@ keystone.catalog = keystone.credential = sql = keystone.credential.backends.sql:Credential +keystone.credential.provider = + fernet = keystone.credential.providers.fernet:Provider + keystone.identity = ldap = keystone.identity.backends.ldap:Identity sql = keystone.identity.backends.sql:Identity