From 1951dc7e9a835ecc6ba12f74413e93e04ff85d95 Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Tue, 26 Jun 2018 12:48:58 +0100 Subject: [PATCH] Add keymaster to fetch root secret from KMIP service Add a new middleware that can be used to fetch an encryption root secret from a KMIP service. The middleware uses a PyKMIP client to interact with a KMIP endpoint. The middleware is configured with a unique identifier for the key to be fetched and options required for the PyKMIP client. Co-Authored-By: Tim Burke Change-Id: Ib0943fb934b347060fc66c091673a33bcfac0a6d --- doc/source/overview_encryption.rst | 93 ++++++++- etc/keymaster.conf-sample | 20 ++ etc/proxy-server.conf-sample | 19 ++ setup.cfg | 4 + .../middleware/crypto/kmip_keymaster.py | 131 ++++++++++++ .../middleware/crypto/test_kmip_keymaster.py | 191 ++++++++++++++++++ 6 files changed, 449 insertions(+), 9 deletions(-) create mode 100644 swift/common/middleware/crypto/kmip_keymaster.py create mode 100644 test/unit/common/middleware/crypto/test_kmip_keymaster.py diff --git a/doc/source/overview_encryption.rst b/doc/source/overview_encryption.rst index d986b6bed0..ea16ea69a9 100644 --- a/doc/source/overview_encryption.rst +++ b/doc/source/overview_encryption.rst @@ -101,7 +101,8 @@ Alternatives to specifying the encryption root secret directly in the `proxy-server.conf` file are storing it in a separate file, or storing it in an :ref:`external key management system ` such as `Barbican -`_. +`_ or a +`KMIP `_ service. .. note:: @@ -184,14 +185,22 @@ re-encrypted when copied. Encryption Root Secret in External Key Management System -------------------------------------------------------- -The benefits of using -a dedicated system for storing the encryption root secret include the -auditing and access control infrastructure that are already in place in such a -system, and the fact that an encryption root secret stored in a key management -system (KMS) may be backed by a hardware security module (HSM) for additional -security. Another significant benefit of storing the root encryption secret in -an external KMS is that it is in this case never stored on a disk in the Swift -cluster. +The benefits of using a dedicated system for storing the encryption root secret +include the auditing and access control infrastructure that are already in +place in such a system, and the fact that an encryption root secret stored in a +key management system (KMS) may be backed by a hardware security module (HSM) +for additional security. Another significant benefit of storing the root +encryption secret in an external KMS is that it is in this case never stored on +a disk in the Swift cluster. + +Swift supports fetching encryption root secrets from a `Barbican +`_ service or a KMIP_ service using the +``kms_keymaster`` or ``kmip_keymaster`` middleware respectively. + +.. _KMIP: https://www.oasis-open.org/committees/kmip/ + +Encryption Root Secret in a Barbican KMS +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Make sure the required dependencies are installed for retrieving an encryption root secret from an external KMS. This can be done when installing Swift (add @@ -308,6 +317,72 @@ For further details on the configuration options, see the `[filter:kms_keymaster]` section in the `proxy-server.conf-sample` file, and the `keymaster.conf-sample` file. + +Encryption Root Secret in a KMIP service +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This middleware enables Swift to fetch a root secret from a KMIP_ service. The +root secret is expected to have been previously created in the KMIP_ service +and is referenced by its unique identifier. The secret should be an AES-256 +symmetric key. + +To use this middleware Swift must be installed with the extra required +dependencies:: + + sudo pip install .[kmip_keymaster] + +Add the ``-e`` flag to install as a development version. + +Edit the swift `proxy-server.conf` file to insert the middleware in the wsgi +pipeline, replacing any other keymaster middleware:: + + [pipeline:main] + pipeline = catch_errors gatekeeper healthcheck proxy-logging \ + kmip_keymaster encryption proxy-logging proxy-server + +and add a new filter section:: + + [filter:kmip_keymaster] + use = egg:swift#kmip_keymaster + key_id = + host = + port = + certfile = /path/to/client/cert.pem + keyfile = /path/to/client/key.pem + ca_certs = /path/to/server/cert.pem + username = + password = + +Apart from ``use`` and ``key_id`` the options are as defined for a PyKMIP +client. The authoritative definition of these options can be found at +``_. + +The value of the ``key_id`` option should be the unique identifier for a secret +that will be retrieved from the KMIP_ service. + +The keymaster configuration can alternatively be defined in a separate config +file by using the ``keymaster_config_path`` option:: + + [filter:kmip_keymaster] + use = egg:swift#kmip_keymaster + keymaster_config_path = /etc/swift/kmip_keymaster.conf + +In this case, the ``filter:kmip_keymaster`` section should contain no other +options than ``use`` and ``keymaster_config_path``. All other options should be +defined in the separate config file in a section named ``kmip_keymaster``. For +example:: + + [kmip_keymaster] + key_id = 1234567890 + host = 127.0.0.1 + port = 5696 + certfile = /etc/swift/kmip_client.crt + keyfile = /etc/swift/kmip_client.key + ca_certs = /etc/swift/kmip_server.crt + username = swift + password = swift_password + + Upgrade Considerations ---------------------- diff --git a/etc/keymaster.conf-sample b/etc/keymaster.conf-sample index ffecc787c4..620740e0d2 100644 --- a/etc/keymaster.conf-sample +++ b/etc/keymaster.conf-sample @@ -74,3 +74,23 @@ # reauthenticate = changeme # domain_id = changeme # domain_name = changeme + +[kmip_keymaster] +# The kmip_keymaster section is used to configure a keymaster that fetches an +# encryption root secret from a KMIP service. + +# The value of the ``key_id`` option should be the unique identifier for a +# secret that will be retrieved from the KMIP service. The secret should be an +# AES-256 symmetric key. +# key_id = + +# The remaining options are used to configure a PyKMIP client and are shown +# below for information. The authoritative definition of these options can be +# found at: https://pykmip.readthedocs.io/en/latest/client.html. +# host = +# port = +# certfile = /path/to/client/cert.pem +# keyfile = /path/to/client/key.pem +# ca_certs = /path/to/server/cert.pem +# username = +# password = diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 8691596821..3f00cdf091 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -1074,6 +1074,25 @@ use = egg:swift#kms_keymaster # options. # keymaster_config_path = +# kmip_keymaster middleware may be used to fetch an encryption root secret from +# a KMIP service. It should replace, in the same position, any other keymaster +# middleware in the proxy-server pipeline, so that the middleware order is as +# shown in this example: +# kmip_keymaster encryption proxy-logging proxy-server +[filter:kmip_keymaster] +use = egg:swift#kmip_keymaster + +# Sets the path from which the keymaster config options should be read. This +# allows multiple processes which need to be encryption-aware (for example, +# proxy-server and container-sync) to share the same config file, ensuring +# that the encryption keys used are the same. As an added benefit the +# keymaster configuration file can have different permissions than the +# `proxy-server.conf` file. The format expected is similar +# to other config files, with a single [kmip_keymaster] section. See the +# keymaster.conf-sample file for details on the kmip_keymaster configuration +# options. +# keymaster_config_path = + [filter:encryption] use = egg:swift#encryption diff --git a/setup.cfg b/setup.cfg index 87eeb4486c..594368dc07 100644 --- a/setup.cfg +++ b/setup.cfg @@ -69,6 +69,9 @@ kms_keymaster = oslo.config>=4.0.0,!=4.3.0,!=4.4.0 # Apache-2.0 castellan>=0.13.0 # Apache-2.0 +kmip_keymaster = + pykmip>=0.7.0 # Apache-2.0 + keystone = keystonemiddleware>=4.17.0 @@ -114,6 +117,7 @@ paste.filter_factory = keymaster = swift.common.middleware.crypto.keymaster:filter_factory encryption = swift.common.middleware.crypto:filter_factory kms_keymaster = swift.common.middleware.crypto.kms_keymaster:filter_factory + kmip_keymaster = swift.common.middleware.crypto.kmip_keymaster:filter_factory listing_formats = swift.common.middleware.listing_formats:filter_factory symlink = swift.common.middleware.symlink:filter_factory s3api = swift.common.middleware.s3api.s3api:filter_factory diff --git a/swift/common/middleware/crypto/kmip_keymaster.py b/swift/common/middleware/crypto/kmip_keymaster.py new file mode 100644 index 0000000000..cff9be3b6e --- /dev/null +++ b/swift/common/middleware/crypto/kmip_keymaster.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2018 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os + +from swift.common.middleware.crypto import keymaster +from swift.common.utils import readconf, get_logger + +from kmip.pie.client import ProxyKmipClient + +""" +This middleware enables Swift to fetch a root secret from a KMIP service. +The root secret is expected to have been previously created in the KMIP service +and is referenced by its unique identifier. The secret should be an AES-256 +symmetric key. + +To use this middleware, edit the swift proxy-server.conf to insert the +middleware in the wsgi pipeline, replacing any other keymaster middleware:: + + [pipeline:main] + pipeline = catch_errors gatekeeper healthcheck proxy-logging \ + kmip_keymaster encryption proxy-logging proxy-server + +and add a new filter section:: + + [filter:kmip_keymaster] + use = egg:swift#kmip_keymaster + key_id = + host = + port = + certfile = /path/to/client/cert.pem + keyfile = /path/to/client/key.pem + ca_certs = /path/to/server/cert.pem + username = + password = + +Apart from ``use`` and ``key_id`` the options are as defined for a PyKMIP +client. The authoritative definition of these options can be found at +`https://pykmip.readthedocs.io/en/latest/client.html`_ + +The value of the ``key_id`` option should be the unique identifier for a secret +that will be retrieved from the KMIP service. + +The keymaster configuration can alternatively be defined in a separate config +file by using the ``keymaster_config_path`` option:: + + [filter:kmip_keymaster] + use = egg:swift#kmip_keymaster + keymaster_config_path=/etc/swift/kmip_keymaster.conf + +In this case, the ``filter:kmip_keymaster`` section should contain no other +options than ``use`` and ``keymaster_config_path``. All other options should be +defined in the separate config file in a section named ``kmip_keymaster``. For +example:: + + [kmip_keymaster] + key_id = 1234567890 + host = 127.0.0.1 + port = 5696 + certfile = /etc/swift/kmip_client.crt + keyfile = /etc/swift/kmip_client.key + ca_certs = /etc/swift/kmip_server.crt + username = swift + password = swift_password +""" + + +class KmipKeyMaster(keymaster.KeyMaster): + def _get_root_secret(self, conf): + self.logger = get_logger(conf, log_route='kmip_keymaster') + if self.keymaster_config_path: + keymaster_opts = ['host', 'port', 'certfile', 'keyfile', + 'ca_certs', 'username', 'password', 'key_id'] + section = 'kmip_keymaster' + if any(opt in conf for opt in keymaster_opts): + raise ValueError('keymaster_config_path is set, but there ' + 'are other config options specified: %s' % + ", ".join(list( + set(keymaster_opts).intersection(conf)))) + conf = readconf(self.keymaster_config_path, section) + else: + section = conf['__name__'] + + if os.path.isdir(conf['__file__']): + raise ValueError( + 'KmipKeyMaster config cannot be read from conf dir %s. Use ' + 'keymaster_config_path option in the proxy server config to ' + 'specify a config file.') + + key_id = conf.get('key_id') + if not key_id: + raise ValueError('key_id option is required') + + kmip_logger = logging.getLogger('kmip') + for handler in self.logger.logger.handlers: + kmip_logger.addHandler(handler) + + with ProxyKmipClient( + config=section, + config_file=conf['__file__'] + ) as client: + secret = client.get(key_id) + if (secret.cryptographic_algorithm.name, + secret.cryptographic_length) != ('AES', 256): + raise ValueError('Expected an AES-256 key, not %s-%d' % ( + secret.cryptographic_algorithm.name, + secret.cryptographic_length)) + return secret.value + + +def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + + def keymaster_filter(app): + return KmipKeyMaster(app, conf) + + return keymaster_filter diff --git a/test/unit/common/middleware/crypto/test_kmip_keymaster.py b/test/unit/common/middleware/crypto/test_kmip_keymaster.py new file mode 100644 index 0000000000..96a05e3116 --- /dev/null +++ b/test/unit/common/middleware/crypto/test_kmip_keymaster.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2018 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import os +import unittest +from tempfile import mkdtemp +from textwrap import dedent +from shutil import rmtree +import sys +sys.modules['kmip'] = mock.Mock() +sys.modules['kmip.pie'] = mock.Mock() +sys.modules['kmip.pie.client'] = mock.Mock() + +from swift.common.middleware.crypto.kmip_keymaster import KmipKeyMaster + + +class MockProxyKmipClient(object): + def __init__(self, secret): + self.secret = secret + self.uid = None + + def get(self, uid): + self.uid = uid + return self.secret + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + pass + + +def create_secret(algorithm_name, length, value): + algorithm = mock.MagicMock() + algorithm.name = algorithm_name + secret = mock.MagicMock(cryptographic_algorithm=algorithm, + cryptographic_length=length, + value=value) + return secret + + +def create_mock_client(secret, calls): + def mock_client(*args, **kwargs): + client = MockProxyKmipClient(secret) + calls.append({'args': args, 'kwargs': kwargs, 'client': client}) + return client + return mock_client + + +class TestKmipKeymaster(unittest.TestCase): + + def setUp(self): + self.tempdir = mkdtemp() + + def tearDown(self): + rmtree(self.tempdir) + + def test_config_in_filter_section(self): + conf = {'__file__': '/etc/swift/proxy-server.conf', + '__name__': 'filter:kmip_keymaster', + 'key_id': '1234'} + secret = create_secret('AES', 256, b'x' * 32) + calls = [] + klass = 'swift.common.middleware.crypto.kmip_keymaster.ProxyKmipClient' + with mock.patch(klass, create_mock_client(secret, calls)): + km = KmipKeyMaster(None, conf) + + self.assertEqual(secret.value, km.root_secret) + self.assertIsNone(km.keymaster_config_path) + self.assertEqual({'config_file': '/etc/swift/proxy-server.conf', + 'config': 'filter:kmip_keymaster'}, + calls[0]['kwargs']) + self.assertEqual('1234', calls[0]['client'].uid) + + def test_config_in_separate_file(self): + km_conf = """ + [kmip_keymaster] + key_id = 4321 + """ + km_config_file = os.path.join(self.tempdir, 'km.conf') + with open(km_config_file, 'wb') as fd: + fd.write(dedent(km_conf)) + + conf = {'__file__': '/etc/swift/proxy-server.conf', + '__name__': 'filter:kmip_keymaster', + 'keymaster_config_path': km_config_file} + secret = create_secret('AES', 256, b'x' * 32) + calls = [] + klass = 'swift.common.middleware.crypto.kmip_keymaster.ProxyKmipClient' + with mock.patch(klass, create_mock_client(secret, calls)): + km = KmipKeyMaster(None, conf) + self.assertEqual(secret.value, km.root_secret) + self.assertEqual(km_config_file, km.keymaster_config_path) + self.assertEqual({'config_file': km_config_file, + 'config': 'kmip_keymaster'}, + calls[0]['kwargs']) + self.assertEqual('4321', calls[0]['client'].uid) + + def test_proxy_server_conf_dir(self): + proxy_server_conf_dir = os.path.join(self.tempdir, 'proxy_server.d') + os.mkdir(proxy_server_conf_dir) + + # KmipClient can't read conf from a dir, so check that is caught early + conf = {'__file__': proxy_server_conf_dir, + '__name__': 'filter:kmip_keymaster', + 'key_id': '789'} + with self.assertRaises(ValueError) as cm: + KmipKeyMaster(None, conf) + self.assertIn('config cannot be read from conf dir', str(cm.exception)) + + # ...but a conf file in a conf dir could point back to itself for the + # KmipClient config + km_config_file = os.path.join(proxy_server_conf_dir, '40.conf') + km_conf = """ + [filter:kmip_keymaster] + keymaster_config_file = %s + + [kmip_keymaster] + key_id = 789 + """ % km_config_file + + with open(km_config_file, 'wb') as fd: + fd.write(dedent(km_conf)) + + conf = {'__file__': proxy_server_conf_dir, + '__name__': 'filter:kmip_keymaster', + 'keymaster_config_path': km_config_file} + secret = create_secret('AES', 256, b'x' * 32) + calls = [] + klass = 'swift.common.middleware.crypto.kmip_keymaster.ProxyKmipClient' + with mock.patch(klass, create_mock_client(secret, calls)): + km = KmipKeyMaster(None, conf) + self.assertEqual(secret.value, km.root_secret) + self.assertEqual(km_config_file, km.keymaster_config_path) + self.assertEqual({'config_file': km_config_file, + 'config': 'kmip_keymaster'}, + calls[0]['kwargs']) + self.assertEqual('789', calls[0]['client'].uid) + + def test_bad_key_length(self): + conf = {'__file__': '/etc/swift/proxy-server.conf', + '__name__': 'filter:kmip_keymaster', + 'key_id': '1234'} + secret = create_secret('AES', 128, b'x' * 16) + calls = [] + klass = 'swift.common.middleware.crypto.kmip_keymaster.ProxyKmipClient' + with mock.patch(klass, create_mock_client(secret, calls)): + with self.assertRaises(ValueError) as cm: + KmipKeyMaster(None, conf) + self.assertIn('Expected an AES-256 key', str(cm.exception)) + self.assertEqual({'config_file': '/etc/swift/proxy-server.conf', + 'config': 'filter:kmip_keymaster'}, + calls[0]['kwargs']) + self.assertEqual('1234', calls[0]['client'].uid) + + def test_bad_key_algorithm(self): + conf = {'__file__': '/etc/swift/proxy-server.conf', + '__name__': 'filter:kmip_keymaster', + 'key_id': '1234'} + secret = create_secret('notAES', 256, b'x' * 32) + calls = [] + klass = 'swift.common.middleware.crypto.kmip_keymaster.ProxyKmipClient' + with mock.patch(klass, create_mock_client(secret, calls)): + with self.assertRaises(ValueError) as cm: + KmipKeyMaster(None, conf) + self.assertIn('Expected an AES-256 key', str(cm.exception)) + self.assertEqual({'config_file': '/etc/swift/proxy-server.conf', + 'config': 'filter:kmip_keymaster'}, + calls[0]['kwargs']) + self.assertEqual('1234', calls[0]['client'].uid) + + def test_missing_key_id(self): + conf = {'__file__': '/etc/swift/proxy-server.conf', + '__name__': 'filter:kmip_keymaster'} + with self.assertRaises(ValueError) as cm: + KmipKeyMaster(None, conf) + self.assertIn('key_id option is required', str(cm.exception))