diff --git a/doc/source/middleware.rst b/doc/source/middleware.rst index a0787472..f636c11f 100644 --- a/doc/source/middleware.rst +++ b/doc/source/middleware.rst @@ -96,6 +96,15 @@ DLO support centers around a user specified filter that matches segments and concatenates them together in object listing order. Please see the DLO docs for :ref:`dlo-doc` further details. +.. _encryption: + +Encryption +========== + +.. automodule:: swift.common.middleware.crypto + :members: + :show-inheritance: + .. _formpost: FormPost @@ -108,7 +117,7 @@ FormPost .. _gatekeeper: GateKeeper -============= +========== .. automodule:: swift.common.middleware.gatekeeper :members: @@ -123,6 +132,13 @@ Healthcheck :members: :show-inheritance: +Keymaster +========= + +.. automodule:: swift.common.middleware.crypto.keymaster + :members: + :show-inheritance: + .. _keystoneauth: KeystoneAuth diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 6a4962ff..aebb8727 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -79,7 +79,7 @@ bind_port = 8080 [pipeline:main] # This sample pipeline uses tempauth and is used for SAIO dev work and # testing. See below for a pipeline using keystone. -pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit tempauth copy container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit tempauth copy container-quotas account-quotas slo dlo versioned_writes keymaster encryption proxy-logging proxy-server # The following pipeline shows keystone integration. Comment out the one # above and uncomment this one. Additional steps for integrating keystone are @@ -765,3 +765,33 @@ use = egg:swift#copy # When object_post_as_copy is set to True, a POST request will be transformed # into a COPY request where source and destination objects are the same. # object_post_as_copy = true + +# Note: To enable encryption, add the following 2 dependent pieces of crypto +# middleware to the proxy-server pipeline. They should be to the right of all +# other middleware apart from the final proxy-logging middleware, and in the +# order shown in this example: +# keymaster encryption proxy-logging proxy-server +[filter:keymaster] +use = egg:swift#keymaster + +# Sets the root secret from which encryption keys are derived. This must be set +# before first use to a value that is a base64 encoding of at least 32 bytes. +# The security of all encrypted data critically depends on this key, therefore +# it should be set to a high-entropy value. For example, a suitable value may +# be obtained by base-64 encoding a 32 byte (or longer) value generated by a +# cryptographically secure random number generator. Changing the root secret is +# likely to result in data loss. +# TODO - STOP SETTING THIS DEFAULT! This is only here while work +# continues on the feature/crypto branch. Later, this will be added +# to the devstack proxy-config so that gate tests can pass. +# base64 encoding of "dontEverUseThisIn_PRODUCTION_xxxxxxxxxxxxxxx" +encryption_root_secret = ZG9udEV2ZXJVc2VUaGlzSW5fUFJPRFVDVElPTl94eHh4eHh4eHh4eHh4eHg= + +[filter:encryption] +use = egg:swift#encryption + +# By default all PUT or POST'ed object data and/or metadata will be encrypted. +# Encryption of new data and/or metadata may be disabled by setting +# disable_encryption to True. However, all encryption middleware should remain +# in the pipeline in order for existing encrypted data to be read. +# disable_encryption = False diff --git a/etc/swift.conf-sample b/etc/swift.conf-sample index 78684730..1d21ba20 100644 --- a/etc/swift.conf-sample +++ b/etc/swift.conf-sample @@ -136,9 +136,10 @@ aliases = yellow, orange # By default the maximum number of allowed headers depends on the number of max -# allowed metadata settings plus a default value of 32 for regular http -# headers. If for some reason this is not enough (custom middleware for -# example) it can be increased with the extra_header_count constraint. +# allowed metadata settings plus a default value of 36 for swift internally +# generated headers and regular http headers. If for some reason this is not +# enough (custom middleware for example) it can be increased with the +# extra_header_count constraint. #extra_header_count = 0 diff --git a/other-requirements.txt b/other-requirements.txt index 394f2b0f..2fef68fd 100644 --- a/other-requirements.txt +++ b/other-requirements.txt @@ -13,3 +13,5 @@ python-dev [platform:dpkg] python-devel [platform:rpm] rsync xfsprogs +libssl-dev [platform:dpkg] +openssl-devel [platform:rpm] diff --git a/requirements.txt b/requirements.txt index 3480d4f3..3c17288b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ pastedeploy>=1.3.3 six>=1.9.0 xattr>=0.4 PyECLib>=1.2.0 # BSD +cryptography>=1.0,!=1.3.0 # BSD/Apache-2.0 diff --git a/setup.cfg b/setup.cfg index 098b6c64..cb4cda44 100644 --- a/setup.cfg +++ b/setup.cfg @@ -97,6 +97,8 @@ paste.filter_factory = xprofile = swift.common.middleware.xprofile:filter_factory versioned_writes = swift.common.middleware.versioned_writes:filter_factory copy = swift.common.middleware.copy:filter_factory + keymaster = swift.common.middleware.crypto.keymaster:filter_factory + encryption = swift.common.middleware.crypto:filter_factory [build_sphinx] all_files = 1 diff --git a/swift/common/constraints.py b/swift/common/constraints.py index 787d2d91..efb70898 100644 --- a/swift/common/constraints.py +++ b/swift/common/constraints.py @@ -110,10 +110,11 @@ FORMAT2CONTENT_TYPE = {'plain': 'text/plain', 'json': 'application/json', # By default the maximum number of allowed headers depends on the number of max -# allowed metadata settings plus a default value of 32 for regular http -# headers. If for some reason this is not enough (custom middleware for -# example) it can be increased with the extra_header_count constraint. -MAX_HEADER_COUNT = MAX_META_COUNT + 32 + max(EXTRA_HEADER_COUNT, 0) +# allowed metadata settings plus a default value of 36 for swift internally +# generated headers and regular http headers. If for some reason this is not +# enough (custom middleware for example) it can be increased with the +# extra_header_count constraint. +MAX_HEADER_COUNT = MAX_META_COUNT + 36 + max(EXTRA_HEADER_COUNT, 0) def check_metadata(req, target_type): diff --git a/swift/common/exceptions.py b/swift/common/exceptions.py index 721ac342..05f972f9 100644 --- a/swift/common/exceptions.py +++ b/swift/common/exceptions.py @@ -207,6 +207,10 @@ class APIVersionError(SwiftException): pass +class EncryptionException(SwiftException): + pass + + class ClientException(Exception): def __init__(self, msg, http_scheme='', http_host='', http_port='', diff --git a/swift/common/middleware/crypto/__init__.py b/swift/common/middleware/crypto/__init__.py new file mode 100644 index 00000000..55fd93a0 --- /dev/null +++ b/swift/common/middleware/crypto/__init__.py @@ -0,0 +1,34 @@ +# Copyright (c) 2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Implements middleware for object encryption which comprises an instance of a +Decrypter combined with an instance of an Encrypter. +""" +from swift.common.middleware.crypto.decrypter import Decrypter +from swift.common.middleware.crypto.encrypter import Encrypter + +from swift.common.utils import config_true_value, register_swift_info + + +def filter_factory(global_conf, **local_conf): + """Provides a factory function for loading encryption middleware.""" + conf = global_conf.copy() + conf.update(local_conf) + enabled = not config_true_value(conf.get('disable_encryption', 'false')) + register_swift_info('encryption', admin=True, enabled=enabled) + + def encryption_filter(app): + return Decrypter(Encrypter(app, conf), conf) + return encryption_filter diff --git a/swift/common/middleware/crypto/crypto_utils.py b/swift/common/middleware/crypto/crypto_utils.py new file mode 100644 index 00000000..4efa1522 --- /dev/null +++ b/swift/common/middleware/crypto/crypto_utils.py @@ -0,0 +1,283 @@ +# Copyright (c) 2015-2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import base64 +import binascii +import collections +import json +import os + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +import six +from six.moves.urllib import parse as urlparse + +from swift import gettext_ as _ +from swift.common.exceptions import EncryptionException +from swift.common.swob import HTTPInternalServerError +from swift.common.utils import get_logger +from swift.common.wsgi import WSGIContext + +CRYPTO_KEY_CALLBACK = 'swift.callback.fetch_crypto_keys' + + +class Crypto(object): + """ + Used by middleware: Calls cryptography library + """ + cipher = 'AES_CTR_256' + # AES will accept several key sizes - we are using 256 bits i.e. 32 bytes + key_length = 32 + iv_length = algorithms.AES.block_size / 8 + + def __init__(self, conf=None): + self.logger = get_logger(conf, log_route="crypto") + # memoize backend to avoid repeated iteration over entry points + self.backend = default_backend() + + def create_encryption_ctxt(self, key, iv): + """ + Creates a crypto context for encrypting + + :param key: 256-bit key + :param iv: 128-bit iv or nonce used for encryption + :raises: ValueError on invalid key or iv + :returns: an instance of an encryptor + """ + self.check_key(key) + engine = Cipher(algorithms.AES(key), modes.CTR(iv), + backend=self.backend) + return engine.encryptor() + + def create_decryption_ctxt(self, key, iv, offset): + """ + Creates a crypto context for decrypting + + :param key: 256-bit key + :param iv: 128-bit iv or nonce used for decryption + :param offset: offset into the message; used for range reads + :returns: an instance of a decryptor + """ + self.check_key(key) + if offset < 0: + raise ValueError('Offset must not be negative') + if offset: + # Adjust IV so that it is correct for decryption at offset. + # The CTR mode offset is incremented for every AES block and taken + # modulo 2^128. + offset_blocks, offset_in_block = divmod(offset, self.iv_length) + ivl = long(binascii.hexlify(iv), 16) + offset_blocks + ivl %= 1 << algorithms.AES.block_size + iv = str(bytearray.fromhex(format( + ivl, '0%dx' % (2 * self.iv_length)))) + else: + offset_in_block = 0 + + engine = Cipher(algorithms.AES(key), modes.CTR(iv), + backend=self.backend) + dec = engine.decryptor() + # Adjust decryption boundary within current AES block + dec.update('*' * offset_in_block) + return dec + + def create_iv(self): + return os.urandom(self.iv_length) + + def create_crypto_meta(self): + # create a set of parameters + return {'iv': self.create_iv(), 'cipher': self.cipher} + + def check_crypto_meta(self, meta): + """ + Check that crypto meta dict has valid items. + + :param meta: a dict + :raises EncryptionException: if an error is found in the crypto meta + """ + try: + if meta['cipher'] != self.cipher: + raise EncryptionException('Bad crypto meta: Cipher must be %s' + % self.cipher) + if len(meta['iv']) != self.iv_length: + raise EncryptionException( + 'Bad crypto meta: IV must be length %s bytes' + % self.iv_length) + except KeyError as err: + raise EncryptionException( + 'Bad crypto meta: Missing %s' % err) + + def create_random_key(self): + # helper method to create random key of correct length + return os.urandom(self.key_length) + + def wrap_key(self, wrapping_key, key_to_wrap): + # we don't use an RFC 3394 key wrap algorithm such as cryptography's + # aes_wrap_key because it's slower and we have iv material readily + # available so don't need a deterministic algorithm + iv = self.create_iv() + encryptor = Cipher(algorithms.AES(wrapping_key), modes.CTR(iv), + backend=self.backend).encryptor() + return {'key': encryptor.update(key_to_wrap), 'iv': iv} + + def unwrap_key(self, wrapping_key, context): + # unwrap a key from dict of form returned by wrap_key + # check the key length early - unwrapping won't change the length + self.check_key(context['key']) + decryptor = Cipher(algorithms.AES(wrapping_key), + modes.CTR(context['iv']), + backend=self.backend).decryptor() + return decryptor.update(context['key']) + + def check_key(self, key): + if len(key) != self.key_length: + raise ValueError("Key must be length %s bytes" % self.key_length) + + +class CryptoWSGIContext(WSGIContext): + """ + Base class for contexts used by crypto middlewares. + """ + def __init__(self, crypto_app, server_type, logger): + super(CryptoWSGIContext, self).__init__(crypto_app.app) + self.crypto = crypto_app.crypto + self.logger = logger + self.server_type = server_type + + def get_keys(self, env, required=None): + # Get the key(s) from the keymaster + required = required if required is not None else [self.server_type] + try: + fetch_crypto_keys = env[CRYPTO_KEY_CALLBACK] + except KeyError: + self.logger.exception(_('ERROR get_keys() missing callback')) + raise HTTPInternalServerError( + "Unable to retrieve encryption keys.") + + try: + keys = fetch_crypto_keys() + except Exception as err: # noqa + self.logger.exception(_( + 'ERROR get_keys(): from callback: %s') % err) + raise HTTPInternalServerError( + "Unable to retrieve encryption keys.") + + for name in required: + try: + key = keys[name] + self.crypto.check_key(key) + continue + except KeyError: + self.logger.exception(_("Missing key for %r") % name) + except TypeError: + self.logger.exception(_("Did not get a keys dict")) + except ValueError as e: + # don't include the key in any messages! + self.logger.exception(_("Bad key for %(name)r: %(err)s") % + {'name': name, 'err': e}) + raise HTTPInternalServerError( + "Unable to retrieve encryption keys.") + + return keys + + +def dump_crypto_meta(crypto_meta): + """ + Serialize crypto meta to a form suitable for including in a header value. + + The crypto-meta is serialized as a json object. The iv and key values are + random bytes and as a result need to be base64 encoded before sending over + the wire. Base64 encoding returns a bytes object in py3, to future proof + the code, decode this data to produce a string, which is what the + json.dumps function expects. + + :param crypto_meta: a dict containing crypto meta items + :returns: a string serialization of a crypto meta dict + """ + def b64_encode_meta(crypto_meta): + return { + name: (base64.b64encode(value).decode() if name in ('iv', 'key') + else b64_encode_meta(value) if isinstance(value, dict) + else value) + for name, value in crypto_meta.items()} + + # use sort_keys=True to make serialized form predictable for testing + return urlparse.quote_plus( + json.dumps(b64_encode_meta(crypto_meta), sort_keys=True)) + + +def load_crypto_meta(value): + """ + Build the crypto_meta from the json object. + + Note that json.loads always produces unicode strings, to ensure the + resultant crypto_meta matches the original object cast all key and value + data to a str except the key and iv which are base64 decoded. This will + work in py3 as well where all strings are unicode implying the cast is + effectively a no-op. + + :param value: a string serialization of a crypto meta dict + :returns: a dict containing crypto meta items + :raises EncryptionException: if an error occurs while parsing the + crypto meta + """ + def b64_decode_meta(crypto_meta): + return { + str(name): (base64.b64decode(val) if name in ('iv', 'key') + else b64_decode_meta(val) if isinstance(val, dict) + else val.encode('utf8')) + for name, val in crypto_meta.items()} + + try: + if not isinstance(value, six.string_types): + raise ValueError('crypto meta not a string') + val = json.loads(urlparse.unquote_plus(value)) + if not isinstance(val, collections.Mapping): + raise ValueError('crypto meta not a Mapping') + return b64_decode_meta(val) + except (KeyError, ValueError, TypeError) as err: + msg = 'Bad crypto meta %r: %s' % (value, err) + raise EncryptionException(msg) + + +def append_crypto_meta(value, crypto_meta): + """ + Serialize and append crypto metadata to an encrypted value. + + :param value: value to which serialized crypto meta will be appended. + :param crypto_meta: a dict of crypto meta + :return: a string of the form ; swift_meta= + """ + return '%s; swift_meta=%s' % (value, dump_crypto_meta(crypto_meta)) + + +def extract_crypto_meta(value): + """ + Extract and deserialize any crypto meta from the end of a value. + + :param value: string that may have crypto meta at end + :return: a tuple of the form: + (, or None) + """ + crypto_meta = None + # we only attempt to extract crypto meta from values that we know were + # encrypted and base64-encoded, or from etag values, so it's safe to split + # on ';' even if it turns out that the value was an unencrypted etag + parts = value.split(';') + if len(parts) == 2: + value, param = parts + crypto_meta_tag = 'swift_meta=' + if param.strip().startswith(crypto_meta_tag): + param = param.strip()[len(crypto_meta_tag):] + crypto_meta = load_crypto_meta(param) + return value, crypto_meta diff --git a/swift/common/middleware/crypto/decrypter.py b/swift/common/middleware/crypto/decrypter.py new file mode 100644 index 00000000..46e2dbc4 --- /dev/null +++ b/swift/common/middleware/crypto/decrypter.py @@ -0,0 +1,449 @@ +# Copyright (c) 2015-2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json +import xml.etree.cElementTree as ElementTree + +from swift import gettext_ as _ +from swift.common.http import is_success +from swift.common.middleware.crypto.crypto_utils import CryptoWSGIContext, \ + load_crypto_meta, extract_crypto_meta, Crypto +from swift.common.exceptions import EncryptionException +from swift.common.request_helpers import get_object_transient_sysmeta, \ + get_listing_content_type, get_sys_meta_prefix, get_user_meta_prefix +from swift.common.swob import Request, HTTPException, HTTPInternalServerError +from swift.common.utils import get_logger, config_true_value, \ + parse_content_range, closing_if_possible, parse_content_type, \ + FileLikeIter, multipart_byteranges_to_document_iters + +DECRYPT_CHUNK_SIZE = 65536 + + +def purge_crypto_sysmeta_headers(headers): + return [h for h in headers if not + h[0].lower().startswith( + (get_object_transient_sysmeta('crypto-'), + get_sys_meta_prefix('object') + 'crypto-'))] + + +class BaseDecrypterContext(CryptoWSGIContext): + def get_crypto_meta(self, header_name): + """ + Extract a crypto_meta dict from a header. + + :param header_name: name of header that may have crypto_meta + :return: A dict containing crypto_meta items + :raises EncryptionException: if an error occurs while parsing the + crypto meta + """ + crypto_meta_json = self._response_header_value(header_name) + + if crypto_meta_json is None: + return None + crypto_meta = load_crypto_meta(crypto_meta_json) + self.crypto.check_crypto_meta(crypto_meta) + return crypto_meta + + def get_unwrapped_key(self, crypto_meta, wrapping_key): + """ + Get a wrapped key from crypto-meta and unwrap it using the provided + wrapping key. + + :param crypto_meta: a dict of crypto-meta + :param wrapping_key: key to be used to decrypt the wrapped key + :return: an unwrapped key + :raises EncryptionException: if the crypto-meta has no wrapped key or + the unwrapped key is invalid + """ + try: + return self.crypto.unwrap_key(wrapping_key, + crypto_meta['body_key']) + except KeyError as err: + err = 'Missing %s' % err + except ValueError as err: + pass + msg = 'Error decrypting %s' % self.server_type + self.logger.error(_('%(msg)s: %(err)s') % {'msg': msg, 'err': err}) + raise HTTPInternalServerError(body=msg, content_type='text/plain') + + def decrypt_value_with_meta(self, value, key, required=False): + """ + Base64-decode and decrypt a value if crypto meta can be extracted from + the value itself, otherwise return the value unmodified. + + A value should either be a string that does not contain the ';' + character or should be of the form: + + ;swift_meta= + + :param value: value to decrypt + :param key: crypto key to use + :param required: if True then the value is required to be decrypted + and an EncryptionException will be raised if the + header cannot be decrypted due to missing crypto meta. + :returns: decrypted value if crypto meta is found, otherwise the + unmodified value + :raises EncryptionException: if an error occurs while parsing crypto + meta or if the header value was required + to be decrypted but crypto meta was not + found. + """ + value, crypto_meta = extract_crypto_meta(value) + if crypto_meta: + self.crypto.check_crypto_meta(crypto_meta) + value = self.decrypt_value(value, key, crypto_meta) + elif required: + raise EncryptionException( + "Missing crypto meta in value %s" % value) + return value + + def decrypt_value(self, value, key, crypto_meta): + """ + Base64-decode and decrypt a value using the crypto_meta provided. + + :param value: a base64-encoded value to decrypt + :param key: crypto key to use + :param crypto_meta: a crypto-meta dict of form returned by + :py:func:`~swift.common.middleware.crypto.Crypto.get_crypto_meta` + :returns: decrypted value + """ + if not value: + return '' + crypto_ctxt = self.crypto.create_decryption_ctxt( + key, crypto_meta['iv'], 0) + return crypto_ctxt.update(base64.b64decode(value)) + + def get_decryption_keys(self, req): + """ + Determine if a response should be decrypted, and if so then fetch keys. + + :param req: a Request object + :returns: a dict of decryption keys + """ + if config_true_value(req.environ.get('swift.crypto.override')): + self.logger.debug('No decryption is necessary because of override') + return None + + return self.get_keys(req.environ) + + +class DecrypterObjContext(BaseDecrypterContext): + def __init__(self, decrypter, logger): + super(DecrypterObjContext, self).__init__(decrypter, 'object', logger) + + def _decrypt_header(self, header, value, key, required=False): + """ + Attempt to decrypt a header value that may be encrypted. + + :param header: the header name + :param value: the header value + :param key: decryption key + :param required: if True then the header is required to be decrypted + and an HTTPInternalServerError will be raised if the + header cannot be decrypted due to missing crypto meta. + :return: decrypted value or the original value if it was not encrypted. + :raises HTTPInternalServerError: if an error occurred during decryption + or if the header value was required to + be decrypted but crypto meta was not + found. + """ + try: + return self.decrypt_value_with_meta(value, key, required) + except EncryptionException as e: + msg = "Error decrypting header" + self.logger.error(_("%(msg)s %(hdr)s: %(e)s") % + {'msg': msg, 'hdr': header, 'e': e}) + raise HTTPInternalServerError(body=msg, content_type='text/plain') + + def decrypt_user_metadata(self, keys): + prefix = get_object_transient_sysmeta('crypto-meta-') + prefix_len = len(prefix) + new_prefix = get_user_meta_prefix(self.server_type).title() + result = [] + for name, val in self._response_headers: + if name.lower().startswith(prefix) and val: + short_name = name[prefix_len:] + decrypted_value = self._decrypt_header( + name, val, keys[self.server_type], required=True) + result.append((new_prefix + short_name, decrypted_value)) + return result + + def decrypt_resp_headers(self, keys): + """ + Find encrypted headers and replace with the decrypted versions. + + :param keys: a dict of decryption keys. + :return: A list of headers with any encrypted headers replaced by their + decrypted values. + :raises HTTPInternalServerError: if any error occurs while decrypting + headers + """ + mod_hdr_pairs = [] + + # Decrypt plaintext etag and place in Etag header for client response + etag_header = 'X-Object-Sysmeta-Crypto-Etag' + encrypted_etag = self._response_header_value(etag_header) + if encrypted_etag: + decrypted_etag = self._decrypt_header( + etag_header, encrypted_etag, keys['object'], required=True) + mod_hdr_pairs.append(('Etag', decrypted_etag)) + + etag_header = 'X-Object-Sysmeta-Container-Update-Override-Etag' + encrypted_etag = self._response_header_value(etag_header) + if encrypted_etag: + decrypted_etag = self._decrypt_header( + etag_header, encrypted_etag, keys['container']) + mod_hdr_pairs.append((etag_header, decrypted_etag)) + + # Decrypt all user metadata. Encrypted user metadata values are stored + # in the x-object-transient-sysmeta-crypto-meta- namespace. Those are + # decrypted and moved back to the x-object-meta- namespace. Prior to + # decryption, the response should have no x-object-meta- headers, but + # if it does then they will be overwritten by any decrypted headers + # that map to the same x-object-meta- header names i.e. decrypted + # headers win over unexpected, unencrypted headers. + mod_hdr_pairs.extend(self.decrypt_user_metadata(keys)) + + mod_hdr_names = {h.lower() for h, v in mod_hdr_pairs} + mod_hdr_pairs.extend([(h, v) for h, v in self._response_headers + if h.lower() not in mod_hdr_names]) + return mod_hdr_pairs + + def multipart_response_iter(self, resp, boundary, body_key, crypto_meta): + """ + Decrypts a multipart mime doc response body. + + :param resp: application response + :param boundary: multipart boundary string + :param keys: a dict of decryption keys. + :param crypto_meta: crypto_meta for the response body + :return: generator for decrypted response body + """ + with closing_if_possible(resp): + parts_iter = multipart_byteranges_to_document_iters( + FileLikeIter(resp), boundary) + for first_byte, last_byte, length, headers, body in parts_iter: + yield "--" + boundary + "\r\n" + + for header_pair in headers: + yield "%s: %s\r\n" % header_pair + + yield "\r\n" + + decrypt_ctxt = self.crypto.create_decryption_ctxt( + body_key, crypto_meta['iv'], first_byte) + for chunk in iter(lambda: body.read(DECRYPT_CHUNK_SIZE), ''): + yield decrypt_ctxt.update(chunk) + + yield "\r\n" + + yield "--" + boundary + "--" + + def response_iter(self, resp, body_key, crypto_meta, offset): + """ + Decrypts a response body. + + :param resp: application response + :param keys: a dict of decryption keys. + :param crypto_meta: crypto_meta for the response body + :param offset: offset into object content at which response body starts + :return: generator for decrypted response body + """ + decrypt_ctxt = self.crypto.create_decryption_ctxt( + body_key, crypto_meta['iv'], offset) + with closing_if_possible(resp): + for chunk in resp: + yield decrypt_ctxt.update(chunk) + + def handle_get(self, req, start_response): + app_resp = self._app_call(req.environ) + + keys = self.get_decryption_keys(req) + if keys is None: + # skip decryption + start_response(self._response_status, self._response_headers, + self._response_exc_info) + return app_resp + + mod_resp_headers = self.decrypt_resp_headers(keys) + + crypto_meta = None + if is_success(self._get_status_int()): + try: + crypto_meta = self.get_crypto_meta( + 'X-Object-Sysmeta-Crypto-Body-Meta') + except EncryptionException as err: + msg = 'Error decrypting object' + self.logger.error(_('%(msg)s: %(err)s') % + {'msg': msg, 'err': err}) + raise HTTPInternalServerError( + body=msg, content_type='text/plain') + + if crypto_meta: + # 2xx response and encrypted body + body_key = self.get_unwrapped_key(crypto_meta, keys['object']) + content_type, content_type_attrs = parse_content_type( + self._response_header_value('Content-Type')) + + if (self._get_status_int() == 206 and + content_type == 'multipart/byteranges'): + boundary = dict(content_type_attrs)["boundary"] + resp_iter = self.multipart_response_iter( + app_resp, boundary, body_key, crypto_meta) + else: + offset = 0 + content_range = self._response_header_value('Content-Range') + if content_range: + # Determine offset within the whole object if ranged GET + offset, end, total = parse_content_range(content_range) + resp_iter = self.response_iter( + app_resp, body_key, crypto_meta, offset) + else: + # don't decrypt body of unencrypted or non-2xx responses + resp_iter = app_resp + + mod_resp_headers = purge_crypto_sysmeta_headers(mod_resp_headers) + start_response(self._response_status, mod_resp_headers, + self._response_exc_info) + + return resp_iter + + def handle_head(self, req, start_response): + app_resp = self._app_call(req.environ) + + keys = self.get_decryption_keys(req) + + if keys is None: + # skip decryption + start_response(self._response_status, self._response_headers, + self._response_exc_info) + else: + mod_resp_headers = self.decrypt_resp_headers(keys) + mod_resp_headers = purge_crypto_sysmeta_headers(mod_resp_headers) + start_response(self._response_status, mod_resp_headers, + self._response_exc_info) + + return app_resp + + +class DecrypterContContext(BaseDecrypterContext): + def __init__(self, decrypter, logger): + super(DecrypterContContext, self).__init__( + decrypter, 'container', logger) + + def handle_get(self, req, start_response): + app_resp = self._app_call(req.environ) + + if is_success(self._get_status_int()): + # only decrypt body of 2xx responses + out_content_type = get_listing_content_type(req) + if out_content_type == 'application/json': + handler = self.process_json_resp + keys = self.get_decryption_keys(req) + elif out_content_type.endswith('/xml'): + handler = self.process_xml_resp + keys = self.get_decryption_keys(req) + else: + handler = keys = None + + if handler and keys: + try: + app_resp = handler(keys['container'], app_resp) + except EncryptionException as err: + msg = "Error decrypting container listing" + self.logger.error(_('%(msg)s: %(err)s') % + {'msg': msg, 'err': err}) + raise HTTPInternalServerError( + body=msg, content_type='text/plain') + + start_response(self._response_status, + self._response_headers, + self._response_exc_info) + + return app_resp + + def update_content_length(self, new_total_len): + self._response_headers = [ + (h, v) for h, v in self._response_headers + if h.lower() != 'content-length'] + self._response_headers.append(('Content-Length', str(new_total_len))) + + def process_json_resp(self, key, resp_iter): + """ + Parses json body listing and decrypt encrypted entries. Updates + Content-Length header with new body length and return a body iter. + """ + with closing_if_possible(resp_iter): + resp_body = ''.join(resp_iter) + body_json = json.loads(resp_body) + new_body = json.dumps([self.decrypt_obj_dict(obj_dict, key) + for obj_dict in body_json]) + self.update_content_length(len(new_body)) + return [new_body] + + def decrypt_obj_dict(self, obj_dict, key): + ciphertext = obj_dict['hash'] + obj_dict['hash'] = self.decrypt_value_with_meta(ciphertext, key) + return obj_dict + + def process_xml_resp(self, key, resp_iter): + """ + Parses xml body listing and decrypt encrypted entries. Updates + Content-Length header with new body length and return a body iter. + """ + with closing_if_possible(resp_iter): + resp_body = ''.join(resp_iter) + tree = ElementTree.fromstring(resp_body) + for elem in tree.iter('hash'): + ciphertext = elem.text.encode('utf8') + plain = self.decrypt_value_with_meta(ciphertext, key) + elem.text = plain.decode('utf8') + new_body = ElementTree.tostring(tree, encoding='UTF-8').replace( + "", + '', 1) + self.update_content_length(len(new_body)) + return [new_body] + + +class Decrypter(object): + """Middleware for decrypting data and user metadata.""" + + def __init__(self, app, conf): + self.app = app + self.logger = get_logger(conf, log_route="decrypter") + self.crypto = Crypto(conf) + + def __call__(self, env, start_response): + req = Request(env) + try: + parts = req.split_path(3, 4, True) + except ValueError: + return self.app(env, start_response) + + if parts[3] and req.method == 'GET': + handler = DecrypterObjContext(self, self.logger).handle_get + elif parts[3] and req.method == 'HEAD': + handler = DecrypterObjContext(self, self.logger).handle_head + elif parts[2] and req.method == 'GET': + handler = DecrypterContContext(self, self.logger).handle_get + else: + # url and/or request verb is not handled by decrypter + return self.app(env, start_response) + + try: + return handler(req, start_response) + except HTTPException as err_resp: + return err_resp(env, start_response) diff --git a/swift/common/middleware/crypto/encrypter.py b/swift/common/middleware/crypto/encrypter.py new file mode 100644 index 00000000..2719d477 --- /dev/null +++ b/swift/common/middleware/crypto/encrypter.py @@ -0,0 +1,369 @@ +# Copyright (c) 2015-2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import base64 +import hashlib +import hmac +from contextlib import contextmanager + +from swift.common.constraints import check_metadata +from swift.common.http import is_success +from swift.common.middleware.crypto.crypto_utils import CryptoWSGIContext, \ + dump_crypto_meta, append_crypto_meta, Crypto +from swift.common.request_helpers import get_object_transient_sysmeta, \ + strip_user_meta_prefix, is_user_meta, update_etag_is_at_header +from swift.common.swob import Request, Match, HTTPException, \ + HTTPUnprocessableEntity +from swift.common.utils import get_logger, config_true_value + + +def encrypt_header_val(crypto, value, key): + """ + Encrypt a header value using the supplied key. + + :param crypto: a Crypto instance + :param value: value to encrypt + :param key: crypto key to use + :returns: a tuple of (encrypted value, crypto_meta) where crypto_meta is a + dict of form returned by + :py:func:`~swift.common.middleware.crypto.Crypto.get_crypto_meta` + """ + if not value: + return '', None + + crypto_meta = crypto.create_crypto_meta() + crypto_ctxt = crypto.create_encryption_ctxt(key, crypto_meta['iv']) + enc_val = base64.b64encode(crypto_ctxt.update(value)) + return enc_val, crypto_meta + + +def _hmac_etag(key, etag): + """ + Compute an HMAC-SHA256 using given key and etag. + + :param key: The starting key for the hash. + :param etag: The etag to hash. + :returns: a Base64-encoded representation of the HMAC + """ + result = hmac.new(key, etag, digestmod=hashlib.sha256).digest() + return base64.b64encode(result).decode() + + +class EncInputWrapper(object): + """File-like object to be swapped in for wsgi.input.""" + def __init__(self, crypto, keys, req, logger): + self.env = req.environ + self.wsgi_input = req.environ['wsgi.input'] + self.path = req.path + self.crypto = crypto + self.body_crypto_ctxt = None + self.keys = keys + self.plaintext_md5 = None + self.ciphertext_md5 = None + self.logger = logger + self.install_footers_callback(req) + + def _init_encryption_context(self): + # do this once when body is first read + if self.body_crypto_ctxt is None: + self.body_crypto_meta = self.crypto.create_crypto_meta() + body_key = self.crypto.create_random_key() + # wrap the body key with object key + self.body_crypto_meta['body_key'] = self.crypto.wrap_key( + self.keys['object'], body_key) + self.body_crypto_meta['key_id'] = self.keys['id'] + self.body_crypto_ctxt = self.crypto.create_encryption_ctxt( + body_key, self.body_crypto_meta.get('iv')) + self.plaintext_md5 = hashlib.md5() + self.ciphertext_md5 = hashlib.md5() + + def install_footers_callback(self, req): + # the proxy controller will call back for footer metadata after + # body has been sent + inner_callback = req.environ.get('swift.callback.update_footers') + # remove any Etag from headers, it won't be valid for ciphertext and + # we'll send the ciphertext Etag later in footer metadata + client_etag = req.headers.pop('etag', None) + container_listing_etag_header = req.headers.get( + 'X-Object-Sysmeta-Container-Update-Override-Etag') + + def footers_callback(footers): + if inner_callback: + # pass on footers dict to any other callback that was + # registered before this one. It may override any footers that + # were set. + inner_callback(footers) + + plaintext_etag = None + if self.body_crypto_ctxt: + plaintext_etag = self.plaintext_md5.hexdigest() + # If client (or other middleware) supplied etag, then validate + # against plaintext etag + etag_to_check = footers.get('Etag') or client_etag + if (etag_to_check is not None and + plaintext_etag != etag_to_check): + raise HTTPUnprocessableEntity(request=Request(self.env)) + + # override any previous notion of etag with the ciphertext etag + footers['Etag'] = self.ciphertext_md5.hexdigest() + + # Encrypt the plaintext etag using the object key and persist + # as sysmeta along with the crypto parameters that were used. + encrypted_etag, etag_crypto_meta = encrypt_header_val( + self.crypto, plaintext_etag, self.keys['object']) + footers['X-Object-Sysmeta-Crypto-Etag'] = \ + append_crypto_meta(encrypted_etag, etag_crypto_meta) + footers['X-Object-Sysmeta-Crypto-Body-Meta'] = \ + dump_crypto_meta(self.body_crypto_meta) + + # Also add an HMAC of the etag for use when evaluating + # conditional requests + footers['X-Object-Sysmeta-Crypto-Etag-Mac'] = _hmac_etag( + self.keys['object'], plaintext_etag) + else: + # No data was read from body, nothing was encrypted, so don't + # set any crypto sysmeta for the body, but do re-instate any + # etag provided in inbound request if other middleware has not + # already set a value. + if client_etag is not None: + footers.setdefault('Etag', client_etag) + + # When deciding on the etag that should appear in container + # listings, look for: + # * override in the footer, otherwise + # * override in the header, and finally + # * MD5 of the plaintext received + # This may be None if no override was set and no data was read + container_listing_etag = footers.get( + 'X-Object-Sysmeta-Container-Update-Override-Etag', + container_listing_etag_header) or plaintext_etag + + if container_listing_etag is not None: + # Encrypt the container-listing etag using the container key + # and a random IV, and use it to override the container update + # value, with the crypto parameters appended. We use the + # container key here so that only that key is required to + # decrypt all etag values in a container listing when handling + # a container GET request. + val, crypto_meta = encrypt_header_val( + self.crypto, container_listing_etag, + self.keys['container']) + crypto_meta['key_id'] = self.keys['id'] + footers['X-Object-Sysmeta-Container-Update-Override-Etag'] = \ + append_crypto_meta(val, crypto_meta) + # else: no override was set and no data was read + + req.environ['swift.callback.update_footers'] = footers_callback + + def read(self, *args, **kwargs): + return self.readChunk(self.wsgi_input.read, *args, **kwargs) + + def readline(self, *args, **kwargs): + return self.readChunk(self.wsgi_input.readline, *args, **kwargs) + + def readChunk(self, read_method, *args, **kwargs): + chunk = read_method(*args, **kwargs) + + if chunk: + self._init_encryption_context() + self.plaintext_md5.update(chunk) + # Encrypt one chunk at a time + ciphertext = self.body_crypto_ctxt.update(chunk) + self.ciphertext_md5.update(ciphertext) + return ciphertext + + return chunk + + +class EncrypterObjContext(CryptoWSGIContext): + def __init__(self, encrypter, logger): + super(EncrypterObjContext, self).__init__( + encrypter, 'object', logger) + + def _check_headers(self, req): + # Check the user-metadata length before encrypting and encoding + error_response = check_metadata(req, self.server_type) + if error_response: + raise error_response + + def encrypt_user_metadata(self, req, keys): + """ + Encrypt user-metadata header values. Replace each x-object-meta- + user metadata header with a corresponding + x-object-transient-sysmeta-crypto-meta- header which has the + crypto metadata required to decrypt appended to the encrypted value. + + :param req: a swob Request + :param keys: a dict of encryption keys + """ + prefix = get_object_transient_sysmeta('crypto-meta-') + user_meta_headers = [h for h in req.headers.items() if + is_user_meta(self.server_type, h[0]) and h[1]] + crypto_meta = None + for name, val in user_meta_headers: + short_name = strip_user_meta_prefix(self.server_type, name) + new_name = prefix + short_name + enc_val, crypto_meta = encrypt_header_val( + self.crypto, val, keys[self.server_type]) + req.headers[new_name] = append_crypto_meta(enc_val, crypto_meta) + req.headers.pop(name) + # store a single copy of the crypto meta items that are common to all + # encrypted user metadata independently of any such meta that is stored + # with the object body because it might change on a POST. This is done + # for future-proofing - the meta stored here is not currently used + # during decryption. + if crypto_meta: + meta = dump_crypto_meta({'cipher': crypto_meta['cipher'], + 'key_id': keys['id']}) + req.headers[get_object_transient_sysmeta('crypto-meta')] = meta + + def handle_put(self, req, start_response): + self._check_headers(req) + keys = self.get_keys(req.environ, required=['object', 'container']) + self.encrypt_user_metadata(req, keys) + + enc_input_proxy = EncInputWrapper(self.crypto, keys, req, self.logger) + req.environ['wsgi.input'] = enc_input_proxy + + resp = self._app_call(req.environ) + + # If an etag is in the response headers and a plaintext etag was + # calculated, then overwrite the response value with the plaintext etag + # provided it matches the ciphertext etag. If it does not match then do + # not overwrite and allow the response value to return to client. + mod_resp_headers = self._response_headers + if (is_success(self._get_status_int()) and + enc_input_proxy.plaintext_md5): + plaintext_etag = enc_input_proxy.plaintext_md5.hexdigest() + ciphertext_etag = enc_input_proxy.ciphertext_md5.hexdigest() + mod_resp_headers = [ + (h, v if (h.lower() != 'etag' or + v.strip('"') != ciphertext_etag) + else plaintext_etag) + for h, v in mod_resp_headers] + + start_response(self._response_status, mod_resp_headers, + self._response_exc_info) + return resp + + def handle_post(self, req, start_response): + """ + Encrypt the new object headers with a new iv and the current crypto. + Note that an object may have encrypted headers while the body may + remain unencrypted. + """ + self._check_headers(req) + keys = self.get_keys(req.environ) + self.encrypt_user_metadata(req, keys) + + resp = self._app_call(req.environ) + start_response(self._response_status, self._response_headers, + self._response_exc_info) + return resp + + @contextmanager + def _mask_conditional_etags(self, req, header_name): + """ + Calculate HMACs of etags in header value and append to existing list. + The HMACs are calculated in the same way as was done for the object + plaintext etag to generate the value of + X-Object-Sysmeta-Crypto-Etag-Mac when the object was PUT. The object + server can therefore use these HMACs to evaluate conditional requests. + + The existing etag values are left in the list of values to match in + case the object was not encrypted when it was PUT. It is unlikely that + a masked etag value would collide with an unmasked value. + + :param req: an instance of swob.Request + :param header_name: name of header that has etags to mask + :return: True if any etags were masked, False otherwise + """ + masked = False + old_etags = req.headers.get(header_name) + if old_etags: + keys = self.get_keys(req.environ) + new_etags = [] + for etag in Match(old_etags).tags: + if etag == '*': + new_etags.append(etag) + continue + masked_etag = _hmac_etag(keys['object'], etag) + new_etags.extend(('"%s"' % etag, '"%s"' % masked_etag)) + masked = True + + req.headers[header_name] = ', '.join(new_etags) + + try: + yield masked + finally: + if old_etags: + req.headers[header_name] = old_etags + + def handle_get_or_head(self, req, start_response): + with self._mask_conditional_etags(req, 'If-Match') as masked1: + with self._mask_conditional_etags(req, 'If-None-Match') as masked2: + if masked1 or masked2: + update_etag_is_at_header( + req, 'X-Object-Sysmeta-Crypto-Etag-Mac') + resp = self._app_call(req.environ) + start_response(self._response_status, self._response_headers, + self._response_exc_info) + return resp + + +class Encrypter(object): + """Middleware for encrypting data and user metadata. + + By default all PUT or POST'ed object data and/or metadata will be + encrypted. Encryption of new data and/or metadata may be disabled by + setting the ``disable_encryption`` option to True. However, this middleware + should remain in the pipeline in order for existing encrypted data to be + read. + """ + + def __init__(self, app, conf): + self.app = app + self.logger = get_logger(conf, log_route="encrypter") + self.crypto = Crypto(conf) + self.disable_encryption = config_true_value( + conf.get('disable_encryption', 'false')) + + def __call__(self, env, start_response): + # If override is set in env, then just pass along + if config_true_value(env.get('swift.crypto.override')): + return self.app(env, start_response) + + req = Request(env) + + if self.disable_encryption and req.method in ('PUT', 'POST'): + return self.app(env, start_response) + try: + req.split_path(4, 4, True) + except ValueError: + return self.app(env, start_response) + + if req.method in ('GET', 'HEAD'): + handler = EncrypterObjContext(self, self.logger).handle_get_or_head + elif req.method == 'PUT': + handler = EncrypterObjContext(self, self.logger).handle_put + elif req.method == 'POST': + handler = EncrypterObjContext(self, self.logger).handle_post + else: + # anything else + return self.app(env, start_response) + + try: + return handler(req, start_response) + except HTTPException as err_resp: + return err_resp(env, start_response) diff --git a/swift/common/middleware/crypto/keymaster.py b/swift/common/middleware/crypto/keymaster.py new file mode 100644 index 00000000..4b6ac71f --- /dev/null +++ b/swift/common/middleware/crypto/keymaster.py @@ -0,0 +1,153 @@ +# Copyright (c) 2015 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import base64 +import hashlib +import hmac +import os + +from swift.common.middleware.crypto.crypto_utils import CRYPTO_KEY_CALLBACK +from swift.common.swob import Request, HTTPException +from swift.common.wsgi import WSGIContext + + +class KeyMasterContext(WSGIContext): + """ + The simple scheme for key derivation is as follows: every path is + associated with a key, where the key is derived from the path itself in a + deterministic fashion such that the key does not need to be stored. + Specifically, the key for any path is an HMAC of a root key and the path + itself, calculated using an SHA256 hash function:: + + = HMAC_SHA256(, ) + """ + def __init__(self, keymaster, account, container, obj): + """ + :param keymaster: a Keymaster instance + :param account: account name + :param container: container name + :param obj: object name + """ + super(KeyMasterContext, self).__init__(keymaster.app) + self.keymaster = keymaster + self.account = account + self.container = container + self.obj = obj + self._keys = None + + def fetch_crypto_keys(self, *args, **kwargs): + """ + Setup container and object keys based on the request path. + + Keys are derived from request path. The 'id' entry in the results dict + includes the part of the path used to derive keys. Other keymaster + implementations may use a different strategy to generate keys and may + include a different type of 'id', so callers should treat the 'id' as + opaque keymaster-specific data. + + :returns: A dict containing encryption keys for 'object' and + 'container' and a key 'id'. + """ + if self._keys: + return self._keys + + self._keys = {} + account_path = os.path.join(os.sep, self.account) + + if self.container: + path = os.path.join(account_path, self.container) + self._keys['container'] = self.keymaster.create_key(path) + + if self.obj: + path = os.path.join(path, self.obj) + self._keys['object'] = self.keymaster.create_key(path) + + # For future-proofing include a keymaster version number and the + # path used to derive keys in the 'id' entry of the results. The + # encrypter will persist this as part of the crypto-meta for + # encrypted data and metadata. If we ever change the way keys are + # generated then the decrypter could pass the persisted 'id' value + # when it calls fetch_crypto_keys to inform the keymaster as to how + # that particular data or metadata had its keys generated. + # Currently we have no need to do that, so we are simply persisting + # this information for future use. + self._keys['id'] = {'v': '1', 'path': path} + + return self._keys + + def handle_request(self, req, start_response): + req.environ[CRYPTO_KEY_CALLBACK] = self.fetch_crypto_keys + resp = self._app_call(req.environ) + start_response(self._response_status, self._response_headers, + self._response_exc_info) + return resp + + +class KeyMaster(object): + """Middleware for providing encryption keys. + + The middleware requires its ``encryption_root_secret`` option to be set. + This is the root secret from which encryption keys are derived. This must + be set before first use to a value that is a base64 encoding of at least 32 + bytes. The security of all encrypted data critically depends on this key, + therefore it should be set to a high-entropy value. For example, a suitable + value may be obtained by base-64 encoding a 32 byte (or longer) value + generated by a cryptographically secure random number generator. Changing + the root secret is likely to result in data loss. + """ + + def __init__(self, app, conf): + self.app = app + self.root_secret = conf.get('encryption_root_secret') + try: + self.root_secret = base64.b64decode(self.root_secret) + if len(self.root_secret) < 32: + raise ValueError + except (TypeError, ValueError): + raise ValueError( + 'encryption_root_secret option in proxy-server.conf must be ' + 'a base64 encoding of at least 32 raw bytes') + + def __call__(self, env, start_response): + req = Request(env) + + try: + parts = req.split_path(2, 4, True) + except ValueError: + return self.app(env, start_response) + + if req.method in ('PUT', 'POST', 'GET', 'HEAD'): + # handle only those request methods that may require keys + km_context = KeyMasterContext(self, *parts[1:]) + try: + return km_context.handle_request(req, start_response) + except HTTPException as err_resp: + return err_resp(env, start_response) + + # anything else + return self.app(env, start_response) + + def create_key(self, key_id): + return hmac.new(self.root_secret, key_id, + digestmod=hashlib.sha256).digest() + + +def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + + def keymaster_filter(app): + return KeyMaster(app, conf) + + return keymaster_filter diff --git a/swift/common/swob.py b/swift/common/swob.py index aa11ec01..f80c1384 100644 --- a/swift/common/swob.py +++ b/swift/common/swob.py @@ -1419,6 +1419,7 @@ HTTPOk = status_map[200] HTTPCreated = status_map[201] HTTPAccepted = status_map[202] HTTPNoContent = status_map[204] +HTTPPartialContent = status_map[206] HTTPMovedPermanently = status_map[301] HTTPFound = status_map[302] HTTPSeeOther = status_map[303] diff --git a/test/functional/__init__.py b/test/functional/__init__.py index 52be849b..0bf324f8 100644 --- a/test/functional/__init__.py +++ b/test/functional/__init__.py @@ -361,6 +361,12 @@ def in_process_setup(the_object_server=object_server): 'allow_account_management': 'true', 'account_autocreate': 'true', 'allow_versions': 'True', + # TODO - Remove encryption_root_secret - this is only necessary while + # encryption middleware is in the default proxy pipeline in + # proxy-server.conf-sample + # base64 encoding of "dontEverUseThisIn_PRODUCTION_xxxxxxxxxxxxxxx" + 'encryption_root_secret': + 'ZG9udEV2ZXJVc2VUaGlzSW5fUFJPRFVDVElPTl94eHh4eHh4eHh4eHh4eHg=', # Below are values used by the functional test framework, as well as # by the various in-process swift servers 'auth_host': '127.0.0.1', diff --git a/test/probe/test_empty_device_handoff.py b/test/probe/test_empty_device_handoff.py index 65338ed8..e1f8ade5 100755 --- a/test/probe/test_empty_device_handoff.py +++ b/test/probe/test_empty_device_handoff.py @@ -73,6 +73,13 @@ class TestEmptyDevice(ReplProbeTest): raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) + # Stash the on disk data from a primary for future comparison with the + # handoff - this may not equal 'VERIFY' if for example the proxy has + # crypto enabled + direct_get_data = direct_client.direct_get_object( + onodes[1], opart, self.account, container, obj, headers={ + 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] + # Kill other two container/obj primary servers # to ensure GET handoff works for node in onodes[1:]: @@ -95,9 +102,7 @@ class TestEmptyDevice(ReplProbeTest): odata = direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] - if odata != 'VERIFY': - raise Exception('Direct object GET did not return VERIFY, instead ' - 'it returned: %s' % repr(odata)) + self.assertEqual(direct_get_data, odata) # Assert container listing (via proxy and directly) has container/obj objs = [o['name'] for o in @@ -155,9 +160,7 @@ class TestEmptyDevice(ReplProbeTest): odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] - if odata != 'VERIFY': - raise Exception('Direct object GET did not return VERIFY, instead ' - 'it returned: %s' % repr(odata)) + self.assertEqual(direct_get_data, odata) # Assert the handoff server no longer has container/obj try: diff --git a/test/probe/test_object_failures.py b/test/probe/test_object_failures.py index ba531777..1850b275 100755 --- a/test/probe/test_object_failures.py +++ b/test/probe/test_object_failures.py @@ -77,6 +77,12 @@ class TestObjectFailures(ReplProbeTest): obj = 'object-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'VERIFY') + # Stash the on disk data for future comparison - this may not equal + # 'VERIFY' if for example the proxy has crypto enabled + backend_data = direct_client.direct_get_object( + onode, opart, self.account, container, obj, headers={ + 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] + metadata = read_metadata(data_file) metadata['ETag'] = 'badetag' write_metadata(data_file, metadata) @@ -84,7 +90,7 @@ class TestObjectFailures(ReplProbeTest): odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] - self.assertEqual(odata, 'VERIFY') + self.assertEqual(odata, backend_data) try: direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ @@ -98,14 +104,19 @@ class TestObjectFailures(ReplProbeTest): obj = 'object-range-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'RANGE') + # Stash the on disk data for future comparison - this may not equal + # 'VERIFY' if for example the proxy has crypto enabled + backend_data = direct_client.direct_get_object( + onode, opart, self.account, container, obj, headers={ + 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] metadata = read_metadata(data_file) metadata['ETag'] = 'badetag' write_metadata(data_file, metadata) base_headers = {'X-Backend-Storage-Policy-Index': self.policy.idx} - for header, result in [({'Range': 'bytes=0-2'}, 'RAN'), - ({'Range': 'bytes=1-11'}, 'ANGE'), - ({'Range': 'bytes=0-11'}, 'RANGE')]: + for header, result in [({'Range': 'bytes=0-2'}, backend_data[0:3]), + ({'Range': 'bytes=1-11'}, backend_data[1:]), + ({'Range': 'bytes=0-11'}, backend_data)]: req_headers = base_headers.copy() req_headers.update(header) odata = direct_client.direct_get_object( diff --git a/test/probe/test_object_handoff.py b/test/probe/test_object_handoff.py index 3808df06..ca0b3d0e 100755 --- a/test/probe/test_object_handoff.py +++ b/test/probe/test_object_handoff.py @@ -55,6 +55,13 @@ class TestObjectHandoff(ReplProbeTest): raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) + # Stash the on disk data from a primary for future comparison with the + # handoff - this may not equal 'VERIFY' if for example the proxy has + # crypto enabled + direct_get_data = direct_client.direct_get_object( + onodes[1], opart, self.account, container, obj, headers={ + 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] + # Kill other two container/obj primary servers # to ensure GET handoff works for node in onodes[1:]: @@ -76,9 +83,7 @@ class TestObjectHandoff(ReplProbeTest): odata = direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] - if odata != 'VERIFY': - raise Exception('Direct object GET did not return VERIFY, instead ' - 'it returned: %s' % repr(odata)) + self.assertEqual(direct_get_data, odata) # drop a tempfile in the handoff's datadir, like it might have # had if there was an rsync failure while it was previously a @@ -143,9 +148,7 @@ class TestObjectHandoff(ReplProbeTest): odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] - if odata != 'VERIFY': - raise Exception('Direct object GET did not return VERIFY, instead ' - 'it returned: %s' % repr(odata)) + self.assertEqual(direct_get_data, odata) # and that it does *not* have a temporary rsync dropping! found_data_filename = False @@ -273,6 +276,14 @@ class TestECObjectHandoffOverwrite(ECProbeTest): # shutdown one of the primary data nodes failed_primary = random.choice(onodes) failed_primary_device_path = self.device_dir('object', failed_primary) + # first read its ec etag value for future reference - this may not + # equal old_contents.etag if for example the proxy has crypto enabled + req_headers = {'X-Backend-Storage-Policy-Index': int(self.policy)} + headers = direct_client.direct_head_object( + failed_primary, opart, self.account, container_name, + object_name, headers=req_headers) + old_backend_etag = headers['X-Object-Sysmeta-EC-Etag'] + self.kill_drive(failed_primary_device_path) # overwrite our object with some new data @@ -290,13 +301,18 @@ class TestECObjectHandoffOverwrite(ECProbeTest): failed_primary, opart, self.account, container_name, object_name, headers=req_headers) self.assertEqual(headers['X-Object-Sysmeta-EC-Etag'], - old_contents.etag) + old_backend_etag) # we have 1 primary with wrong old etag, and we should have 5 with # new etag plus a handoff with the new etag, so killing 2 other # primaries forces proxy to try to GET from all primaries plus handoff. other_nodes = [n for n in onodes if n != failed_primary] random.shuffle(other_nodes) + # grab the value of the new content's ec etag for future reference + headers = direct_client.direct_head_object( + other_nodes[0], opart, self.account, container_name, + object_name, headers=req_headers) + new_backend_etag = headers['X-Object-Sysmeta-EC-Etag'] for node in other_nodes[:2]: self.kill_drive(self.device_dir('object', node)) @@ -314,8 +330,8 @@ class TestECObjectHandoffOverwrite(ECProbeTest): continue found_frags[headers['X-Object-Sysmeta-EC-Etag']] += 1 self.assertEqual(found_frags, { - new_contents.etag: 4, # this should be enough to rebuild! - old_contents.etag: 1, + new_backend_etag: 4, # this should be enough to rebuild! + old_backend_etag: 1, }) # clear node error limiting diff --git a/test/unit/common/middleware/crypto/__init__.py b/test/unit/common/middleware/crypto/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/unit/common/middleware/crypto/crypto_helpers.py b/test/unit/common/middleware/crypto/crypto_helpers.py new file mode 100644 index 00000000..0af7d3e8 --- /dev/null +++ b/test/unit/common/middleware/crypto/crypto_helpers.py @@ -0,0 +1,54 @@ +# Copyright (c) 2015-2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import base64 +import hashlib + +from swift.common.middleware.crypto.crypto_utils import Crypto + + +def fetch_crypto_keys(): + return {'account': 'This is an account key 012345678', + 'container': 'This is a container key 01234567', + 'object': 'This is an object key 0123456789', + 'id': {'v': 'fake', 'path': '/a/c/fake'}} + + +def md5hex(s): + return hashlib.md5(s).hexdigest() + + +def encrypt(val, key=None, iv=None, ctxt=None): + if ctxt is None: + ctxt = Crypto({}).create_encryption_ctxt(key, iv) + enc_val = ctxt.update(val) + return enc_val + + +def decrypt(key, iv, enc_val): + dec_ctxt = Crypto({}).create_decryption_ctxt(key, iv, 0) + dec_val = dec_ctxt.update(enc_val) + return dec_val + + +FAKE_IV = "This is an IV123" +# do not use this example encryption_root_secret in production, use a randomly +# generated value with high entropy +TEST_KEYMASTER_CONF = {'encryption_root_secret': base64.b64encode(b'x' * 32)} + + +def fake_get_crypto_meta(**kwargs): + meta = {'iv': FAKE_IV, 'cipher': Crypto.cipher} + meta.update(kwargs) + return meta diff --git a/test/unit/common/middleware/crypto/test_crypto.py b/test/unit/common/middleware/crypto/test_crypto.py new file mode 100644 index 00000000..c5f6cd0c --- /dev/null +++ b/test/unit/common/middleware/crypto/test_crypto.py @@ -0,0 +1,39 @@ +# Copyright (c) 2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from swift.common import utils +from swift.common.middleware import crypto + + +class TestCrypto(unittest.TestCase): + def test_filter_factory(self): + factory = crypto.filter_factory({}) + self.assertTrue(callable(factory)) + self.assertIsInstance(factory({}), crypto.decrypter.Decrypter) + self.assertIsInstance(factory({}).app, crypto.encrypter.Encrypter) + self.assertIn('encryption', utils._swift_admin_info) + self.assertDictEqual( + {'enabled': True}, utils._swift_admin_info['encryption']) + self.assertNotIn('encryption', utils._swift_info) + + factory = crypto.filter_factory({'disable_encryption': True}) + self.assertTrue(callable(factory)) + self.assertIsInstance(factory({}), crypto.decrypter.Decrypter) + self.assertIsInstance(factory({}).app, crypto.encrypter.Encrypter) + self.assertIn('encryption', utils._swift_admin_info) + self.assertDictEqual( + {'enabled': False}, utils._swift_admin_info['encryption']) + self.assertNotIn('encryption', utils._swift_info) diff --git a/test/unit/common/middleware/crypto/test_crypto_utils.py b/test/unit/common/middleware/crypto/test_crypto_utils.py new file mode 100644 index 00000000..56aca2ea --- /dev/null +++ b/test/unit/common/middleware/crypto/test_crypto_utils.py @@ -0,0 +1,495 @@ +# Copyright (c) 2015-2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest + +import mock +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + +from swift.common.exceptions import EncryptionException +from swift.common.middleware.crypto import crypto_utils +from swift.common.middleware.crypto.crypto_utils import ( + CRYPTO_KEY_CALLBACK, Crypto, CryptoWSGIContext) +from swift.common.swob import HTTPException +from test.unit import FakeLogger +from test.unit.common.middleware.crypto.crypto_helpers import fetch_crypto_keys + + +class TestCryptoWsgiContext(unittest.TestCase): + def setUp(self): + class FakeFilter(object): + app = None + crypto = Crypto({}) + + self.fake_logger = FakeLogger() + self.crypto_context = CryptoWSGIContext( + FakeFilter(), 'object', self.fake_logger) + + def test_get_keys(self): + # ok + env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + keys = self.crypto_context.get_keys(env) + self.assertDictEqual(fetch_crypto_keys(), keys) + + # only default required keys are checked + subset_keys = {'object': fetch_crypto_keys()['object']} + env = {CRYPTO_KEY_CALLBACK: lambda: subset_keys} + keys = self.crypto_context.get_keys(env) + self.assertDictEqual(subset_keys, keys) + + # only specified required keys are checked + subset_keys = {'container': fetch_crypto_keys()['container']} + env = {CRYPTO_KEY_CALLBACK: lambda: subset_keys} + keys = self.crypto_context.get_keys(env, required=['container']) + self.assertDictEqual(subset_keys, keys) + + subset_keys = {'object': fetch_crypto_keys()['object'], + 'container': fetch_crypto_keys()['container']} + env = {CRYPTO_KEY_CALLBACK: lambda: subset_keys} + keys = self.crypto_context.get_keys( + env, required=['object', 'container']) + self.assertDictEqual(subset_keys, keys) + + def test_get_keys_missing_callback(self): + with self.assertRaises(HTTPException) as cm: + self.crypto_context.get_keys({}) + self.assertIn('500 Internal Error', cm.exception.message) + self.assertIn('missing callback', + self.fake_logger.get_lines_for_level('error')[0]) + self.assertIn('Unable to retrieve encryption keys.', cm.exception.body) + + def test_get_keys_callback_exception(self): + def callback(): + raise Exception('boom') + with self.assertRaises(HTTPException) as cm: + self.crypto_context.get_keys({CRYPTO_KEY_CALLBACK: callback}) + self.assertIn('500 Internal Error', cm.exception.message) + self.assertIn('from callback: boom', + self.fake_logger.get_lines_for_level('error')[0]) + self.assertIn('Unable to retrieve encryption keys.', cm.exception.body) + + def test_get_keys_missing_key_for_default_required_list(self): + bad_keys = dict(fetch_crypto_keys()) + bad_keys.pop('object') + with self.assertRaises(HTTPException) as cm: + self.crypto_context.get_keys( + {CRYPTO_KEY_CALLBACK: lambda: bad_keys}) + self.assertIn('500 Internal Error', cm.exception.message) + self.assertIn("Missing key for 'object'", + self.fake_logger.get_lines_for_level('error')[0]) + self.assertIn('Unable to retrieve encryption keys.', cm.exception.body) + + def test_get_keys_missing_object_key_for_specified_required_list(self): + bad_keys = dict(fetch_crypto_keys()) + bad_keys.pop('object') + with self.assertRaises(HTTPException) as cm: + self.crypto_context.get_keys( + {CRYPTO_KEY_CALLBACK: lambda: bad_keys}, + required=['object', 'container']) + self.assertIn('500 Internal Error', cm.exception.message) + self.assertIn("Missing key for 'object'", + self.fake_logger.get_lines_for_level('error')[0]) + self.assertIn('Unable to retrieve encryption keys.', cm.exception.body) + + def test_get_keys_missing_container_key_for_specified_required_list(self): + bad_keys = dict(fetch_crypto_keys()) + bad_keys.pop('container') + with self.assertRaises(HTTPException) as cm: + self.crypto_context.get_keys( + {CRYPTO_KEY_CALLBACK: lambda: bad_keys}, + required=['object', 'container']) + self.assertIn('500 Internal Error', cm.exception.message) + self.assertIn("Missing key for 'container'", + self.fake_logger.get_lines_for_level('error')[0]) + self.assertIn('Unable to retrieve encryption keys.', cm.exception.body) + + def test_bad_object_key_for_default_required_list(self): + bad_keys = dict(fetch_crypto_keys()) + bad_keys['object'] = 'the minor key' + with self.assertRaises(HTTPException) as cm: + self.crypto_context.get_keys( + {CRYPTO_KEY_CALLBACK: lambda: bad_keys}) + self.assertIn('500 Internal Error', cm.exception.message) + self.assertIn("Bad key for 'object'", + self.fake_logger.get_lines_for_level('error')[0]) + self.assertIn('Unable to retrieve encryption keys.', cm.exception.body) + + def test_bad_container_key_for_default_required_list(self): + bad_keys = dict(fetch_crypto_keys()) + bad_keys['container'] = 'the major key' + with self.assertRaises(HTTPException) as cm: + self.crypto_context.get_keys( + {CRYPTO_KEY_CALLBACK: lambda: bad_keys}, + required=['object', 'container']) + self.assertIn('500 Internal Error', cm.exception.message) + self.assertIn("Bad key for 'container'", + self.fake_logger.get_lines_for_level('error')[0]) + self.assertIn('Unable to retrieve encryption keys.', cm.exception.body) + + def test_get_keys_not_a_dict(self): + with self.assertRaises(HTTPException) as cm: + self.crypto_context.get_keys( + {CRYPTO_KEY_CALLBACK: lambda: ['key', 'quay', 'qui']}) + self.assertIn('500 Internal Error', cm.exception.message) + self.assertIn("Did not get a keys dict", + self.fake_logger.get_lines_for_level('error')[0]) + self.assertIn('Unable to retrieve encryption keys.', cm.exception.body) + + +class TestModuleMethods(unittest.TestCase): + meta = {'iv': '0123456789abcdef', 'cipher': 'AES_CTR_256'} + serialized_meta = '%7B%22cipher%22%3A+%22AES_CTR_256%22%2C+%22' \ + 'iv%22%3A+%22MDEyMzQ1Njc4OWFiY2RlZg%3D%3D%22%7D' + + meta_with_key = {'iv': '0123456789abcdef', 'cipher': 'AES_CTR_256', + 'body_key': {'key': 'fedcba9876543210fedcba9876543210', + 'iv': 'fedcba9876543210'}} + serialized_meta_with_key = '%7B%22body_key%22%3A+%7B%22iv%22%3A+%22ZmVkY' \ + '2JhOTg3NjU0MzIxMA%3D%3D%22%2C+%22key%22%3A+%' \ + '22ZmVkY2JhOTg3NjU0MzIxMGZlZGNiYTk4NzY1NDMyMT' \ + 'A%3D%22%7D%2C+%22cipher%22%3A+%22AES_CTR_256' \ + '%22%2C+%22iv%22%3A+%22MDEyMzQ1Njc4OWFiY2RlZg' \ + '%3D%3D%22%7D' + + def test_dump_crypto_meta(self): + actual = crypto_utils.dump_crypto_meta(self.meta) + self.assertEqual(self.serialized_meta, actual) + + actual = crypto_utils.dump_crypto_meta(self.meta_with_key) + self.assertEqual(self.serialized_meta_with_key, actual) + + def test_load_crypto_meta(self): + actual = crypto_utils.load_crypto_meta(self.serialized_meta) + self.assertEqual(self.meta, actual) + + actual = crypto_utils.load_crypto_meta(self.serialized_meta_with_key) + self.assertEqual(self.meta_with_key, actual) + + def assert_raises(value, message): + with self.assertRaises(EncryptionException) as cm: + crypto_utils.load_crypto_meta(value) + self.assertIn('Bad crypto meta %r' % value, cm.exception.message) + self.assertIn(message, cm.exception.message) + + assert_raises(None, 'crypto meta not a string') + assert_raises(99, 'crypto meta not a string') + assert_raises('', 'No JSON object could be decoded') + assert_raises('abc', 'No JSON object could be decoded') + assert_raises('[]', 'crypto meta not a Mapping') + assert_raises('{"iv": "abcdef"}', 'Incorrect padding') + assert_raises('{"iv": []}', 'must be string or buffer') + assert_raises('{"iv": {}}', 'must be string or buffer') + assert_raises('{"iv": 99}', 'must be string or buffer') + assert_raises('{"key": "abcdef"}', 'Incorrect padding') + assert_raises('{"key": []}', 'must be string or buffer') + assert_raises('{"key": {}}', 'must be string or buffer') + assert_raises('{"key": 99}', 'must be string or buffer') + assert_raises('{"body_key": {"iv": "abcdef"}}', 'Incorrect padding') + assert_raises('{"body_key": {"iv": []}}', 'must be string or buffer') + assert_raises('{"body_key": {"iv": {}}}', 'must be string or buffer') + assert_raises('{"body_key": {"iv": 99}}', 'must be string or buffer') + assert_raises('{"body_key": {"key": "abcdef"}}', 'Incorrect padding') + assert_raises('{"body_key": {"key": []}}', 'must be string or buffer') + assert_raises('{"body_key": {"key": {}}}', 'must be string or buffer') + assert_raises('{"body_key": {"key": 99}}', 'must be string or buffer') + + def test_dump_then_load_crypto_meta(self): + actual = crypto_utils.load_crypto_meta( + crypto_utils.dump_crypto_meta(self.meta)) + self.assertEqual(self.meta, actual) + + actual = crypto_utils.load_crypto_meta( + crypto_utils.dump_crypto_meta(self.meta_with_key)) + self.assertEqual(self.meta_with_key, actual) + + def test_append_crypto_meta(self): + actual = crypto_utils.append_crypto_meta('abc', self.meta) + expected = 'abc; swift_meta=%s' % self.serialized_meta + self.assertEqual(actual, expected) + + actual = crypto_utils.append_crypto_meta('abc', self.meta_with_key) + expected = 'abc; swift_meta=%s' % self.serialized_meta_with_key + self.assertEqual(actual, expected) + + def test_extract_crypto_meta(self): + val, meta = crypto_utils.extract_crypto_meta( + 'abc; swift_meta=%s' % self.serialized_meta) + self.assertEqual('abc', val) + self.assertDictEqual(self.meta, meta) + + val, meta = crypto_utils.extract_crypto_meta( + 'abc; swift_meta=%s' % self.serialized_meta_with_key) + self.assertEqual('abc', val) + self.assertDictEqual(self.meta_with_key, meta) + + val, meta = crypto_utils.extract_crypto_meta('abc') + self.assertEqual('abc', val) + self.assertIsNone(meta) + + # other param names will be ignored + val, meta = crypto_utils.extract_crypto_meta('abc; foo=bar') + self.assertEqual('abc', val) + self.assertIsNone(meta) + + def test_append_then_extract_crypto_meta(self): + val = 'abc' + actual = crypto_utils.extract_crypto_meta( + crypto_utils.append_crypto_meta(val, self.meta)) + self.assertEqual((val, self.meta), actual) + + +class TestCrypto(unittest.TestCase): + + def setUp(self): + self.crypto = Crypto({}) + + def test_create_encryption_context(self): + value = 'encrypt me' * 100 # more than one cipher block + key = os.urandom(32) + iv = os.urandom(16) + ctxt = self.crypto.create_encryption_ctxt(key, iv) + expected = Cipher( + algorithms.AES(key), modes.CTR(iv), + backend=default_backend()).encryptor().update(value) + self.assertEqual(expected, ctxt.update(value)) + + for bad_iv in ('a little too long', 'too short'): + self.assertRaises( + ValueError, self.crypto.create_encryption_ctxt, key, bad_iv) + + for bad_key in ('objKey', 'a' * 31, 'a' * 33, 'a' * 16, 'a' * 24): + self.assertRaises( + ValueError, self.crypto.create_encryption_ctxt, bad_key, iv) + + def test_create_decryption_context(self): + value = 'decrypt me' * 100 # more than one cipher block + key = os.urandom(32) + iv = os.urandom(16) + ctxt = self.crypto.create_decryption_ctxt(key, iv, 0) + expected = Cipher( + algorithms.AES(key), modes.CTR(iv), + backend=default_backend()).decryptor().update(value) + self.assertEqual(expected, ctxt.update(value)) + + for bad_iv in ('a little too long', 'too short'): + self.assertRaises( + ValueError, self.crypto.create_decryption_ctxt, key, bad_iv, 0) + + for bad_key in ('objKey', 'a' * 31, 'a' * 33, 'a' * 16, 'a' * 24): + self.assertRaises( + ValueError, self.crypto.create_decryption_ctxt, bad_key, iv, 0) + + with self.assertRaises(ValueError) as cm: + self.crypto.create_decryption_ctxt(key, iv, -1) + self.assertEqual("Offset must not be negative", cm.exception.message) + + def test_enc_dec_small_chunks(self): + self.enc_dec_chunks(['encrypt me', 'because I', 'am sensitive']) + + def test_enc_dec_large_chunks(self): + self.enc_dec_chunks([os.urandom(65536), os.urandom(65536)]) + + def enc_dec_chunks(self, chunks): + key = 'objL7wjV6L79Sfs4y7dy41273l0k6Wki' + iv = self.crypto.create_iv() + enc_ctxt = self.crypto.create_encryption_ctxt(key, iv) + enc_val = [enc_ctxt.update(chunk) for chunk in chunks] + self.assertTrue(''.join(enc_val) != chunks) + dec_ctxt = self.crypto.create_decryption_ctxt(key, iv, 0) + dec_val = [dec_ctxt.update(chunk) for chunk in enc_val] + self.assertEqual(''.join(chunks), ''.join(dec_val), + 'Expected value {%s} but got {%s}' % + (''.join(chunks), ''.join(dec_val))) + + def test_decrypt_range(self): + chunks = ['0123456789abcdef', 'ghijklmnopqrstuv'] + key = 'objL7wjV6L79Sfs4y7dy41273l0k6Wki' + iv = self.crypto.create_iv() + enc_ctxt = self.crypto.create_encryption_ctxt(key, iv) + enc_val = [enc_ctxt.update(chunk) for chunk in chunks] + self.assertTrue(''.join(enc_val) != chunks) + + # Simulate a ranged GET from byte 19 to 32 : 'jklmnopqrstuv' + dec_ctxt = self.crypto.create_decryption_ctxt(key, iv, 19) + ranged_chunks = [enc_val[1][3:]] + dec_val = [dec_ctxt.update(chunk) for chunk in ranged_chunks] + self.assertEqual('jklmnopqrstuv', ''.join(dec_val), + 'Expected value {%s} but got {%s}' % + ('jklmnopqrstuv', ''.join(dec_val))) + + def test_create_decryption_context_non_zero_offset(self): + # Verify that iv increments for each 16 bytes of offset. + # For a ranged GET we pass a non-zero offset so that the decrypter + # counter is incremented to the correct value to start decrypting at + # that offset into the object body. The counter should increment by one + # from the starting IV value for every 16 bytes offset into the object + # body, until it reaches 2^128 -1 when it should wrap to zero. We check + # that is happening by verifying a decrypted value using various + # offsets. + key = 'objL7wjV6L79Sfs4y7dy41273l0k6Wki' + + def do_test(): + for offset, exp_iv in mappings.items(): + dec_ctxt = self.crypto.create_decryption_ctxt(key, iv, offset) + offset_in_block = offset % 16 + cipher = Cipher(algorithms.AES(key), + modes.CTR(exp_iv), + backend=default_backend()) + expected = cipher.decryptor().update( + 'p' * offset_in_block + 'ciphertext') + actual = dec_ctxt.update('ciphertext') + expected = expected[offset % 16:] + self.assertEqual(expected, actual, + 'Expected %r but got %r, iv=%s and offset=%s' + % (expected, actual, iv, offset)) + + iv = '0000000010000000' + mappings = { + 2: '0000000010000000', + 16: '0000000010000001', + 19: '0000000010000001', + 48: '0000000010000003', + 1024: '000000001000000p', + 5119: '000000001000001o' + } + do_test() + + # choose max iv value and test that it wraps to zero + iv = chr(0xff) * 16 + mappings = { + 2: iv, + 16: str(bytearray.fromhex('00' * 16)), # iv wraps to 0 + 19: str(bytearray.fromhex('00' * 16)), + 48: str(bytearray.fromhex('00' * 15 + '02')), + 1024: str(bytearray.fromhex('00' * 15 + '3f')), + 5119: str(bytearray.fromhex('00' * 14 + '013E')) + } + do_test() + + iv = chr(0x0) * 16 + mappings = { + 2: iv, + 16: str(bytearray.fromhex('00' * 15 + '01')), + 19: str(bytearray.fromhex('00' * 15 + '01')), + 48: str(bytearray.fromhex('00' * 15 + '03')), + 1024: str(bytearray.fromhex('00' * 15 + '40')), + 5119: str(bytearray.fromhex('00' * 14 + '013F')) + } + do_test() + + iv = chr(0x0) * 8 + chr(0xff) * 8 + mappings = { + 2: iv, + 16: str(bytearray.fromhex('00' * 7 + '01' + '00' * 8)), + 19: str(bytearray.fromhex('00' * 7 + '01' + '00' * 8)), + 48: str(bytearray.fromhex('00' * 7 + '01' + '00' * 7 + '02')), + 1024: str(bytearray.fromhex('00' * 7 + '01' + '00' * 7 + '3F')), + 5119: str(bytearray.fromhex('00' * 7 + '01' + '00' * 6 + '013E')) + } + do_test() + + def test_check_key(self): + for key in ('objKey', 'a' * 31, 'a' * 33, 'a' * 16, 'a' * 24): + with self.assertRaises(ValueError) as cm: + self.crypto.check_key(key) + self.assertEqual("Key must be length 32 bytes", + cm.exception.message) + + def test_check_crypto_meta(self): + meta = {'cipher': 'AES_CTR_256'} + with self.assertRaises(EncryptionException) as cm: + self.crypto.check_crypto_meta(meta) + self.assertEqual("Bad crypto meta: Missing 'iv'", + cm.exception.message) + + for bad_iv in ('a little too long', 'too short'): + meta['iv'] = bad_iv + with self.assertRaises(EncryptionException) as cm: + self.crypto.check_crypto_meta(meta) + self.assertEqual("Bad crypto meta: IV must be length 16 bytes", + cm.exception.message) + + meta = {'iv': os.urandom(16)} + with self.assertRaises(EncryptionException) as cm: + self.crypto.check_crypto_meta(meta) + self.assertEqual("Bad crypto meta: Missing 'cipher'", + cm.exception.message) + + meta['cipher'] = 'Mystery cipher' + with self.assertRaises(EncryptionException) as cm: + self.crypto.check_crypto_meta(meta) + self.assertEqual("Bad crypto meta: Cipher must be AES_CTR_256", + cm.exception.message) + + def test_create_iv(self): + self.assertEqual(16, len(self.crypto.create_iv())) + # crude check that we get back different values on each call + self.assertNotEqual(self.crypto.create_iv(), self.crypto.create_iv()) + + def test_get_crypto_meta(self): + meta = self.crypto.create_crypto_meta() + self.assertIsInstance(meta, dict) + # this is deliberately brittle so that if new items are added then the + # test will need to be updated + self.assertEqual(2, len(meta)) + self.assertIn('iv', meta) + self.assertEqual(16, len(meta['iv'])) + self.assertIn('cipher', meta) + self.assertEqual('AES_CTR_256', meta['cipher']) + self.crypto.check_crypto_meta(meta) # sanity check + meta2 = self.crypto.create_crypto_meta() + self.assertNotEqual(meta['iv'], meta2['iv']) # crude sanity check + + def test_create_random_key(self): + # crude check that we get unique keys on each call + keys = set() + for i in range(10): + key = self.crypto.create_random_key() + self.assertEqual(32, len(key)) + keys.add(key) + self.assertEqual(10, len(keys)) + + def test_wrap_unwrap_key(self): + wrapping_key = os.urandom(32) + key_to_wrap = os.urandom(32) + iv = os.urandom(16) + with mock.patch( + 'swift.common.middleware.crypto.crypto_utils.Crypto.create_iv', + return_value=iv): + wrapped = self.crypto.wrap_key(wrapping_key, key_to_wrap) + cipher = Cipher(algorithms.AES(wrapping_key), modes.CTR(iv), + backend=default_backend()) + expected = {'key': cipher.encryptor().update(key_to_wrap), + 'iv': iv} + self.assertEqual(expected, wrapped) + + unwrapped = self.crypto.unwrap_key(wrapping_key, wrapped) + self.assertEqual(key_to_wrap, unwrapped) + + def test_unwrap_bad_key(self): + # verify that ValueError is raised if unwrapped key is invalid + wrapping_key = os.urandom(32) + for length in (0, 16, 24, 31, 33): + key_to_wrap = os.urandom(length) + wrapped = self.crypto.wrap_key(wrapping_key, key_to_wrap) + with self.assertRaises(ValueError) as cm: + self.crypto.unwrap_key(wrapping_key, wrapped) + self.assertEqual( + cm.exception.message, 'Key must be length 32 bytes') + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/middleware/crypto/test_decrypter.py b/test/unit/common/middleware/crypto/test_decrypter.py new file mode 100644 index 00000000..b70d6502 --- /dev/null +++ b/test/unit/common/middleware/crypto/test_decrypter.py @@ -0,0 +1,1119 @@ +# Copyright (c) 2015-2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import base64 +import json +import os +import unittest +from xml.dom import minidom + +import mock + +from swift.common.header_key_dict import HeaderKeyDict +from swift.common.middleware.crypto import decrypter +from swift.common.middleware.crypto.crypto_utils import CRYPTO_KEY_CALLBACK, \ + dump_crypto_meta, Crypto +from swift.common.swob import Request, HTTPException, HTTPOk, \ + HTTPPreconditionFailed, HTTPNotFound, HTTPPartialContent + +from test.unit import FakeLogger +from test.unit.common.middleware.crypto.crypto_helpers import md5hex, \ + fetch_crypto_keys, FAKE_IV, encrypt, fake_get_crypto_meta +from test.unit.common.middleware.helpers import FakeSwift, FakeAppThatExcepts + + +def get_crypto_meta_header(crypto_meta=None): + if crypto_meta is None: + crypto_meta = fake_get_crypto_meta() + return dump_crypto_meta(crypto_meta) + + +def encrypt_and_append_meta(value, key, crypto_meta=None): + return '%s; swift_meta=%s' % ( + base64.b64encode(encrypt(value, key, FAKE_IV)), + get_crypto_meta_header(crypto_meta)) + + +class TestDecrypterObjectRequests(unittest.TestCase): + def setUp(self): + self.app = FakeSwift() + self.decrypter = decrypter.Decrypter(self.app, {}) + self.decrypter.logger = FakeLogger() + + def _make_response_headers(self, content_length, plaintext_etag, keys, + body_key): + # helper method to make a typical set of response headers for a GET or + # HEAD request + cont_key = keys['container'] + object_key = keys['object'] + body_key_meta = {'key': encrypt(body_key, object_key, FAKE_IV), + 'iv': FAKE_IV} + body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta) + return HeaderKeyDict({ + 'Etag': 'hashOfCiphertext', + 'content-type': 'text/plain', + 'content-length': content_length, + 'X-Object-Sysmeta-Crypto-Etag': '%s; swift_meta=%s' % ( + base64.b64encode(encrypt(plaintext_etag, object_key, FAKE_IV)), + get_crypto_meta_header()), + 'X-Object-Sysmeta-Crypto-Body-Meta': + get_crypto_meta_header(body_crypto_meta), + 'x-object-transient-sysmeta-crypto-meta-test': + base64.b64encode(encrypt('encrypt me', object_key, FAKE_IV)) + + ';swift_meta=' + get_crypto_meta_header(), + 'x-object-sysmeta-container-update-override-etag': + encrypt_and_append_meta('encrypt me, too', cont_key), + 'x-object-sysmeta-test': 'do not encrypt me', + }) + + def _test_request_success(self, method, body): + env = {'REQUEST_METHOD': method, + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env) + plaintext_etag = md5hex(body) + body_key = os.urandom(32) + enc_body = encrypt(body, body_key, FAKE_IV) + hdrs = self._make_response_headers( + len(enc_body), plaintext_etag, fetch_crypto_keys(), body_key) + + # there shouldn't be any x-object-meta- headers, but if there are + # then the decrypted header will win where there is a name clash... + hdrs.update({ + 'x-object-meta-test': 'unexpected, overwritten by decrypted value', + 'x-object-meta-distinct': 'unexpected but distinct from encrypted' + }) + self.app.register( + method, '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs) + resp = req.get_response(self.decrypter) + self.assertEqual('200 OK', resp.status) + self.assertEqual(plaintext_etag, resp.headers['Etag']) + self.assertEqual('text/plain', resp.headers['Content-Type']) + self.assertEqual('encrypt me', resp.headers['x-object-meta-test']) + self.assertEqual('unexpected but distinct from encrypted', + resp.headers['x-object-meta-distinct']) + self.assertEqual('do not encrypt me', + resp.headers['x-object-sysmeta-test']) + self.assertEqual( + 'encrypt me, too', + resp.headers['X-Object-Sysmeta-Container-Update-Override-Etag']) + self.assertNotIn('X-Object-Sysmeta-Crypto-Body-Meta', resp.headers) + self.assertNotIn('X-Object-Sysmeta-Crypto-Etag', resp.headers) + return resp + + def test_GET_success(self): + body = 'FAKE APP' + resp = self._test_request_success('GET', body) + self.assertEqual(body, resp.body) + + def test_HEAD_success(self): + body = 'FAKE APP' + resp = self._test_request_success('HEAD', body) + self.assertEqual('', resp.body) + + def test_headers_case(self): + body = 'fAkE ApP' + req = Request.blank('/v1/a/c/o', body='FaKe') + req.environ[CRYPTO_KEY_CALLBACK] = fetch_crypto_keys + plaintext_etag = md5hex(body) + body_key = os.urandom(32) + enc_body = encrypt(body, body_key, FAKE_IV) + hdrs = self._make_response_headers( + len(enc_body), plaintext_etag, fetch_crypto_keys(), body_key) + + hdrs.update({ + 'x-Object-mEta-ignoRes-caSe': 'thIs pArt WilL bE cOol', + }) + self.app.register( + 'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs) + + status, headers, app_iter = req.call_application(self.decrypter) + self.assertEqual(status, '200 OK') + expected = { + 'Etag': '7f7837924188f7b511a9e3881a9f77a8', + 'X-Object-Sysmeta-Container-Update-Override-Etag': + 'encrypt me, too', + 'X-Object-Meta-Test': 'encrypt me', + 'Content-Length': '8', + 'X-Object-Meta-Ignores-Case': 'thIs pArt WilL bE cOol', + 'X-Object-Sysmeta-Test': 'do not encrypt me', + 'Content-Type': 'text/plain', + } + self.assertEqual(dict(headers), expected) + self.assertEqual('fAkE ApP', ''.join(app_iter)) + + def _test_412_response(self, method): + # simulate a 412 response to a conditional GET which has an Etag header + data = 'the object content' + env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env, method=method) + resp_body = 'I am sorry, you have failed to meet a precondition' + hdrs = self._make_response_headers( + len(resp_body), md5hex(data), fetch_crypto_keys(), 'not used') + self.app.register(method, '/v1/a/c/o', HTTPPreconditionFailed, + body=resp_body, headers=hdrs) + resp = req.get_response(self.decrypter) + + self.assertEqual('412 Precondition Failed', resp.status) + # the response body should not be decrypted, it is already plaintext + self.assertEqual(resp_body if method == 'GET' else '', resp.body) + # whereas the Etag and other headers should be decrypted + self.assertEqual(md5hex(data), resp.headers['Etag']) + self.assertEqual('text/plain', resp.headers['Content-Type']) + self.assertEqual('encrypt me', resp.headers['x-object-meta-test']) + self.assertEqual('do not encrypt me', + resp.headers['x-object-sysmeta-test']) + + def test_GET_412_response(self): + self._test_412_response('GET') + + def test_HEAD_412_response(self): + self._test_412_response('HEAD') + + def _test_404_response(self, method): + # simulate a 404 response, sanity check response headers + env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env, method=method) + resp_body = 'You still have not found what you are looking for' + hdrs = {'content-type': 'text/plain', + 'content-length': len(resp_body)} + self.app.register(method, '/v1/a/c/o', HTTPNotFound, + body=resp_body, headers=hdrs) + resp = req.get_response(self.decrypter) + + self.assertEqual('404 Not Found', resp.status) + # the response body should not be decrypted, it is already plaintext + self.assertEqual(resp_body if method == 'GET' else '', resp.body) + # there should be no etag header inserted by decrypter + self.assertNotIn('Etag', resp.headers) + self.assertEqual('text/plain', resp.headers['Content-Type']) + + def test_GET_404_response(self): + self._test_404_response('GET') + + def test_HEAD_404_response(self): + self._test_404_response('HEAD') + + def test_GET_missing_etag_crypto_meta(self): + env = {'REQUEST_METHOD': 'GET', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env) + body = 'FAKE APP' + key = fetch_crypto_keys()['object'] + enc_body = encrypt(body, key, FAKE_IV) + hdrs = self._make_response_headers( + len(body), md5hex(body), fetch_crypto_keys(), 'not used') + # simulate missing crypto meta from encrypted etag + hdrs['X-Object-Sysmeta-Crypto-Etag'] = \ + base64.b64encode(encrypt(md5hex(body), key, FAKE_IV)) + self.app.register('GET', '/v1/a/c/o', HTTPOk, body=enc_body, + headers=hdrs) + resp = req.get_response(self.decrypter) + self.assertEqual('500 Internal Error', resp.status) + self.assertIn('Error decrypting header', resp.body) + self.assertIn('Error decrypting header X-Object-Sysmeta-Crypto-Etag', + self.decrypter.logger.get_lines_for_level('error')[0]) + + def _test_override_etag_bad_meta(self, method, bad_crypto_meta): + env = {'REQUEST_METHOD': method, + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env) + body = 'FAKE APP' + key = fetch_crypto_keys()['object'] + enc_body = encrypt(body, key, FAKE_IV) + hdrs = self._make_response_headers( + len(body), md5hex(body), fetch_crypto_keys(), 'not used') + # simulate missing crypto meta from encrypted override etag + hdrs['X-Object-Sysmeta-Container-Update-Override-Etag'] = \ + encrypt_and_append_meta( + md5hex(body), key, crypto_meta=bad_crypto_meta) + self.app.register(method, '/v1/a/c/o', HTTPOk, body=enc_body, + headers=hdrs) + resp = req.get_response(self.decrypter) + self.assertEqual('500 Internal Error', resp.status) + self.assertIn('Error decrypting header ' + 'X-Object-Sysmeta-Container-Update-Override-Etag', + self.decrypter.logger.get_lines_for_level('error')[0]) + return resp + + def test_GET_override_etag_bad_iv(self): + bad_crypto_meta = fake_get_crypto_meta() + bad_crypto_meta['iv'] = 'bad_iv' + resp = self._test_override_etag_bad_meta('GET', bad_crypto_meta) + self.assertIn('Error decrypting header', resp.body) + + def test_HEAD_override_etag_bad_iv(self): + bad_crypto_meta = fake_get_crypto_meta() + bad_crypto_meta['iv'] = 'bad_iv' + resp = self._test_override_etag_bad_meta('HEAD', bad_crypto_meta) + self.assertEqual('', resp.body) + + def test_GET_override_etag_bad_cipher(self): + bad_crypto_meta = fake_get_crypto_meta() + bad_crypto_meta['cipher'] = 'unknown cipher' + resp = self._test_override_etag_bad_meta('GET', bad_crypto_meta) + self.assertIn('Error decrypting header', resp.body) + + def test_HEAD_override_etag_bad_cipher(self): + bad_crypto_meta = fake_get_crypto_meta() + bad_crypto_meta['cipher'] = 'unknown cipher' + resp = self._test_override_etag_bad_meta('HEAD', bad_crypto_meta) + self.assertEqual('', resp.body) + + def _test_bad_key(self, method): + # use bad key + def bad_fetch_crypto_keys(): + keys = fetch_crypto_keys() + keys['object'] = 'bad key' + return keys + + env = {'REQUEST_METHOD': method, + CRYPTO_KEY_CALLBACK: bad_fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env) + body = 'FAKE APP' + key = fetch_crypto_keys()['object'] + enc_body = encrypt(body, key, FAKE_IV) + hdrs = self._make_response_headers( + len(body), md5hex(body), fetch_crypto_keys(), 'not used') + self.app.register(method, '/v1/a/c/o', HTTPOk, body=enc_body, + headers=hdrs) + return req.get_response(self.decrypter) + + def test_HEAD_with_bad_key(self): + resp = self._test_bad_key('HEAD') + self.assertEqual('500 Internal Error', resp.status) + self.assertIn("Bad key for 'object'", + self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_GET_with_bad_key(self): + resp = self._test_bad_key('GET') + self.assertEqual('500 Internal Error', resp.status) + self.assertEqual('Unable to retrieve encryption keys.', + resp.body) + self.assertIn("Bad key for 'object'", + self.decrypter.logger.get_lines_for_level('error')[0]) + + def _test_bad_crypto_meta_for_user_metadata(self, method, bad_crypto_meta): + # use bad iv for metadata headers + env = {'REQUEST_METHOD': method, + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env) + body = 'FAKE APP' + key = fetch_crypto_keys()['object'] + enc_body = encrypt(body, key, FAKE_IV) + hdrs = self._make_response_headers( + len(body), md5hex(body), fetch_crypto_keys(), 'not used') + enc_val = base64.b64encode(encrypt('encrypt me', key, FAKE_IV)) + if bad_crypto_meta: + enc_val += ';swift_meta=' + get_crypto_meta_header( + crypto_meta=bad_crypto_meta) + hdrs['x-object-transient-sysmeta-crypto-meta-test'] = enc_val + self.app.register(method, '/v1/a/c/o', HTTPOk, body=enc_body, + headers=hdrs) + resp = req.get_response(self.decrypter) + self.assertEqual('500 Internal Error', resp.status) + self.assertIn( + 'Error decrypting header X-Object-Transient-Sysmeta-Crypto-Meta-' + 'Test', self.decrypter.logger.get_lines_for_level('error')[0]) + return resp + + def test_HEAD_with_missing_crypto_meta_for_user_metadata(self): + self._test_bad_crypto_meta_for_user_metadata('HEAD', None) + self.assertIn('Missing crypto meta in value', + self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_GET_with_missing_crypto_meta_for_user_metadata(self): + self._test_bad_crypto_meta_for_user_metadata('GET', None) + self.assertIn('Missing crypto meta in value', + self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_HEAD_with_bad_iv_for_user_metadata(self): + bad_crypto_meta = fake_get_crypto_meta() + bad_crypto_meta['iv'] = 'bad_iv' + self._test_bad_crypto_meta_for_user_metadata('HEAD', bad_crypto_meta) + self.assertIn('IV must be length 16', + self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_HEAD_with_missing_iv_for_user_metadata(self): + bad_crypto_meta = fake_get_crypto_meta() + bad_crypto_meta.pop('iv') + self._test_bad_crypto_meta_for_user_metadata('HEAD', bad_crypto_meta) + self.assertIn( + 'iv', self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_GET_with_bad_iv_for_user_metadata(self): + bad_crypto_meta = fake_get_crypto_meta() + bad_crypto_meta['iv'] = 'bad_iv' + resp = self._test_bad_crypto_meta_for_user_metadata( + 'GET', bad_crypto_meta) + self.assertEqual('Error decrypting header', resp.body) + self.assertIn('IV must be length 16', + self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_GET_with_missing_iv_for_user_metadata(self): + bad_crypto_meta = fake_get_crypto_meta() + bad_crypto_meta.pop('iv') + resp = self._test_bad_crypto_meta_for_user_metadata( + 'GET', bad_crypto_meta) + self.assertEqual('Error decrypting header', resp.body) + self.assertIn( + 'iv', self.decrypter.logger.get_lines_for_level('error')[0]) + + def _test_GET_with_bad_crypto_meta_for_object_body(self, bad_crypto_meta): + # use bad iv for object body + env = {'REQUEST_METHOD': 'GET', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env) + body = 'FAKE APP' + key = fetch_crypto_keys()['object'] + enc_body = encrypt(body, key, FAKE_IV) + hdrs = self._make_response_headers( + len(body), md5hex(body), fetch_crypto_keys(), 'not used') + hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \ + get_crypto_meta_header(crypto_meta=bad_crypto_meta) + self.app.register('GET', '/v1/a/c/o', HTTPOk, body=enc_body, + headers=hdrs) + resp = req.get_response(self.decrypter) + self.assertEqual('500 Internal Error', resp.status) + self.assertEqual('Error decrypting object', resp.body) + self.assertIn('Error decrypting object', + self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_GET_with_bad_iv_for_object_body(self): + bad_crypto_meta = fake_get_crypto_meta(key=os.urandom(32)) + bad_crypto_meta['iv'] = 'bad_iv' + self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta) + self.assertIn('IV must be length 16', + self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_GET_with_missing_iv_for_object_body(self): + bad_crypto_meta = fake_get_crypto_meta(key=os.urandom(32)) + bad_crypto_meta.pop('iv') + self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta) + self.assertIn("Missing 'iv'", + self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_GET_with_bad_body_key_for_object_body(self): + body_key_meta = {'key': 'wrapped too short key', 'iv': FAKE_IV} + bad_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta) + self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta) + self.assertIn('Key must be length 32', + self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_GET_with_missing_body_key_for_object_body(self): + bad_crypto_meta = fake_get_crypto_meta() # no key by default + self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta) + self.assertIn("Missing 'body_key'", + self.decrypter.logger.get_lines_for_level('error')[0]) + + def _test_req_metadata_not_encrypted(self, method): + # check that metadata is not decrypted if it does not have crypto meta; + # testing for case of an unencrypted POST to an object. + env = {'REQUEST_METHOD': method, + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env) + body = 'FAKE APP' + plaintext_etag = md5hex(body) + body_key = os.urandom(32) + enc_body = encrypt(body, body_key, FAKE_IV) + hdrs = self._make_response_headers( + len(body), plaintext_etag, fetch_crypto_keys(), body_key) + hdrs.pop('x-object-transient-sysmeta-crypto-meta-test') + hdrs['x-object-meta-test'] = 'plaintext not encrypted' + self.app.register( + method, '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs) + resp = req.get_response(self.decrypter) + self.assertEqual('200 OK', resp.status) + self.assertEqual(plaintext_etag, resp.headers['Etag']) + self.assertEqual('text/plain', resp.headers['Content-Type']) + self.assertEqual('plaintext not encrypted', + resp.headers['x-object-meta-test']) + + def test_HEAD_metadata_not_encrypted(self): + self._test_req_metadata_not_encrypted('HEAD') + + def test_GET_metadata_not_encrypted(self): + self._test_req_metadata_not_encrypted('GET') + + def test_GET_unencrypted_data(self): + # testing case of an unencrypted object with encrypted metadata from + # a later POST + env = {'REQUEST_METHOD': 'GET', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env) + body = 'FAKE APP' + obj_key = fetch_crypto_keys()['object'] + hdrs = {'Etag': md5hex(body), + 'content-type': 'text/plain', + 'content-length': len(body), + 'x-object-transient-sysmeta-crypto-meta-test': + base64.b64encode(encrypt('encrypt me', obj_key, FAKE_IV)) + + ';swift_meta=' + get_crypto_meta_header(), + 'x-object-sysmeta-test': 'do not encrypt me'} + self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs) + resp = req.get_response(self.decrypter) + self.assertEqual(body, resp.body) + self.assertEqual('200 OK', resp.status) + self.assertEqual(md5hex(body), resp.headers['Etag']) + self.assertEqual('text/plain', resp.headers['Content-Type']) + # POSTed user meta was encrypted + self.assertEqual('encrypt me', resp.headers['x-object-meta-test']) + # PUT sysmeta was not encrypted + self.assertEqual('do not encrypt me', + resp.headers['x-object-sysmeta-test']) + + def test_GET_multiseg(self): + env = {'REQUEST_METHOD': 'GET', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env) + chunks = ['some', 'chunks', 'of data'] + body = ''.join(chunks) + plaintext_etag = md5hex(body) + body_key = os.urandom(32) + ctxt = Crypto().create_encryption_ctxt(body_key, FAKE_IV) + enc_body = [encrypt(chunk, ctxt=ctxt) for chunk in chunks] + hdrs = self._make_response_headers( + sum(map(len, enc_body)), plaintext_etag, fetch_crypto_keys(), + body_key) + self.app.register( + 'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs) + resp = req.get_response(self.decrypter) + self.assertEqual(body, resp.body) + self.assertEqual('200 OK', resp.status) + self.assertEqual(plaintext_etag, resp.headers['Etag']) + self.assertEqual('text/plain', resp.headers['Content-Type']) + + def test_GET_multiseg_with_range(self): + env = {'REQUEST_METHOD': 'GET', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env) + req.headers['Content-Range'] = 'bytes 3-10/17' + chunks = ['0123', '45678', '9abcdef'] + body = ''.join(chunks) + plaintext_etag = md5hex(body) + body_key = os.urandom(32) + ctxt = Crypto().create_encryption_ctxt(body_key, FAKE_IV) + enc_body = [encrypt(chunk, ctxt=ctxt) for chunk in chunks] + enc_body = [enc_body[0][3:], enc_body[1], enc_body[2][:2]] + hdrs = self._make_response_headers( + sum(map(len, enc_body)), plaintext_etag, fetch_crypto_keys(), + body_key) + hdrs['content-range'] = req.headers['Content-Range'] + self.app.register( + 'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs) + resp = req.get_response(self.decrypter) + self.assertEqual('3456789a', resp.body) + self.assertEqual('200 OK', resp.status) + self.assertEqual(plaintext_etag, resp.headers['Etag']) + self.assertEqual('text/plain', resp.headers['Content-Type']) + + # Force the decrypter context updates to be less than one of our range + # sizes to check that the decrypt context offset is setup correctly with + # offset to first byte of range for first update and then re-used. + # Do mocking here to have the mocked value have effect in the generator + # function. + @mock.patch.object(decrypter, 'DECRYPT_CHUNK_SIZE', 4) + def test_GET_multipart_ciphertext(self): + # build fake multipart response body + body_key = os.urandom(32) + plaintext = 'Cwm fjord veg balks nth pyx quiz' + plaintext_etag = md5hex(plaintext) + ciphertext = encrypt(plaintext, body_key, FAKE_IV) + parts = ((0, 3, 'text/plain'), + (4, 9, 'text/plain; charset=us-ascii'), + (24, 32, 'text/plain')) + length = len(ciphertext) + body = '' + for start, end, ctype in parts: + body += '--multipartboundary\r\n' + body += 'Content-Type: %s\r\n' % ctype + body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length) + body += '\r\n\r\n' + ciphertext[start:end] + '\r\n' + body += '--multipartboundary--' + + # register request with fake swift + hdrs = self._make_response_headers( + len(body), plaintext_etag, fetch_crypto_keys(), body_key) + hdrs['content-type'] = \ + 'multipart/byteranges;boundary=multipartboundary' + self.app.register('GET', '/v1/a/c/o', HTTPPartialContent, body=body, + headers=hdrs) + + # issue request + env = {'REQUEST_METHOD': 'GET', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env) + resp = req.get_response(self.decrypter) + + self.assertEqual('206 Partial Content', resp.status) + self.assertEqual(plaintext_etag, resp.headers['Etag']) + self.assertEqual(len(body), int(resp.headers['Content-Length'])) + self.assertEqual('multipart/byteranges;boundary=multipartboundary', + resp.headers['Content-Type']) + + # the multipart headers could be re-ordered, so parse response body to + # verify expected content + resp_lines = resp.body.split('\r\n') + resp_lines.reverse() + for start, end, ctype in parts: + self.assertEqual('--multipartboundary', resp_lines.pop()) + expected_header_lines = { + 'Content-Type: %s' % ctype, + 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length)} + resp_header_lines = {resp_lines.pop(), resp_lines.pop()} + self.assertEqual(expected_header_lines, resp_header_lines) + self.assertEqual('', resp_lines.pop()) + self.assertEqual(plaintext[start:end], resp_lines.pop()) + self.assertEqual('--multipartboundary--', resp_lines.pop()) + + # we should have consumed the whole response body + self.assertFalse(resp_lines) + + def test_GET_multipart_content_type(self): + # *just* having multipart content type shouldn't trigger the mime doc + # code path + body_key = os.urandom(32) + plaintext = 'Cwm fjord veg balks nth pyx quiz' + plaintext_etag = md5hex(plaintext) + ciphertext = encrypt(plaintext, body_key, FAKE_IV) + + # register request with fake swift + hdrs = self._make_response_headers( + len(ciphertext), plaintext_etag, fetch_crypto_keys(), body_key) + hdrs['content-type'] = \ + 'multipart/byteranges;boundary=multipartboundary' + self.app.register('GET', '/v1/a/c/o', HTTPOk, body=ciphertext, + headers=hdrs) + + # issue request + env = {'REQUEST_METHOD': 'GET', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env) + resp = req.get_response(self.decrypter) + + self.assertEqual('200 OK', resp.status) + self.assertEqual(plaintext_etag, resp.headers['Etag']) + self.assertEqual(len(plaintext), int(resp.headers['Content-Length'])) + self.assertEqual('multipart/byteranges;boundary=multipartboundary', + resp.headers['Content-Type']) + self.assertEqual(plaintext, resp.body) + + def test_GET_multipart_no_body_crypto_meta(self): + # build fake multipart response body + plaintext = 'Cwm fjord veg balks nth pyx quiz' + plaintext_etag = md5hex(plaintext) + parts = ((0, 3, 'text/plain'), + (4, 9, 'text/plain; charset=us-ascii'), + (24, 32, 'text/plain')) + length = len(plaintext) + body = '' + for start, end, ctype in parts: + body += '--multipartboundary\r\n' + body += 'Content-Type: %s\r\n' % ctype + body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length) + body += '\r\n\r\n' + plaintext[start:end] + '\r\n' + body += '--multipartboundary--' + + # register request with fake swift + hdrs = { + 'Etag': plaintext_etag, + 'content-type': 'multipart/byteranges;boundary=multipartboundary', + 'content-length': len(body)} + self.app.register('GET', '/v1/a/c/o', HTTPPartialContent, body=body, + headers=hdrs) + + # issue request + env = {'REQUEST_METHOD': 'GET', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env) + resp = req.get_response(self.decrypter) + + self.assertEqual('206 Partial Content', resp.status) + self.assertEqual(plaintext_etag, resp.headers['Etag']) + self.assertEqual(len(body), int(resp.headers['Content-Length'])) + self.assertEqual('multipart/byteranges;boundary=multipartboundary', + resp.headers['Content-Type']) + + # the multipart response body should be unchanged + self.assertEqual(body, resp.body) + + def _test_GET_multipart_bad_body_crypto_meta(self, bad_crypto_meta): + # build fake multipart response body + key = fetch_crypto_keys()['object'] + ctxt = Crypto().create_encryption_ctxt(key, FAKE_IV) + plaintext = 'Cwm fjord veg balks nth pyx quiz' + plaintext_etag = md5hex(plaintext) + ciphertext = encrypt(plaintext, ctxt=ctxt) + parts = ((0, 3, 'text/plain'), + (4, 9, 'text/plain; charset=us-ascii'), + (24, 32, 'text/plain')) + length = len(ciphertext) + body = '' + for start, end, ctype in parts: + body += '--multipartboundary\r\n' + body += 'Content-Type: %s\r\n' % ctype + body += 'Content-Range: bytes %s-%s/%s' % (start, end - 1, length) + body += '\r\n\r\n' + ciphertext[start:end] + '\r\n' + body += '--multipartboundary--' + + # register request with fake swift + hdrs = self._make_response_headers( + len(body), plaintext_etag, fetch_crypto_keys(), 'not used') + hdrs['content-type'] = \ + 'multipart/byteranges;boundary=multipartboundary' + hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \ + get_crypto_meta_header(bad_crypto_meta) + self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs) + + # issue request + env = {'REQUEST_METHOD': 'GET', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env) + resp = req.get_response(self.decrypter) + + self.assertEqual('500 Internal Error', resp.status) + self.assertEqual('Error decrypting object', resp.body) + self.assertIn('Error decrypting object', + self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_GET_multipart_bad_body_cipher(self): + self._test_GET_multipart_bad_body_crypto_meta( + {'cipher': 'Mystery cipher', 'iv': '1234567887654321'}) + self.assertIn('Cipher must be AES_CTR_256', + self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_GET_multipart_missing_body_cipher(self): + self._test_GET_multipart_bad_body_crypto_meta( + {'iv': '1234567887654321'}) + self.assertIn('cipher', + self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_GET_multipart_too_short_body_iv(self): + self._test_GET_multipart_bad_body_crypto_meta( + {'cipher': 'AES_CTR_256', 'iv': 'too short'}) + self.assertIn('IV must be length 16', + self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_GET_multipart_too_long_body_iv(self): + self._test_GET_multipart_bad_body_crypto_meta( + {'cipher': 'AES_CTR_256', 'iv': 'a little too long'}) + self.assertIn('IV must be length 16', + self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_GET_multipart_missing_body_iv(self): + self._test_GET_multipart_bad_body_crypto_meta( + {'cipher': 'AES_CTR_256'}) + self.assertIn('iv', + self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_GET_missing_key_callback(self): + # Do not provide keys, and do not set override flag + env = {'REQUEST_METHOD': 'GET'} + req = Request.blank('/v1/a/c/o', environ=env) + body = 'FAKE APP' + enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV) + hdrs = self._make_response_headers( + len(body), md5hex('not the body'), fetch_crypto_keys(), 'not used') + self.app.register( + 'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs) + resp = req.get_response(self.decrypter) + self.assertEqual('500 Internal Error', resp.status) + self.assertEqual('Unable to retrieve encryption keys.', + resp.body) + self.assertIn('missing callback', + self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_GET_error_in_key_callback(self): + def raise_exc(): + raise Exception('Testing') + + env = {'REQUEST_METHOD': 'GET', + CRYPTO_KEY_CALLBACK: raise_exc} + req = Request.blank('/v1/a/c/o', environ=env) + body = 'FAKE APP' + enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV) + hdrs = self._make_response_headers( + len(body), md5hex(body), fetch_crypto_keys(), 'not used') + self.app.register( + 'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs) + resp = req.get_response(self.decrypter) + self.assertEqual('500 Internal Error', resp.status) + self.assertEqual('Unable to retrieve encryption keys.', + resp.body) + self.assertIn('from callback: Testing', + self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_GET_cipher_mismatch_for_body(self): + # Cipher does not match + env = {'REQUEST_METHOD': 'GET', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env) + body = 'FAKE APP' + enc_body = encrypt(body, fetch_crypto_keys()['object'], FAKE_IV) + bad_crypto_meta = fake_get_crypto_meta() + bad_crypto_meta['cipher'] = 'unknown_cipher' + hdrs = self._make_response_headers( + len(enc_body), md5hex(body), fetch_crypto_keys(), 'not used') + hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] = \ + get_crypto_meta_header(crypto_meta=bad_crypto_meta) + self.app.register( + 'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs) + resp = req.get_response(self.decrypter) + self.assertEqual('500 Internal Error', resp.status) + self.assertEqual('Error decrypting object', resp.body) + self.assertIn('Error decrypting object', + self.decrypter.logger.get_lines_for_level('error')[0]) + self.assertIn('Bad crypto meta: Cipher', + self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_GET_cipher_mismatch_for_metadata(self): + # Cipher does not match + env = {'REQUEST_METHOD': 'GET', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env) + body = 'FAKE APP' + key = fetch_crypto_keys()['object'] + enc_body = encrypt(body, key, FAKE_IV) + bad_crypto_meta = fake_get_crypto_meta() + bad_crypto_meta['cipher'] = 'unknown_cipher' + hdrs = self._make_response_headers( + len(enc_body), md5hex(body), fetch_crypto_keys(), 'not used') + hdrs.update({'x-object-transient-sysmeta-crypto-meta-test': + base64.b64encode(encrypt('encrypt me', key, FAKE_IV)) + + ';swift_meta=' + + get_crypto_meta_header(crypto_meta=bad_crypto_meta)}) + self.app.register( + 'GET', '/v1/a/c/o', HTTPOk, body=enc_body, headers=hdrs) + resp = req.get_response(self.decrypter) + self.assertEqual('500 Internal Error', resp.status) + self.assertEqual('Error decrypting header', resp.body) + self.assertIn( + 'Error decrypting header X-Object-Transient-Sysmeta-Crypto-Meta-' + 'Test', self.decrypter.logger.get_lines_for_level('error')[0]) + + def test_GET_decryption_override(self): + # This covers the case of an old un-encrypted object + env = {'REQUEST_METHOD': 'GET', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys, + 'swift.crypto.override': True} + req = Request.blank('/v1/a/c/o', environ=env) + body = 'FAKE APP' + hdrs = {'Etag': md5hex(body), + 'content-type': 'text/plain', + 'content-length': len(body), + 'x-object-meta-test': 'do not encrypt me', + 'x-object-sysmeta-test': 'do not encrypt me'} + self.app.register('GET', '/v1/a/c/o', HTTPOk, body=body, headers=hdrs) + resp = req.get_response(self.decrypter) + self.assertEqual(body, resp.body) + self.assertEqual('200 OK', resp.status) + self.assertEqual(md5hex(body), resp.headers['Etag']) + self.assertEqual('text/plain', resp.headers['Content-Type']) + self.assertEqual('do not encrypt me', + resp.headers['x-object-meta-test']) + self.assertEqual('do not encrypt me', + resp.headers['x-object-sysmeta-test']) + + +class TestDecrypterContainerRequests(unittest.TestCase): + def setUp(self): + self.app = FakeSwift() + self.decrypter = decrypter.Decrypter(self.app, {}) + self.decrypter.logger = FakeLogger() + + def _make_cont_get_req(self, resp_body, format, override=False, + callback=fetch_crypto_keys): + path = '/v1/a/c' + content_type = 'text/plain' + if format: + path = '%s/?format=%s' % (path, format) + content_type = 'application/' + format + env = {'REQUEST_METHOD': 'GET', + CRYPTO_KEY_CALLBACK: callback} + if override: + env['swift.crypto.override'] = True + req = Request.blank(path, environ=env) + hdrs = {'content-type': content_type} + self.app.register('GET', path, HTTPOk, body=resp_body, headers=hdrs) + return req.get_response(self.decrypter) + + def test_GET_container_success(self): + # no format requested, listing has names only + fake_body = 'testfile1\ntestfile2\n' + calls = [0] + + def wrapped_fetch_crypto_keys(): + calls[0] += 1 + return fetch_crypto_keys() + + resp = self._make_cont_get_req(fake_body, None, + callback=wrapped_fetch_crypto_keys) + + self.assertEqual('200 OK', resp.status) + names = resp.body.split('\n') + self.assertEqual(3, len(names)) + self.assertIn('testfile1', names) + self.assertIn('testfile2', names) + self.assertIn('', names) + self.assertEqual(0, calls[0]) + + def test_GET_container_json(self): + content_type_1 = u'\uF10F\uD20D\uB30B\u9409' + content_type_2 = 'text/plain; param=foo' + pt_etag1 = 'c6e8196d7f0fff6444b90861fe8d609d' + pt_etag2 = 'ac0374ed4d43635f803c82469d0b5a10' + key = fetch_crypto_keys()['container'] + + obj_dict_1 = {"bytes": 16, + "last_modified": "2015-04-14T23:33:06.439040", + "hash": encrypt_and_append_meta( + pt_etag1.encode('utf-8'), key), + "name": "testfile", + "content_type": content_type_1} + + obj_dict_2 = {"bytes": 24, + "last_modified": "2015-04-14T23:33:06.519020", + "hash": encrypt_and_append_meta( + pt_etag2.encode('utf-8'), key), + "name": "testfile2", + "content_type": content_type_2} + + listing = [obj_dict_1, obj_dict_2] + fake_body = json.dumps(listing) + + resp = self._make_cont_get_req(fake_body, 'json') + + self.assertEqual('200 OK', resp.status) + body = resp.body + self.assertEqual(len(body), int(resp.headers['Content-Length'])) + body_json = json.loads(body) + self.assertEqual(2, len(body_json)) + obj_dict_1['hash'] = pt_etag1 + self.assertDictEqual(obj_dict_1, body_json[0]) + obj_dict_2['hash'] = pt_etag2 + self.assertDictEqual(obj_dict_2, body_json[1]) + + def test_GET_container_json_with_crypto_override(self): + content_type_1 = 'image/jpeg' + content_type_2 = 'text/plain; param=foo' + pt_etag1 = 'c6e8196d7f0fff6444b90861fe8d609d' + pt_etag2 = 'ac0374ed4d43635f803c82469d0b5a10' + + obj_dict_1 = {"bytes": 16, + "last_modified": "2015-04-14T23:33:06.439040", + "hash": pt_etag1, + "name": "testfile", + "content_type": content_type_1} + + obj_dict_2 = {"bytes": 24, + "last_modified": "2015-04-14T23:33:06.519020", + "hash": pt_etag2, + "name": "testfile2", + "content_type": content_type_2} + + listing = [obj_dict_1, obj_dict_2] + fake_body = json.dumps(listing) + + resp = self._make_cont_get_req(fake_body, 'json', override=True) + + self.assertEqual('200 OK', resp.status) + body = resp.body + self.assertEqual(len(body), int(resp.headers['Content-Length'])) + body_json = json.loads(body) + self.assertEqual(2, len(body_json)) + self.assertDictEqual(obj_dict_1, body_json[0]) + self.assertDictEqual(obj_dict_2, body_json[1]) + + def test_cont_get_json_req_with_cipher_mismatch(self): + bad_crypto_meta = fake_get_crypto_meta() + bad_crypto_meta['cipher'] = 'unknown_cipher' + key = fetch_crypto_keys()['container'] + pt_etag = 'c6e8196d7f0fff6444b90861fe8d609d' + ct_etag = encrypt_and_append_meta(pt_etag, key, + crypto_meta=bad_crypto_meta) + + obj_dict_1 = {"bytes": 16, + "last_modified": "2015-04-14T23:33:06.439040", + "hash": ct_etag, + "name": "testfile", + "content_type": "image/jpeg"} + + listing = [obj_dict_1] + fake_body = json.dumps(listing) + + resp = self._make_cont_get_req(fake_body, 'json') + + self.assertEqual('500 Internal Error', resp.status) + self.assertEqual('Error decrypting container listing', resp.body) + self.assertIn("Cipher must be AES_CTR_256", + self.decrypter.logger.get_lines_for_level('error')[0]) + + def _assert_element_contains_dict(self, expected, element): + for k, v in expected.items(): + entry = element.getElementsByTagName(k) + self.assertIsNotNone(entry, 'Key %s not found' % k) + actual = entry[0].childNodes[0].nodeValue + self.assertEqual(v, actual, + "Expected %s but got %s for key %s" + % (v, actual, k)) + + def test_GET_container_xml(self): + content_type_1 = u'\uF10F\uD20D\uB30B\u9409' + content_type_2 = 'text/plain; param=foo' + pt_etag1 = 'c6e8196d7f0fff6444b90861fe8d609d' + pt_etag2 = 'ac0374ed4d43635f803c82469d0b5a10' + key = fetch_crypto_keys()['container'] + + fake_body = ''' +\ +\ +''' + encrypt_and_append_meta(pt_etag1.encode('utf8'), key) + '''\ +\ +''' + content_type_1 + '''\ +testfile16\ +2015-04-19T02:37:39.601660\ +\ +''' + encrypt_and_append_meta(pt_etag2.encode('utf8'), key) + '''\ +\ +''' + content_type_2 + '''\ +testfile224\ +2015-04-19T02:37:39.684740\ +''' + + resp = self._make_cont_get_req(fake_body, 'xml') + self.assertEqual('200 OK', resp.status) + body = resp.body + self.assertEqual(len(body), int(resp.headers['Content-Length'])) + + tree = minidom.parseString(body) + containers = tree.getElementsByTagName('container') + self.assertEqual(1, len(containers)) + self.assertEqual('testc', + containers[0].attributes.getNamedItem("name").value) + + objs = tree.getElementsByTagName('object') + self.assertEqual(2, len(objs)) + + obj_dict_1 = {"bytes": "16", + "last_modified": "2015-04-19T02:37:39.601660", + "hash": pt_etag1, + "name": "testfile", + "content_type": content_type_1} + self._assert_element_contains_dict(obj_dict_1, objs[0]) + obj_dict_2 = {"bytes": "24", + "last_modified": "2015-04-19T02:37:39.684740", + "hash": pt_etag2, + "name": "testfile2", + "content_type": content_type_2} + self._assert_element_contains_dict(obj_dict_2, objs[1]) + + def test_GET_container_xml_with_crypto_override(self): + content_type_1 = 'image/jpeg' + content_type_2 = 'text/plain; param=foo' + + fake_body = ''' +\ +c6e8196d7f0fff6444b90861fe8d609d\ +''' + content_type_1 + '''\ +testfile16\ +2015-04-19T02:37:39.601660\ +ac0374ed4d43635f803c82469d0b5a10\ +''' + content_type_2 + '''\ +testfile224\ +2015-04-19T02:37:39.684740\ +''' + + resp = self._make_cont_get_req(fake_body, 'xml', override=True) + + self.assertEqual('200 OK', resp.status) + body = resp.body + self.assertEqual(len(body), int(resp.headers['Content-Length'])) + + tree = minidom.parseString(body) + containers = tree.getElementsByTagName('container') + self.assertEqual(1, len(containers)) + self.assertEqual('testc', + containers[0].attributes.getNamedItem("name").value) + + objs = tree.getElementsByTagName('object') + self.assertEqual(2, len(objs)) + + obj_dict_1 = {"bytes": "16", + "last_modified": "2015-04-19T02:37:39.601660", + "hash": "c6e8196d7f0fff6444b90861fe8d609d", + "name": "testfile", + "content_type": content_type_1} + self._assert_element_contains_dict(obj_dict_1, objs[0]) + obj_dict_2 = {"bytes": "24", + "last_modified": "2015-04-19T02:37:39.684740", + "hash": "ac0374ed4d43635f803c82469d0b5a10", + "name": "testfile2", + "content_type": content_type_2} + self._assert_element_contains_dict(obj_dict_2, objs[1]) + + def test_cont_get_xml_req_with_cipher_mismatch(self): + bad_crypto_meta = fake_get_crypto_meta() + bad_crypto_meta['cipher'] = 'unknown_cipher' + + fake_body = ''' +\ +''' + encrypt_and_append_meta('c6e8196d7f0fff6444b90861fe8d609d', + fetch_crypto_keys()['container'], + crypto_meta=bad_crypto_meta) + '''\ +\ +image/jpeg\ +testfile16\ +2015-04-19T02:37:39.601660\ +''' + + resp = self._make_cont_get_req(fake_body, 'xml') + + self.assertEqual('500 Internal Error', resp.status) + self.assertEqual('Error decrypting container listing', resp.body) + self.assertIn("Cipher must be AES_CTR_256", + self.decrypter.logger.get_lines_for_level('error')[0]) + + +class TestModuleMethods(unittest.TestCase): + def test_purge_crypto_sysmeta_headers(self): + retained_headers = {'x-object-sysmeta-test1': 'keep', + 'x-object-meta-test2': 'retain', + 'x-object-transient-sysmeta-test3': 'leave intact', + 'etag': 'hold onto', + 'x-other': 'cherish', + 'x-object-not-meta': 'do not remove'} + purged_headers = {'x-object-sysmeta-crypto-test1': 'remove', + 'x-object-transient-sysmeta-crypto-test2': 'purge'} + test_headers = retained_headers.copy() + test_headers.update(purged_headers) + actual = decrypter.purge_crypto_sysmeta_headers(test_headers.items()) + + for k, v in actual: + k = k.lower() + self.assertNotIn(k, purged_headers) + self.assertEqual(retained_headers[k], v) + retained_headers.pop(k) + self.assertFalse(retained_headers) + + +class TestDecrypter(unittest.TestCase): + def test_app_exception(self): + app = decrypter.Decrypter(FakeAppThatExcepts(HTTPException), {}) + req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) + with self.assertRaises(HTTPException) as catcher: + req.get_response(app) + self.assertEqual(FakeAppThatExcepts.MESSAGE, catcher.exception.body) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/middleware/crypto/test_encrypter.py b/test/unit/common/middleware/crypto/test_encrypter.py new file mode 100644 index 00000000..0f9553ca --- /dev/null +++ b/test/unit/common/middleware/crypto/test_encrypter.py @@ -0,0 +1,820 @@ +# Copyright (c) 2015-2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import base64 +import hashlib +import hmac +import json +import os +import unittest +import urllib + +import mock + +from swift.common.middleware.crypto import encrypter +from swift.common.middleware.crypto.crypto_utils import ( + CRYPTO_KEY_CALLBACK, Crypto) +from swift.common.swob import ( + Request, HTTPException, HTTPCreated, HTTPAccepted, HTTPOk, HTTPBadRequest) +from swift.common.utils import FileLikeIter + +from test.unit import FakeLogger, EMPTY_ETAG +from test.unit.common.middleware.crypto.crypto_helpers import ( + fetch_crypto_keys, md5hex, FAKE_IV, encrypt) +from test.unit.common.middleware.helpers import FakeSwift, FakeAppThatExcepts + + +@mock.patch('swift.common.middleware.crypto.crypto_utils.Crypto.create_iv', + lambda *args: FAKE_IV) +class TestEncrypter(unittest.TestCase): + def setUp(self): + self.app = FakeSwift() + self.encrypter = encrypter.Encrypter(self.app, {}) + self.encrypter.logger = FakeLogger() + + def _verify_user_metadata(self, req_hdrs, name, value, key): + # verify encrypted version of user metadata + self.assertNotIn('X-Object-Meta-' + name, req_hdrs) + expected_hdr = 'X-Object-Transient-Sysmeta-Crypto-Meta-' + name + self.assertIn(expected_hdr, req_hdrs) + enc_val, param = req_hdrs[expected_hdr].split(';') + param = param.strip() + self.assertTrue(param.startswith('swift_meta=')) + actual_meta = json.loads( + urllib.unquote_plus(param[len('swift_meta='):])) + self.assertEqual(Crypto.cipher, actual_meta['cipher']) + meta_iv = base64.b64decode(actual_meta['iv']) + self.assertEqual(FAKE_IV, meta_iv) + self.assertEqual( + base64.b64encode(encrypt(value, key, meta_iv)), + enc_val) + # if there is any encrypted user metadata then this header should exist + self.assertIn('X-Object-Transient-Sysmeta-Crypto-Meta', req_hdrs) + common_meta = json.loads(urllib.unquote_plus( + req_hdrs['X-Object-Transient-Sysmeta-Crypto-Meta'])) + self.assertDictEqual({'cipher': Crypto.cipher, + 'key_id': {'v': 'fake', 'path': '/a/c/fake'}}, + common_meta) + + def test_PUT_req(self): + body_key = os.urandom(32) + object_key = fetch_crypto_keys()['object'] + plaintext = 'FAKE APP' + plaintext_etag = md5hex(plaintext) + ciphertext = encrypt(plaintext, body_key, FAKE_IV) + ciphertext_etag = md5hex(ciphertext) + + env = {'REQUEST_METHOD': 'PUT', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + hdrs = {'etag': plaintext_etag, + 'content-type': 'text/plain', + 'content-length': str(len(plaintext)), + 'x-object-meta-etag': 'not to be confused with the Etag!', + 'x-object-meta-test': 'encrypt me', + 'x-object-sysmeta-test': 'do not encrypt me'} + req = Request.blank( + '/v1/a/c/o', environ=env, body=plaintext, headers=hdrs) + self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {}) + with mock.patch( + 'swift.common.middleware.crypto.crypto_utils.' + 'Crypto.create_random_key', + return_value=body_key): + resp = req.get_response(self.encrypter) + self.assertEqual('201 Created', resp.status) + self.assertEqual(plaintext_etag, resp.headers['Etag']) + + # verify metadata items + self.assertEqual(1, len(self.app.calls), self.app.calls) + self.assertEqual('PUT', self.app.calls[0][0]) + req_hdrs = self.app.headers[0] + + # verify body crypto meta + actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] + actual = json.loads(urllib.unquote_plus(actual)) + self.assertEqual(Crypto().cipher, actual['cipher']) + self.assertEqual(FAKE_IV, base64.b64decode(actual['iv'])) + + # verify wrapped body key + expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV) + self.assertEqual(expected_wrapped_key, + base64.b64decode(actual['body_key']['key'])) + self.assertEqual(FAKE_IV, + base64.b64decode(actual['body_key']['iv'])) + self.assertEqual(fetch_crypto_keys()['id'], actual['key_id']) + + # verify etag + self.assertEqual(ciphertext_etag, req_hdrs['Etag']) + + encrypted_etag, _junk, etag_meta = \ + req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=') + # verify crypto_meta was appended to this etag + self.assertTrue(etag_meta) + actual_meta = json.loads(urllib.unquote_plus(etag_meta)) + self.assertEqual(Crypto().cipher, actual_meta['cipher']) + + # verify encrypted version of plaintext etag + actual = base64.b64decode(encrypted_etag) + etag_iv = base64.b64decode(actual_meta['iv']) + enc_etag = encrypt(plaintext_etag, object_key, etag_iv) + self.assertEqual(enc_etag, actual) + + # verify etag MAC for conditional requests + actual_hmac = base64.b64decode( + req_hdrs['X-Object-Sysmeta-Crypto-Etag-Mac']) + self.assertEqual(actual_hmac, hmac.new( + object_key, plaintext_etag, hashlib.sha256).digest()) + + # verify encrypted etag for container update + self.assertIn( + 'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs) + parts = req_hdrs[ + 'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1) + self.assertEqual(2, len(parts)) + + # extract crypto_meta from end of etag for container update + param = parts[1].strip() + crypto_meta_tag = 'swift_meta=' + self.assertTrue(param.startswith(crypto_meta_tag), param) + actual_meta = json.loads( + urllib.unquote_plus(param[len(crypto_meta_tag):])) + self.assertEqual(Crypto().cipher, actual_meta['cipher']) + self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id']) + + cont_key = fetch_crypto_keys()['container'] + cont_etag_iv = base64.b64decode(actual_meta['iv']) + self.assertEqual(FAKE_IV, cont_etag_iv) + self.assertEqual(encrypt(plaintext_etag, cont_key, cont_etag_iv), + base64.b64decode(parts[0])) + + # content-type is not encrypted + self.assertEqual('text/plain', req_hdrs['Content-Type']) + + # user meta is encrypted + self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', object_key) + self._verify_user_metadata( + req_hdrs, 'Etag', 'not to be confused with the Etag!', object_key) + + # sysmeta is not encrypted + self.assertEqual('do not encrypt me', + req_hdrs['X-Object-Sysmeta-Test']) + + # verify object is encrypted by getting direct from the app + get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + resp = get_req.get_response(self.app) + self.assertEqual(ciphertext, resp.body) + self.assertEqual(ciphertext_etag, resp.headers['Etag']) + + def test_PUT_zero_size_object(self): + # object body encryption should be skipped for zero sized object body + object_key = fetch_crypto_keys()['object'] + plaintext_etag = EMPTY_ETAG + + env = {'REQUEST_METHOD': 'PUT', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + hdrs = {'etag': EMPTY_ETAG, + 'content-type': 'text/plain', + 'content-length': '0', + 'x-object-meta-etag': 'not to be confused with the Etag!', + 'x-object-meta-test': 'encrypt me', + 'x-object-sysmeta-test': 'do not encrypt me'} + req = Request.blank( + '/v1/a/c/o', environ=env, body='', headers=hdrs) + self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {}) + + resp = req.get_response(self.encrypter) + + self.assertEqual('201 Created', resp.status) + self.assertEqual(plaintext_etag, resp.headers['Etag']) + self.assertEqual(1, len(self.app.calls), self.app.calls) + self.assertEqual('PUT', self.app.calls[0][0]) + req_hdrs = self.app.headers[0] + + # verify that there is no body crypto meta + self.assertNotIn('X-Object-Sysmeta-Crypto-Meta', req_hdrs) + # verify etag is md5 of plaintext + self.assertEqual(EMPTY_ETAG, req_hdrs['Etag']) + # verify there is no etag crypto meta + self.assertNotIn('X-Object-Sysmeta-Crypto-Etag', req_hdrs) + # verify there is no container update override for etag + self.assertNotIn( + 'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs) + + # user meta is still encrypted + self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', object_key) + self._verify_user_metadata( + req_hdrs, 'Etag', 'not to be confused with the Etag!', object_key) + + # sysmeta is not encrypted + self.assertEqual('do not encrypt me', + req_hdrs['X-Object-Sysmeta-Test']) + + # verify object is empty by getting direct from the app + get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + resp = get_req.get_response(self.app) + self.assertEqual('', resp.body) + self.assertEqual(EMPTY_ETAG, resp.headers['Etag']) + + def test_PUT_with_other_footers(self): + # verify handling of another middleware's footer callback + cont_key = fetch_crypto_keys()['container'] + body_key = os.urandom(32) + object_key = fetch_crypto_keys()['object'] + plaintext = 'FAKE APP' + plaintext_etag = md5hex(plaintext) + ciphertext = encrypt(plaintext, body_key, FAKE_IV) + ciphertext_etag = md5hex(ciphertext) + other_footers = { + 'Etag': plaintext_etag, + 'X-Object-Sysmeta-Other': 'other sysmeta', + 'X-Object-Sysmeta-Container-Update-Override-Size': + 'other override', + 'X-Object-Sysmeta-Container-Update-Override-Etag': + 'final etag'} + + env = {'REQUEST_METHOD': 'PUT', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys, + 'swift.callback.update_footers': + lambda footers: footers.update(other_footers)} + hdrs = {'content-type': 'text/plain', + 'content-length': str(len(plaintext)), + 'Etag': 'correct etag is in footers'} + req = Request.blank( + '/v1/a/c/o', environ=env, body=plaintext, headers=hdrs) + self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {}) + + with mock.patch( + 'swift.common.middleware.crypto.crypto_utils.' + 'Crypto.create_random_key', + lambda *args: body_key): + resp = req.get_response(self.encrypter) + + self.assertEqual('201 Created', resp.status) + self.assertEqual(plaintext_etag, resp.headers['Etag']) + + # verify metadata items + self.assertEqual(1, len(self.app.calls), self.app.calls) + self.assertEqual('PUT', self.app.calls[0][0]) + req_hdrs = self.app.headers[0] + + # verify that other middleware's footers made it to app, including any + # container update overrides but nothing Etag-related + other_footers.pop('Etag') + other_footers.pop('X-Object-Sysmeta-Container-Update-Override-Etag') + for k, v in other_footers.items(): + self.assertEqual(v, req_hdrs[k]) + + # verify encryption footers are ok + encrypted_etag, _junk, etag_meta = \ + req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=') + self.assertTrue(etag_meta) + actual_meta = json.loads(urllib.unquote_plus(etag_meta)) + self.assertEqual(Crypto().cipher, actual_meta['cipher']) + + self.assertEqual(ciphertext_etag, req_hdrs['Etag']) + actual = base64.b64decode(encrypted_etag) + etag_iv = base64.b64decode(actual_meta['iv']) + self.assertEqual(encrypt(plaintext_etag, object_key, etag_iv), actual) + + # verify encrypted etag for container update + self.assertIn( + 'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs) + parts = req_hdrs[ + 'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1) + self.assertEqual(2, len(parts)) + + # extract crypto_meta from end of etag for container update + param = parts[1].strip() + crypto_meta_tag = 'swift_meta=' + self.assertTrue(param.startswith(crypto_meta_tag), param) + actual_meta = json.loads( + urllib.unquote_plus(param[len(crypto_meta_tag):])) + self.assertEqual(Crypto().cipher, actual_meta['cipher']) + + cont_key = fetch_crypto_keys()['container'] + cont_etag_iv = base64.b64decode(actual_meta['iv']) + self.assertEqual(FAKE_IV, cont_etag_iv) + self.assertEqual(encrypt('final etag', cont_key, cont_etag_iv), + base64.b64decode(parts[0])) + + # verify body crypto meta + actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta'] + actual = json.loads(urllib.unquote_plus(actual)) + self.assertEqual(Crypto().cipher, actual['cipher']) + self.assertEqual(FAKE_IV, base64.b64decode(actual['iv'])) + + # verify wrapped body key + expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV) + self.assertEqual(expected_wrapped_key, + base64.b64decode(actual['body_key']['key'])) + self.assertEqual(FAKE_IV, + base64.b64decode(actual['body_key']['iv'])) + self.assertEqual(fetch_crypto_keys()['id'], actual['key_id']) + + def test_PUT_with_etag_override_in_headers(self): + # verify handling of another middleware's + # container-update-override-etag in headers + plaintext = 'FAKE APP' + plaintext_etag = md5hex(plaintext) + + env = {'REQUEST_METHOD': 'PUT', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + hdrs = {'content-type': 'text/plain', + 'content-length': str(len(plaintext)), + 'Etag': plaintext_etag, + 'X-Object-Sysmeta-Container-Update-Override-Etag': + 'final etag'} + req = Request.blank( + '/v1/a/c/o', environ=env, body=plaintext, headers=hdrs) + self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {}) + resp = req.get_response(self.encrypter) + + self.assertEqual('201 Created', resp.status) + self.assertEqual(plaintext_etag, resp.headers['Etag']) + + # verify metadata items + self.assertEqual(1, len(self.app.calls), self.app.calls) + self.assertEqual(('PUT', '/v1/a/c/o'), self.app.calls[0]) + req_hdrs = self.app.headers[0] + + # verify encrypted etag for container update + self.assertIn( + 'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs) + parts = req_hdrs[ + 'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1) + self.assertEqual(2, len(parts)) + cont_key = fetch_crypto_keys()['container'] + + # extract crypto_meta from end of etag for container update + param = parts[1].strip() + crypto_meta_tag = 'swift_meta=' + self.assertTrue(param.startswith(crypto_meta_tag), param) + actual_meta = json.loads( + urllib.unquote_plus(param[len(crypto_meta_tag):])) + self.assertEqual(Crypto().cipher, actual_meta['cipher']) + self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id']) + + cont_etag_iv = base64.b64decode(actual_meta['iv']) + self.assertEqual(FAKE_IV, cont_etag_iv) + self.assertEqual(encrypt('final etag', cont_key, cont_etag_iv), + base64.b64decode(parts[0])) + + def test_PUT_with_bad_etag_in_other_footers(self): + # verify that etag supplied in footers from other middleware overrides + # header etag when validating inbound plaintext etags + plaintext = 'FAKE APP' + plaintext_etag = md5hex(plaintext) + other_footers = { + 'Etag': 'bad etag', + 'X-Object-Sysmeta-Other': 'other sysmeta', + 'X-Object-Sysmeta-Container-Update-Override-Etag': + 'other override'} + + env = {'REQUEST_METHOD': 'PUT', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys, + 'swift.callback.update_footers': + lambda footers: footers.update(other_footers)} + hdrs = {'content-type': 'text/plain', + 'content-length': str(len(plaintext)), + 'Etag': plaintext_etag} + req = Request.blank( + '/v1/a/c/o', environ=env, body=plaintext, headers=hdrs) + self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {}) + resp = req.get_response(self.encrypter) + self.assertEqual('422 Unprocessable Entity', resp.status) + self.assertNotIn('Etag', resp.headers) + + def test_PUT_with_bad_etag_in_headers_and_other_footers(self): + # verify that etag supplied in headers from other middleware is used if + # none is supplied in footers when validating inbound plaintext etags + plaintext = 'FAKE APP' + other_footers = { + 'X-Object-Sysmeta-Other': 'other sysmeta', + 'X-Object-Sysmeta-Container-Update-Override-Etag': + 'other override'} + + env = {'REQUEST_METHOD': 'PUT', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys, + 'swift.callback.update_footers': + lambda footers: footers.update(other_footers)} + hdrs = {'content-type': 'text/plain', + 'content-length': str(len(plaintext)), + 'Etag': 'bad etag'} + req = Request.blank( + '/v1/a/c/o', environ=env, body=plaintext, headers=hdrs) + self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {}) + resp = req.get_response(self.encrypter) + self.assertEqual('422 Unprocessable Entity', resp.status) + self.assertNotIn('Etag', resp.headers) + + def test_PUT_nothing_read(self): + # simulate an artificial scenario of a downstream filter/app not + # actually reading the input stream from encrypter. + class NonReadingApp(object): + def __call__(self, env, start_response): + # note: no read from wsgi.input + req = Request(env) + env['swift.callback.update_footers'](req.headers) + call_headers.append(req.headers) + resp = HTTPCreated(req=req, headers={'Etag': 'response etag'}) + return resp(env, start_response) + + env = {'REQUEST_METHOD': 'PUT', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + hdrs = {'content-type': 'text/plain', + 'content-length': 0, + 'etag': 'etag from client'} + req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs) + + call_headers = [] + resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {})) + self.assertEqual('201 Created', resp.status) + self.assertEqual('response etag', resp.headers['Etag']) + self.assertEqual(1, len(call_headers)) + self.assertEqual('etag from client', call_headers[0]['etag']) + # verify no encryption footers + for k in call_headers[0]: + self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-')) + + # check that an upstream footer callback gets called + other_footers = { + 'Etag': 'other etag', + 'X-Object-Sysmeta-Other': 'other sysmeta', + 'X-Backend-Container-Update-Override-Etag': 'other override'} + env.update({'swift.callback.update_footers': + lambda footers: footers.update(other_footers)}) + req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs) + + call_headers = [] + resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {})) + + self.assertEqual('201 Created', resp.status) + self.assertEqual('response etag', resp.headers['Etag']) + self.assertEqual(1, len(call_headers)) + # verify that other middleware's footers made it to app + for k, v in other_footers.items(): + self.assertEqual(v, call_headers[0][k]) + # verify no encryption footers + for k in call_headers[0]: + self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-')) + + def test_POST_req(self): + body = 'FAKE APP' + env = {'REQUEST_METHOD': 'POST', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + hdrs = {'x-object-meta-test': 'encrypt me', + 'x-object-sysmeta-test': 'do not encrypt me'} + req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs) + key = fetch_crypto_keys()['object'] + self.app.register('POST', '/v1/a/c/o', HTTPAccepted, {}) + resp = req.get_response(self.encrypter) + self.assertEqual('202 Accepted', resp.status) + self.assertNotIn('Etag', resp.headers) + + # verify metadata items + self.assertEqual(1, len(self.app.calls), self.app.calls) + self.assertEqual('POST', self.app.calls[0][0]) + req_hdrs = self.app.headers[0] + + # user meta is encrypted + self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', key) + + # sysmeta is not encrypted + self.assertEqual('do not encrypt me', + req_hdrs['X-Object-Sysmeta-Test']) + + def _test_no_user_metadata(self, method): + # verify that x-object-transient-sysmeta-crypto-meta is not set when + # there is no user metadata + env = {'REQUEST_METHOD': method, + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank('/v1/a/c/o', environ=env, body='body') + self.app.register(method, '/v1/a/c/o', HTTPAccepted, {}) + resp = req.get_response(self.encrypter) + self.assertEqual('202 Accepted', resp.status) + self.assertEqual(1, len(self.app.calls), self.app.calls) + self.assertEqual(method, self.app.calls[0][0]) + self.assertNotIn('x-object-transient-sysmeta-crypto-meta', + self.app.headers[0]) + + def test_PUT_no_user_metadata(self): + self._test_no_user_metadata('PUT') + + def test_POST_no_user_metadata(self): + self._test_no_user_metadata('POST') + + def _test_if_match(self, method, match_header_name): + def do_test(method, plain_etags, expected_plain_etags=None): + env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + match_header_value = ', '.join(plain_etags) + req = Request.blank( + '/v1/a/c/o', environ=env, method=method, + headers={match_header_name: match_header_value}) + app = FakeSwift() + app.register(method, '/v1/a/c/o', HTTPOk, {}) + resp = req.get_response(encrypter.Encrypter(app, {})) + self.assertEqual('200 OK', resp.status) + + self.assertEqual(1, len(app.calls), app.calls) + self.assertEqual(method, app.calls[0][0]) + actual_headers = app.headers[0] + + # verify the alternate etag location has been specified + if match_header_value and match_header_value != '*': + self.assertIn('X-Backend-Etag-Is-At', actual_headers) + self.assertEqual('X-Object-Sysmeta-Crypto-Etag-Mac', + actual_headers['X-Backend-Etag-Is-At']) + + # verify etags have been supplemented with masked values + self.assertIn(match_header_name, actual_headers) + actual_etags = set(actual_headers[match_header_name].split(', ')) + key = fetch_crypto_keys()['object'] + masked_etags = [ + '"%s"' % base64.b64encode(hmac.new( + key, etag.strip('"'), hashlib.sha256).digest()) + for etag in plain_etags if etag not in ('*', '')] + expected_etags = set((expected_plain_etags or plain_etags) + + masked_etags) + self.assertEqual(expected_etags, actual_etags) + # check that the request environ was returned to original state + self.assertEqual(set(plain_etags), + set(req.headers[match_header_name].split(', '))) + + do_test(method, ['']) + do_test(method, ['"an etag"']) + do_test(method, ['"an etag"', '"another_etag"']) + do_test(method, ['*']) + # rfc2616 does not allow wildcard *and* etag but test it anyway + do_test(method, ['*', '"an etag"']) + # etags should be quoted but check we can cope if they are not + do_test( + method, ['*', 'an etag', 'another_etag'], + expected_plain_etags=['*', '"an etag"', '"another_etag"']) + + def test_GET_if_match(self): + self._test_if_match('GET', 'If-Match') + + def test_HEAD_if_match(self): + self._test_if_match('HEAD', 'If-Match') + + def test_GET_if_none_match(self): + self._test_if_match('GET', 'If-None-Match') + + def test_HEAD_if_none_match(self): + self._test_if_match('HEAD', 'If-None-Match') + + def _test_existing_etag_is_at_header(self, method, match_header_name): + # if another middleware has already set X-Backend-Etag-Is-At then + # encrypter should not override that value + env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank( + '/v1/a/c/o', environ=env, method=method, + headers={match_header_name: "an etag", + 'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Other-Etag'}) + self.app.register(method, '/v1/a/c/o', HTTPOk, {}) + resp = req.get_response(self.encrypter) + self.assertEqual('200 OK', resp.status) + + self.assertEqual(1, len(self.app.calls), self.app.calls) + self.assertEqual(method, self.app.calls[0][0]) + actual_headers = self.app.headers[0] + self.assertIn('X-Backend-Etag-Is-At', actual_headers) + self.assertEqual( + 'X-Object-Sysmeta-Other-Etag,X-Object-Sysmeta-Crypto-Etag-Mac', + actual_headers['X-Backend-Etag-Is-At']) + actual_etags = set(actual_headers[match_header_name].split(', ')) + self.assertIn('"an etag"', actual_etags) + + def test_GET_if_match_with_existing_etag_is_at_header(self): + self._test_existing_etag_is_at_header('GET', 'If-Match') + + def test_HEAD_if_match_with_existing_etag_is_at_header(self): + self._test_existing_etag_is_at_header('HEAD', 'If-Match') + + def test_GET_if_none_match_with_existing_etag_is_at_header(self): + self._test_existing_etag_is_at_header('GET', 'If-None-Match') + + def test_HEAD_if_none_match_with_existing_etag_is_at_header(self): + self._test_existing_etag_is_at_header('HEAD', 'If-None-Match') + + def _test_etag_is_at_not_duplicated(self, method): + # verify only one occurrence of X-Object-Sysmeta-Crypto-Etag-Mac in + # X-Backend-Etag-Is-At + key = fetch_crypto_keys()['object'] + env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + req = Request.blank( + '/v1/a/c/o', environ=env, method=method, + headers={'If-Match': '"an etag"', + 'If-None-Match': '"another etag"'}) + self.app.register(method, '/v1/a/c/o', HTTPOk, {}) + resp = req.get_response(self.encrypter) + self.assertEqual('200 OK', resp.status) + + self.assertEqual(1, len(self.app.calls), self.app.calls) + self.assertEqual(method, self.app.calls[0][0]) + actual_headers = self.app.headers[0] + self.assertIn('X-Backend-Etag-Is-At', actual_headers) + self.assertEqual('X-Object-Sysmeta-Crypto-Etag-Mac', + actual_headers['X-Backend-Etag-Is-At']) + + self.assertIn('"%s"' % base64.b64encode( + hmac.new(key, 'an etag', hashlib.sha256).digest()), + actual_headers['If-Match']) + self.assertIn('"another etag"', actual_headers['If-None-Match']) + self.assertIn('"%s"' % base64.b64encode( + hmac.new(key, 'another etag', hashlib.sha256).digest()), + actual_headers['If-None-Match']) + + def test_GET_etag_is_at_not_duplicated(self): + self._test_etag_is_at_not_duplicated('GET') + + def test_HEAD_etag_is_at_not_duplicated(self): + self._test_etag_is_at_not_duplicated('HEAD') + + def test_PUT_response_inconsistent_etag_is_not_replaced(self): + # if response is success but etag does not match the ciphertext md5 + # then verify that we do *not* replace it with the plaintext etag + body = 'FAKE APP' + env = {'REQUEST_METHOD': 'PUT', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + hdrs = {'content-type': 'text/plain', + 'content-length': str(len(body))} + req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs) + self.app.register('PUT', '/v1/a/c/o', HTTPCreated, + {'Etag': 'not the ciphertext etag'}) + resp = req.get_response(self.encrypter) + self.assertEqual('201 Created', resp.status) + self.assertEqual('not the ciphertext etag', resp.headers['Etag']) + + def test_PUT_multiseg_no_client_etag(self): + body_key = os.urandom(32) + chunks = ['some', 'chunks', 'of data'] + body = ''.join(chunks) + env = {'REQUEST_METHOD': 'PUT', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys, + 'wsgi.input': FileLikeIter(chunks)} + hdrs = {'content-type': 'text/plain', + 'content-length': str(len(body))} + req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs) + self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {}) + + with mock.patch( + 'swift.common.middleware.crypto.crypto_utils.' + 'Crypto.create_random_key', + lambda *args: body_key): + resp = req.get_response(self.encrypter) + + self.assertEqual('201 Created', resp.status) + # verify object is encrypted by getting direct from the app + get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + self.assertEqual(encrypt(body, body_key, FAKE_IV), + get_req.get_response(self.app).body) + + def test_PUT_multiseg_good_client_etag(self): + body_key = os.urandom(32) + chunks = ['some', 'chunks', 'of data'] + body = ''.join(chunks) + env = {'REQUEST_METHOD': 'PUT', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys, + 'wsgi.input': FileLikeIter(chunks)} + hdrs = {'content-type': 'text/plain', + 'content-length': str(len(body)), + 'Etag': md5hex(body)} + req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs) + self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {}) + + with mock.patch( + 'swift.common.middleware.crypto.crypto_utils.' + 'Crypto.create_random_key', + lambda *args: body_key): + resp = req.get_response(self.encrypter) + + self.assertEqual('201 Created', resp.status) + # verify object is encrypted by getting direct from the app + get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + self.assertEqual(encrypt(body, body_key, FAKE_IV), + get_req.get_response(self.app).body) + + def test_PUT_multiseg_bad_client_etag(self): + chunks = ['some', 'chunks', 'of data'] + body = ''.join(chunks) + env = {'REQUEST_METHOD': 'PUT', + CRYPTO_KEY_CALLBACK: fetch_crypto_keys, + 'wsgi.input': FileLikeIter(chunks)} + hdrs = {'content-type': 'text/plain', + 'content-length': str(len(body)), + 'Etag': 'badclientetag'} + req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs) + self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {}) + resp = req.get_response(self.encrypter) + self.assertEqual('422 Unprocessable Entity', resp.status) + + def test_PUT_missing_key_callback(self): + body = 'FAKE APP' + env = {'REQUEST_METHOD': 'PUT'} + hdrs = {'content-type': 'text/plain', + 'content-length': str(len(body))} + req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs) + resp = req.get_response(self.encrypter) + self.assertEqual('500 Internal Error', resp.status) + self.assertIn('missing callback', + self.encrypter.logger.get_lines_for_level('error')[0]) + self.assertEqual('Unable to retrieve encryption keys.', resp.body) + + def test_PUT_error_in_key_callback(self): + def raise_exc(): + raise Exception('Testing') + + body = 'FAKE APP' + env = {'REQUEST_METHOD': 'PUT', + CRYPTO_KEY_CALLBACK: raise_exc} + hdrs = {'content-type': 'text/plain', + 'content-length': str(len(body))} + req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs) + resp = req.get_response(self.encrypter) + self.assertEqual('500 Internal Error', resp.status) + self.assertIn('from callback: Testing', + self.encrypter.logger.get_lines_for_level('error')[0]) + self.assertEqual('Unable to retrieve encryption keys.', resp.body) + + def test_PUT_encryption_override(self): + # set crypto override to disable encryption. + # simulate another middleware wanting to set footers + other_footers = { + 'Etag': 'other etag', + 'X-Object-Sysmeta-Other': 'other sysmeta', + 'X-Object-Sysmeta-Container-Update-Override-Etag': + 'other override'} + body = 'FAKE APP' + env = {'REQUEST_METHOD': 'PUT', + 'swift.crypto.override': True, + 'swift.callback.update_footers': + lambda footers: footers.update(other_footers)} + hdrs = {'content-type': 'text/plain', + 'content-length': str(len(body))} + req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs) + self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {}) + resp = req.get_response(self.encrypter) + self.assertEqual('201 Created', resp.status) + + # verify that other middleware's footers made it to app + req_hdrs = self.app.headers[0] + for k, v in other_footers.items(): + self.assertEqual(v, req_hdrs[k]) + + # verify object is NOT encrypted by getting direct from the app + get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) + self.assertEqual(body, get_req.get_response(self.app).body) + + def _test_constraints_checking(self, method): + # verify that the check_metadata function is called on PUT and POST + body = 'FAKE APP' + env = {'REQUEST_METHOD': method, + CRYPTO_KEY_CALLBACK: fetch_crypto_keys} + hdrs = {'content-type': 'text/plain', + 'content-length': str(len(body))} + req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs) + mocked_func = 'swift.common.middleware.crypto.encrypter.check_metadata' + with mock.patch(mocked_func) as mocked: + mocked.side_effect = [HTTPBadRequest('testing')] + resp = req.get_response(self.encrypter) + self.assertEqual('400 Bad Request', resp.status) + self.assertEqual(1, mocked.call_count) + mocked.assert_called_once_with(mock.ANY, 'object') + self.assertEqual(req.headers, + mocked.call_args_list[0][0][0].headers) + + def test_PUT_constraints_checking(self): + self._test_constraints_checking('PUT') + + def test_POST_constraints_checking(self): + self._test_constraints_checking('POST') + + def test_config_true_value_on_disable_encryption(self): + app = FakeSwift() + self.assertFalse(encrypter.Encrypter(app, {}).disable_encryption) + for val in ('true', '1', 'yes', 'on', 't', 'y'): + app = encrypter.Encrypter(app, + {'disable_encryption': val}) + self.assertTrue(app.disable_encryption) + + def test_PUT_app_exception(self): + app = encrypter.Encrypter(FakeAppThatExcepts(HTTPException), {}) + req = Request.blank('/', environ={'REQUEST_METHOD': 'PUT'}) + with self.assertRaises(HTTPException) as catcher: + req.get_response(app) + self.assertEqual(FakeAppThatExcepts.MESSAGE, catcher.exception.body) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/middleware/crypto/test_encryption.py b/test/unit/common/middleware/crypto/test_encryption.py new file mode 100644 index 00000000..e984a5f0 --- /dev/null +++ b/test/unit/common/middleware/crypto/test_encryption.py @@ -0,0 +1,631 @@ +# Copyright (c) 2015-2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import base64 +import hashlib +import hmac +import json +import unittest +import uuid + +from swift.common import storage_policy, constraints +from swift.common.middleware import copy +from swift.common.middleware import crypto +from swift.common.middleware.crypto import keymaster +from swift.common.middleware.crypto.crypto_utils import ( + load_crypto_meta, Crypto) +from swift.common.ring import Ring +from swift.common.swob import Request +from swift.obj import diskfile + +from test.unit import FakeLogger +from test.unit.common.middleware.crypto.crypto_helpers import ( + md5hex, encrypt, TEST_KEYMASTER_CONF) +from test.unit.helpers import setup_servers, teardown_servers + + +class TestCryptoPipelineChanges(unittest.TestCase): + # Tests the consequences of crypto middleware being in/out of the pipeline + # or having encryption disabled for PUT/GET requests on same object. Uses + # real backend servers so that the handling of headers and sysmeta is + # verified to diskfile and back. + _test_context = None + + @classmethod + def setUpClass(cls): + cls._test_context = setup_servers() + cls.proxy_app = cls._test_context["test_servers"][0] + + @classmethod + def tearDownClass(cls): + if cls._test_context is not None: + teardown_servers(cls._test_context) + cls._test_context = None + + def setUp(self): + self.plaintext = 'unencrypted body content' + self.plaintext_etag = md5hex(self.plaintext) + self._setup_crypto_app() + + def _setup_crypto_app(self, disable_encryption=False): + # Set up a pipeline of crypto middleware ending in the proxy app so + # that tests can make requests to either the proxy server directly or + # via the crypto middleware. Make a fresh instance for each test to + # avoid any state coupling. + conf = {'disable_encryption': disable_encryption} + self.encryption = crypto.filter_factory(conf)(self.proxy_app) + self.km = keymaster.KeyMaster(self.encryption, TEST_KEYMASTER_CONF) + self.crypto_app = self.km # for clarity + + def _create_container(self, app, policy_name='one', container_path=None): + if not container_path: + # choose new container name so that the policy can be specified + self.container_name = uuid.uuid4().hex + self.container_path = 'http://foo:8080/v1/a/' + self.container_name + self.object_name = 'o' + self.object_path = self.container_path + '/' + self.object_name + container_path = self.container_path + req = Request.blank( + container_path, method='PUT', + headers={'X-Storage-Policy': policy_name}) + resp = req.get_response(app) + self.assertEqual('201 Created', resp.status) + # sanity check + req = Request.blank( + container_path, method='HEAD', + headers={'X-Storage-Policy': policy_name}) + resp = req.get_response(app) + self.assertEqual(policy_name, resp.headers['X-Storage-Policy']) + + def _put_object(self, app, body): + req = Request.blank(self.object_path, method='PUT', body=body, + headers={'Content-Type': 'application/test'}) + resp = req.get_response(app) + self.assertEqual('201 Created', resp.status) + self.assertEqual(self.plaintext_etag, resp.headers['Etag']) + return resp + + def _post_object(self, app): + req = Request.blank(self.object_path, method='POST', + headers={'Content-Type': 'application/test', + 'X-Object-Meta-Fruit': 'Kiwi'}) + resp = req.get_response(app) + self.assertEqual('202 Accepted', resp.status) + return resp + + def _copy_object(self, app, destination): + req = Request.blank(self.object_path, method='COPY', + headers={'Destination': destination}) + resp = req.get_response(app) + self.assertEqual('201 Created', resp.status) + self.assertEqual(self.plaintext_etag, resp.headers['Etag']) + return resp + + def _check_GET_and_HEAD(self, app, object_path=None): + object_path = object_path or self.object_path + req = Request.blank(object_path, method='GET') + resp = req.get_response(app) + self.assertEqual('200 OK', resp.status) + self.assertEqual(self.plaintext, resp.body) + self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit']) + + req = Request.blank(object_path, method='HEAD') + resp = req.get_response(app) + self.assertEqual('200 OK', resp.status) + self.assertEqual('', resp.body) + self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit']) + + def _check_match_requests(self, method, app, object_path=None): + object_path = object_path or self.object_path + # verify conditional match requests + expected_body = self.plaintext if method == 'GET' else '' + + # If-Match matches + req = Request.blank(object_path, method=method, + headers={'If-Match': '"%s"' % self.plaintext_etag}) + resp = req.get_response(app) + self.assertEqual('200 OK', resp.status) + self.assertEqual(expected_body, resp.body) + self.assertEqual(self.plaintext_etag, resp.headers['Etag']) + self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit']) + + # If-Match wildcard + req = Request.blank(object_path, method=method, + headers={'If-Match': '*'}) + resp = req.get_response(app) + self.assertEqual('200 OK', resp.status) + self.assertEqual(expected_body, resp.body) + self.assertEqual(self.plaintext_etag, resp.headers['Etag']) + self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit']) + + # If-Match does not match + req = Request.blank(object_path, method=method, + headers={'If-Match': '"not the etag"'}) + resp = req.get_response(app) + self.assertEqual('412 Precondition Failed', resp.status) + self.assertEqual('', resp.body) + self.assertEqual(self.plaintext_etag, resp.headers['Etag']) + + # If-None-Match matches + req = Request.blank( + object_path, method=method, + headers={'If-None-Match': '"%s"' % self.plaintext_etag}) + resp = req.get_response(app) + self.assertEqual('304 Not Modified', resp.status) + self.assertEqual('', resp.body) + self.assertEqual(self.plaintext_etag, resp.headers['Etag']) + + # If-None-Match wildcard + req = Request.blank(object_path, method=method, + headers={'If-None-Match': '*'}) + resp = req.get_response(app) + self.assertEqual('304 Not Modified', resp.status) + self.assertEqual('', resp.body) + self.assertEqual(self.plaintext_etag, resp.headers['Etag']) + + # If-None-Match does not match + req = Request.blank(object_path, method=method, + headers={'If-None-Match': '"not the etag"'}) + resp = req.get_response(app) + self.assertEqual('200 OK', resp.status) + self.assertEqual(expected_body, resp.body) + self.assertEqual(self.plaintext_etag, resp.headers['Etag']) + self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit']) + + def _check_listing(self, app, expect_mismatch=False, container_path=None): + container_path = container_path or self.container_path + req = Request.blank( + container_path, method='GET', query_string='format=json') + resp = req.get_response(app) + self.assertEqual('200 OK', resp.status) + listing = json.loads(resp.body) + self.assertEqual(1, len(listing)) + self.assertEqual(self.object_name, listing[0]['name']) + self.assertEqual(len(self.plaintext), listing[0]['bytes']) + if expect_mismatch: + self.assertNotEqual(self.plaintext_etag, listing[0]['hash']) + else: + self.assertEqual(self.plaintext_etag, listing[0]['hash']) + + def test_write_with_crypto_and_override_headers(self): + self._create_container(self.proxy_app, policy_name='one') + + def verify_overrides(): + # verify object sysmeta + req = Request.blank( + self.object_path, method='GET') + resp = req.get_response(self.crypto_app) + for k, v in overrides.items(): + self.assertIn(k, resp.headers) + self.assertEqual(overrides[k], resp.headers[k]) + + # check container listing + req = Request.blank( + self.container_path, method='GET', query_string='format=json') + resp = req.get_response(self.crypto_app) + self.assertEqual('200 OK', resp.status) + listing = json.loads(resp.body) + self.assertEqual(1, len(listing)) + self.assertEqual('o', listing[0]['name']) + self.assertEqual( + overrides['x-object-sysmeta-container-update-override-size'], + str(listing[0]['bytes'])) + self.assertEqual( + overrides['x-object-sysmeta-container-update-override-etag'], + listing[0]['hash']) + + # include overrides in headers + overrides = {'x-object-sysmeta-container-update-override-etag': 'foo', + 'x-object-sysmeta-container-update-override-size': + str(len(self.plaintext) + 1)} + req = Request.blank(self.object_path, method='PUT', + body=self.plaintext, headers=overrides.copy()) + resp = req.get_response(self.crypto_app) + self.assertEqual('201 Created', resp.status) + self.assertEqual(self.plaintext_etag, resp.headers['Etag']) + verify_overrides() + + # include overrides in footers + overrides = {'x-object-sysmeta-container-update-override-etag': 'bar', + 'x-object-sysmeta-container-update-override-size': + str(len(self.plaintext) + 2)} + + def callback(footers): + footers.update(overrides) + + req = Request.blank( + self.object_path, method='PUT', body=self.plaintext) + req.environ['swift.callback.update_footers'] = callback + resp = req.get_response(self.crypto_app) + self.assertEqual('201 Created', resp.status) + self.assertEqual(self.plaintext_etag, resp.headers['Etag']) + verify_overrides() + + def test_write_with_crypto_read_with_crypto(self): + self._create_container(self.proxy_app, policy_name='one') + self._put_object(self.crypto_app, self.plaintext) + self._post_object(self.crypto_app) + self._check_GET_and_HEAD(self.crypto_app) + self._check_match_requests('GET', self.crypto_app) + self._check_match_requests('HEAD', self.crypto_app) + self._check_listing(self.crypto_app) + + def test_write_with_crypto_read_with_crypto_ec(self): + self._create_container(self.proxy_app, policy_name='ec') + self._put_object(self.crypto_app, self.plaintext) + self._post_object(self.crypto_app) + self._check_GET_and_HEAD(self.crypto_app) + self._check_match_requests('GET', self.crypto_app) + self._check_match_requests('HEAD', self.crypto_app) + self._check_listing(self.crypto_app) + + def test_put_without_crypto_post_with_crypto_read_with_crypto(self): + self._create_container(self.proxy_app, policy_name='one') + self._put_object(self.proxy_app, self.plaintext) + self._post_object(self.crypto_app) + self._check_GET_and_HEAD(self.crypto_app) + self._check_match_requests('GET', self.crypto_app) + self._check_match_requests('HEAD', self.crypto_app) + self._check_listing(self.crypto_app) + + def test_write_without_crypto_read_with_crypto(self): + self._create_container(self.proxy_app, policy_name='one') + self._put_object(self.proxy_app, self.plaintext) + self._post_object(self.proxy_app) + self._check_GET_and_HEAD(self.proxy_app) # sanity check + self._check_GET_and_HEAD(self.crypto_app) + self._check_match_requests('GET', self.proxy_app) # sanity check + self._check_match_requests('GET', self.crypto_app) + self._check_match_requests('HEAD', self.proxy_app) # sanity check + self._check_match_requests('HEAD', self.crypto_app) + self._check_listing(self.crypto_app) + + def test_write_without_crypto_read_with_crypto_ec(self): + self._create_container(self.proxy_app, policy_name='ec') + self._put_object(self.proxy_app, self.plaintext) + self._post_object(self.proxy_app) + self._check_GET_and_HEAD(self.proxy_app) # sanity check + self._check_GET_and_HEAD(self.crypto_app) + self._check_match_requests('GET', self.proxy_app) # sanity check + self._check_match_requests('GET', self.crypto_app) + self._check_match_requests('HEAD', self.proxy_app) # sanity check + self._check_match_requests('HEAD', self.crypto_app) + self._check_listing(self.crypto_app) + + def _check_GET_and_HEAD_not_decrypted(self, app): + req = Request.blank(self.object_path, method='GET') + resp = req.get_response(app) + self.assertEqual('200 OK', resp.status) + self.assertNotEqual(self.plaintext, resp.body) + self.assertEqual('%s' % len(self.plaintext), + resp.headers['Content-Length']) + self.assertNotEqual('Kiwi', resp.headers['X-Object-Meta-Fruit']) + + req = Request.blank(self.object_path, method='HEAD') + resp = req.get_response(app) + self.assertEqual('200 OK', resp.status) + self.assertEqual('', resp.body) + self.assertNotEqual('Kiwi', resp.headers['X-Object-Meta-Fruit']) + + def test_write_with_crypto_read_without_crypto(self): + self._create_container(self.proxy_app, policy_name='one') + self._put_object(self.crypto_app, self.plaintext) + self._post_object(self.crypto_app) + self._check_GET_and_HEAD(self.crypto_app) # sanity check + # without crypto middleware, GET and HEAD returns ciphertext + self._check_GET_and_HEAD_not_decrypted(self.proxy_app) + self._check_listing(self.proxy_app, expect_mismatch=True) + + def test_write_with_crypto_read_without_crypto_ec(self): + self._create_container(self.proxy_app, policy_name='ec') + self._put_object(self.crypto_app, self.plaintext) + self._post_object(self.crypto_app) + self._check_GET_and_HEAD(self.crypto_app) # sanity check + # without crypto middleware, GET and HEAD returns ciphertext + self._check_GET_and_HEAD_not_decrypted(self.proxy_app) + self._check_listing(self.proxy_app, expect_mismatch=True) + + def test_disable_encryption_config_option(self): + # check that on disable_encryption = true, object is not encrypted + self._setup_crypto_app(disable_encryption=True) + self._create_container(self.proxy_app, policy_name='one') + self._put_object(self.crypto_app, self.plaintext) + self._post_object(self.crypto_app) + self._check_GET_and_HEAD(self.crypto_app) + # check as if no crypto middleware exists + self._check_GET_and_HEAD(self.proxy_app) + self._check_match_requests('GET', self.crypto_app) + self._check_match_requests('HEAD', self.crypto_app) + self._check_match_requests('GET', self.proxy_app) + self._check_match_requests('HEAD', self.proxy_app) + + def test_write_with_crypto_read_with_disable_encryption_conf(self): + self._create_container(self.proxy_app, policy_name='one') + self._put_object(self.crypto_app, self.plaintext) + self._post_object(self.crypto_app) + self._check_GET_and_HEAD(self.crypto_app) # sanity check + # turn on disable_encryption config option + self._setup_crypto_app(disable_encryption=True) + # GET and HEAD of encrypted objects should still work + self._check_GET_and_HEAD(self.crypto_app) + self._check_listing(self.crypto_app, expect_mismatch=False) + self._check_match_requests('GET', self.crypto_app) + self._check_match_requests('HEAD', self.crypto_app) + + def _test_ondisk_data_after_write_with_crypto(self, policy_name): + policy = storage_policy.POLICIES.get_by_name(policy_name) + self._create_container(self.proxy_app, policy_name=policy_name) + self._put_object(self.crypto_app, self.plaintext) + self._post_object(self.crypto_app) + + # Verify container listing etag is encrypted by direct GET to container + # server. We can use any server for all nodes since they all share same + # devices dir. + cont_server = self._test_context['test_servers'][3] + cont_ring = Ring(self._test_context['testdir'], ring_name='container') + part, nodes = cont_ring.get_nodes('a', self.container_name) + for node in nodes: + req = Request.blank('/%s/%s/a/%s' + % (node['device'], part, self.container_name), + method='GET', query_string='format=json') + resp = req.get_response(cont_server) + listing = json.loads(resp.body) + # sanity checks... + self.assertEqual(1, len(listing)) + self.assertEqual('o', listing[0]['name']) + self.assertEqual('application/test', listing[0]['content_type']) + # verify encrypted etag value + parts = listing[0]['hash'].rsplit(';', 1) + crypto_meta_param = parts[1].strip() + crypto_meta = crypto_meta_param[len('swift_meta='):] + listing_etag_iv = load_crypto_meta(crypto_meta)['iv'] + exp_enc_listing_etag = base64.b64encode( + encrypt(self.plaintext_etag, + self.km.create_key('/a/%s' % self.container_name), + listing_etag_iv)) + self.assertEqual(exp_enc_listing_etag, parts[0]) + + # Verify diskfile data and metadata is encrypted + ring_object = self.proxy_app.get_object_ring(int(policy)) + partition, nodes = ring_object.get_nodes('a', self.container_name, 'o') + conf = {'devices': self._test_context["testdir"], + 'mount_check': 'false'} + df_mgr = diskfile.DiskFileRouter(conf, FakeLogger())[policy] + ondisk_data = [] + exp_enc_body = None + for node_index, node in enumerate(nodes): + df = df_mgr.get_diskfile(node['device'], partition, + 'a', self.container_name, 'o', + policy=policy) + with df.open(): + meta = df.get_metadata() + contents = ''.join(df.reader()) + metadata = dict((k.lower(), v) for k, v in meta.items()) + # verify on disk data - body + body_iv = load_crypto_meta( + metadata['x-object-sysmeta-crypto-body-meta'])['iv'] + body_key_meta = load_crypto_meta( + metadata['x-object-sysmeta-crypto-body-meta'])['body_key'] + obj_key = self.km.create_key('/a/%s/o' % self.container_name) + body_key = Crypto().unwrap_key(obj_key, body_key_meta) + exp_enc_body = encrypt(self.plaintext, body_key, body_iv) + ondisk_data.append((node, contents)) + + # verify on disk user metadata + enc_val, meta = metadata[ + 'x-object-transient-sysmeta-crypto-meta-fruit'].split(';') + meta = meta.strip()[len('swift_meta='):] + metadata_iv = load_crypto_meta(meta)['iv'] + exp_enc_meta = base64.b64encode(encrypt('Kiwi', obj_key, + metadata_iv)) + self.assertEqual(exp_enc_meta, enc_val) + self.assertNotIn('x-object-meta-fruit', metadata) + + self.assertIn( + 'x-object-transient-sysmeta-crypto-meta', metadata) + meta = load_crypto_meta( + metadata['x-object-transient-sysmeta-crypto-meta']) + self.assertIn('key_id', meta) + self.assertIn('path', meta['key_id']) + self.assertEqual( + '/a/%s/%s' % (self.container_name, self.object_name), + meta['key_id']['path']) + self.assertIn('v', meta['key_id']) + self.assertEqual('1', meta['key_id']['v']) + self.assertIn('cipher', meta) + self.assertEqual(Crypto.cipher, meta['cipher']) + + # verify etag + actual_enc_etag, _junk, actual_etag_meta = metadata[ + 'x-object-sysmeta-crypto-etag'].partition('; swift_meta=') + etag_iv = load_crypto_meta(actual_etag_meta)['iv'] + exp_enc_etag = base64.b64encode(encrypt(self.plaintext_etag, + obj_key, etag_iv)) + self.assertEqual(exp_enc_etag, actual_enc_etag) + + # verify etag hmac + exp_etag_mac = hmac.new( + obj_key, self.plaintext_etag, digestmod=hashlib.sha256) + exp_etag_mac = base64.b64encode(exp_etag_mac.digest()) + self.assertEqual(exp_etag_mac, + metadata['x-object-sysmeta-crypto-etag-mac']) + + # verify etag override for container updates + override = 'x-object-sysmeta-container-update-override-etag' + parts = metadata[override].rsplit(';', 1) + crypto_meta_param = parts[1].strip() + crypto_meta = crypto_meta_param[len('swift_meta='):] + listing_etag_iv = load_crypto_meta(crypto_meta)['iv'] + cont_key = self.km.create_key('/a/%s' % self.container_name) + exp_enc_listing_etag = base64.b64encode( + encrypt(self.plaintext_etag, cont_key, + listing_etag_iv)) + self.assertEqual(exp_enc_listing_etag, parts[0]) + + self._check_GET_and_HEAD(self.crypto_app) + return exp_enc_body, ondisk_data + + def test_ondisk_data_after_write_with_crypto(self): + exp_body, ondisk_data = self._test_ondisk_data_after_write_with_crypto( + policy_name='one') + for node, body in ondisk_data: + self.assertEqual(exp_body, body) + + def test_ondisk_data_after_write_with_crypto_ec(self): + exp_body, ondisk_data = self._test_ondisk_data_after_write_with_crypto( + policy_name='ec') + policy = storage_policy.POLICIES.get_by_name('ec') + for frag_selection in (ondisk_data[:2], ondisk_data[1:]): + frags = [frag for node, frag in frag_selection] + self.assertEqual(exp_body, policy.pyeclib_driver.decode(frags)) + + def _test_copy_encrypted_to_encrypted( + self, src_policy_name, dest_policy_name): + self._create_container(self.proxy_app, policy_name=src_policy_name) + self._put_object(self.crypto_app, self.plaintext) + self._post_object(self.crypto_app) + + copy_crypto_app = copy.ServerSideCopyMiddleware(self.crypto_app, {}) + + dest_container = uuid.uuid4().hex + dest_container_path = 'http://localhost:8080/v1/a/' + dest_container + self._create_container(copy_crypto_app, policy_name=dest_policy_name, + container_path=dest_container_path) + dest_obj_path = dest_container_path + '/o' + dest = '/%s/%s' % (dest_container, 'o') + self._copy_object(copy_crypto_app, dest) + + self._check_GET_and_HEAD(copy_crypto_app, object_path=dest_obj_path) + self._check_listing( + copy_crypto_app, container_path=dest_container_path) + self._check_match_requests( + 'GET', copy_crypto_app, object_path=dest_obj_path) + self._check_match_requests( + 'HEAD', copy_crypto_app, object_path=dest_obj_path) + + def test_copy_encrypted_to_encrypted(self): + self._test_copy_encrypted_to_encrypted('ec', 'ec') + self._test_copy_encrypted_to_encrypted('one', 'ec') + self._test_copy_encrypted_to_encrypted('ec', 'one') + self._test_copy_encrypted_to_encrypted('one', 'one') + + def _test_copy_encrypted_to_unencrypted( + self, src_policy_name, dest_policy_name): + self._create_container(self.proxy_app, policy_name=src_policy_name) + self._put_object(self.crypto_app, self.plaintext) + self._post_object(self.crypto_app) + + # make a pipeline with encryption disabled, use it to copy object + self._setup_crypto_app(disable_encryption=True) + copy_app = copy.ServerSideCopyMiddleware(self.crypto_app, {}) + + dest_container = uuid.uuid4().hex + dest_container_path = 'http://localhost:8080/v1/a/' + dest_container + self._create_container(self.crypto_app, policy_name=dest_policy_name, + container_path=dest_container_path) + dest_obj_path = dest_container_path + '/o' + dest = '/%s/%s' % (dest_container, 'o') + self._copy_object(copy_app, dest) + + self._check_GET_and_HEAD(copy_app, object_path=dest_obj_path) + self._check_GET_and_HEAD(self.proxy_app, object_path=dest_obj_path) + self._check_listing(copy_app, container_path=dest_container_path) + self._check_listing(self.proxy_app, container_path=dest_container_path) + self._check_match_requests( + 'GET', self.proxy_app, object_path=dest_obj_path) + self._check_match_requests( + 'HEAD', self.proxy_app, object_path=dest_obj_path) + + def test_copy_encrypted_to_unencrypted(self): + self._test_copy_encrypted_to_unencrypted('ec', 'ec') + self._test_copy_encrypted_to_unencrypted('one', 'ec') + self._test_copy_encrypted_to_unencrypted('ec', 'one') + self._test_copy_encrypted_to_unencrypted('one', 'one') + + def _test_copy_unencrypted_to_encrypted( + self, src_policy_name, dest_policy_name): + self._create_container(self.proxy_app, policy_name=src_policy_name) + self._put_object(self.proxy_app, self.plaintext) + self._post_object(self.proxy_app) + + copy_crypto_app = copy.ServerSideCopyMiddleware(self.crypto_app, {}) + + dest_container = uuid.uuid4().hex + dest_container_path = 'http://localhost:8080/v1/a/' + dest_container + self._create_container(copy_crypto_app, policy_name=dest_policy_name, + container_path=dest_container_path) + dest_obj_path = dest_container_path + '/o' + dest = '/%s/%s' % (dest_container, 'o') + self._copy_object(copy_crypto_app, dest) + + self._check_GET_and_HEAD(copy_crypto_app, object_path=dest_obj_path) + self._check_listing( + copy_crypto_app, container_path=dest_container_path) + self._check_match_requests( + 'GET', copy_crypto_app, object_path=dest_obj_path) + self._check_match_requests( + 'HEAD', copy_crypto_app, object_path=dest_obj_path) + + def test_copy_unencrypted_to_encrypted(self): + self._test_copy_unencrypted_to_encrypted('ec', 'ec') + self._test_copy_unencrypted_to_encrypted('one', 'ec') + self._test_copy_unencrypted_to_encrypted('ec', 'one') + self._test_copy_unencrypted_to_encrypted('one', 'one') + + def test_crypto_max_length_path(self): + # the path is stashed in the key_id in crypto meta; check that a long + # path is ok + self.container_name = 'c' * constraints.MAX_CONTAINER_NAME_LENGTH + self.object_name = 'o' * constraints.MAX_OBJECT_NAME_LENGTH + self.container_path = 'http://foo:8080/v1/a/' + self.container_name + self.object_path = '%s/%s' % (self.container_path, self.object_name) + + self._create_container(self.proxy_app, policy_name='one', + container_path=self.container_path) + + self._put_object(self.crypto_app, self.plaintext) + self._post_object(self.crypto_app) + self._check_GET_and_HEAD(self.crypto_app) + self._check_match_requests('GET', self.crypto_app) + self._check_match_requests('HEAD', self.crypto_app) + self._check_listing(self.crypto_app) + + def test_crypto_UTF8_path(self): + # check that UTF8 path is ok + self.container_name = self.object_name = u'\u010brypto' + self.container_path = 'http://foo:8080/v1/a/' + self.container_name + self.object_path = '%s/%s' % (self.container_path, self.object_name) + + self._create_container(self.proxy_app, policy_name='one', + container_path=self.container_path) + + self._put_object(self.crypto_app, self.plaintext) + self._post_object(self.crypto_app) + self._check_GET_and_HEAD(self.crypto_app) + self._check_match_requests('GET', self.crypto_app) + self._check_match_requests('HEAD', self.crypto_app) + self._check_listing(self.crypto_app) + + +class TestCryptoPipelineChangesFastPost(TestCryptoPipelineChanges): + @classmethod + def setUpClass(cls): + # set proxy config to use fast post + extra_conf = {'object_post_as_copy': 'False'} + cls._test_context = setup_servers(extra_conf=extra_conf) + cls.proxy_app = cls._test_context["test_servers"][0] + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/middleware/crypto/test_keymaster.py b/test/unit/common/middleware/crypto/test_keymaster.py new file mode 100644 index 00000000..2f8a1db4 --- /dev/null +++ b/test/unit/common/middleware/crypto/test_keymaster.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2015 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import base64 +import os + +import unittest + +from swift.common import swob +from swift.common.middleware.crypto import keymaster +from swift.common.middleware.crypto.crypto_utils import CRYPTO_KEY_CALLBACK +from swift.common.swob import Request +from test.unit.common.middleware.helpers import FakeSwift, FakeAppThatExcepts +from test.unit.common.middleware.crypto.crypto_helpers import ( + TEST_KEYMASTER_CONF) + + +def capture_start_response(): + calls = [] + + def start_response(*args): + calls.append(args) + return start_response, calls + + +class TestKeymaster(unittest.TestCase): + + def setUp(self): + super(TestKeymaster, self).setUp() + self.swift = FakeSwift() + self.app = keymaster.KeyMaster(self.swift, TEST_KEYMASTER_CONF) + + def test_object_path(self): + self.verify_keys_for_path( + '/a/c/o', expected_keys=('object', 'container')) + + def test_container_path(self): + self.verify_keys_for_path( + '/a/c', expected_keys=('container',)) + + def verify_keys_for_path(self, path, expected_keys): + put_keys = None + for method, resp_class, status in ( + ('PUT', swob.HTTPCreated, '201'), + ('POST', swob.HTTPAccepted, '202'), + ('GET', swob.HTTPOk, '200'), + ('HEAD', swob.HTTPNoContent, '204')): + resp_headers = {} + self.swift.register( + method, '/v1' + path, resp_class, resp_headers, '') + req = Request.blank( + '/v1' + path, environ={'REQUEST_METHOD': method}) + start_response, calls = capture_start_response() + self.app(req.environ, start_response) + self.assertEqual(1, len(calls)) + self.assertTrue(calls[0][0].startswith(status)) + self.assertNotIn('swift.crypto.override', req.environ) + self.assertIn(CRYPTO_KEY_CALLBACK, req.environ, + '%s not set in env' % CRYPTO_KEY_CALLBACK) + keys = req.environ.get(CRYPTO_KEY_CALLBACK)() + self.assertIn('id', keys) + id = keys.pop('id') + self.assertEqual(path, id['path']) + self.assertEqual('1', id['v']) + self.assertListEqual(sorted(expected_keys), sorted(keys.keys()), + '%s %s got keys %r, but expected %r' + % (method, path, keys.keys(), expected_keys)) + if put_keys is not None: + # check all key sets were consistent for this path + self.assertDictEqual(put_keys, keys) + else: + put_keys = keys + return put_keys + + def test_key_uniqueness(self): + # a rudimentary check that different keys are made for different paths + ref_path_parts = ('a1', 'c1', 'o1') + path = '/' + '/'.join(ref_path_parts) + ref_keys = self.verify_keys_for_path( + path, expected_keys=('object', 'container')) + + # for same path and for each differing path check that keys are unique + # when path to object or container is unique and vice-versa + for path_parts in [(a, c, o) for a in ('a1', 'a2') + for c in ('c1', 'c2') + for o in ('o1', 'o2')]: + path = '/' + '/'.join(path_parts) + keys = self.verify_keys_for_path( + path, expected_keys=('object', 'container')) + # object keys should only be equal when complete paths are equal + self.assertEqual(path_parts == ref_path_parts, + keys['object'] == ref_keys['object'], + 'Path %s keys:\n%s\npath %s keys\n%s' % + (ref_path_parts, ref_keys, path_parts, keys)) + # container keys should only be equal when paths to container are + # equal + self.assertEqual(path_parts[:2] == ref_path_parts[:2], + keys['container'] == ref_keys['container'], + 'Path %s keys:\n%s\npath %s keys\n%s' % + (ref_path_parts, ref_keys, path_parts, keys)) + + def test_filter(self): + factory = keymaster.filter_factory(TEST_KEYMASTER_CONF) + self.assertTrue(callable(factory)) + self.assertTrue(callable(factory(self.swift))) + + def test_app_exception(self): + app = keymaster.KeyMaster( + FakeAppThatExcepts(), TEST_KEYMASTER_CONF) + req = Request.blank('/', environ={'REQUEST_METHOD': 'PUT'}) + start_response, _ = capture_start_response() + self.assertRaises(Exception, app, req.environ, start_response) + + def test_root_secret(self): + for secret in (os.urandom(32), os.urandom(33), os.urandom(50)): + encoded_secret = base64.b64encode(secret) + try: + app = keymaster.KeyMaster( + self.swift, {'encryption_root_secret': + bytes(encoded_secret)}) + self.assertEqual(secret, app.root_secret) + except AssertionError as err: + self.fail(str(err) + ' for secret %s' % secret) + try: + app = keymaster.KeyMaster( + self.swift, {'encryption_root_secret': + unicode(encoded_secret)}) + self.assertEqual(secret, app.root_secret) + except AssertionError as err: + self.fail(str(err) + ' for secret %s' % secret) + + def test_invalid_root_secret(self): + for secret in (bytes(base64.b64encode(os.urandom(31))), # too short + unicode(base64.b64encode(os.urandom(31))), + u'?' * 44, b'?' * 44, # not base64 + u'a' * 45, b'a' * 45, # bad padding + 99, None): + conf = {'encryption_root_secret': secret} + try: + with self.assertRaises(ValueError) as err: + keymaster.KeyMaster(self.swift, conf) + self.assertEqual( + 'encryption_root_secret option in proxy-server.conf ' + 'must be a base64 encoding of at least 32 raw bytes', + err.exception.message) + except AssertionError as err: + self.fail(str(err) + ' for conf %s' % str(conf)) + + +if __name__ == '__main__': + unittest.main()