openstack/deb-swift
Code Issues Proposed changes

Enable object body and metadata encryption

Browse Source 
Adds encryption middlewares.

All object servers and proxy servers should be upgraded before
introducing encryption middleware.

Encryption middleware should be first introduced with the
encryption middleware disable_encryption option set to True.
Once all proxies have encryption middleware installed this
option may be set to False (the default).

Increases constraints.py:MAX_HEADER_COUNT by 4 to allow for
headers generated by encryption-related middleware.

Co-Authored-By: Tim Burke <tim.burke@gmail.com>
Co-Authored-By: Christian Cachin <cca@zurich.ibm.com>
Co-Authored-By: Mahati Chamarthy <mahati.chamarthy@gmail.com>
Co-Authored-By: Peter Chng <pchng@ca.ibm.com>
Co-Authored-By: Alistair Coles <alistair.coles@hpe.com>
Co-Authored-By: Jonathan Hinson <jlhinson@us.ibm.com>
Co-Authored-By: Hamdi Roumani <roumani@ca.ibm.com>

UpgradeImpact

Change-Id: Ie6db22697ceb1021baaa6bddcf8e41ae3acb5376
This commit is contained in:
Janie Richling 2016-06-07 15:01:32 +01:00 committed by Clay Gerrard
parent 3ad003cf51
commit 96a0e07753
26 changed files with 4731 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
doc/source/middleware.rst
View File

@ -96,6 +96,15 @@ DLO support centers around a user specified filter that matches
segments and concatenates them together in object listing order. Please see
the DLO docs for :ref:`dlo-doc` further details.
.. _encryption:
Encryption
==========
.. automodule:: swift.common.middleware.crypto
:members:
:show-inheritance:
.. _formpost:
FormPost
@ -108,7 +117,7 @@ FormPost
.. _gatekeeper:
GateKeeper
=============
==========
.. automodule:: swift.common.middleware.gatekeeper
:members:
@ -123,6 +132,13 @@ Healthcheck
:members:
:show-inheritance:
Keymaster
=========
.. automodule:: swift.common.middleware.crypto.keymaster
:members:
:show-inheritance:
.. _keystoneauth:
KeystoneAuth

32
etc/proxy-server.conf-sample
View File

@ -79,7 +79,7 @@ bind_port = 8080
[pipeline:main]
# This sample pipeline uses tempauth and is used for SAIO dev work and
# testing. See below for a pipeline using keystone.
pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit tempauth copy container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server
pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit tempauth copy container-quotas account-quotas slo dlo versioned_writes keymaster encryption proxy-logging proxy-server
# The following pipeline shows keystone integration. Comment out the one
# above and uncomment this one. Additional steps for integrating keystone are
@ -765,3 +765,33 @@ use = egg:swift#copy
# When object_post_as_copy is set to True, a POST request will be transformed
# into a COPY request where source and destination objects are the same.
# object_post_as_copy = true
# Note: To enable encryption, add the following 2 dependent pieces of crypto
# middleware to the proxy-server pipeline. They should be to the right of all
# other middleware apart from the final proxy-logging middleware, and in the
# order shown in this example:
# <other middleware> keymaster encryption proxy-logging proxy-server
[filter:keymaster]
use = egg:swift#keymaster
# Sets the root secret from which encryption keys are derived. This must be set
# before first use to a value that is a base64 encoding of at least 32 bytes.
# The security of all encrypted data critically depends on this key, therefore
# it should be set to a high-entropy value. For example, a suitable value may
# be obtained by base-64 encoding a 32 byte (or longer) value generated by a
# cryptographically secure random number generator. Changing the root secret is
# likely to result in data loss.
# TODO - STOP SETTING THIS DEFAULT! This is only here while work
# continues on the feature/crypto branch. Later, this will be added
# to the devstack proxy-config so that gate tests can pass.
# base64 encoding of "dontEverUseThisIn_PRODUCTION_xxxxxxxxxxxxxxx"
encryption_root_secret = ZG9udEV2ZXJVc2VUaGlzSW5fUFJPRFVDVElPTl94eHh4eHh4eHh4eHh4eHg=
[filter:encryption]
use = egg:swift#encryption
# By default all PUT or POST'ed object data and/or metadata will be encrypted.
# Encryption of new data and/or metadata may be disabled by setting
# disable_encryption to True. However, all encryption middleware should remain
# in the pipeline in order for existing encrypted data to be read.
# disable_encryption = False

7
etc/swift.conf-sample
View File

@ -136,9 +136,10 @@ aliases = yellow, orange
# By default the maximum number of allowed headers depends on the number of max
# allowed metadata settings plus a default value of 32 for regular http
# headers. If for some reason this is not enough (custom middleware for
# example) it can be increased with the extra_header_count constraint.
# allowed metadata settings plus a default value of 36 for swift internally
# generated headers and regular http headers. If for some reason this is not
# enough (custom middleware for example) it can be increased with the
# extra_header_count constraint.
#extra_header_count = 0

2
other-requirements.txt
View File

@ -13,3 +13,5 @@ python-dev [platform:dpkg]
python-devel [platform:rpm]
rsync
xfsprogs
libssl-dev [platform:dpkg]
openssl-devel [platform:rpm]

1
requirements.txt
View File

@ -11,3 +11,4 @@ pastedeploy>=1.3.3
six>=1.9.0
xattr>=0.4
PyECLib>=1.2.0 # BSD
cryptography>=1.0,!=1.3.0 # BSD/Apache-2.0

2
setup.cfg
View File

@ -97,6 +97,8 @@ paste.filter_factory =
xprofile = swift.common.middleware.xprofile:filter_factory
versioned_writes = swift.common.middleware.versioned_writes:filter_factory
copy = swift.common.middleware.copy:filter_factory
keymaster = swift.common.middleware.crypto.keymaster:filter_factory
encryption = swift.common.middleware.crypto:filter_factory
[build_sphinx]
all_files = 1

9
swift/common/constraints.py
View File

@ -110,10 +110,11 @@ FORMAT2CONTENT_TYPE = {'plain': 'text/plain', 'json': 'application/json',
# By default the maximum number of allowed headers depends on the number of max
# allowed metadata settings plus a default value of 32 for regular http
# headers. If for some reason this is not enough (custom middleware for
# example) it can be increased with the extra_header_count constraint.
MAX_HEADER_COUNT = MAX_META_COUNT + 32 + max(EXTRA_HEADER_COUNT, 0)
# allowed metadata settings plus a default value of 36 for swift internally
# generated headers and regular http headers. If for some reason this is not
# enough (custom middleware for example) it can be increased with the
# extra_header_count constraint.
MAX_HEADER_COUNT = MAX_META_COUNT + 36 + max(EXTRA_HEADER_COUNT, 0)
def check_metadata(req, target_type):

4
swift/common/exceptions.py
View File

@ -207,6 +207,10 @@ class APIVersionError(SwiftException):
pass
class EncryptionException(SwiftException):
pass
class ClientException(Exception):
def __init__(self, msg, http_scheme='', http_host='', http_port='',

34
swift/common/middleware/crypto/__init__.py Normal file
View File

@ -0,0 +1,34 @@
# Copyright (c) 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Implements middleware for object encryption which comprises an instance of a
Decrypter combined with an instance of an Encrypter.
"""
from swift.common.middleware.crypto.decrypter import Decrypter
from swift.common.middleware.crypto.encrypter import Encrypter
from swift.common.utils import config_true_value, register_swift_info
def filter_factory(global_conf, **local_conf):
"""Provides a factory function for loading encryption middleware."""
conf = global_conf.copy()
conf.update(local_conf)
enabled = not config_true_value(conf.get('disable_encryption', 'false'))
register_swift_info('encryption', admin=True, enabled=enabled)
def encryption_filter(app):
return Decrypter(Encrypter(app, conf), conf)
return encryption_filter

283
swift/common/middleware/crypto/crypto_utils.py Normal file
View File

@ -0,0 +1,283 @@
# Copyright (c) 2015-2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
import binascii
import collections
import json
import os
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
import six
from six.moves.urllib import parse as urlparse
from swift import gettext_ as _
from swift.common.exceptions import EncryptionException
from swift.common.swob import HTTPInternalServerError
from swift.common.utils import get_logger
from swift.common.wsgi import WSGIContext
CRYPTO_KEY_CALLBACK = 'swift.callback.fetch_crypto_keys'
class Crypto(object):
"""
Used by middleware: Calls cryptography library
"""
cipher = 'AES_CTR_256'
# AES will accept several key sizes - we are using 256 bits i.e. 32 bytes
key_length = 32
iv_length = algorithms.AES.block_size / 8
def __init__(self, conf=None):
self.logger = get_logger(conf, log_route="crypto")
# memoize backend to avoid repeated iteration over entry points
self.backend = default_backend()
def create_encryption_ctxt(self, key, iv):
"""
Creates a crypto context for encrypting
:param key: 256-bit key
:param iv: 128-bit iv or nonce used for encryption
:raises: ValueError on invalid key or iv
:returns: an instance of an encryptor
"""
self.check_key(key)
engine = Cipher(algorithms.AES(key), modes.CTR(iv),
backend=self.backend)
return engine.encryptor()
def create_decryption_ctxt(self, key, iv, offset):
"""
Creates a crypto context for decrypting
:param key: 256-bit key
:param iv: 128-bit iv or nonce used for decryption
:param offset: offset into the message; used for range reads
:returns: an instance of a decryptor
"""
self.check_key(key)
if offset < 0:
raise ValueError('Offset must not be negative')
if offset:
# Adjust IV so that it is correct for decryption at offset.
# The CTR mode offset is incremented for every AES block and taken
# modulo 2^128.
offset_blocks, offset_in_block = divmod(offset, self.iv_length)
ivl = long(binascii.hexlify(iv), 16) + offset_blocks
ivl %= 1 << algorithms.AES.block_size
iv = str(bytearray.fromhex(format(
ivl, '0%dx' % (2 * self.iv_length))))
else:
offset_in_block = 0
engine = Cipher(algorithms.AES(key), modes.CTR(iv),
backend=self.backend)
dec = engine.decryptor()
# Adjust decryption boundary within current AES block
dec.update('*' * offset_in_block)
return dec
def create_iv(self):
return os.urandom(self.iv_length)
def create_crypto_meta(self):
# create a set of parameters
return {'iv': self.create_iv(), 'cipher': self.cipher}
def check_crypto_meta(self, meta):
"""
Check that crypto meta dict has valid items.
:param meta: a dict
:raises EncryptionException: if an error is found in the crypto meta
"""
try:
if meta['cipher'] != self.cipher:
raise EncryptionException('Bad crypto meta: Cipher must be %s'
% self.cipher)
if len(meta['iv']) != self.iv_length:
raise EncryptionException(
'Bad crypto meta: IV must be length %s bytes'
% self.iv_length)
except KeyError as err:
raise EncryptionException(
'Bad crypto meta: Missing %s' % err)
def create_random_key(self):
# helper method to create random key of correct length
return os.urandom(self.key_length)
def wrap_key(self, wrapping_key, key_to_wrap):
# we don't use an RFC 3394 key wrap algorithm such as cryptography's
# aes_wrap_key because it's slower and we have iv material readily
# available so don't need a deterministic algorithm
iv = self.create_iv()
encryptor = Cipher(algorithms.AES(wrapping_key), modes.CTR(iv),
backend=self.backend).encryptor()
return {'key': encryptor.update(key_to_wrap), 'iv': iv}
def unwrap_key(self, wrapping_key, context):
# unwrap a key from dict of form returned by wrap_key
# check the key length early - unwrapping won't change the length
self.check_key(context['key'])
decryptor = Cipher(algorithms.AES(wrapping_key),
modes.CTR(context['iv']),
backend=self.backend).decryptor()
return decryptor.update(context['key'])
def check_key(self, key):
if len(key) != self.key_length:
raise ValueError("Key must be length %s bytes" % self.key_length)
class CryptoWSGIContext(WSGIContext):
"""
Base class for contexts used by crypto middlewares.
"""
def __init__(self, crypto_app, server_type, logger):
super(CryptoWSGIContext, self).__init__(crypto_app.app)
self.crypto = crypto_app.crypto
self.logger = logger
self.server_type = server_type
def get_keys(self, env, required=None):
# Get the key(s) from the keymaster
required = required if required is not None else [self.server_type]
try:
fetch_crypto_keys = env[CRYPTO_KEY_CALLBACK]
except KeyError:
self.logger.exception(_('ERROR get_keys() missing callback'))
raise HTTPInternalServerError(
"Unable to retrieve encryption keys.")
try:
keys = fetch_crypto_keys()
except Exception as err: # noqa
self.logger.exception(_(
'ERROR get_keys(): from callback: %s') % err)
raise HTTPInternalServerError(
"Unable to retrieve encryption keys.")
for name in required:
try:
key = keys[name]
self.crypto.check_key(key)
continue
except KeyError:
self.logger.exception(_("Missing key for %r") % name)
except TypeError:
self.logger.exception(_("Did not get a keys dict"))
except ValueError as e:
# don't include the key in any messages!
self.logger.exception(_("Bad key for %(name)r: %(err)s") %
{'name': name, 'err': e})
raise HTTPInternalServerError(
"Unable to retrieve encryption keys.")
return keys
def dump_crypto_meta(crypto_meta):
"""
Serialize crypto meta to a form suitable for including in a header value.
The crypto-meta is serialized as a json object. The iv and key values are
random bytes and as a result need to be base64 encoded before sending over
the wire. Base64 encoding returns a bytes object in py3, to future proof
the code, decode this data to produce a string, which is what the
json.dumps function expects.
:param crypto_meta: a dict containing crypto meta items
:returns: a string serialization of a crypto meta dict
"""
def b64_encode_meta(crypto_meta):
return {
name: (base64.b64encode(value).decode() if name in ('iv', 'key')
else b64_encode_meta(value) if isinstance(value, dict)
else value)
for name, value in crypto_meta.items()}
# use sort_keys=True to make serialized form predictable for testing
return urlparse.quote_plus(
json.dumps(b64_encode_meta(crypto_meta), sort_keys=True))
def load_crypto_meta(value):
"""
Build the crypto_meta from the json object.
Note that json.loads always produces unicode strings, to ensure the
resultant crypto_meta matches the original object cast all key and value
data to a str except the key and iv which are base64 decoded. This will
work in py3 as well where all strings are unicode implying the cast is
effectively a no-op.
:param value: a string serialization of a crypto meta dict
:returns: a dict containing crypto meta items
:raises EncryptionException: if an error occurs while parsing the
crypto meta
"""
def b64_decode_meta(crypto_meta):
return {
str(name): (base64.b64decode(val) if name in ('iv', 'key')
else b64_decode_meta(val) if isinstance(val, dict)
else val.encode('utf8'))
for name, val in crypto_meta.items()}
try:
if not isinstance(value, six.string_types):
raise ValueError('crypto meta not a string')
val = json.loads(urlparse.unquote_plus(value))
if not isinstance(val, collections.Mapping):
raise ValueError('crypto meta not a Mapping')
return b64_decode_meta(val)
except (KeyError, ValueError, TypeError) as err:
msg = 'Bad crypto meta %r: %s' % (value, err)
raise EncryptionException(msg)
def append_crypto_meta(value, crypto_meta):
"""
Serialize and append crypto metadata to an encrypted value.
:param value: value to which serialized crypto meta will be appended.
:param crypto_meta: a dict of crypto meta
:return: a string of the form <value>; swift_meta=<serialized crypto meta>
"""
return '%s; swift_meta=%s' % (value, dump_crypto_meta(crypto_meta))
def extract_crypto_meta(value):
"""
Extract and deserialize any crypto meta from the end of a value.
:param value: string that may have crypto meta at end
:return: a tuple of the form:
(<value without crypto meta>, <deserialized crypto meta> or None)
"""
crypto_meta = None
# we only attempt to extract crypto meta from values that we know were
# encrypted and base64-encoded, or from etag values, so it's safe to split
# on ';' even if it turns out that the value was an unencrypted etag
parts = value.split(';')
if len(parts) == 2:
value, param = parts
crypto_meta_tag = 'swift_meta='
if param.strip().startswith(crypto_meta_tag):
param = param.strip()[len(crypto_meta_tag):]
crypto_meta = load_crypto_meta(param)
return value, crypto_meta

449
swift/common/middleware/crypto/decrypter.py Normal file
View File

@ -0,0 +1,449 @@
# Copyright (c) 2015-2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
import json
import xml.etree.cElementTree as ElementTree
from swift import gettext_ as _
from swift.common.http import is_success
from swift.common.middleware.crypto.crypto_utils import CryptoWSGIContext, \
load_crypto_meta, extract_crypto_meta, Crypto
from swift.common.exceptions import EncryptionException
from swift.common.request_helpers import get_object_transient_sysmeta, \
get_listing_content_type, get_sys_meta_prefix, get_user_meta_prefix
from swift.common.swob import Request, HTTPException, HTTPInternalServerError
from swift.common.utils import get_logger, config_true_value, \
parse_content_range, closing_if_possible, parse_content_type, \
FileLikeIter, multipart_byteranges_to_document_iters
DECRYPT_CHUNK_SIZE = 65536
def purge_crypto_sysmeta_headers(headers):
return [h for h in headers if not
h[0].lower().startswith(
(get_object_transient_sysmeta('crypto-'),
get_sys_meta_prefix('object') + 'crypto-'))]
class BaseDecrypterContext(CryptoWSGIContext):
def get_crypto_meta(self, header_name):
"""
Extract a crypto_meta dict from a header.
:param header_name: name of header that may have crypto_meta
:return: A dict containing crypto_meta items
:raises EncryptionException: if an error occurs while parsing the
crypto meta
"""
crypto_meta_json = self._response_header_value(header_name)
if crypto_meta_json is None:
return None
crypto_meta = load_crypto_meta(crypto_meta_json)
self.crypto.check_crypto_meta(crypto_meta)
return crypto_meta
def get_unwrapped_key(self, crypto_meta, wrapping_key):
"""
Get a wrapped key from crypto-meta and unwrap it using the provided
wrapping key.
:param crypto_meta: a dict of crypto-meta
:param wrapping_key: key to be used to decrypt the wrapped key
:return: an unwrapped key
:raises EncryptionException: if the crypto-meta has no wrapped key or
the unwrapped key is invalid
"""
try:
return self.crypto.unwrap_key(wrapping_key,
crypto_meta['body_key'])
except KeyError as err:
err = 'Missing %s' % err
except ValueError as err:
pass
msg = 'Error decrypting %s' % self.server_type
self.logger.error(_('%(msg)s: %(err)s') % {'msg': msg, 'err': err})
raise HTTPInternalServerError(body=msg, content_type='text/plain')
def decrypt_value_with_meta(self, value, key, required=False):
"""
Base64-decode and decrypt a value if crypto meta can be extracted from
the value itself, otherwise return the value unmodified.
A value should either be a string that does not contain the ';'
character or should be of the form:
<base64-encoded ciphertext>;swift_meta=<crypto meta>
:param value: value to decrypt
:param key: crypto key to use
:param required: if True then the value is required to be decrypted
and an EncryptionException will be raised if the
header cannot be decrypted due to missing crypto meta.
:returns: decrypted value if crypto meta is found, otherwise the
unmodified value
:raises EncryptionException: if an error occurs while parsing crypto
meta or if the header value was required
to be decrypted but crypto meta was not
found.
"""
value, crypto_meta = extract_crypto_meta(value)
if crypto_meta:
self.crypto.check_crypto_meta(crypto_meta)
value = self.decrypt_value(value, key, crypto_meta)
elif required:
raise EncryptionException(
"Missing crypto meta in value %s" % value)
return value
def decrypt_value(self, value, key, crypto_meta):
"""
Base64-decode and decrypt a value using the crypto_meta provided.
:param value: a base64-encoded value to decrypt
:param key: crypto key to use
:param crypto_meta: a crypto-meta dict of form returned by
:py:func:`~swift.common.middleware.crypto.Crypto.get_crypto_meta`
:returns: decrypted value
"""
if not value:
return ''
crypto_ctxt = self.crypto.create_decryption_ctxt(
key, crypto_meta['iv'], 0)
return crypto_ctxt.update(base64.b64decode(value))
def get_decryption_keys(self, req):
"""
Determine if a response should be decrypted, and if so then fetch keys.
:param req: a Request object
:returns: a dict of decryption keys
"""
if config_true_value(req.environ.get('swift.crypto.override')):
self.logger.debug('No decryption is necessary because of override')
return None
return self.get_keys(req.environ)
class DecrypterObjContext(BaseDecrypterContext):
def __init__(self, decrypter, logger):
super(DecrypterObjContext, self).__init__(decrypter, 'object', logger)
def _decrypt_header(self, header, value, key, required=False):
"""
Attempt to decrypt a header value that may be encrypted.
:param header: the header name
:param value: the header value
:param key: decryption key
:param required: if True then the header is required to be decrypted
and an HTTPInternalServerError will be raised if the
header cannot be decrypted due to missing crypto meta.
:return: decrypted value or the original value if it was not encrypted.
:raises HTTPInternalServerError: if an error occurred during decryption
or if the header value was required to
be decrypted but crypto meta was not
found.
"""
try:
return self.decrypt_value_with_meta(value, key, required)
except EncryptionException as e:
msg = "Error decrypting header"
self.logger.error(_("%(msg)s %(hdr)s: %(e)s") %
{'msg': msg, 'hdr': header, 'e': e})
raise HTTPInternalServerError(body=msg, content_type='text/plain')
def decrypt_user_metadata(self, keys):
prefix = get_object_transient_sysmeta('crypto-meta-')
prefix_len = len(prefix)
new_prefix = get_user_meta_prefix(self.server_type).title()
result = []
for name, val in self._response_headers:
if name.lower().startswith(prefix) and val:
short_name = name[prefix_len:]
decrypted_value = self._decrypt_header(
name, val, keys[self.server_type], required=True)
result.append((new_prefix + short_name, decrypted_value))
return result
def decrypt_resp_headers(self, keys):
"""
Find encrypted headers and replace with the decrypted versions.
:param keys: a dict of decryption keys.
:return: A list of headers with any encrypted headers replaced by their
decrypted values.
:raises HTTPInternalServerError: if any error occurs while decrypting
headers
"""
mod_hdr_pairs = []
# Decrypt plaintext etag and place in Etag header for client response
etag_header = 'X-Object-Sysmeta-Crypto-Etag'
encrypted_etag = self._response_header_value(etag_header)
if encrypted_etag:
decrypted_etag = self._decrypt_header(
etag_header, encrypted_etag, keys['object'], required=True)
mod_hdr_pairs.append(('Etag', decrypted_etag))
etag_header = 'X-Object-Sysmeta-Container-Update-Override-Etag'
encrypted_etag = self._response_header_value(etag_header)
if encrypted_etag:
decrypted_etag = self._decrypt_header(
etag_header, encrypted_etag, keys['container'])
mod_hdr_pairs.append((etag_header, decrypted_etag))
# Decrypt all user metadata. Encrypted user metadata values are stored
# in the x-object-transient-sysmeta-crypto-meta- namespace. Those are
# decrypted and moved back to the x-object-meta- namespace. Prior to
# decryption, the response should have no x-object-meta- headers, but
# if it does then they will be overwritten by any decrypted headers
# that map to the same x-object-meta- header names i.e. decrypted
# headers win over unexpected, unencrypted headers.
mod_hdr_pairs.extend(self.decrypt_user_metadata(keys))
mod_hdr_names = {h.lower() for h, v in mod_hdr_pairs}
mod_hdr_pairs.extend([(h, v) for h, v in self._response_headers
if h.lower() not in mod_hdr_names])
return mod_hdr_pairs
def multipart_response_iter(self, resp, boundary, body_key, crypto_meta):
"""
Decrypts a multipart mime doc response body.
:param resp: application response
:param boundary: multipart boundary string
:param keys: a dict of decryption keys.
:param crypto_meta: crypto_meta for the response body
:return: generator for decrypted response body
"""
with closing_if_possible(resp):
parts_iter = multipart_byteranges_to_document_iters(
FileLikeIter(resp), boundary)
for first_byte, last_byte, length, headers, body in parts_iter:
yield "--" + boundary + "\r\n"
for header_pair in headers:
yield "%s: %s\r\n" % header_pair
yield "\r\n"
decrypt_ctxt = self.crypto.create_decryption_ctxt(
body_key, crypto_meta['iv'], first_byte)
for chunk in iter(lambda: body.read(DECRYPT_CHUNK_SIZE), ''):
yield decrypt_ctxt.update(chunk)
yield "\r\n"
yield "--" + boundary + "--"
def response_iter(self, resp, body_key, crypto_meta, offset):
"""
Decrypts a response body.
:param resp: application response
:param keys: a dict of decryption keys.
:param crypto_meta: crypto_meta for the response body
:param offset: offset into object content at which response body starts
:return: generator for decrypted response body
"""
decrypt_ctxt = self.crypto.create_decryption_ctxt(
body_key, crypto_meta['iv'], offset)
with closing_if_possible(resp):
for chunk in resp:
yield decrypt_ctxt.update(chunk)
def handle_get(self, req, start_response):
app_resp = self._app_call(req.environ)
keys = self.get_decryption_keys(req)
if keys is None:
# skip decryption
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return app_resp
mod_resp_headers = self.decrypt_resp_headers(keys)
crypto_meta = None
if is_success(self._get_status_int()):
try:
crypto_meta = self.get_crypto_meta(
'X-Object-Sysmeta-Crypto-Body-Meta')
except EncryptionException as err:
msg = 'Error decrypting object'
self.logger.error(_('%(msg)s: %(err)s') %
{'msg': msg, 'err': err})
raise HTTPInternalServerError(
body=msg, content_type='text/plain')
if crypto_meta:
# 2xx response and encrypted body
body_key = self.get_unwrapped_key(crypto_meta, keys['object'])
content_type, content_type_attrs = parse_content_type(
self._response_header_value('Content-Type'))
if (self._get_status_int() == 206 and
content_type == 'multipart/byteranges'):
boundary = dict(content_type_attrs)["boundary"]
resp_iter = self.multipart_response_iter(
app_resp, boundary, body_key, crypto_meta)
else:
offset = 0
content_range = self._response_header_value('Content-Range')
if content_range:
# Determine offset within the whole object if ranged GET
offset, end, total = parse_content_range(content_range)
resp_iter = self.response_iter(
app_resp, body_key, crypto_meta, offset)
else:
# don't decrypt body of unencrypted or non-2xx responses
resp_iter = app_resp
mod_resp_headers = purge_crypto_sysmeta_headers(mod_resp_headers)
start_response(self._response_status, mod_resp_headers,
self._response_exc_info)
return resp_iter
def handle_head(self, req, start_response):
app_resp = self._app_call(req.environ)
keys = self.get_decryption_keys(req)
if keys is None:
# skip decryption
start_response(self._response_status, self._response_headers,
self._response_exc_info)
else:
mod_resp_headers = self.decrypt_resp_headers(keys)
mod_resp_headers = purge_crypto_sysmeta_headers(mod_resp_headers)
start_response(self._response_status, mod_resp_headers,
self._response_exc_info)
return app_resp
class DecrypterContContext(BaseDecrypterContext):
def __init__(self, decrypter, logger):
super(DecrypterContContext, self).__init__(
decrypter, 'container', logger)
def handle_get(self, req, start_response):
app_resp = self._app_call(req.environ)
if is_success(self._get_status_int()):
# only decrypt body of 2xx responses
out_content_type = get_listing_content_type(req)
if out_content_type == 'application/json':
handler = self.process_json_resp
keys = self.get_decryption_keys(req)
elif out_content_type.endswith('/xml'):
handler = self.process_xml_resp
keys = self.get_decryption_keys(req)
else:
handler = keys = None
if handler and keys:
try:
app_resp = handler(keys['container'], app_resp)
except EncryptionException as err:
msg = "Error decrypting container listing"
self.logger.error(_('%(msg)s: %(err)s') %
{'msg': msg, 'err': err})
raise HTTPInternalServerError(
body=msg, content_type='text/plain')
start_response(self._response_status,
self._response_headers,
self._response_exc_info)
return app_resp
def update_content_length(self, new_total_len):
self._response_headers = [
(h, v) for h, v in self._response_headers
if h.lower() != 'content-length']
self._response_headers.append(('Content-Length', str(new_total_len)))
def process_json_resp(self, key, resp_iter):
"""
Parses json body listing and decrypt encrypted entries. Updates
Content-Length header with new body length and return a body iter.
"""
with closing_if_possible(resp_iter):
resp_body = ''.join(resp_iter)
body_json = json.loads(resp_body)
new_body = json.dumps([self.decrypt_obj_dict(obj_dict, key)
for obj_dict in body_json])
self.update_content_length(len(new_body))
return [new_body]
def decrypt_obj_dict(self, obj_dict, key):
ciphertext = obj_dict['hash']
obj_dict['hash'] = self.decrypt_value_with_meta(ciphertext, key)
return obj_dict
def process_xml_resp(self, key, resp_iter):
"""
Parses xml body listing and decrypt encrypted entries. Updates
Content-Length header with new body length and return a body iter.
"""
with closing_if_possible(resp_iter):
resp_body = ''.join(resp_iter)
tree = ElementTree.fromstring(resp_body)
for elem in tree.iter('hash'):
ciphertext = elem.text.encode('utf8')
plain = self.decrypt_value_with_meta(ciphertext, key)
elem.text = plain.decode('utf8')
new_body = ElementTree.tostring(tree, encoding='UTF-8').replace(
"<?xml version='1.0' encoding='UTF-8'?>",
'<?xml version="1.0" encoding="UTF-8"?>', 1)
self.update_content_length(len(new_body))
return [new_body]
class Decrypter(object):
"""Middleware for decrypting data and user metadata."""
def __init__(self, app, conf):
self.app = app
self.logger = get_logger(conf, log_route="decrypter")
self.crypto = Crypto(conf)
def __call__(self, env, start_response):
req = Request(env)
try:
parts = req.split_path(3, 4, True)
except ValueError:
return self.app(env, start_response)
if parts[3] and req.method == 'GET':
handler = DecrypterObjContext(self, self.logger).handle_get
elif parts[3] and req.method == 'HEAD':
handler = DecrypterObjContext(self, self.logger).handle_head
elif parts[2] and req.method == 'GET':
handler = DecrypterContContext(self, self.logger).handle_get
else:
# url and/or request verb is not handled by decrypter
return self.app(env, start_response)
try:
return handler(req, start_response)
except HTTPException as err_resp:
return err_resp(env, start_response)

369
swift/common/middleware/crypto/encrypter.py Normal file
View File

@ -0,0 +1,369 @@
# Copyright (c) 2015-2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
import hashlib
import hmac
from contextlib import contextmanager
from swift.common.constraints import check_metadata
from swift.common.http import is_success
from swift.common.middleware.crypto.crypto_utils import CryptoWSGIContext, \
dump_crypto_meta, append_crypto_meta, Crypto
from swift.common.request_helpers import get_object_transient_sysmeta, \
strip_user_meta_prefix, is_user_meta, update_etag_is_at_header
from swift.common.swob import Request, Match, HTTPException, \
HTTPUnprocessableEntity
from swift.common.utils import get_logger, config_true_value
def encrypt_header_val(crypto, value, key):
"""
Encrypt a header value using the supplied key.
:param crypto: a Crypto instance
:param value: value to encrypt
:param key: crypto key to use
:returns: a tuple of (encrypted value, crypto_meta) where crypto_meta is a
dict of form returned by
:py:func:`~swift.common.middleware.crypto.Crypto.get_crypto_meta`
"""
if not value:
return '', None
crypto_meta = crypto.create_crypto_meta()
crypto_ctxt = crypto.create_encryption_ctxt(key, crypto_meta['iv'])
enc_val = base64.b64encode(crypto_ctxt.update(value))
return enc_val, crypto_meta
def _hmac_etag(key, etag):
"""
Compute an HMAC-SHA256 using given key and etag.
:param key: The starting key for the hash.
:param etag: The etag to hash.
:returns: a Base64-encoded representation of the HMAC
"""
result = hmac.new(key, etag, digestmod=hashlib.sha256).digest()
return base64.b64encode(result).decode()
class EncInputWrapper(object):
"""File-like object to be swapped in for wsgi.input."""
def __init__(self, crypto, keys, req, logger):
self.env = req.environ
self.wsgi_input = req.environ['wsgi.input']
self.path = req.path
self.crypto = crypto
self.body_crypto_ctxt = None
self.keys = keys
self.plaintext_md5 = None
self.ciphertext_md5 = None
self.logger = logger
self.install_footers_callback(req)
def _init_encryption_context(self):
# do this once when body is first read
if self.body_crypto_ctxt is None:
self.body_crypto_meta = self.crypto.create_crypto_meta()
body_key = self.crypto.create_random_key()
# wrap the body key with object key
self.body_crypto_meta['body_key'] = self.crypto.wrap_key(
self.keys['object'], body_key)
self.body_crypto_meta['key_id'] = self.keys['id']
self.body_crypto_ctxt = self.crypto.create_encryption_ctxt(
body_key, self.body_crypto_meta.get('iv'))
self.plaintext_md5 = hashlib.md5()
self.ciphertext_md5 = hashlib.md5()
def install_footers_callback(self, req):
# the proxy controller will call back for footer metadata after
# body has been sent
inner_callback = req.environ.get('swift.callback.update_footers')
# remove any Etag from headers, it won't be valid for ciphertext and
# we'll send the ciphertext Etag later in footer metadata
client_etag = req.headers.pop('etag', None)
container_listing_etag_header = req.headers.get(
'X-Object-Sysmeta-Container-Update-Override-Etag')
def footers_callback(footers):
if inner_callback:
# pass on footers dict to any other callback that was
# registered before this one. It may override any footers that
# were set.
inner_callback(footers)
plaintext_etag = None
if self.body_crypto_ctxt:
plaintext_etag = self.plaintext_md5.hexdigest()
# If client (or other middleware) supplied etag, then validate
# against plaintext etag
etag_to_check = footers.get('Etag') or client_etag
if (etag_to_check is not None and
plaintext_etag != etag_to_check):
raise HTTPUnprocessableEntity(request=Request(self.env))
# override any previous notion of etag with the ciphertext etag
footers['Etag'] = self.ciphertext_md5.hexdigest()
# Encrypt the plaintext etag using the object key and persist
# as sysmeta along with the crypto parameters that were used.
encrypted_etag, etag_crypto_meta = encrypt_header_val(
self.crypto, plaintext_etag, self.keys['object'])
footers['X-Object-Sysmeta-Crypto-Etag'] = \
append_crypto_meta(encrypted_etag, etag_crypto_meta)
footers['X-Object-Sysmeta-Crypto-Body-Meta'] = \
dump_crypto_meta(self.body_crypto_meta)
# Also add an HMAC of the etag for use when evaluating
# conditional requests
footers['X-Object-Sysmeta-Crypto-Etag-Mac'] = _hmac_etag(
self.keys['object'], plaintext_etag)
else:
# No data was read from body, nothing was encrypted, so don't
# set any crypto sysmeta for the body, but do re-instate any
# etag provided in inbound request if other middleware has not
# already set a value.
if client_etag is not None:
footers.setdefault('Etag', client_etag)
# When deciding on the etag that should appear in container
# listings, look for:
# * override in the footer, otherwise
# * override in the header, and finally
# * MD5 of the plaintext received
# This may be None if no override was set and no data was read
container_listing_etag = footers.get(
'X-Object-Sysmeta-Container-Update-Override-Etag',
container_listing_etag_header) or plaintext_etag
if container_listing_etag is not None:
# Encrypt the container-listing etag using the container key
# and a random IV, and use it to override the container update
# value, with the crypto parameters appended. We use the
# container key here so that only that key is required to
# decrypt all etag values in a container listing when handling
# a container GET request.
val, crypto_meta = encrypt_header_val(
self.crypto, container_listing_etag,
self.keys['container'])
crypto_meta['key_id'] = self.keys['id']
footers['X-Object-Sysmeta-Container-Update-Override-Etag'] = \
append_crypto_meta(val, crypto_meta)
# else: no override was set and no data was read
req.environ['swift.callback.update_footers'] = footers_callback
def read(self, *args, **kwargs):
return self.readChunk(self.wsgi_input.read, *args, **kwargs)
def readline(self, *args, **kwargs):
return self.readChunk(self.wsgi_input.readline, *args, **kwargs)
def readChunk(self, read_method, *args, **kwargs):
chunk = read_method(*args, **kwargs)
if chunk:
self._init_encryption_context()
self.plaintext_md5.update(chunk)
# Encrypt one chunk at a time
ciphertext = self.body_crypto_ctxt.update(chunk)
self.ciphertext_md5.update(ciphertext)
return ciphertext
return chunk
class EncrypterObjContext(CryptoWSGIContext):
def __init__(self, encrypter, logger):
super(EncrypterObjContext, self).__init__(
encrypter, 'object', logger)
def _check_headers(self, req):
# Check the user-metadata length before encrypting and encoding
error_response = check_metadata(req, self.server_type)
if error_response:
raise error_response
def encrypt_user_metadata(self, req, keys):
"""
Encrypt user-metadata header values. Replace each x-object-meta-<key>
user metadata header with a corresponding
x-object-transient-sysmeta-crypto-meta-<key> header which has the
crypto metadata required to decrypt appended to the encrypted value.
:param req: a swob Request
:param keys: a dict of encryption keys
"""
prefix = get_object_transient_sysmeta('crypto-meta-')
user_meta_headers = [h for h in req.headers.items() if
is_user_meta(self.server_type, h[0]) and h[1]]
crypto_meta = None
for name, val in user_meta_headers:
short_name = strip_user_meta_prefix(self.server_type, name)
new_name = prefix + short_name
enc_val, crypto_meta = encrypt_header_val(
self.crypto, val, keys[self.server_type])
req.headers[new_name] = append_crypto_meta(enc_val, crypto_meta)
req.headers.pop(name)
# store a single copy of the crypto meta items that are common to all
# encrypted user metadata independently of any such meta that is stored
# with the object body because it might change on a POST. This is done
# for future-proofing - the meta stored here is not currently used
# during decryption.
if crypto_meta:
meta = dump_crypto_meta({'cipher': crypto_meta['cipher'],
'key_id': keys['id']})
req.headers[get_object_transient_sysmeta('crypto-meta')] = meta
def handle_put(self, req, start_response):
self._check_headers(req)
keys = self.get_keys(req.environ, required=['object', 'container'])
self.encrypt_user_metadata(req, keys)
enc_input_proxy = EncInputWrapper(self.crypto, keys, req, self.logger)
req.environ['wsgi.input'] = enc_input_proxy
resp = self._app_call(req.environ)
# If an etag is in the response headers and a plaintext etag was
# calculated, then overwrite the response value with the plaintext etag
# provided it matches the ciphertext etag. If it does not match then do
# not overwrite and allow the response value to return to client.
mod_resp_headers = self._response_headers
if (is_success(self._get_status_int()) and
enc_input_proxy.plaintext_md5):
plaintext_etag = enc_input_proxy.plaintext_md5.hexdigest()
ciphertext_etag = enc_input_proxy.ciphertext_md5.hexdigest()
mod_resp_headers = [
(h, v if (h.lower() != 'etag' or
v.strip('"') != ciphertext_etag)
else plaintext_etag)
for h, v in mod_resp_headers]
start_response(self._response_status, mod_resp_headers,
self._response_exc_info)
return resp
def handle_post(self, req, start_response):
"""
Encrypt the new object headers with a new iv and the current crypto.
Note that an object may have encrypted headers while the body may
remain unencrypted.
"""
self._check_headers(req)
keys = self.get_keys(req.environ)
self.encrypt_user_metadata(req, keys)
resp = self._app_call(req.environ)
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return resp
@contextmanager
def _mask_conditional_etags(self, req, header_name):
"""
Calculate HMACs of etags in header value and append to existing list.
The HMACs are calculated in the same way as was done for the object
plaintext etag to generate the value of
X-Object-Sysmeta-Crypto-Etag-Mac when the object was PUT. The object
server can therefore use these HMACs to evaluate conditional requests.
The existing etag values are left in the list of values to match in
case the object was not encrypted when it was PUT. It is unlikely that
a masked etag value would collide with an unmasked value.
:param req: an instance of swob.Request
:param header_name: name of header that has etags to mask
:return: True if any etags were masked, False otherwise
"""
masked = False
old_etags = req.headers.get(header_name)
if old_etags:
keys = self.get_keys(req.environ)
new_etags = []
for etag in Match(old_etags).tags:
if etag == '*':
new_etags.append(etag)
continue
masked_etag = _hmac_etag(keys['object'], etag)
new_etags.extend(('"%s"' % etag, '"%s"' % masked_etag))
masked = True
req.headers[header_name] = ', '.join(new_etags)
try:
yield masked
finally:
if old_etags:
req.headers[header_name] = old_etags
def handle_get_or_head(self, req, start_response):
with self._mask_conditional_etags(req, 'If-Match') as masked1:
with self._mask_conditional_etags(req, 'If-None-Match') as masked2:
if masked1 or masked2:
update_etag_is_at_header(
req, 'X-Object-Sysmeta-Crypto-Etag-Mac')
resp = self._app_call(req.environ)
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return resp
class Encrypter(object):
"""Middleware for encrypting data and user metadata.
By default all PUT or POST'ed object data and/or metadata will be
encrypted. Encryption of new data and/or metadata may be disabled by
setting the ``disable_encryption`` option to True. However, this middleware
should remain in the pipeline in order for existing encrypted data to be
read.
"""
def __init__(self, app, conf):
self.app = app
self.logger = get_logger(conf, log_route="encrypter")
self.crypto = Crypto(conf)
self.disable_encryption = config_true_value(
conf.get('disable_encryption', 'false'))
def __call__(self, env, start_response):
# If override is set in env, then just pass along
if config_true_value(env.get('swift.crypto.override')):
return self.app(env, start_response)
req = Request(env)
if self.disable_encryption and req.method in ('PUT', 'POST'):
return self.app(env, start_response)
try:
req.split_path(4, 4, True)
except ValueError:
return self.app(env, start_response)
if req.method in ('GET', 'HEAD'):
handler = EncrypterObjContext(self, self.logger).handle_get_or_head
elif req.method == 'PUT':
handler = EncrypterObjContext(self, self.logger).handle_put
elif req.method == 'POST':
handler = EncrypterObjContext(self, self.logger).handle_post
else:
# anything else
return self.app(env, start_response)
try:
return handler(req, start_response)
except HTTPException as err_resp:
return err_resp(env, start_response)

153
swift/common/middleware/crypto/keymaster.py Normal file
View File

@ -0,0 +1,153 @@
# Copyright (c) 2015 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
import hashlib
import hmac
import os
from swift.common.middleware.crypto.crypto_utils import CRYPTO_KEY_CALLBACK
from swift.common.swob import Request, HTTPException
from swift.common.wsgi import WSGIContext
class KeyMasterContext(WSGIContext):
"""
The simple scheme for key derivation is as follows: every path is
associated with a key, where the key is derived from the path itself in a
deterministic fashion such that the key does not need to be stored.
Specifically, the key for any path is an HMAC of a root key and the path
itself, calculated using an SHA256 hash function::
<path_key> = HMAC_SHA256(<root_secret>, <path>)
"""
def __init__(self, keymaster, account, container, obj):
"""
:param keymaster: a Keymaster instance
:param account: account name
:param container: container name
:param obj: object name
"""
super(KeyMasterContext, self).__init__(keymaster.app)
self.keymaster = keymaster
self.account = account
self.container = container
self.obj = obj
self._keys = None
def fetch_crypto_keys(self, *args, **kwargs):
"""
Setup container and object keys based on the request path.
Keys are derived from request path. The 'id' entry in the results dict
includes the part of the path used to derive keys. Other keymaster
implementations may use a different strategy to generate keys and may
include a different type of 'id', so callers should treat the 'id' as
opaque keymaster-specific data.
:returns: A dict containing encryption keys for 'object' and
'container' and a key 'id'.
"""
if self._keys:
return self._keys
self._keys = {}
account_path = os.path.join(os.sep, self.account)
if self.container:
path = os.path.join(account_path, self.container)
self._keys['container'] = self.keymaster.create_key(path)
if self.obj:
path = os.path.join(path, self.obj)
self._keys['object'] = self.keymaster.create_key(path)
# For future-proofing include a keymaster version number and the
# path used to derive keys in the 'id' entry of the results. The
# encrypter will persist this as part of the crypto-meta for
# encrypted data and metadata. If we ever change the way keys are
# generated then the decrypter could pass the persisted 'id' value
# when it calls fetch_crypto_keys to inform the keymaster as to how
# that particular data or metadata had its keys generated.
# Currently we have no need to do that, so we are simply persisting
# this information for future use.
self._keys['id'] = {'v': '1', 'path': path}
return self._keys
def handle_request(self, req, start_response):
req.environ[CRYPTO_KEY_CALLBACK] = self.fetch_crypto_keys
resp = self._app_call(req.environ)
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return resp
class KeyMaster(object):
"""Middleware for providing encryption keys.
The middleware requires its ``encryption_root_secret`` option to be set.
This is the root secret from which encryption keys are derived. This must
be set before first use to a value that is a base64 encoding of at least 32
bytes. The security of all encrypted data critically depends on this key,
therefore it should be set to a high-entropy value. For example, a suitable
value may be obtained by base-64 encoding a 32 byte (or longer) value
generated by a cryptographically secure random number generator. Changing
the root secret is likely to result in data loss.
"""
def __init__(self, app, conf):
self.app = app
self.root_secret = conf.get('encryption_root_secret')
try:
self.root_secret = base64.b64decode(self.root_secret)
if len(self.root_secret) < 32:
raise ValueError
except (TypeError, ValueError):
raise ValueError(
'encryption_root_secret option in proxy-server.conf must be '
'a base64 encoding of at least 32 raw bytes')
def __call__(self, env, start_response):
req = Request(env)
try:
parts = req.split_path(2, 4, True)
except ValueError:
return self.app(env, start_response)
if req.method in ('PUT', 'POST', 'GET', 'HEAD'):
# handle only those request methods that may require keys
km_context = KeyMasterContext(self, *parts[1:])
try:
return km_context.handle_request(req, start_response)
except HTTPException as err_resp:
return err_resp(env, start_response)
# anything else
return self.app(env, start_response)
def create_key(self, key_id):
return hmac.new(self.root_secret, key_id,
digestmod=hashlib.sha256).digest()
def filter_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
def keymaster_filter(app):
return KeyMaster(app, conf)
return keymaster_filter

1
swift/common/swob.py
View File

@ -1419,6 +1419,7 @@ HTTPOk = status_map[200]
HTTPCreated = status_map[201]
HTTPAccepted = status_map[202]
HTTPNoContent = status_map[204]
HTTPPartialContent = status_map[206]
HTTPMovedPermanently = status_map[301]
HTTPFound = status_map[302]
HTTPSeeOther = status_map[303]

6
test/functional/__init__.py
View File

@ -361,6 +361,12 @@ def in_process_setup(the_object_server=object_server):
'allow_account_management': 'true',
'account_autocreate': 'true',
'allow_versions': 'True',
# TODO - Remove encryption_root_secret - this is only necessary while
# encryption middleware is in the default proxy pipeline in
# proxy-server.conf-sample
# base64 encoding of "dontEverUseThisIn_PRODUCTION_xxxxxxxxxxxxxxx"
'encryption_root_secret':
'ZG9udEV2ZXJVc2VUaGlzSW5fUFJPRFVDVElPTl94eHh4eHh4eHh4eHh4eHg=',
# Below are values used by the functional test framework, as well as
# by the various in-process swift servers
'auth_host': '127.0.0.1',

15
test/probe/test_empty_device_handoff.py
View File

@ -73,6 +73,13 @@ class TestEmptyDevice(ReplProbeTest):
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Stash the on disk data from a primary for future comparison with the
# handoff - this may not equal 'VERIFY' if for example the proxy has
# crypto enabled
direct_get_data = direct_client.direct_get_object(
onodes[1], opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
# Kill other two container/obj primary servers
# to ensure GET handoff works
for node in onodes[1:]:
@ -95,9 +102,7 @@ class TestEmptyDevice(ReplProbeTest):
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj,
headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
self.assertEqual(direct_get_data, odata)
# Assert container listing (via proxy and directly) has container/obj
objs = [o['name'] for o in
@ -155,9 +160,7 @@ class TestEmptyDevice(ReplProbeTest):
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
self.assertEqual(direct_get_data, odata)
# Assert the handoff server no longer has container/obj
try:

19
test/probe/test_object_failures.py
View File

@ -77,6 +77,12 @@ class TestObjectFailures(ReplProbeTest):
obj = 'object-%s' % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj,
'VERIFY')
# Stash the on disk data for future comparison - this may not equal
# 'VERIFY' if for example the proxy has crypto enabled
backend_data = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
metadata = read_metadata(data_file)
metadata['ETag'] = 'badetag'
write_metadata(data_file, metadata)
@ -84,7 +90,7 @@ class TestObjectFailures(ReplProbeTest):
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
self.assertEqual(odata, 'VERIFY')
self.assertEqual(odata, backend_data)
try:
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
@ -98,14 +104,19 @@ class TestObjectFailures(ReplProbeTest):
obj = 'object-range-%s' % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj,
'RANGE')
# Stash the on disk data for future comparison - this may not equal
# 'VERIFY' if for example the proxy has crypto enabled
backend_data = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
metadata = read_metadata(data_file)
metadata['ETag'] = 'badetag'
write_metadata(data_file, metadata)
base_headers = {'X-Backend-Storage-Policy-Index': self.policy.idx}
for header, result in [({'Range': 'bytes=0-2'}, 'RAN'),
({'Range': 'bytes=1-11'}, 'ANGE'),
({'Range': 'bytes=0-11'}, 'RANGE')]:
for header, result in [({'Range': 'bytes=0-2'}, backend_data[0:3]),
({'Range': 'bytes=1-11'}, backend_data[1:]),
({'Range': 'bytes=0-11'}, backend_data)]:
req_headers = base_headers.copy()
req_headers.update(header)
odata = direct_client.direct_get_object(

34
test/probe/test_object_handoff.py
View File

@ -55,6 +55,13 @@ class TestObjectHandoff(ReplProbeTest):
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Stash the on disk data from a primary for future comparison with the
# handoff - this may not equal 'VERIFY' if for example the proxy has
# crypto enabled
direct_get_data = direct_client.direct_get_object(
onodes[1], opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
# Kill other two container/obj primary servers
# to ensure GET handoff works
for node in onodes[1:]:
@ -76,9 +83,7 @@ class TestObjectHandoff(ReplProbeTest):
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
self.assertEqual(direct_get_data, odata)
# drop a tempfile in the handoff's datadir, like it might have
# had if there was an rsync failure while it was previously a
@ -143,9 +148,7 @@ class TestObjectHandoff(ReplProbeTest):
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
'X-Backend-Storage-Policy-Index': self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
self.assertEqual(direct_get_data, odata)
# and that it does *not* have a temporary rsync dropping!
found_data_filename = False
@ -273,6 +276,14 @@ class TestECObjectHandoffOverwrite(ECProbeTest):
# shutdown one of the primary data nodes
failed_primary = random.choice(onodes)
failed_primary_device_path = self.device_dir('object', failed_primary)
# first read its ec etag value for future reference - this may not
# equal old_contents.etag if for example the proxy has crypto enabled
req_headers = {'X-Backend-Storage-Policy-Index': int(self.policy)}
headers = direct_client.direct_head_object(
failed_primary, opart, self.account, container_name,
object_name, headers=req_headers)
old_backend_etag = headers['X-Object-Sysmeta-EC-Etag']
self.kill_drive(failed_primary_device_path)
# overwrite our object with some new data
@ -290,13 +301,18 @@ class TestECObjectHandoffOverwrite(ECProbeTest):
failed_primary, opart, self.account, container_name,
object_name, headers=req_headers)
self.assertEqual(headers['X-Object-Sysmeta-EC-Etag'],
old_contents.etag)
old_backend_etag)
# we have 1 primary with wrong old etag, and we should have 5 with
# new etag plus a handoff with the new etag, so killing 2 other
# primaries forces proxy to try to GET from all primaries plus handoff.
other_nodes = [n for n in onodes if n != failed_primary]
random.shuffle(other_nodes)
# grab the value of the new content's ec etag for future reference
headers = direct_client.direct_head_object(
other_nodes[0], opart, self.account, container_name,
object_name, headers=req_headers)
new_backend_etag = headers['X-Object-Sysmeta-EC-Etag']
for node in other_nodes[:2]:
self.kill_drive(self.device_dir('object', node))
@ -314,8 +330,8 @@ class TestECObjectHandoffOverwrite(ECProbeTest):
continue
found_frags[headers['X-Object-Sysmeta-EC-Etag']] += 1
self.assertEqual(found_frags, {
new_contents.etag: 4, # this should be enough to rebuild!
old_contents.etag: 1,
new_backend_etag: 4, # this should be enough to rebuild!
old_backend_etag: 1,
})
# clear node error limiting

0
test/unit/common/middleware/crypto/__init__.py Normal file
View File

54
test/unit/common/middleware/crypto/crypto_helpers.py Normal file
View File

@ -0,0 +1,54 @@
# Copyright (c) 2015-2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
import hashlib
from swift.common.middleware.crypto.crypto_utils import Crypto
def fetch_crypto_keys():
return {'account': 'This is an account key 012345678',
'container': 'This is a container key 01234567',
'object': 'This is an object key 0123456789',
'id': {'v': 'fake', 'path': '/a/c/fake'}}
def md5hex(s):
return hashlib.md5(s).hexdigest()
def encrypt(val, key=None, iv=None, ctxt=None):
if ctxt is None:
ctxt = Crypto({}).create_encryption_ctxt(key, iv)
enc_val = ctxt.update(val)
return enc_val
def decrypt(key, iv, enc_val):
dec_ctxt = Crypto({}).create_decryption_ctxt(key, iv, 0)
dec_val = dec_ctxt.update(enc_val)
return dec_val
FAKE_IV = "This is an IV123"
# do not use this example encryption_root_secret in production, use a randomly
# generated value with high entropy
TEST_KEYMASTER_CONF = {'encryption_root_secret': base64.b64encode(b'x' * 32)}
def fake_get_crypto_meta(**kwargs):
meta = {'iv': FAKE_IV, 'cipher': Crypto.cipher}
meta.update(kwargs)
return meta

39
test/unit/common/middleware/crypto/test_crypto.py Normal file
View File

@ -0,0 +1,39 @@
# Copyright (c) 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from swift.common import utils
from swift.common.middleware import crypto
class TestCrypto(unittest.TestCase):
def test_filter_factory(self):
factory = crypto.filter_factory({})
self.assertTrue(callable(factory))
self.assertIsInstance(factory({}), crypto.decrypter.Decrypter)
self.assertIsInstance(factory({}).app, crypto.encrypter.Encrypter)
self.assertIn('encryption', utils._swift_admin_info)
self.assertDictEqual(
{'enabled': True}, utils._swift_admin_info['encryption'])
self.assertNotIn('encryption', utils._swift_info)
factory = crypto.filter_factory({'disable_encryption': True})
self.assertTrue(callable(factory))
self.assertIsInstance(factory({}), crypto.decrypter.Decrypter)
self.assertIsInstance(factory({}).app, crypto.encrypter.Encrypter)
self.assertIn('encryption', utils._swift_admin_info)
self.assertDictEqual(
{'enabled': False}, utils._swift_admin_info['encryption'])
self.assertNotIn('encryption', utils._swift_info)

495
test/unit/common/middleware/crypto/test_crypto_utils.py Normal file
View File

@ -0,0 +1,495 @@
# Copyright (c) 2015-2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import unittest
import mock
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from swift.common.exceptions import EncryptionException
from swift.common.middleware.crypto import crypto_utils
from swift.common.middleware.crypto.crypto_utils import (
CRYPTO_KEY_CALLBACK, Crypto, CryptoWSGIContext)
from swift.common.swob import HTTPException
from test.unit import FakeLogger
from test.unit.common.middleware.crypto.crypto_helpers import fetch_crypto_keys
class TestCryptoWsgiContext(unittest.TestCase):
def setUp(self):
class FakeFilter(object):
app = None
crypto = Crypto({})
self.fake_logger = FakeLogger()
self.crypto_context = CryptoWSGIContext(
FakeFilter(), 'object', self.fake_logger)
def test_get_keys(self):
# ok
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
keys = self.crypto_context.get_keys(env)
self.assertDictEqual(fetch_crypto_keys(), keys)
# only default required keys are checked
subset_keys = {'object': fetch_crypto_keys()['object']}
env = {CRYPTO_KEY_CALLBACK: lambda: subset_keys}
keys = self.crypto_context.get_keys(env)
self.assertDictEqual(subset_keys, keys)
# only specified required keys are checked
subset_keys = {'container': fetch_crypto_keys()['container']}
env = {CRYPTO_KEY_CALLBACK: lambda: subset_keys}
keys = self.crypto_context.get_keys(env, required=['container'])
self.assertDictEqual(subset_keys, keys)
subset_keys = {'object': fetch_crypto_keys()['object'],
'container': fetch_crypto_keys()['container']}
env = {CRYPTO_KEY_CALLBACK: lambda: subset_keys}
keys = self.crypto_context.get_keys(
env, required=['object', 'container'])
self.assertDictEqual(subset_keys, keys)
def test_get_keys_missing_callback(self):
with self.assertRaises(HTTPException) as cm:
self.crypto_context.get_keys({})
self.assertIn('500 Internal Error', cm.exception.message)
self.assertIn('missing callback',
self.fake_logger.get_lines_for_level('error')[0])
self.assertIn('Unable to retrieve encryption keys.', cm.exception.body)
def test_get_keys_callback_exception(self):
def callback():
raise Exception('boom')
with self.assertRaises(HTTPException) as cm:
self.crypto_context.get_keys({CRYPTO_KEY_CALLBACK: callback})
self.assertIn('500 Internal Error', cm.exception.message)
self.assertIn('from callback: boom',
self.fake_logger.get_lines_for_level('error')[0])
self.assertIn('Unable to retrieve encryption keys.', cm.exception.body)
def test_get_keys_missing_key_for_default_required_list(self):
bad_keys = dict(fetch_crypto_keys())
bad_keys.pop('object')
with self.assertRaises(HTTPException) as cm:
self.crypto_context.get_keys(
{CRYPTO_KEY_CALLBACK: lambda: bad_keys})
self.assertIn('500 Internal Error', cm.exception.message)
self.assertIn("Missing key for 'object'",
self.fake_logger.get_lines_for_level('error')[0])
self.assertIn('Unable to retrieve encryption keys.', cm.exception.body)
def test_get_keys_missing_object_key_for_specified_required_list(self):
bad_keys = dict(fetch_crypto_keys())
bad_keys.pop('object')
with self.assertRaises(HTTPException) as cm:
self.crypto_context.get_keys(
{CRYPTO_KEY_CALLBACK: lambda: bad_keys},
required=['object', 'container'])
self.assertIn('500 Internal Error', cm.exception.message)
self.assertIn("Missing key for 'object'",
self.fake_logger.get_lines_for_level('error')[0])
self.assertIn('Unable to retrieve encryption keys.', cm.exception.body)
def test_get_keys_missing_container_key_for_specified_required_list(self):
bad_keys = dict(fetch_crypto_keys())
bad_keys.pop('container')
with self.assertRaises(HTTPException) as cm:
self.crypto_context.get_keys(
{CRYPTO_KEY_CALLBACK: lambda: bad_keys},
required=['object', 'container'])
self.assertIn('500 Internal Error', cm.exception.message)
self.assertIn("Missing key for 'container'",
self.fake_logger.get_lines_for_level('error')[0])
self.assertIn('Unable to retrieve encryption keys.', cm.exception.body)
def test_bad_object_key_for_default_required_list(self):
bad_keys = dict(fetch_crypto_keys())
bad_keys['object'] = 'the minor key'
with self.assertRaises(HTTPException) as cm:
self.crypto_context.get_keys(
{CRYPTO_KEY_CALLBACK: lambda: bad_keys})
self.assertIn('500 Internal Error', cm.exception.message)
self.assertIn("Bad key for 'object'",
self.fake_logger.get_lines_for_level('error')[0])
self.assertIn('Unable to retrieve encryption keys.', cm.exception.body)
def test_bad_container_key_for_default_required_list(self):
bad_keys = dict(fetch_crypto_keys())
bad_keys['container'] = 'the major key'
with self.assertRaises(HTTPException) as cm:
self.crypto_context.get_keys(
{CRYPTO_KEY_CALLBACK: lambda: bad_keys},
required=['object', 'container'])
self.assertIn('500 Internal Error', cm.exception.message)
self.assertIn("Bad key for 'container'",
self.fake_logger.get_lines_for_level('error')[0])
self.assertIn('Unable to retrieve encryption keys.', cm.exception.body)
def test_get_keys_not_a_dict(self):
with self.assertRaises(HTTPException) as cm:
self.crypto_context.get_keys(
{CRYPTO_KEY_CALLBACK: lambda: ['key', 'quay', 'qui']})
self.assertIn('500 Internal Error', cm.exception.message)
self.assertIn("Did not get a keys dict",
self.fake_logger.get_lines_for_level('error')[0])
self.assertIn('Unable to retrieve encryption keys.', cm.exception.body)
class TestModuleMethods(unittest.TestCase):
meta = {'iv': '0123456789abcdef', 'cipher': 'AES_CTR_256'}
serialized_meta = '%7B%22cipher%22%3A+%22AES_CTR_256%22%2C+%22' \
'iv%22%3A+%22MDEyMzQ1Njc4OWFiY2RlZg%3D%3D%22%7D'
meta_with_key = {'iv': '0123456789abcdef', 'cipher': 'AES_CTR_256',
'body_key': {'key': 'fedcba9876543210fedcba9876543210',
'iv': 'fedcba9876543210'}}
serialized_meta_with_key = '%7B%22body_key%22%3A+%7B%22iv%22%3A+%22ZmVkY' \
'2JhOTg3NjU0MzIxMA%3D%3D%22%2C+%22key%22%3A+%' \
'22ZmVkY2JhOTg3NjU0MzIxMGZlZGNiYTk4NzY1NDMyMT' \
'A%3D%22%7D%2C+%22cipher%22%3A+%22AES_CTR_256' \
'%22%2C+%22iv%22%3A+%22MDEyMzQ1Njc4OWFiY2RlZg' \
'%3D%3D%22%7D'
def test_dump_crypto_meta(self):
actual = crypto_utils.dump_crypto_meta(self.meta)
self.assertEqual(self.serialized_meta, actual)
actual = crypto_utils.dump_crypto_meta(self.meta_with_key)
self.assertEqual(self.serialized_meta_with_key, actual)
def test_load_crypto_meta(self):
actual = crypto_utils.load_crypto_meta(self.serialized_meta)
self.assertEqual(self.meta, actual)
actual = crypto_utils.load_crypto_meta(self.serialized_meta_with_key)
self.assertEqual(self.meta_with_key, actual)
def assert_raises(value, message):
with self.assertRaises(EncryptionException) as cm:
crypto_utils.load_crypto_meta(value)
self.assertIn('Bad crypto meta %r' % value, cm.exception.message)
self.assertIn(message, cm.exception.message)
assert_raises(None, 'crypto meta not a string')
assert_raises(99, 'crypto meta not a string')
assert_raises('', 'No JSON object could be decoded')
assert_raises('abc', 'No JSON object could be decoded')
assert_raises('[]', 'crypto meta not a Mapping')
assert_raises('{"iv": "abcdef"}', 'Incorrect padding')
assert_raises('{"iv": []}', 'must be string or buffer')
assert_raises('{"iv": {}}', 'must be string or buffer')
assert_raises('{"iv": 99}', 'must be string or buffer')
assert_raises('{"key": "abcdef"}', 'Incorrect padding')
assert_raises('{"key": []}', 'must be string or buffer')
assert_raises('{"key": {}}', 'must be string or buffer')
assert_raises('{"key": 99}', 'must be string or buffer')
assert_raises('{"body_key": {"iv": "abcdef"}}', 'Incorrect padding')
assert_raises('{"body_key": {"iv": []}}', 'must be string or buffer')
assert_raises('{"body_key": {"iv": {}}}', 'must be string or buffer')
assert_raises('{"body_key": {"iv": 99}}', 'must be string or buffer')
assert_raises('{"body_key": {"key": "abcdef"}}', 'Incorrect padding')
assert_raises('{"body_key": {"key": []}}', 'must be string or buffer')
assert_raises('{"body_key": {"key": {}}}', 'must be string or buffer')
assert_raises('{"body_key": {"key": 99}}', 'must be string or buffer')
def test_dump_then_load_crypto_meta(self):
actual = crypto_utils.load_crypto_meta(
crypto_utils.dump_crypto_meta(self.meta))
self.assertEqual(self.meta, actual)
actual = crypto_utils.load_crypto_meta(
crypto_utils.dump_crypto_meta(self.meta_with_key))
self.assertEqual(self.meta_with_key, actual)
def test_append_crypto_meta(self):
actual = crypto_utils.append_crypto_meta('abc', self.meta)
expected = 'abc; swift_meta=%s' % self.serialized_meta
self.assertEqual(actual, expected)
actual = crypto_utils.append_crypto_meta('abc', self.meta_with_key)
expected = 'abc; swift_meta=%s' % self.serialized_meta_with_key
self.assertEqual(actual, expected)
def test_extract_crypto_meta(self):
val, meta = crypto_utils.extract_crypto_meta(
'abc; swift_meta=%s' % self.serialized_meta)
self.assertEqual('abc', val)
self.assertDictEqual(self.meta, meta)
val, meta = crypto_utils.extract_crypto_meta(
'abc; swift_meta=%s' % self.serialized_meta_with_key)
self.assertEqual('abc', val)
self.assertDictEqual(self.meta_with_key, meta)
val, meta = crypto_utils.extract_crypto_meta('abc')
self.assertEqual('abc', val)
self.assertIsNone(meta)
# other param names will be ignored
val, meta = crypto_utils.extract_crypto_meta('abc; foo=bar')
self.assertEqual('abc', val)
self.assertIsNone(meta)
def test_append_then_extract_crypto_meta(self):
val = 'abc'
actual = crypto_utils.extract_crypto_meta(
crypto_utils.append_crypto_meta(val, self.meta))
self.assertEqual((val, self.meta), actual)
class TestCrypto(unittest.TestCase):
def setUp(self):
self.crypto = Crypto({})
def test_create_encryption_context(self):
value = 'encrypt me' * 100 # more than one cipher block
key = os.urandom(32)
iv = os.urandom(16)
ctxt = self.crypto.create_encryption_ctxt(key, iv)
expected = Cipher(
algorithms.AES(key), modes.CTR(iv),
backend=default_backend()).encryptor().update(value)
self.assertEqual(expected, ctxt.update(value))
for bad_iv in ('a little too long', 'too short'):
self.assertRaises(
ValueError, self.crypto.create_encryption_ctxt, key, bad_iv)
for bad_key in ('objKey', 'a' * 31, 'a' * 33, 'a' * 16, 'a' * 24):
self.assertRaises(
ValueError, self.crypto.create_encryption_ctxt, bad_key, iv)
def test_create_decryption_context(self):
value = 'decrypt me' * 100 # more than one cipher block
key = os.urandom(32)
iv = os.urandom(16)
ctxt = self.crypto.create_decryption_ctxt(key, iv, 0)
expected = Cipher(
algorithms.AES(key), modes.CTR(iv),
backend=default_backend()).decryptor().update(value)
self.assertEqual(expected, ctxt.update(value))
for bad_iv in ('a little too long', 'too short'):
self.assertRaises(
ValueError, self.crypto.create_decryption_ctxt, key, bad_iv, 0)
for bad_key in ('objKey', 'a' * 31, 'a' * 33, 'a' * 16, 'a' * 24):
self.assertRaises(
ValueError, self.crypto.create_decryption_ctxt, bad_key, iv, 0)
with self.assertRaises(ValueError) as cm:
self.crypto.create_decryption_ctxt(key, iv, -1)
self.assertEqual("Offset must not be negative", cm.exception.message)
def test_enc_dec_small_chunks(self):
self.enc_dec_chunks(['encrypt me', 'because I', 'am sensitive'])
def test_enc_dec_large_chunks(self):
self.enc_dec_chunks([os.urandom(65536), os.urandom(65536)])
def enc_dec_chunks(self, chunks):
key = 'objL7wjV6L79Sfs4y7dy41273l0k6Wki'
iv = self.crypto.create_iv()
enc_ctxt = self.crypto.create_encryption_ctxt(key, iv)
enc_val = [enc_ctxt.update(chunk) for chunk in chunks]
self.assertTrue(''.join(enc_val) != chunks)
dec_ctxt = self.crypto.create_decryption_ctxt(key, iv, 0)
dec_val = [dec_ctxt.update(chunk) for chunk in enc_val]
self.assertEqual(''.join(chunks), ''.join(dec_val),
'Expected value {%s} but got {%s}' %
(''.join(chunks), ''.join(dec_val)))
def test_decrypt_range(self):
chunks = ['0123456789abcdef', 'ghijklmnopqrstuv']
key = 'objL7wjV6L79Sfs4y7dy41273l0k6Wki'
iv = self.crypto.create_iv()
enc_ctxt = self.crypto.create_encryption_ctxt(key, iv)
enc_val = [enc_ctxt.update(chunk) for chunk in chunks]
self.assertTrue(''.join(enc_val) != chunks)
# Simulate a ranged GET from byte 19 to 32 : 'jklmnopqrstuv'
dec_ctxt = self.crypto.create_decryption_ctxt(key, iv, 19)
ranged_chunks = [enc_val[1][3:]]
dec_val = [dec_ctxt.update(chunk) for chunk in ranged_chunks]
self.assertEqual('jklmnopqrstuv', ''.join(dec_val),
'Expected value {%s} but got {%s}' %
('jklmnopqrstuv', ''.join(dec_val)))
def test_create_decryption_context_non_zero_offset(self):
# Verify that iv increments for each 16 bytes of offset.
# For a ranged GET we pass a non-zero offset so that the decrypter
# counter is incremented to the correct value to start decrypting at
# that offset into the object body. The counter should increment by one
# from the starting IV value for every 16 bytes offset into the object
# body, until it reaches 2^128 -1 when it should wrap to zero. We check
# that is happening by verifying a decrypted value using various
# offsets.
key = 'objL7wjV6L79Sfs4y7dy41273l0k6Wki'
def do_test():
for offset, exp_iv in mappings.items():
dec_ctxt = self.crypto.create_decryption_ctxt(key, iv, offset)
offset_in_block = offset % 16
cipher = Cipher(algorithms.AES(key),
modes.CTR(exp_iv),
backend=default_backend())
expected = cipher.decryptor().update(
'p' * offset_in_block + 'ciphertext')
actual = dec_ctxt.update('ciphertext')
expected = expected[offset % 16:]
self.assertEqual(expected, actual,
'Expected %r but got %r, iv=%s and offset=%s'
% (expected, actual, iv, offset))
iv = '0000000010000000'
mappings = {
2: '0000000010000000',
16: '0000000010000001',
19: '0000000010000001',
48: '0000000010000003',
1024: '000000001000000p',
5119: '000000001000001o'
}
do_test()
# choose max iv value and test that it wraps to zero
iv = chr(0xff) * 16
mappings = {
2: iv,
16: str(bytearray.fromhex('00' * 16)), # iv wraps to 0
19: str(bytearray.fromhex('00' * 16)),
48: str(bytearray.fromhex('00' * 15 + '02')),
1024: str(bytearray.fromhex('00' * 15 + '3f')),
5119: str(bytearray.fromhex('00' * 14 + '013E'))
}
do_test()
iv = chr(0x0) * 16
mappings = {
2: iv,
16: str(bytearray.fromhex('00' * 15 + '01')),
19: str(bytearray.fromhex('00' * 15 + '01')),
48: str(bytearray.fromhex('00' * 15 + '03')),
1024: str(bytearray.fromhex('00' * 15 + '40')),
5119: str(bytearray.fromhex('00' * 14 + '013F'))
}
do_test()
iv = chr(0x0) * 8 + chr(0xff) * 8
mappings = {
2: iv,
16: str(bytearray.fromhex('00' * 7 + '01' + '00' * 8)),
19: str(bytearray.fromhex('00' * 7 + '01' + '00' * 8)),
48: str(bytearray.fromhex('00' * 7 + '01' + '00' * 7 + '02')),
1024: str(bytearray.fromhex('00' * 7 + '01' + '00' * 7 + '3F')),
5119: str(bytearray.fromhex('00' * 7 + '01' + '00' * 6 + '013E'))
}
do_test()
def test_check_key(self):
for key in ('objKey', 'a' * 31, 'a' * 33, 'a' * 16, 'a' * 24):
with self.assertRaises(ValueError) as cm:
self.crypto.check_key(key)
self.assertEqual("Key must be length 32 bytes",
cm.exception.message)
def test_check_crypto_meta(self):
meta = {'cipher': 'AES_CTR_256'}
with self.assertRaises(EncryptionException) as cm:
self.crypto.check_crypto_meta(meta)
self.assertEqual("Bad crypto meta: Missing 'iv'",
cm.exception.message)
for bad_iv in ('a little too long', 'too short'):
meta['iv'] = bad_iv
with self.assertRaises(EncryptionException) as cm:
self.crypto.check_crypto_meta(meta)
self.assertEqual("Bad crypto meta: IV must be length 16 bytes",
cm.exception.message)
meta = {'iv': os.urandom(16)}
with self.assertRaises(EncryptionException) as cm:
self.crypto.check_crypto_meta(meta)
self.assertEqual("Bad crypto meta: Missing 'cipher'",
cm.exception.message)
meta['cipher'] = 'Mystery cipher'
with self.assertRaises(EncryptionException) as cm:
self.crypto.check_crypto_meta(meta)
self.assertEqual("Bad crypto meta: Cipher must be AES_CTR_256",
cm.exception.message)
def test_create_iv(self):
self.assertEqual(16, len(self.crypto.create_iv()))
# crude check that we get back different values on each call
self.assertNotEqual(self.crypto.create_iv(), self.crypto.create_iv())
def test_get_crypto_meta(self):
meta = self.crypto.create_crypto_meta()
self.assertIsInstance(meta, dict)
# this is deliberately brittle so that if new items are added then the
# test will need to be updated
self.assertEqual(2, len(meta))
self.assertIn('iv', meta)
self.assertEqual(16, len(meta['iv']))
self.assertIn('cipher', meta)
self.assertEqual('AES_CTR_256', meta['cipher'])
self.crypto.check_crypto_meta(meta) # sanity check
meta2 = self.crypto.create_crypto_meta()
self.assertNotEqual(meta['iv'], meta2['iv']) # crude sanity check
def test_create_random_key(self):
# crude check that we get unique keys on each call
keys = set()
for i in range(10):
key = self.crypto.create_random_key()
self.assertEqual(32, len(key))
keys.add(key)
self.assertEqual(10, len(keys))
def test_wrap_unwrap_key(self):
wrapping_key = os.urandom(32)
key_to_wrap = os.urandom(32)
iv = os.urandom(16)
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.Crypto.create_iv',
return_value=iv):
wrapped = self.crypto.wrap_key(wrapping_key, key_to_wrap)
cipher = Cipher(algorithms.AES(wrapping_key), modes.CTR(iv),
backend=default_backend())
expected = {'key': cipher.encryptor().update(key_to_wrap),
'iv': iv}
self.assertEqual(expected, wrapped)
unwrapped = self.crypto.unwrap_key(wrapping_key, wrapped)
self.assertEqual(key_to_wrap, unwrapped)
def test_unwrap_bad_key(self):
# verify that ValueError is raised if unwrapped key is invalid
wrapping_key = os.urandom(32)
for length in (0, 16, 24, 31, 33):
key_to_wrap = os.urandom(length)
wrapped = self.crypto.wrap_key(wrapping_key, key_to_wrap)
with self.assertRaises(ValueError) as cm:
self.crypto.unwrap_key(wrapping_key, wrapped)
self.assertEqual(
cm.exception.message, 'Key must be length 32 bytes')
if __name__ == '__main__':
unittest.main()

1119
test/unit/common/middleware/crypto/test_decrypter.py Normal file
View File

File diff suppressed because it is too large Load Diff

820
test/unit/common/middleware/crypto/test_encrypter.py Normal file
View File

@ -0,0 +1,820 @@
# Copyright (c) 2015-2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
import hashlib
import hmac
import json
import os
import unittest
import urllib
import mock
from swift.common.middleware.crypto import encrypter
from swift.common.middleware.crypto.crypto_utils import (
CRYPTO_KEY_CALLBACK, Crypto)
from swift.common.swob import (
Request, HTTPException, HTTPCreated, HTTPAccepted, HTTPOk, HTTPBadRequest)
from swift.common.utils import FileLikeIter
from test.unit import FakeLogger, EMPTY_ETAG
from test.unit.common.middleware.crypto.crypto_helpers import (
fetch_crypto_keys, md5hex, FAKE_IV, encrypt)
from test.unit.common.middleware.helpers import FakeSwift, FakeAppThatExcepts
@mock.patch('swift.common.middleware.crypto.crypto_utils.Crypto.create_iv',
lambda *args: FAKE_IV)
class TestEncrypter(unittest.TestCase):
def setUp(self):
self.app = FakeSwift()
self.encrypter = encrypter.Encrypter(self.app, {})
self.encrypter.logger = FakeLogger()
def _verify_user_metadata(self, req_hdrs, name, value, key):
# verify encrypted version of user metadata
self.assertNotIn('X-Object-Meta-' + name, req_hdrs)
expected_hdr = 'X-Object-Transient-Sysmeta-Crypto-Meta-' + name
self.assertIn(expected_hdr, req_hdrs)
enc_val, param = req_hdrs[expected_hdr].split(';')
param = param.strip()
self.assertTrue(param.startswith('swift_meta='))
actual_meta = json.loads(
urllib.unquote_plus(param[len('swift_meta='):]))
self.assertEqual(Crypto.cipher, actual_meta['cipher'])
meta_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, meta_iv)
self.assertEqual(
base64.b64encode(encrypt(value, key, meta_iv)),
enc_val)
# if there is any encrypted user metadata then this header should exist
self.assertIn('X-Object-Transient-Sysmeta-Crypto-Meta', req_hdrs)
common_meta = json.loads(urllib.unquote_plus(
req_hdrs['X-Object-Transient-Sysmeta-Crypto-Meta']))
self.assertDictEqual({'cipher': Crypto.cipher,
'key_id': {'v': 'fake', 'path': '/a/c/fake'}},
common_meta)
def test_PUT_req(self):
body_key = os.urandom(32)
object_key = fetch_crypto_keys()['object']
plaintext = 'FAKE APP'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
ciphertext_etag = md5hex(ciphertext)
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'etag': plaintext_etag,
'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'x-object-meta-etag': 'not to be confused with the Etag!',
'x-object-meta-test': 'encrypt me',
'x-object-sysmeta-test': 'do not encrypt me'}
req = Request.blank(
'/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key',
return_value=body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('PUT', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# verify body crypto meta
actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta']
actual = json.loads(urllib.unquote_plus(actual))
self.assertEqual(Crypto().cipher, actual['cipher'])
self.assertEqual(FAKE_IV, base64.b64decode(actual['iv']))
# verify wrapped body key
expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
self.assertEqual(expected_wrapped_key,
base64.b64decode(actual['body_key']['key']))
self.assertEqual(FAKE_IV,
base64.b64decode(actual['body_key']['iv']))
self.assertEqual(fetch_crypto_keys()['id'], actual['key_id'])
# verify etag
self.assertEqual(ciphertext_etag, req_hdrs['Etag'])
encrypted_etag, _junk, etag_meta = \
req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=')
# verify crypto_meta was appended to this etag
self.assertTrue(etag_meta)
actual_meta = json.loads(urllib.unquote_plus(etag_meta))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
# verify encrypted version of plaintext etag
actual = base64.b64decode(encrypted_etag)
etag_iv = base64.b64decode(actual_meta['iv'])
enc_etag = encrypt(plaintext_etag, object_key, etag_iv)
self.assertEqual(enc_etag, actual)
# verify etag MAC for conditional requests
actual_hmac = base64.b64decode(
req_hdrs['X-Object-Sysmeta-Crypto-Etag-Mac'])
self.assertEqual(actual_hmac, hmac.new(
object_key, plaintext_etag, hashlib.sha256).digest())
# verify encrypted etag for container update
self.assertIn(
'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
parts = req_hdrs[
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urllib.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])
cont_key = fetch_crypto_keys()['container']
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt(plaintext_etag, cont_key, cont_etag_iv),
base64.b64decode(parts[0]))
# content-type is not encrypted
self.assertEqual('text/plain', req_hdrs['Content-Type'])
# user meta is encrypted
self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', object_key)
self._verify_user_metadata(
req_hdrs, 'Etag', 'not to be confused with the Etag!', object_key)
# sysmeta is not encrypted
self.assertEqual('do not encrypt me',
req_hdrs['X-Object-Sysmeta-Test'])
# verify object is encrypted by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
resp = get_req.get_response(self.app)
self.assertEqual(ciphertext, resp.body)
self.assertEqual(ciphertext_etag, resp.headers['Etag'])
def test_PUT_zero_size_object(self):
# object body encryption should be skipped for zero sized object body
object_key = fetch_crypto_keys()['object']
plaintext_etag = EMPTY_ETAG
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'etag': EMPTY_ETAG,
'content-type': 'text/plain',
'content-length': '0',
'x-object-meta-etag': 'not to be confused with the Etag!',
'x-object-meta-test': 'encrypt me',
'x-object-sysmeta-test': 'do not encrypt me'}
req = Request.blank(
'/v1/a/c/o', environ=env, body='', headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('PUT', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# verify that there is no body crypto meta
self.assertNotIn('X-Object-Sysmeta-Crypto-Meta', req_hdrs)
# verify etag is md5 of plaintext
self.assertEqual(EMPTY_ETAG, req_hdrs['Etag'])
# verify there is no etag crypto meta
self.assertNotIn('X-Object-Sysmeta-Crypto-Etag', req_hdrs)
# verify there is no container update override for etag
self.assertNotIn(
'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
# user meta is still encrypted
self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', object_key)
self._verify_user_metadata(
req_hdrs, 'Etag', 'not to be confused with the Etag!', object_key)
# sysmeta is not encrypted
self.assertEqual('do not encrypt me',
req_hdrs['X-Object-Sysmeta-Test'])
# verify object is empty by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
resp = get_req.get_response(self.app)
self.assertEqual('', resp.body)
self.assertEqual(EMPTY_ETAG, resp.headers['Etag'])
def test_PUT_with_other_footers(self):
# verify handling of another middleware's footer callback
cont_key = fetch_crypto_keys()['container']
body_key = os.urandom(32)
object_key = fetch_crypto_keys()['object']
plaintext = 'FAKE APP'
plaintext_etag = md5hex(plaintext)
ciphertext = encrypt(plaintext, body_key, FAKE_IV)
ciphertext_etag = md5hex(ciphertext)
other_footers = {
'Etag': plaintext_etag,
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Size':
'other override',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'final etag'}
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': 'correct etag is in footers'}
req = Request.blank(
'/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key',
lambda *args: body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('PUT', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# verify that other middleware's footers made it to app, including any
# container update overrides but nothing Etag-related
other_footers.pop('Etag')
other_footers.pop('X-Object-Sysmeta-Container-Update-Override-Etag')
for k, v in other_footers.items():
self.assertEqual(v, req_hdrs[k])
# verify encryption footers are ok
encrypted_etag, _junk, etag_meta = \
req_hdrs['X-Object-Sysmeta-Crypto-Etag'].partition('; swift_meta=')
self.assertTrue(etag_meta)
actual_meta = json.loads(urllib.unquote_plus(etag_meta))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(ciphertext_etag, req_hdrs['Etag'])
actual = base64.b64decode(encrypted_etag)
etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(encrypt(plaintext_etag, object_key, etag_iv), actual)
# verify encrypted etag for container update
self.assertIn(
'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
parts = req_hdrs[
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urllib.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
cont_key = fetch_crypto_keys()['container']
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt('final etag', cont_key, cont_etag_iv),
base64.b64decode(parts[0]))
# verify body crypto meta
actual = req_hdrs['X-Object-Sysmeta-Crypto-Body-Meta']
actual = json.loads(urllib.unquote_plus(actual))
self.assertEqual(Crypto().cipher, actual['cipher'])
self.assertEqual(FAKE_IV, base64.b64decode(actual['iv']))
# verify wrapped body key
expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
self.assertEqual(expected_wrapped_key,
base64.b64decode(actual['body_key']['key']))
self.assertEqual(FAKE_IV,
base64.b64decode(actual['body_key']['iv']))
self.assertEqual(fetch_crypto_keys()['id'], actual['key_id'])
def test_PUT_with_etag_override_in_headers(self):
# verify handling of another middleware's
# container-update-override-etag in headers
plaintext = 'FAKE APP'
plaintext_etag = md5hex(plaintext)
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': plaintext_etag,
'X-Object-Sysmeta-Container-Update-Override-Etag':
'final etag'}
req = Request.blank(
'/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual(plaintext_etag, resp.headers['Etag'])
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(('PUT', '/v1/a/c/o'), self.app.calls[0])
req_hdrs = self.app.headers[0]
# verify encrypted etag for container update
self.assertIn(
'X-Object-Sysmeta-Container-Update-Override-Etag', req_hdrs)
parts = req_hdrs[
'X-Object-Sysmeta-Container-Update-Override-Etag'].rsplit(';', 1)
self.assertEqual(2, len(parts))
cont_key = fetch_crypto_keys()['container']
# extract crypto_meta from end of etag for container update
param = parts[1].strip()
crypto_meta_tag = 'swift_meta='
self.assertTrue(param.startswith(crypto_meta_tag), param)
actual_meta = json.loads(
urllib.unquote_plus(param[len(crypto_meta_tag):]))
self.assertEqual(Crypto().cipher, actual_meta['cipher'])
self.assertEqual(fetch_crypto_keys()['id'], actual_meta['key_id'])
cont_etag_iv = base64.b64decode(actual_meta['iv'])
self.assertEqual(FAKE_IV, cont_etag_iv)
self.assertEqual(encrypt('final etag', cont_key, cont_etag_iv),
base64.b64decode(parts[0]))
def test_PUT_with_bad_etag_in_other_footers(self):
# verify that etag supplied in footers from other middleware overrides
# header etag when validating inbound plaintext etags
plaintext = 'FAKE APP'
plaintext_etag = md5hex(plaintext)
other_footers = {
'Etag': 'bad etag',
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'other override'}
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': plaintext_etag}
req = Request.blank(
'/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('422 Unprocessable Entity', resp.status)
self.assertNotIn('Etag', resp.headers)
def test_PUT_with_bad_etag_in_headers_and_other_footers(self):
# verify that etag supplied in headers from other middleware is used if
# none is supplied in footers when validating inbound plaintext etags
plaintext = 'FAKE APP'
other_footers = {
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'other override'}
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(plaintext)),
'Etag': 'bad etag'}
req = Request.blank(
'/v1/a/c/o', environ=env, body=plaintext, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('422 Unprocessable Entity', resp.status)
self.assertNotIn('Etag', resp.headers)
def test_PUT_nothing_read(self):
# simulate an artificial scenario of a downstream filter/app not
# actually reading the input stream from encrypter.
class NonReadingApp(object):
def __call__(self, env, start_response):
# note: no read from wsgi.input
req = Request(env)
env['swift.callback.update_footers'](req.headers)
call_headers.append(req.headers)
resp = HTTPCreated(req=req, headers={'Etag': 'response etag'})
return resp(env, start_response)
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'content-type': 'text/plain',
'content-length': 0,
'etag': 'etag from client'}
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
call_headers = []
resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
self.assertEqual('201 Created', resp.status)
self.assertEqual('response etag', resp.headers['Etag'])
self.assertEqual(1, len(call_headers))
self.assertEqual('etag from client', call_headers[0]['etag'])
# verify no encryption footers
for k in call_headers[0]:
self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))
# check that an upstream footer callback gets called
other_footers = {
'Etag': 'other etag',
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Backend-Container-Update-Override-Etag': 'other override'}
env.update({'swift.callback.update_footers':
lambda footers: footers.update(other_footers)})
req = Request.blank('/v1/a/c/o', environ=env, body='', headers=hdrs)
call_headers = []
resp = req.get_response(encrypter.Encrypter(NonReadingApp(), {}))
self.assertEqual('201 Created', resp.status)
self.assertEqual('response etag', resp.headers['Etag'])
self.assertEqual(1, len(call_headers))
# verify that other middleware's footers made it to app
for k, v in other_footers.items():
self.assertEqual(v, call_headers[0][k])
# verify no encryption footers
for k in call_headers[0]:
self.assertFalse(k.lower().startswith('x-object-sysmeta-crypto-'))
def test_POST_req(self):
body = 'FAKE APP'
env = {'REQUEST_METHOD': 'POST',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'x-object-meta-test': 'encrypt me',
'x-object-sysmeta-test': 'do not encrypt me'}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
key = fetch_crypto_keys()['object']
self.app.register('POST', '/v1/a/c/o', HTTPAccepted, {})
resp = req.get_response(self.encrypter)
self.assertEqual('202 Accepted', resp.status)
self.assertNotIn('Etag', resp.headers)
# verify metadata items
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual('POST', self.app.calls[0][0])
req_hdrs = self.app.headers[0]
# user meta is encrypted
self._verify_user_metadata(req_hdrs, 'Test', 'encrypt me', key)
# sysmeta is not encrypted
self.assertEqual('do not encrypt me',
req_hdrs['X-Object-Sysmeta-Test'])
def _test_no_user_metadata(self, method):
# verify that x-object-transient-sysmeta-crypto-meta is not set when
# there is no user metadata
env = {'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank('/v1/a/c/o', environ=env, body='body')
self.app.register(method, '/v1/a/c/o', HTTPAccepted, {})
resp = req.get_response(self.encrypter)
self.assertEqual('202 Accepted', resp.status)
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(method, self.app.calls[0][0])
self.assertNotIn('x-object-transient-sysmeta-crypto-meta',
self.app.headers[0])
def test_PUT_no_user_metadata(self):
self._test_no_user_metadata('PUT')
def test_POST_no_user_metadata(self):
self._test_no_user_metadata('POST')
def _test_if_match(self, method, match_header_name):
def do_test(method, plain_etags, expected_plain_etags=None):
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
match_header_value = ', '.join(plain_etags)
req = Request.blank(
'/v1/a/c/o', environ=env, method=method,
headers={match_header_name: match_header_value})
app = FakeSwift()
app.register(method, '/v1/a/c/o', HTTPOk, {})
resp = req.get_response(encrypter.Encrypter(app, {}))
self.assertEqual('200 OK', resp.status)
self.assertEqual(1, len(app.calls), app.calls)
self.assertEqual(method, app.calls[0][0])
actual_headers = app.headers[0]
# verify the alternate etag location has been specified
if match_header_value and match_header_value != '*':
self.assertIn('X-Backend-Etag-Is-At', actual_headers)
self.assertEqual('X-Object-Sysmeta-Crypto-Etag-Mac',
actual_headers['X-Backend-Etag-Is-At'])
# verify etags have been supplemented with masked values
self.assertIn(match_header_name, actual_headers)
actual_etags = set(actual_headers[match_header_name].split(', '))
key = fetch_crypto_keys()['object']
masked_etags = [
'"%s"' % base64.b64encode(hmac.new(
key, etag.strip('"'), hashlib.sha256).digest())
for etag in plain_etags if etag not in ('*', '')]
expected_etags = set((expected_plain_etags or plain_etags) +
masked_etags)
self.assertEqual(expected_etags, actual_etags)
# check that the request environ was returned to original state
self.assertEqual(set(plain_etags),
set(req.headers[match_header_name].split(', ')))
do_test(method, [''])
do_test(method, ['"an etag"'])
do_test(method, ['"an etag"', '"another_etag"'])
do_test(method, ['*'])
# rfc2616 does not allow wildcard *and* etag but test it anyway
do_test(method, ['*', '"an etag"'])
# etags should be quoted but check we can cope if they are not
do_test(
method, ['*', 'an etag', 'another_etag'],
expected_plain_etags=['*', '"an etag"', '"another_etag"'])
def test_GET_if_match(self):
self._test_if_match('GET', 'If-Match')
def test_HEAD_if_match(self):
self._test_if_match('HEAD', 'If-Match')
def test_GET_if_none_match(self):
self._test_if_match('GET', 'If-None-Match')
def test_HEAD_if_none_match(self):
self._test_if_match('HEAD', 'If-None-Match')
def _test_existing_etag_is_at_header(self, method, match_header_name):
# if another middleware has already set X-Backend-Etag-Is-At then
# encrypter should not override that value
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank(
'/v1/a/c/o', environ=env, method=method,
headers={match_header_name: "an etag",
'X-Backend-Etag-Is-At': 'X-Object-Sysmeta-Other-Etag'})
self.app.register(method, '/v1/a/c/o', HTTPOk, {})
resp = req.get_response(self.encrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(method, self.app.calls[0][0])
actual_headers = self.app.headers[0]
self.assertIn('X-Backend-Etag-Is-At', actual_headers)
self.assertEqual(
'X-Object-Sysmeta-Other-Etag,X-Object-Sysmeta-Crypto-Etag-Mac',
actual_headers['X-Backend-Etag-Is-At'])
actual_etags = set(actual_headers[match_header_name].split(', '))
self.assertIn('"an etag"', actual_etags)
def test_GET_if_match_with_existing_etag_is_at_header(self):
self._test_existing_etag_is_at_header('GET', 'If-Match')
def test_HEAD_if_match_with_existing_etag_is_at_header(self):
self._test_existing_etag_is_at_header('HEAD', 'If-Match')
def test_GET_if_none_match_with_existing_etag_is_at_header(self):
self._test_existing_etag_is_at_header('GET', 'If-None-Match')
def test_HEAD_if_none_match_with_existing_etag_is_at_header(self):
self._test_existing_etag_is_at_header('HEAD', 'If-None-Match')
def _test_etag_is_at_not_duplicated(self, method):
# verify only one occurrence of X-Object-Sysmeta-Crypto-Etag-Mac in
# X-Backend-Etag-Is-At
key = fetch_crypto_keys()['object']
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank(
'/v1/a/c/o', environ=env, method=method,
headers={'If-Match': '"an etag"',
'If-None-Match': '"another etag"'})
self.app.register(method, '/v1/a/c/o', HTTPOk, {})
resp = req.get_response(self.encrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(method, self.app.calls[0][0])
actual_headers = self.app.headers[0]
self.assertIn('X-Backend-Etag-Is-At', actual_headers)
self.assertEqual('X-Object-Sysmeta-Crypto-Etag-Mac',
actual_headers['X-Backend-Etag-Is-At'])
self.assertIn('"%s"' % base64.b64encode(
hmac.new(key, 'an etag', hashlib.sha256).digest()),
actual_headers['If-Match'])
self.assertIn('"another etag"', actual_headers['If-None-Match'])
self.assertIn('"%s"' % base64.b64encode(
hmac.new(key, 'another etag', hashlib.sha256).digest()),
actual_headers['If-None-Match'])
def test_GET_etag_is_at_not_duplicated(self):
self._test_etag_is_at_not_duplicated('GET')
def test_HEAD_etag_is_at_not_duplicated(self):
self._test_etag_is_at_not_duplicated('HEAD')
def test_PUT_response_inconsistent_etag_is_not_replaced(self):
# if response is success but etag does not match the ciphertext md5
# then verify that we do *not* replace it with the plaintext etag
body = 'FAKE APP'
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated,
{'Etag': 'not the ciphertext etag'})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
self.assertEqual('not the ciphertext etag', resp.headers['Etag'])
def test_PUT_multiseg_no_client_etag(self):
body_key = os.urandom(32)
chunks = ['some', 'chunks', 'of data']
body = ''.join(chunks)
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'wsgi.input': FileLikeIter(chunks)}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key',
lambda *args: body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
# verify object is encrypted by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.assertEqual(encrypt(body, body_key, FAKE_IV),
get_req.get_response(self.app).body)
def test_PUT_multiseg_good_client_etag(self):
body_key = os.urandom(32)
chunks = ['some', 'chunks', 'of data']
body = ''.join(chunks)
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'wsgi.input': FileLikeIter(chunks)}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(body)),
'Etag': md5hex(body)}
req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key',
lambda *args: body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
# verify object is encrypted by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.assertEqual(encrypt(body, body_key, FAKE_IV),
get_req.get_response(self.app).body)
def test_PUT_multiseg_bad_client_etag(self):
chunks = ['some', 'chunks', 'of data']
body = ''.join(chunks)
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'wsgi.input': FileLikeIter(chunks)}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(body)),
'Etag': 'badclientetag'}
req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('422 Unprocessable Entity', resp.status)
def test_PUT_missing_key_callback(self):
body = 'FAKE APP'
env = {'REQUEST_METHOD': 'PUT'}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
resp = req.get_response(self.encrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn('missing callback',
self.encrypter.logger.get_lines_for_level('error')[0])
self.assertEqual('Unable to retrieve encryption keys.', resp.body)
def test_PUT_error_in_key_callback(self):
def raise_exc():
raise Exception('Testing')
body = 'FAKE APP'
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: raise_exc}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
resp = req.get_response(self.encrypter)
self.assertEqual('500 Internal Error', resp.status)
self.assertIn('from callback: Testing',
self.encrypter.logger.get_lines_for_level('error')[0])
self.assertEqual('Unable to retrieve encryption keys.', resp.body)
def test_PUT_encryption_override(self):
# set crypto override to disable encryption.
# simulate another middleware wanting to set footers
other_footers = {
'Etag': 'other etag',
'X-Object-Sysmeta-Other': 'other sysmeta',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'other override'}
body = 'FAKE APP'
env = {'REQUEST_METHOD': 'PUT',
'swift.crypto.override': True,
'swift.callback.update_footers':
lambda footers: footers.update(other_footers)}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
# verify that other middleware's footers made it to app
req_hdrs = self.app.headers[0]
for k, v in other_footers.items():
self.assertEqual(v, req_hdrs[k])
# verify object is NOT encrypted by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.assertEqual(body, get_req.get_response(self.app).body)
def _test_constraints_checking(self, method):
# verify that the check_metadata function is called on PUT and POST
body = 'FAKE APP'
env = {'REQUEST_METHOD': method,
CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(body))}
req = Request.blank('/v1/a/c/o', environ=env, body=body, headers=hdrs)
mocked_func = 'swift.common.middleware.crypto.encrypter.check_metadata'
with mock.patch(mocked_func) as mocked:
mocked.side_effect = [HTTPBadRequest('testing')]
resp = req.get_response(self.encrypter)
self.assertEqual('400 Bad Request', resp.status)
self.assertEqual(1, mocked.call_count)
mocked.assert_called_once_with(mock.ANY, 'object')
self.assertEqual(req.headers,
mocked.call_args_list[0][0][0].headers)
def test_PUT_constraints_checking(self):
self._test_constraints_checking('PUT')
def test_POST_constraints_checking(self):
self._test_constraints_checking('POST')
def test_config_true_value_on_disable_encryption(self):
app = FakeSwift()
self.assertFalse(encrypter.Encrypter(app, {}).disable_encryption)
for val in ('true', '1', 'yes', 'on', 't', 'y'):
app = encrypter.Encrypter(app,
{'disable_encryption': val})
self.assertTrue(app.disable_encryption)
def test_PUT_app_exception(self):
app = encrypter.Encrypter(FakeAppThatExcepts(HTTPException), {})
req = Request.blank('/', environ={'REQUEST_METHOD': 'PUT'})
with self.assertRaises(HTTPException) as catcher:
req.get_response(app)
self.assertEqual(FakeAppThatExcepts.MESSAGE, catcher.exception.body)
if __name__ == '__main__':
unittest.main()

631
test/unit/common/middleware/crypto/test_encryption.py Normal file
View File

@ -0,0 +1,631 @@
# Copyright (c) 2015-2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
import hashlib
import hmac
import json
import unittest
import uuid
from swift.common import storage_policy, constraints
from swift.common.middleware import copy
from swift.common.middleware import crypto
from swift.common.middleware.crypto import keymaster
from swift.common.middleware.crypto.crypto_utils import (
load_crypto_meta, Crypto)
from swift.common.ring import Ring
from swift.common.swob import Request
from swift.obj import diskfile
from test.unit import FakeLogger
from test.unit.common.middleware.crypto.crypto_helpers import (
md5hex, encrypt, TEST_KEYMASTER_CONF)
from test.unit.helpers import setup_servers, teardown_servers
class TestCryptoPipelineChanges(unittest.TestCase):
# Tests the consequences of crypto middleware being in/out of the pipeline
# or having encryption disabled for PUT/GET requests on same object. Uses
# real backend servers so that the handling of headers and sysmeta is
# verified to diskfile and back.
_test_context = None
@classmethod
def setUpClass(cls):
cls._test_context = setup_servers()
cls.proxy_app = cls._test_context["test_servers"][0]
@classmethod
def tearDownClass(cls):
if cls._test_context is not None:
teardown_servers(cls._test_context)
cls._test_context = None
def setUp(self):
self.plaintext = 'unencrypted body content'
self.plaintext_etag = md5hex(self.plaintext)
self._setup_crypto_app()
def _setup_crypto_app(self, disable_encryption=False):
# Set up a pipeline of crypto middleware ending in the proxy app so
# that tests can make requests to either the proxy server directly or
# via the crypto middleware. Make a fresh instance for each test to
# avoid any state coupling.
conf = {'disable_encryption': disable_encryption}
self.encryption = crypto.filter_factory(conf)(self.proxy_app)
self.km = keymaster.KeyMaster(self.encryption, TEST_KEYMASTER_CONF)
self.crypto_app = self.km # for clarity
def _create_container(self, app, policy_name='one', container_path=None):
if not container_path:
# choose new container name so that the policy can be specified
self.container_name = uuid.uuid4().hex
self.container_path = 'http://foo:8080/v1/a/' + self.container_name
self.object_name = 'o'
self.object_path = self.container_path + '/' + self.object_name
container_path = self.container_path
req = Request.blank(
container_path, method='PUT',
headers={'X-Storage-Policy': policy_name})
resp = req.get_response(app)
self.assertEqual('201 Created', resp.status)
# sanity check
req = Request.blank(
container_path, method='HEAD',
headers={'X-Storage-Policy': policy_name})
resp = req.get_response(app)
self.assertEqual(policy_name, resp.headers['X-Storage-Policy'])
def _put_object(self, app, body):
req = Request.blank(self.object_path, method='PUT', body=body,
headers={'Content-Type': 'application/test'})
resp = req.get_response(app)
self.assertEqual('201 Created', resp.status)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
return resp
def _post_object(self, app):
req = Request.blank(self.object_path, method='POST',
headers={'Content-Type': 'application/test',
'X-Object-Meta-Fruit': 'Kiwi'})
resp = req.get_response(app)
self.assertEqual('202 Accepted', resp.status)
return resp
def _copy_object(self, app, destination):
req = Request.blank(self.object_path, method='COPY',
headers={'Destination': destination})
resp = req.get_response(app)
self.assertEqual('201 Created', resp.status)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
return resp
def _check_GET_and_HEAD(self, app, object_path=None):
object_path = object_path or self.object_path
req = Request.blank(object_path, method='GET')
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertEqual(self.plaintext, resp.body)
self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
req = Request.blank(object_path, method='HEAD')
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertEqual('', resp.body)
self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
def _check_match_requests(self, method, app, object_path=None):
object_path = object_path or self.object_path
# verify conditional match requests
expected_body = self.plaintext if method == 'GET' else ''
# If-Match matches
req = Request.blank(object_path, method=method,
headers={'If-Match': '"%s"' % self.plaintext_etag})
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertEqual(expected_body, resp.body)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
# If-Match wildcard
req = Request.blank(object_path, method=method,
headers={'If-Match': '*'})
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertEqual(expected_body, resp.body)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
# If-Match does not match
req = Request.blank(object_path, method=method,
headers={'If-Match': '"not the etag"'})
resp = req.get_response(app)
self.assertEqual('412 Precondition Failed', resp.status)
self.assertEqual('', resp.body)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
# If-None-Match matches
req = Request.blank(
object_path, method=method,
headers={'If-None-Match': '"%s"' % self.plaintext_etag})
resp = req.get_response(app)
self.assertEqual('304 Not Modified', resp.status)
self.assertEqual('', resp.body)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
# If-None-Match wildcard
req = Request.blank(object_path, method=method,
headers={'If-None-Match': '*'})
resp = req.get_response(app)
self.assertEqual('304 Not Modified', resp.status)
self.assertEqual('', resp.body)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
# If-None-Match does not match
req = Request.blank(object_path, method=method,
headers={'If-None-Match': '"not the etag"'})
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertEqual(expected_body, resp.body)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
self.assertEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
def _check_listing(self, app, expect_mismatch=False, container_path=None):
container_path = container_path or self.container_path
req = Request.blank(
container_path, method='GET', query_string='format=json')
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
listing = json.loads(resp.body)
self.assertEqual(1, len(listing))
self.assertEqual(self.object_name, listing[0]['name'])
self.assertEqual(len(self.plaintext), listing[0]['bytes'])
if expect_mismatch:
self.assertNotEqual(self.plaintext_etag, listing[0]['hash'])
else:
self.assertEqual(self.plaintext_etag, listing[0]['hash'])
def test_write_with_crypto_and_override_headers(self):
self._create_container(self.proxy_app, policy_name='one')
def verify_overrides():
# verify object sysmeta
req = Request.blank(
self.object_path, method='GET')
resp = req.get_response(self.crypto_app)
for k, v in overrides.items():
self.assertIn(k, resp.headers)
self.assertEqual(overrides[k], resp.headers[k])
# check container listing
req = Request.blank(
self.container_path, method='GET', query_string='format=json')
resp = req.get_response(self.crypto_app)
self.assertEqual('200 OK', resp.status)
listing = json.loads(resp.body)
self.assertEqual(1, len(listing))
self.assertEqual('o', listing[0]['name'])
self.assertEqual(
overrides['x-object-sysmeta-container-update-override-size'],
str(listing[0]['bytes']))
self.assertEqual(
overrides['x-object-sysmeta-container-update-override-etag'],
listing[0]['hash'])
# include overrides in headers
overrides = {'x-object-sysmeta-container-update-override-etag': 'foo',
'x-object-sysmeta-container-update-override-size':
str(len(self.plaintext) + 1)}
req = Request.blank(self.object_path, method='PUT',
body=self.plaintext, headers=overrides.copy())
resp = req.get_response(self.crypto_app)
self.assertEqual('201 Created', resp.status)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
verify_overrides()
# include overrides in footers
overrides = {'x-object-sysmeta-container-update-override-etag': 'bar',
'x-object-sysmeta-container-update-override-size':
str(len(self.plaintext) + 2)}
def callback(footers):
footers.update(overrides)
req = Request.blank(
self.object_path, method='PUT', body=self.plaintext)
req.environ['swift.callback.update_footers'] = callback
resp = req.get_response(self.crypto_app)
self.assertEqual('201 Created', resp.status)
self.assertEqual(self.plaintext_etag, resp.headers['Etag'])
verify_overrides()
def test_write_with_crypto_read_with_crypto(self):
self._create_container(self.proxy_app, policy_name='one')
self._put_object(self.crypto_app, self.plaintext)
self._post_object(self.crypto_app)
self._check_GET_and_HEAD(self.crypto_app)
self._check_match_requests('GET', self.crypto_app)
self._check_match_requests('HEAD', self.crypto_app)
self._check_listing(self.crypto_app)
def test_write_with_crypto_read_with_crypto_ec(self):
self._create_container(self.proxy_app, policy_name='ec')
self._put_object(self.crypto_app, self.plaintext)
self._post_object(self.crypto_app)
self._check_GET_and_HEAD(self.crypto_app)
self._check_match_requests('GET', self.crypto_app)
self._check_match_requests('HEAD', self.crypto_app)
self._check_listing(self.crypto_app)
def test_put_without_crypto_post_with_crypto_read_with_crypto(self):
self._create_container(self.proxy_app, policy_name='one')
self._put_object(self.proxy_app, self.plaintext)
self._post_object(self.crypto_app)
self._check_GET_and_HEAD(self.crypto_app)
self._check_match_requests('GET', self.crypto_app)
self._check_match_requests('HEAD', self.crypto_app)
self._check_listing(self.crypto_app)
def test_write_without_crypto_read_with_crypto(self):
self._create_container(self.proxy_app, policy_name='one')
self._put_object(self.proxy_app, self.plaintext)
self._post_object(self.proxy_app)
self._check_GET_and_HEAD(self.proxy_app) # sanity check
self._check_GET_and_HEAD(self.crypto_app)
self._check_match_requests('GET', self.proxy_app) # sanity check
self._check_match_requests('GET', self.crypto_app)
self._check_match_requests('HEAD', self.proxy_app) # sanity check
self._check_match_requests('HEAD', self.crypto_app)
self._check_listing(self.crypto_app)
def test_write_without_crypto_read_with_crypto_ec(self):
self._create_container(self.proxy_app, policy_name='ec')
self._put_object(self.proxy_app, self.plaintext)
self._post_object(self.proxy_app)
self._check_GET_and_HEAD(self.proxy_app) # sanity check
self._check_GET_and_HEAD(self.crypto_app)
self._check_match_requests('GET', self.proxy_app) # sanity check
self._check_match_requests('GET', self.crypto_app)
self._check_match_requests('HEAD', self.proxy_app) # sanity check
self._check_match_requests('HEAD', self.crypto_app)
self._check_listing(self.crypto_app)
def _check_GET_and_HEAD_not_decrypted(self, app):
req = Request.blank(self.object_path, method='GET')
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertNotEqual(self.plaintext, resp.body)
self.assertEqual('%s' % len(self.plaintext),
resp.headers['Content-Length'])
self.assertNotEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
req = Request.blank(self.object_path, method='HEAD')
resp = req.get_response(app)
self.assertEqual('200 OK', resp.status)
self.assertEqual('', resp.body)
self.assertNotEqual('Kiwi', resp.headers['X-Object-Meta-Fruit'])
def test_write_with_crypto_read_without_crypto(self):
self._create_container(self.proxy_app, policy_name='one')
self._put_object(self.crypto_app, self.plaintext)
self._post_object(self.crypto_app)
self._check_GET_and_HEAD(self.crypto_app) # sanity check
# without crypto middleware, GET and HEAD returns ciphertext
self._check_GET_and_HEAD_not_decrypted(self.proxy_app)
self._check_listing(self.proxy_app, expect_mismatch=True)
def test_write_with_crypto_read_without_crypto_ec(self):
self._create_container(self.proxy_app, policy_name='ec')
self._put_object(self.crypto_app, self.plaintext)
self._post_object(self.crypto_app)
self._check_GET_and_HEAD(self.crypto_app) # sanity check
# without crypto middleware, GET and HEAD returns ciphertext
self._check_GET_and_HEAD_not_decrypted(self.proxy_app)
self._check_listing(self.proxy_app, expect_mismatch=True)
def test_disable_encryption_config_option(self):
# check that on disable_encryption = true, object is not encrypted
self._setup_crypto_app(disable_encryption=True)
self._create_container(self.proxy_app, policy_name='one')
self._put_object(self.crypto_app, self.plaintext)
self._post_object(self.crypto_app)
self._check_GET_and_HEAD(self.crypto_app)
# check as if no crypto middleware exists
self._check_GET_and_HEAD(self.proxy_app)
self._check_match_requests('GET', self.crypto_app)
self._check_match_requests('HEAD', self.crypto_app)
self._check_match_requests('GET', self.proxy_app)
self._check_match_requests('HEAD', self.proxy_app)
def test_write_with_crypto_read_with_disable_encryption_conf(self):
self._create_container(self.proxy_app, policy_name='one')
self._put_object(self.crypto_app, self.plaintext)
self._post_object(self.crypto_app)
self._check_GET_and_HEAD(self.crypto_app) # sanity check
# turn on disable_encryption config option
self._setup_crypto_app(disable_encryption=True)
# GET and HEAD of encrypted objects should still work
self._check_GET_and_HEAD(self.crypto_app)
self._check_listing(self.crypto_app, expect_mismatch=False)
self._check_match_requests('GET', self.crypto_app)
self._check_match_requests('HEAD', self.crypto_app)
def _test_ondisk_data_after_write_with_crypto(self, policy_name):
policy = storage_policy.POLICIES.get_by_name(policy_name)
self._create_container(self.proxy_app, policy_name=policy_name)
self._put_object(self.crypto_app, self.plaintext)
self._post_object(self.crypto_app)
# Verify container listing etag is encrypted by direct GET to container
# server. We can use any server for all nodes since they all share same
# devices dir.
cont_server = self._test_context['test_servers'][3]
cont_ring = Ring(self._test_context['testdir'], ring_name='container')
part, nodes = cont_ring.get_nodes('a', self.container_name)
for node in nodes:
req = Request.blank('/%s/%s/a/%s'
% (node['device'], part, self.container_name),
method='GET', query_string='format=json')
resp = req.get_response(cont_server)
listing = json.loads(resp.body)
# sanity checks...
self.assertEqual(1, len(listing))
self.assertEqual('o', listing[0]['name'])
self.assertEqual('application/test', listing[0]['content_type'])
# verify encrypted etag value
parts = listing[0]['hash'].rsplit(';', 1)
crypto_meta_param = parts[1].strip()
crypto_meta = crypto_meta_param[len('swift_meta='):]
listing_etag_iv = load_crypto_meta(crypto_meta)['iv']
exp_enc_listing_etag = base64.b64encode(
encrypt(self.plaintext_etag,
self.km.create_key('/a/%s' % self.container_name),
listing_etag_iv))
self.assertEqual(exp_enc_listing_etag, parts[0])
# Verify diskfile data and metadata is encrypted
ring_object = self.proxy_app.get_object_ring(int(policy))
partition, nodes = ring_object.get_nodes('a', self.container_name, 'o')
conf = {'devices': self._test_context["testdir"],
'mount_check': 'false'}
df_mgr = diskfile.DiskFileRouter(conf, FakeLogger())[policy]
ondisk_data = []
exp_enc_body = None
for node_index, node in enumerate(nodes):
df = df_mgr.get_diskfile(node['device'], partition,
'a', self.container_name, 'o',
policy=policy)
with df.open():
meta = df.get_metadata()
contents = ''.join(df.reader())
metadata = dict((k.lower(), v) for k, v in meta.items())
# verify on disk data - body
body_iv = load_crypto_meta(
metadata['x-object-sysmeta-crypto-body-meta'])['iv']
body_key_meta = load_crypto_meta(
metadata['x-object-sysmeta-crypto-body-meta'])['body_key']
obj_key = self.km.create_key('/a/%s/o' % self.container_name)
body_key = Crypto().unwrap_key(obj_key, body_key_meta)
exp_enc_body = encrypt(self.plaintext, body_key, body_iv)
ondisk_data.append((node, contents))
# verify on disk user metadata
enc_val, meta = metadata[
'x-object-transient-sysmeta-crypto-meta-fruit'].split(';')
meta = meta.strip()[len('swift_meta='):]
metadata_iv = load_crypto_meta(meta)['iv']
exp_enc_meta = base64.b64encode(encrypt('Kiwi', obj_key,
metadata_iv))
self.assertEqual(exp_enc_meta, enc_val)
self.assertNotIn('x-object-meta-fruit', metadata)
self.assertIn(
'x-object-transient-sysmeta-crypto-meta', metadata)
meta = load_crypto_meta(
metadata['x-object-transient-sysmeta-crypto-meta'])
self.assertIn('key_id', meta)
self.assertIn('path', meta['key_id'])
self.assertEqual(
'/a/%s/%s' % (self.container_name, self.object_name),
meta['key_id']['path'])
self.assertIn('v', meta['key_id'])
self.assertEqual('1', meta['key_id']['v'])
self.assertIn('cipher', meta)
self.assertEqual(Crypto.cipher, meta['cipher'])
# verify etag
actual_enc_etag, _junk, actual_etag_meta = metadata[
'x-object-sysmeta-crypto-etag'].partition('; swift_meta=')
etag_iv = load_crypto_meta(actual_etag_meta)['iv']
exp_enc_etag = base64.b64encode(encrypt(self.plaintext_etag,
obj_key, etag_iv))
self.assertEqual(exp_enc_etag, actual_enc_etag)
# verify etag hmac
exp_etag_mac = hmac.new(
obj_key, self.plaintext_etag, digestmod=hashlib.sha256)
exp_etag_mac = base64.b64encode(exp_etag_mac.digest())
self.assertEqual(exp_etag_mac,
metadata['x-object-sysmeta-crypto-etag-mac'])
# verify etag override for container updates
override = 'x-object-sysmeta-container-update-override-etag'
parts = metadata[override].rsplit(';', 1)
crypto_meta_param = parts[1].strip()
crypto_meta = crypto_meta_param[len('swift_meta='):]
listing_etag_iv = load_crypto_meta(crypto_meta)['iv']
cont_key = self.km.create_key('/a/%s' % self.container_name)
exp_enc_listing_etag = base64.b64encode(
encrypt(self.plaintext_etag, cont_key,
listing_etag_iv))
self.assertEqual(exp_enc_listing_etag, parts[0])
self._check_GET_and_HEAD(self.crypto_app)
return exp_enc_body, ondisk_data
def test_ondisk_data_after_write_with_crypto(self):
exp_body, ondisk_data = self._test_ondisk_data_after_write_with_crypto(
policy_name='one')
for node, body in ondisk_data:
self.assertEqual(exp_body, body)
def test_ondisk_data_after_write_with_crypto_ec(self):
exp_body, ondisk_data = self._test_ondisk_data_after_write_with_crypto(
policy_name='ec')
policy = storage_policy.POLICIES.get_by_name('ec')
for frag_selection in (ondisk_data[:2], ondisk_data[1:]):
frags = [frag for node, frag in frag_selection]
self.assertEqual(exp_body, policy.pyeclib_driver.decode(frags))
def _test_copy_encrypted_to_encrypted(
self, src_policy_name, dest_policy_name):
self._create_container(self.proxy_app, policy_name=src_policy_name)
self._put_object(self.crypto_app, self.plaintext)
self._post_object(self.crypto_app)
copy_crypto_app = copy.ServerSideCopyMiddleware(self.crypto_app, {})
dest_container = uuid.uuid4().hex
dest_container_path = 'http://localhost:8080/v1/a/' + dest_container
self._create_container(copy_crypto_app, policy_name=dest_policy_name,
container_path=dest_container_path)
dest_obj_path = dest_container_path + '/o'
dest = '/%s/%s' % (dest_container, 'o')
self._copy_object(copy_crypto_app, dest)
self._check_GET_and_HEAD(copy_crypto_app, object_path=dest_obj_path)
self._check_listing(
copy_crypto_app, container_path=dest_container_path)
self._check_match_requests(
'GET', copy_crypto_app, object_path=dest_obj_path)
self._check_match_requests(
'HEAD', copy_crypto_app, object_path=dest_obj_path)
def test_copy_encrypted_to_encrypted(self):
self._test_copy_encrypted_to_encrypted('ec', 'ec')
self._test_copy_encrypted_to_encrypted('one', 'ec')
self._test_copy_encrypted_to_encrypted('ec', 'one')
self._test_copy_encrypted_to_encrypted('one', 'one')
def _test_copy_encrypted_to_unencrypted(
self, src_policy_name, dest_policy_name):
self._create_container(self.proxy_app, policy_name=src_policy_name)
self._put_object(self.crypto_app, self.plaintext)
self._post_object(self.crypto_app)
# make a pipeline with encryption disabled, use it to copy object
self._setup_crypto_app(disable_encryption=True)
copy_app = copy.ServerSideCopyMiddleware(self.crypto_app, {})
dest_container = uuid.uuid4().hex
dest_container_path = 'http://localhost:8080/v1/a/' + dest_container
self._create_container(self.crypto_app, policy_name=dest_policy_name,
container_path=dest_container_path)
dest_obj_path = dest_container_path + '/o'
dest = '/%s/%s' % (dest_container, 'o')
self._copy_object(copy_app, dest)
self._check_GET_and_HEAD(copy_app, object_path=dest_obj_path)
self._check_GET_and_HEAD(self.proxy_app, object_path=dest_obj_path)
self._check_listing(copy_app, container_path=dest_container_path)
self._check_listing(self.proxy_app, container_path=dest_container_path)
self._check_match_requests(
'GET', self.proxy_app, object_path=dest_obj_path)
self._check_match_requests(
'HEAD', self.proxy_app, object_path=dest_obj_path)
def test_copy_encrypted_to_unencrypted(self):
self._test_copy_encrypted_to_unencrypted('ec', 'ec')
self._test_copy_encrypted_to_unencrypted('one', 'ec')
self._test_copy_encrypted_to_unencrypted('ec', 'one')
self._test_copy_encrypted_to_unencrypted('one', 'one')
def _test_copy_unencrypted_to_encrypted(
self, src_policy_name, dest_policy_name):
self._create_container(self.proxy_app, policy_name=src_policy_name)
self._put_object(self.proxy_app, self.plaintext)
self._post_object(self.proxy_app)
copy_crypto_app = copy.ServerSideCopyMiddleware(self.crypto_app, {})
dest_container = uuid.uuid4().hex
dest_container_path = 'http://localhost:8080/v1/a/' + dest_container
self._create_container(copy_crypto_app, policy_name=dest_policy_name,
container_path=dest_container_path)
dest_obj_path = dest_container_path + '/o'
dest = '/%s/%s' % (dest_container, 'o')
self._copy_object(copy_crypto_app, dest)
self._check_GET_and_HEAD(copy_crypto_app, object_path=dest_obj_path)
self._check_listing(
copy_crypto_app, container_path=dest_container_path)
self._check_match_requests(
'GET', copy_crypto_app, object_path=dest_obj_path)
self._check_match_requests(
'HEAD', copy_crypto_app, object_path=dest_obj_path)
def test_copy_unencrypted_to_encrypted(self):
self._test_copy_unencrypted_to_encrypted('ec', 'ec')
self._test_copy_unencrypted_to_encrypted('one', 'ec')
self._test_copy_unencrypted_to_encrypted('ec', 'one')
self._test_copy_unencrypted_to_encrypted('one', 'one')
def test_crypto_max_length_path(self):
# the path is stashed in the key_id in crypto meta; check that a long
# path is ok
self.container_name = 'c' * constraints.MAX_CONTAINER_NAME_LENGTH
self.object_name = 'o' * constraints.MAX_OBJECT_NAME_LENGTH
self.container_path = 'http://foo:8080/v1/a/' + self.container_name
self.object_path = '%s/%s' % (self.container_path, self.object_name)
self._create_container(self.proxy_app, policy_name='one',
container_path=self.container_path)
self._put_object(self.crypto_app, self.plaintext)
self._post_object(self.crypto_app)
self._check_GET_and_HEAD(self.crypto_app)
self._check_match_requests('GET', self.crypto_app)
self._check_match_requests('HEAD', self.crypto_app)
self._check_listing(self.crypto_app)
def test_crypto_UTF8_path(self):
# check that UTF8 path is ok
self.container_name = self.object_name = u'\u010brypto'
self.container_path = 'http://foo:8080/v1/a/' + self.container_name
self.object_path = '%s/%s' % (self.container_path, self.object_name)
self._create_container(self.proxy_app, policy_name='one',
container_path=self.container_path)
self._put_object(self.crypto_app, self.plaintext)
self._post_object(self.crypto_app)
self._check_GET_and_HEAD(self.crypto_app)
self._check_match_requests('GET', self.crypto_app)
self._check_match_requests('HEAD', self.crypto_app)
self._check_listing(self.crypto_app)
class TestCryptoPipelineChangesFastPost(TestCryptoPipelineChanges):
@classmethod
def setUpClass(cls):
# set proxy config to use fast post
extra_conf = {'object_post_as_copy': 'False'}
cls._test_context = setup_servers(extra_conf=extra_conf)
cls.proxy_app = cls._test_context["test_servers"][0]
if __name__ == '__main__':
unittest.main()

163
test/unit/common/middleware/crypto/test_keymaster.py Normal file
View File

@ -0,0 +1,163 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2015 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
import os
import unittest
from swift.common import swob
from swift.common.middleware.crypto import keymaster
from swift.common.middleware.crypto.crypto_utils import CRYPTO_KEY_CALLBACK
from swift.common.swob import Request
from test.unit.common.middleware.helpers import FakeSwift, FakeAppThatExcepts
from test.unit.common.middleware.crypto.crypto_helpers import (
TEST_KEYMASTER_CONF)
def capture_start_response():
calls = []
def start_response(*args):
calls.append(args)
return start_response, calls
class TestKeymaster(unittest.TestCase):
def setUp(self):
super(TestKeymaster, self).setUp()
self.swift = FakeSwift()
self.app = keymaster.KeyMaster(self.swift, TEST_KEYMASTER_CONF)
def test_object_path(self):
self.verify_keys_for_path(
'/a/c/o', expected_keys=('object', 'container'))
def test_container_path(self):
self.verify_keys_for_path(
'/a/c', expected_keys=('container',))
def verify_keys_for_path(self, path, expected_keys):
put_keys = None
for method, resp_class, status in (
('PUT', swob.HTTPCreated, '201'),
('POST', swob.HTTPAccepted, '202'),
('GET', swob.HTTPOk, '200'),
('HEAD', swob.HTTPNoContent, '204')):
resp_headers = {}
self.swift.register(
method, '/v1' + path, resp_class, resp_headers, '')
req = Request.blank(
'/v1' + path, environ={'REQUEST_METHOD': method})
start_response, calls = capture_start_response()
self.app(req.environ, start_response)
self.assertEqual(1, len(calls))
self.assertTrue(calls[0][0].startswith(status))
self.assertNotIn('swift.crypto.override', req.environ)
self.assertIn(CRYPTO_KEY_CALLBACK, req.environ,
'%s not set in env' % CRYPTO_KEY_CALLBACK)
keys = req.environ.get(CRYPTO_KEY_CALLBACK)()
self.assertIn('id', keys)
id = keys.pop('id')
self.assertEqual(path, id['path'])
self.assertEqual('1', id['v'])
self.assertListEqual(sorted(expected_keys), sorted(keys.keys()),
'%s %s got keys %r, but expected %r'
% (method, path, keys.keys(), expected_keys))
if put_keys is not None:
# check all key sets were consistent for this path
self.assertDictEqual(put_keys, keys)
else:
put_keys = keys
return put_keys
def test_key_uniqueness(self):
# a rudimentary check that different keys are made for different paths
ref_path_parts = ('a1', 'c1', 'o1')
path = '/' + '/'.join(ref_path_parts)
ref_keys = self.verify_keys_for_path(
path, expected_keys=('object', 'container'))
# for same path and for each differing path check that keys are unique
# when path to object or container is unique and vice-versa
for path_parts in [(a, c, o) for a in ('a1', 'a2')
for c in ('c1', 'c2')
for o in ('o1', 'o2')]:
path = '/' + '/'.join(path_parts)
keys = self.verify_keys_for_path(
path, expected_keys=('object', 'container'))
# object keys should only be equal when complete paths are equal
self.assertEqual(path_parts == ref_path_parts,
keys['object'] == ref_keys['object'],
'Path %s keys:\n%s\npath %s keys\n%s' %
(ref_path_parts, ref_keys, path_parts, keys))
# container keys should only be equal when paths to container are
# equal
self.assertEqual(path_parts[:2] == ref_path_parts[:2],
keys['container'] == ref_keys['container'],
'Path %s keys:\n%s\npath %s keys\n%s' %
(ref_path_parts, ref_keys, path_parts, keys))
def test_filter(self):
factory = keymaster.filter_factory(TEST_KEYMASTER_CONF)
self.assertTrue(callable(factory))
self.assertTrue(callable(factory(self.swift)))
def test_app_exception(self):
app = keymaster.KeyMaster(
FakeAppThatExcepts(), TEST_KEYMASTER_CONF)
req = Request.blank('/', environ={'REQUEST_METHOD': 'PUT'})
start_response, _ = capture_start_response()
self.assertRaises(Exception, app, req.environ, start_response)
def test_root_secret(self):
for secret in (os.urandom(32), os.urandom(33), os.urandom(50)):
encoded_secret = base64.b64encode(secret)
try:
app = keymaster.KeyMaster(
self.swift, {'encryption_root_secret':
bytes(encoded_secret)})
self.assertEqual(secret, app.root_secret)
except AssertionError as err:
self.fail(str(err) + ' for secret %s' % secret)
try:
app = keymaster.KeyMaster(
self.swift, {'encryption_root_secret':
unicode(encoded_secret)})
self.assertEqual(secret, app.root_secret)
except AssertionError as err:
self.fail(str(err) + ' for secret %s' % secret)
def test_invalid_root_secret(self):
for secret in (bytes(base64.b64encode(os.urandom(31))), # too short
unicode(base64.b64encode(os.urandom(31))),
u'?' * 44, b'?' * 44, # not base64
u'a' * 45, b'a' * 45, # bad padding
99, None):
conf = {'encryption_root_secret': secret}
try:
with self.assertRaises(ValueError) as err:
keymaster.KeyMaster(self.swift, conf)
self.assertEqual(
'encryption_root_secret option in proxy-server.conf '
'must be a base64 encoding of at least 32 raw bytes',
err.exception.message)
except AssertionError as err:
self.fail(str(err) + ' for conf %s' % str(conf))
if __name__ == '__main__':
unittest.main()