Decrypting Container Listing

See encryption spec in_progress/at_rest_encryption.html

Decrypts the content-type in the container-listing.
The content-length is modified since decoding decreases length.
When there is "footers" support, the etag will also be decrypted.

Depends on having keymaster middleware and cryptography module.
You can point the following vagrant to this patch:
https://github.com/swiftstack/vagrant-swift-all-in-one/tree/crypto
Alternatively, please follow the instructions
in the proxy-server.conf-sample.  You will also need to add the
fake_footers middleware as well to get the correct functionality.

Co-Authored-By: Alistair Coles <alistair.coles@hp.com>

Change-Id: If7830ed0fc17deb3939492a02a3e07a20f1018c7
This commit is contained in:
Janie Richling 2015-08-18 22:42:09 -05:00 committed by Alistair Coles
parent 8cb5505bf1
commit f0e5ed1b7a
7 changed files with 527 additions and 111 deletions

View File

@ -15,15 +15,21 @@
import base64
import json
from swift.common.utils import get_logger, config_true_value, \
parse_content_range, closing_if_possible
import urllib
try:
import xml.etree.cElementTree as ElementTree
except ImportError:
import xml.etree.ElementTree as ElementTree
from swift.common.http import is_success
from swift.common.swob import Request, HTTPException, HTTPInternalServerError
from swift.common.middleware.crypto import Crypto
from swift.common.crypto_utils import CryptoWSGIContext
from swift.common.exceptions import EncryptionException
from swift.common.middleware.crypto import Crypto
from swift.common.request_helpers import strip_user_meta_prefix, is_user_meta,\
get_obj_persisted_sysmeta_prefix
get_obj_persisted_sysmeta_prefix, get_listing_content_type
from swift.common.swob import Request, HTTPException, HTTPInternalServerError
from swift.common.utils import get_logger, config_true_value, \
parse_content_range, closing_if_possible
def _load_crypto_meta(value):
@ -38,16 +44,24 @@ def _load_crypto_meta(value):
:param value: a string serialization of a crypto meta dict
:returns: a dict containing crypto meta items
"""
return {str(name): (base64.b64decode(value)
if name == 'iv' else str(value))
for name, value in json.loads(value).items()}
value = urllib.unquote_plus(value)
try:
crypto_meta = {str(name): (base64.b64decode(value)
if name == 'iv' else str(value))
for name, value in json.loads(value).items()}
except (ValueError, TypeError) as e:
msg = "Could not decrypt. Bad crypto_meta: %r : %s" % (value, e)
raise HTTPInternalServerError(body=msg, content_type='text/plain')
if 'iv' not in crypto_meta or 'cipher' not in crypto_meta:
msg = "Could not decrypt. Missing iv and/or cipher: %r" % value
raise HTTPInternalServerError(body=msg, content_type='text/plain')
return crypto_meta
class DecrypterObjContext(CryptoWSGIContext):
class BaseDecrypterContext(CryptoWSGIContext):
def __init__(self, decrypter, logger):
super(DecrypterObjContext, self).__init__(decrypter, logger)
self.server_type = 'object'
self.body_crypto_ctxt = None
super(BaseDecrypterContext, self).__init__(decrypter, logger)
def _check_cipher(self, cipher):
"""
@ -74,7 +88,7 @@ class DecrypterObjContext(CryptoWSGIContext):
return _load_crypto_meta(crypto_meta_json)
def decrypt_header_val(self, value, key, crypto_meta):
def decrypt_value(self, value, key, crypto_meta):
"""
Decrypt a value if suitable crypto_meta is provided or can be extracted
from the value itself.
@ -116,6 +130,30 @@ class DecrypterObjContext(CryptoWSGIContext):
key, crypto_meta['iv'], 0)
return crypto_ctxt.update(base64.b64decode(value))
def process_resp(self, req):
"""
Determine if a response should be decrypted, and if so then fetch keys.
:param req: a Request object
:returns: a dict if decryption keys
"""
# Only proceed processing if an error has not occurred
if not is_success(self._get_status_int()):
return None
if config_true_value(req.environ.get('swift.crypto.override')):
self.logger.debug('No decryption is necessary because of override')
return None
return self.get_keys(req.environ)
class DecrypterObjContext(BaseDecrypterContext):
def __init__(self, decrypter, logger):
super(DecrypterObjContext, self).__init__(decrypter, logger)
self.server_type = 'object'
self.body_crypto_ctxt = None
def decrypt_user_metadata(self, keys):
prefix = "%scrypto-meta-" % get_obj_persisted_sysmeta_prefix()
result = []
@ -130,7 +168,7 @@ class DecrypterObjContext(CryptoWSGIContext):
% name)
continue
# the corresponding value must have been encrypted/encoded
value = self.decrypt_header_val(
value = self.decrypt_value(
val, keys[self.server_type], crypto_meta)
result.append((name, value))
@ -154,12 +192,12 @@ class DecrypterObjContext(CryptoWSGIContext):
crypto_meta = self.get_sysmeta_crypto_meta(
'X-Object-Sysmeta-Crypto-Meta-Etag')
if crypto_meta:
mod_hdr_pairs.append(('Etag', self.decrypt_header_val(
mod_hdr_pairs.append(('Etag', self.decrypt_value(
etag, keys[self.server_type], crypto_meta)))
# Decrypt content-type
ctype = self._response_header_value('Content-Type')
mod_hdr_pairs.append(('Content-Type', self.decrypt_header_val(
mod_hdr_pairs.append(('Content-Type', self.decrypt_value(
ctype, keys[self.server_type], None)))
# Decrypt all user metadata
@ -174,23 +212,6 @@ class DecrypterObjContext(CryptoWSGIContext):
return mod_resp_headers
def process_resp(self, req):
"""
Determine if a response should be decrypted, and if so then fetch keys.
:param req: a Request object
:returns: a dict if decryption keys
"""
# Only proceed processing if an error has not occurred
if not is_success(self._get_status_int()):
return None
if config_true_value(req.environ.get('swift.crypto.override')):
self.logger.debug('No decryption is necessary because of override')
return None
return self.get_keys(req.environ)
def make_decryption_context(self, keys):
body_crypto_meta = self.get_sysmeta_crypto_meta(
'X-Object-Sysmeta-Crypto-Meta')
@ -262,6 +283,75 @@ class DecrypterObjContext(CryptoWSGIContext):
return app_resp
class DecrypterContContext(BaseDecrypterContext):
def __init__(self, decrypter, logger):
super(DecrypterContContext, self).__init__(decrypter, logger)
def GET(self, req, start_response):
app_resp = self._app_call(req.environ)
keys = self.process_resp(req)
if keys:
out_content_type = get_listing_content_type(req)
if out_content_type == 'application/json':
app_resp = self.process_json_resp(keys, app_resp)
elif out_content_type.endswith('/xml'):
app_resp = self.process_xml_resp(keys, app_resp)
start_response(self._response_status,
self._response_headers,
self._response_exc_info)
return app_resp
def update_content_length(self, new_total_len):
self._response_headers = [
(h, v) for h, v in self._response_headers
if h.lower() != 'content-length']
self._response_headers.append(('Content-Length', str(new_total_len)))
def process_json_resp(self, keys, resp_iter):
"""
Parses json body listing and decrypt content-type entries. Updates
Content-Length header with new body length and return a body iter.
"""
with closing_if_possible(resp_iter):
resp_body = ''.join(resp_iter)
body_json = json.loads(resp_body)
new_body = json.dumps([self.decrypt_obj_dict(obj_dict, keys)
for obj_dict in body_json])
self.update_content_length(len(new_body))
return [new_body]
def decrypt_obj_dict(self, obj_dict, keys):
ciphertext = obj_dict['content_type']
obj_dict['content_type'] = self.decrypt_value(
ciphertext, keys['container'], None)
# TODO - decode/decrypt etag when not using FakeFooters
# if etag and (len(etag) > constraints.ETAG_LENGTH):
return obj_dict
def process_xml_resp(self, keys, resp_iter):
"""
Parses xml body listing and decrypt content-type entries. Updates
Content-Length header with new body length and return a body iter.
"""
with closing_if_possible(resp_iter):
resp_body = ''.join(resp_iter)
tree = ElementTree.fromstring(resp_body)
for elem in tree.iter('content_type'):
ciphertext = elem.text.encode('utf8')
plaintext = self.decrypt_value(ciphertext, keys['container'], None)
elem.text = plaintext.decode('utf8')
new_body = ElementTree.tostring(tree, encoding='UTF-8').replace(
"<?xml version='1.0' encoding='UTF-8'?>",
'<?xml version="1.0" encoding="UTF-8"?>', 1)
self.update_content_length(len(new_body))
return [new_body]
class Decrypter(object):
def __init__(self, app, conf):
@ -274,19 +364,24 @@ class Decrypter(object):
req = Request(env)
try:
req.split_path(4, 4, True)
parts = req.split_path(3, 4, True)
except ValueError:
return self.app(env, start_response)
if hasattr(DecrypterObjContext, req.method):
# handle only those request methods that may require keys
km_context = DecrypterObjContext(self, self.logger)
if parts[3] and hasattr(DecrypterObjContext, req.method):
dec_context = DecrypterObjContext(self, self.logger)
elif parts[2] and hasattr(DecrypterContContext, req.method):
dec_context = DecrypterContContext(self, self.logger)
else:
# url and/or request verb is not handled by decrypter
dec_context = None
if dec_context:
try:
return getattr(km_context, req.method)(req, start_response)
return getattr(dec_context, req.method)(req, start_response)
except HTTPException as err_resp:
return err_resp(env, start_response)
# anything else
return self.app(env, start_response)

View File

@ -16,8 +16,10 @@
from hashlib import md5
import base64
import json
import urllib
from swift.common.crypto_utils import CryptoWSGIContext
from swift.common.utils import get_logger, config_true_value
from swift.common.utils import get_logger, config_true_value, \
extract_swift_bytes
from swift.common.request_helpers import get_obj_persisted_sysmeta_prefix, \
strip_user_meta_prefix, is_user_meta, update_content_type
from swift.common.swob import Request, HTTPException, HTTPUnprocessableEntity
@ -38,9 +40,9 @@ def _dump_crypto_meta(crypto_meta):
:param crypto_meta: a dict containing crypto meta items
:returns: a string serialization of a crypto meta dict
"""
return json.dumps({
return urllib.quote_plus(json.dumps({
name: (base64.b64encode(value).decode() if name == 'iv' else value)
for name, value in crypto_meta.items()})
for name, value in crypto_meta.items()}))
def encrypt_header_val(crypto, value, key, append_crypto_meta=False):
@ -181,35 +183,40 @@ class EncrypterObjContext(CryptoWSGIContext):
% (name, req.headers[name]))
def encrypt_req_headers(self, req, keys):
# update content type in case it is missing
update_content_type(req)
content_type = req.headers.get('Content-Type')
if 'container' not in keys:
# TODO fail somewhere else, earlier, or not at all
self.logger.error('Error: no container key to encrypt')
raise HTTPUnprocessableEntity(request=req)
# update content-type in case it is missing
update_content_type(req)
# Encrypt the plaintext content-type using the object key and
# persist as sysmeta along with the crypto parameters that were
# used. Do this for PUT and POST because object_post_as_copy mode
# allows content-type to be updated on a POST.
req.headers['Content-Type'], crypto_meta = encrypt_header_val(
self.crypto, content_type, keys[self.server_type],
# Separate out any swift_bytes param that may have been set by SLO
# because that should not be encrypted.
ctype, swift_bytes = extract_swift_bytes(req.headers['Content-Type'])
enc_ctype, crypto_meta = encrypt_header_val(
self.crypto, ctype, keys[self.server_type],
append_crypto_meta=True)
if swift_bytes:
enc_ctype = '%s; swift_bytes=%s' % (enc_ctype, swift_bytes)
req.headers['Content-Type'] = enc_ctype
self.logger.debug("encrypted for object Content-Type: %s using %s"
% (req.headers['Content-Type'], crypto_meta))
# TODO: Encrypt the plaintext content-type using the container
# Encrypt the plaintext content-type using the container
# key and use it to override the container update value, with the
# crypto parameters appended.
# val, _ = encrypt_header_val(
# self.crypto, ct, keys['container'], append_crypto_meta=True)
val = content_type
enc_ctype, _ = encrypt_header_val(
self.crypto, ctype, keys['container'], append_crypto_meta=True)
if swift_bytes:
enc_ctype = '%s; swift_bytes=%s' % (enc_ctype, swift_bytes)
name = 'X-Backend-Container-Update-Override-Content-Type'
req.headers[name] = val
req.headers[name] = enc_ctype
self.logger.debug("encrypted for container %s: %s" %
(name, val))
(name, enc_ctype))
error_resp = self.encrypt_user_metadata(req, keys)
if error_resp:
@ -281,9 +288,9 @@ class Encrypter(object):
if hasattr(EncrypterObjContext, req.method):
# handle only those request methods that may require keys
km_context = EncrypterObjContext(self, self.logger)
enc_context = EncrypterObjContext(self, self.logger)
try:
return getattr(km_context, req.method)(req, start_response)
return getattr(enc_context, req.method)(req, start_response)
except HTTPException as err_resp:
return err_resp(env, start_response)

View File

@ -3311,12 +3311,35 @@ def override_bytes_from_content_type(listing_dict, logger=None):
listing_dict['content_type'] = content_type
def extract_swift_bytes(content_type):
"""
Parse content_type value and extract any param with name 'swift_bytes'.
For example::
extract_swift_bytes('text/plain; swift_bytes=2048)
would return a tuple ('text/plain', 2048).
:param content_type: string value of content-type
:return: a tuple (content_type, swift_bytes) where content_type is the
original content_type value with any swift_bytes param removed and
swift_bytes is the value of the swift_bytes param, if any, or None
"""
content_type, params = parse_content_type(content_type)
swift_bytes = None
for key, value in params:
if key == 'swift_bytes':
swift_bytes = value
else:
content_type += '; %s=%s' % (key, value)
return content_type, swift_bytes
def clean_content_type(value):
if ';' in value:
left, right = value.rsplit(';', 1)
if right.lstrip().startswith('swift_bytes='):
return left
return value
"""
Remove any param with name 'swift_bytes' from a content-type value.
"""
return extract_swift_bytes(value)[0]
def quote(value, safe='/'):

View File

@ -2277,6 +2277,13 @@ class TestSlo(Base):
self.assertEqual('d', file_contents[-2])
self.assertEqual('e', file_contents[-1])
def test_slo_container_listing(self):
files = self.env.container.files(parms={'format': 'json'})
for f in files:
if f['name'] == 'manifest-abcde':
self.assertEqual(4 * 1024 * 1024 + 1, f['bytes'])
self.assertEqual('application/octet-stream', f['content_type'])
def test_slo_get_nested_manifest(self):
file_item = self.env.container.file('manifest-abcde-submanifest')
file_contents = file_item.read()

View File

@ -14,9 +14,11 @@
# limitations under the License.
import unittest
from xml.dom import minidom
import mock
import base64
import json
import urllib
from swift.common.middleware import decrypter
from swift.common.swob import Request, HTTPException, HTTPOk, \
@ -53,17 +55,24 @@ def get_crypto_meta():
def get_crypto_meta_header(crypto_meta=None):
if crypto_meta is None:
crypto_meta = get_crypto_meta()
return json.dumps({key: (base64.b64encode(value).decode()
if key == 'iv' else value)
for key, value in crypto_meta.items()})
return urllib.quote_plus(
json.dumps({key: (base64.b64encode(value).decode()
if key == 'iv' else value)
for key, value in crypto_meta.items()}))
def get_content_type():
return 'text/plain'
def encrypt_and_append_meta(value, crypto_meta=None):
return '%s; meta=%s' % (
base64.b64encode(fake_encrypt(value)),
get_crypto_meta_header(crypto_meta))
@mock.patch('swift.common.middleware.decrypter.Crypto', FakeCrypto)
class TestDecrypter(unittest.TestCase):
class TestDecrypterObjectRequests(unittest.TestCase):
def test_basic_get_req(self):
env = {'REQUEST_METHOD': 'GET',
@ -71,9 +80,7 @@ class TestDecrypter(unittest.TestCase):
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = fake_encrypt(body)
content_type = '%s; meta=%s' % (
base64.b64encode(fake_encrypt('text/plain')),
get_crypto_meta_header())
content_type = encrypt_and_append_meta('text/plain')
app = FakeSwift()
hdrs = {'Etag': 'hashOfCiphertext',
'content-type': content_type,
@ -127,9 +134,7 @@ class TestDecrypter(unittest.TestCase):
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = fake_encrypt(body)
content_type = '%s; meta=%s' % (
base64.b64encode(fake_encrypt('text/plain')),
get_crypto_meta_header())
content_type = encrypt_and_append_meta('text/plain')
app = FakeSwift()
hdrs = {'Etag': 'hashOfCiphertext',
'content-type': content_type,
@ -157,9 +162,7 @@ class TestDecrypter(unittest.TestCase):
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = fake_encrypt(body)
content_type = '%s; meta=%s' % (
base64.b64encode(fake_encrypt('text/plain')),
get_crypto_meta_header())
content_type = encrypt_and_append_meta('text/plain')
app = FakeSwift()
hdrs = {'Etag': 'hashOfCiphertext',
'etag': 'hashOfCiphertext',
@ -220,9 +223,7 @@ class TestDecrypter(unittest.TestCase):
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = fake_encrypt(body)
content_type = '%s; meta=%s' % (
base64.b64encode(fake_encrypt('text/plain')),
get_crypto_meta_header())
content_type = encrypt_and_append_meta('text/plain')
app = FakeSwift()
hdrs = {'Etag': 'hashOfCiphertext',
'etag': 'hashOfCiphertext',
@ -267,7 +268,9 @@ class TestDecrypter(unittest.TestCase):
self.assertEqual(resp.status, '200 OK')
self.assertEqual(resp.headers['Etag'], md5hex(body))
self.assertEqual(resp.headers['Content-Type'], 'text/plain')
# POSTed user meta was encrypted
self.assertEqual(resp.headers['x-object-meta-test'], 'encrypt me')
# PUT sysmeta was not encrypted
self.assertEqual(resp.headers['x-object-sysmeta-test'],
'do not encrypt me')
@ -278,9 +281,7 @@ class TestDecrypter(unittest.TestCase):
chunks = ['some', 'chunks', 'of data']
body = ''.join(chunks)
enc_body = [fake_encrypt(chunk) for chunk in chunks]
content_type = '%s; meta=%s' % (
base64.b64encode(fake_encrypt('text/plain')),
get_crypto_meta_header())
content_type = encrypt_and_append_meta('text/plain')
app = FakeSwift()
hdrs = {'Etag': 'hashOfCiphertext',
'content-type': content_type,
@ -305,9 +306,7 @@ class TestDecrypter(unittest.TestCase):
body = ''.join(chunks)
enc_body = [fake_encrypt(chunk) for chunk in chunks]
enc_body = [enc_body[0][3:], enc_body[1], enc_body[2][:2]]
content_type = '%s; meta=%s' % (
base64.b64encode(fake_encrypt('text/plain')),
get_crypto_meta_header())
content_type = encrypt_and_append_meta('text/plain')
app = FakeSwift()
hdrs = {'Etag': 'hashOfCiphertext',
'content-type': content_type,
@ -332,9 +331,7 @@ class TestDecrypter(unittest.TestCase):
req = Request.blank('/v1/a/c/o', environ=env)
body = 'FAKE APP'
enc_body = fake_encrypt(body)
content_type = base64.b64encode('%s; meta=%s'
% (fake_encrypt('text/plain'),
get_crypto_meta_header()))
content_type = encrypt_and_append_meta('text/plain')
app = FakeSwift()
hdrs = {'Etag': 'hashOfCiphertext',
'content-type': content_type,
@ -415,9 +412,8 @@ class TestDecrypter(unittest.TestCase):
enc_body = fake_encrypt(body)
bad_crypto_meta = get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown_cipher'
content_type = '%s; meta=%s' % (
base64.b64encode(fake_encrypt('text/plain')),
get_crypto_meta_header(crypto_meta=bad_crypto_meta))
content_type = encrypt_and_append_meta('text/plain',
crypto_meta=bad_crypto_meta)
app = FakeSwift()
hdrs = {'Etag': 'hashOfCiphertext',
'content-type': content_type,
@ -438,9 +434,7 @@ class TestDecrypter(unittest.TestCase):
enc_body = fake_encrypt(body)
bad_crypto_meta = get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown_cipher'
content_type = '%s; meta=%s' % (
base64.b64encode(fake_encrypt('text/plain')),
get_crypto_meta_header())
content_type = encrypt_and_append_meta('text/plain')
app = FakeSwift()
hdrs = {'Etag': 'hashOfCiphertext',
'content-type': content_type,
@ -480,11 +474,249 @@ class TestDecrypter(unittest.TestCase):
self.assertEqual(resp.headers['x-object-sysmeta-test'],
'do not encrypt me')
@mock.patch('swift.common.middleware.decrypter.Crypto', FakeCrypto)
class TestDecrypterContainerRequests(unittest.TestCase):
# TODO - update these tests to have etag to be encrypted and have
# crypto-meta in response, and verify that the etag gets decrypted.
def _make_cont_get_req(self, resp_body, format, override=False):
path = '/v1/a/c'
content_type = 'text/plain'
if format:
path = '%s/?format=%s' % (path, format)
content_type = 'application/' + format
env = {'REQUEST_METHOD': 'GET',
'swift.crypto.fetch_crypto_keys': fetch_crypto_keys}
if override:
env['swift.crypto.override'] = True
req = Request.blank(path, environ=env)
app = FakeSwift()
hdrs = {'content-type': content_type}
app.register('GET', path, HTTPOk, body=resp_body, headers=hdrs)
return req.get_response(decrypter.Decrypter(app, {}))
def test_cont_get_simple_req(self):
# no format requested, listing has names only
fake_body = 'testfile1\ntestfile2\n'
resp = self._make_cont_get_req(fake_body, None)
self.assertEqual('200 OK', resp.status)
names = resp.body.split('\n')
self.assertEqual(3, len(names))
self.assertIn('testfile1', names)
self.assertIn('testfile2', names)
self.assertIn('', names)
def test_cont_get_json_req(self):
content_type_1 = u'\uF10F\uD20D\uB30B\u9409'
content_type_2 = 'text/plain; param=foo'
obj_dict_1 = {"bytes": 16,
"last_modified": "2015-04-14T23:33:06.439040",
"hash": "c6e8196d7f0fff6444b90861fe8d609d",
"name": "testfile",
"content_type":
encrypt_and_append_meta(content_type_1.encode('utf8'))}
obj_dict_2 = {"bytes": 24,
"last_modified": "2015-04-14T23:33:06.519020",
"hash": "ac0374ed4d43635f803c82469d0b5a10",
"name": "testfile2",
"content_type":
encrypt_and_append_meta(content_type_2.encode('utf8'))}
listing = [obj_dict_1, obj_dict_2]
fake_body = json.dumps(listing)
resp = self._make_cont_get_req(fake_body, 'json')
self.assertEqual('200 OK', resp.status)
body = resp.body
self.assertEqual(len(body), int(resp.headers['Content-Length']))
body_json = json.loads(body)
self.assertEqual(2, len(body_json))
obj_dict_1['content_type'] = content_type_1
self.assertDictEqual(obj_dict_1, body_json[0])
obj_dict_2['content_type'] = content_type_2
self.assertDictEqual(obj_dict_2, body_json[1])
def test_cont_get_json_req_with_crypto_override(self):
content_type_1 = 'image/jpeg'
content_type_2 = 'text/plain; param=foo'
obj_dict_1 = {"bytes": 16,
"last_modified": "2015-04-14T23:33:06.439040",
"hash": "c6e8196d7f0fff6444b90861fe8d609d",
"name": "testfile",
"content_type": content_type_1}
obj_dict_2 = {"bytes": 24,
"last_modified": "2015-04-14T23:33:06.519020",
"hash": "ac0374ed4d43635f803c82469d0b5a10",
"name": "testfile2",
"content_type": content_type_2}
listing = [obj_dict_1, obj_dict_2]
fake_body = json.dumps(listing)
resp = self._make_cont_get_req(fake_body, 'json', override=True)
self.assertEqual('200 OK', resp.status)
body = resp.body
self.assertEqual(len(body), int(resp.headers['Content-Length']))
body_json = json.loads(body)
self.assertEqual(2, len(body_json))
obj_dict_1['content_type'] = content_type_1
self.assertDictEqual(obj_dict_1, body_json[0])
obj_dict_2['content_type'] = content_type_2
self.assertDictEqual(obj_dict_2, body_json[1])
def test_cont_get_json_req_with_cipher_mismatch(self):
content_type = 'image/jpeg'
bad_crypto_meta = get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown_cipher'
obj_dict_1 = {"bytes": 16,
"last_modified": "2015-04-14T23:33:06.439040",
"hash": "c6e8196d7f0fff6444b90861fe8d609d",
"name": "testfile",
"content_type":
encrypt_and_append_meta(content_type,
crypto_meta=bad_crypto_meta)}
listing = [obj_dict_1]
fake_body = json.dumps(listing)
resp = self._make_cont_get_req(fake_body, 'json')
self.assertEqual('500 Internal Error', resp.status)
# TODO: this error message is not appropriate, change
self.assertEqual('Error decrypting header value', resp.body)
def _assert_element_contains_dict(self, expected, element):
for k, v in expected.items():
entry = element.getElementsByTagName(k)
self.assertIsNotNone(entry, 'Key %s not found' % k)
actual = entry[0].childNodes[0].nodeValue
self.assertEqual(v, actual,
"Expected %s but got %s for key %s"
% (v, actual, k))
def test_cont_get_xml_req(self):
content_type_1 = u'\uF10F\uD20D\uB30B\u9409'
content_type_2 = 'text/plain; param=foo'
fake_body = '''<?xml version="1.0" encoding="UTF-8"?>
<container name="testc">\
<object><hash>c6e8196d7f0fff6444b90861fe8d609d</hash><content_type>\
''' + encrypt_and_append_meta(content_type_1.encode('utf8')) + '''\
</content_type><name>testfile</name><bytes>16</bytes>\
<last_modified>2015-04-19T02:37:39.601660</last_modified></object>\
<object><hash>ac0374ed4d43635f803c82469d0b5a10</hash><content_type>\
''' + encrypt_and_append_meta(content_type_2.encode('utf8')) + '''\
</content_type><name>testfile2</name><bytes>24</bytes>\
<last_modified>2015-04-19T02:37:39.684740</last_modified></object>\
</container>'''
resp = self._make_cont_get_req(fake_body, 'xml')
self.assertEqual('200 OK', resp.status)
body = resp.body
self.assertEqual(len(body), int(resp.headers['Content-Length']))
tree = minidom.parseString(body)
containers = tree.getElementsByTagName('container')
self.assertEqual(1, len(containers))
self.assertEqual('testc',
containers[0].attributes.getNamedItem("name").value)
objs = tree.getElementsByTagName('object')
self.assertEqual(2, len(objs))
obj_dict_1 = {"bytes": "16",
"last_modified": "2015-04-19T02:37:39.601660",
"hash": "c6e8196d7f0fff6444b90861fe8d609d",
"name": "testfile",
"content_type": content_type_1}
self._assert_element_contains_dict(obj_dict_1, objs[0])
obj_dict_2 = {"bytes": "24",
"last_modified": "2015-04-19T02:37:39.684740",
"hash": "ac0374ed4d43635f803c82469d0b5a10",
"name": "testfile2",
"content_type": content_type_2}
self._assert_element_contains_dict(obj_dict_2, objs[1])
def test_cont_get_xml_req_with_crypto_override(self):
content_type_1 = 'image/jpeg'
content_type_2 = 'text/plain; param=foo'
fake_body = '''<?xml version="1.0" encoding="UTF-8"?>
<container name="testc">\
<object><hash>c6e8196d7f0fff6444b90861fe8d609d</hash>\
<content_type>''' + content_type_1 + '''\
</content_type><name>testfile</name><bytes>16</bytes>\
<last_modified>2015-04-19T02:37:39.601660</last_modified></object>\
<object><hash>ac0374ed4d43635f803c82469d0b5a10</hash>\
<content_type>''' + content_type_2 + '''\
</content_type><name>testfile2</name><bytes>24</bytes>\
<last_modified>2015-04-19T02:37:39.684740</last_modified></object>\
</container>'''
resp = self._make_cont_get_req(fake_body, 'xml', override=True)
self.assertEqual('200 OK', resp.status)
body = resp.body
self.assertEqual(len(body), int(resp.headers['Content-Length']))
tree = minidom.parseString(body)
containers = tree.getElementsByTagName('container')
self.assertEqual(1, len(containers))
self.assertEqual('testc',
containers[0].attributes.getNamedItem("name").value)
objs = tree.getElementsByTagName('object')
self.assertEqual(2, len(objs))
obj_dict_1 = {"bytes": "16",
"last_modified": "2015-04-19T02:37:39.601660",
"hash": "c6e8196d7f0fff6444b90861fe8d609d",
"name": "testfile",
"content_type": content_type_1}
self._assert_element_contains_dict(obj_dict_1, objs[0])
obj_dict_2 = {"bytes": "24",
"last_modified": "2015-04-19T02:37:39.684740",
"hash": "ac0374ed4d43635f803c82469d0b5a10",
"name": "testfile2",
"content_type": content_type_2}
self._assert_element_contains_dict(obj_dict_2, objs[1])
def test_cont_get_xml_req_with_cipher_mismatch(self):
content_type = 'image/jpeg'
bad_crypto_meta = get_crypto_meta()
bad_crypto_meta['cipher'] = 'unknown_cipher'
fake_body = '''<?xml version="1.0" encoding="UTF-8"?>
<container name="testc">\
<object><hash>c6e8196d7f0fff6444b90861fe8d609d</hash><content_type>\
''' + encrypt_and_append_meta(content_type, crypto_meta=bad_crypto_meta) + '''\
</content_type><name>testfile</name><bytes>16</bytes>\
<last_modified>2015-04-19T02:37:39.601660</last_modified></object>\
</container>'''
resp = self._make_cont_get_req(fake_body, 'xml')
self.assertEqual('500 Internal Error', resp.status)
self.assertEqual('Error decrypting header value', resp.body)
class TestModuleMethods(unittest.TestCase):
def test_filter_factory(self):
factory = decrypter.filter_factory({})
self.assertTrue(callable(factory))
self.assertIsInstance(factory(None), decrypter.Decrypter)
class TestDecrypter(unittest.TestCase):
def test_app_exception(self):
app = decrypter.Decrypter(
FakeAppThatExcepts(), {})

View File

@ -14,6 +14,7 @@
# limitations under the License.
import base64
import json
import urllib
import unittest
import mock
@ -60,7 +61,8 @@ class TestEncrypter(unittest.TestCase):
# encrypted version of plaintext etag
actual = base64.b64decode(req_hdrs['X-Object-Sysmeta-Crypto-Etag'])
self.assertEqual(fake_encrypt(md5hex(body)), actual)
actual = json.loads(req_hdrs['X-Object-Sysmeta-Crypto-Meta-Etag'])
actual = json.loads(urllib.unquote_plus(
req_hdrs['X-Object-Sysmeta-Crypto-Meta-Etag']))
self.assertEqual('test_cipher', actual['cipher'])
self.assertEqual('test_iv', base64.b64decode(actual['iv']))
@ -70,7 +72,7 @@ class TestEncrypter(unittest.TestCase):
# TODO: uncomment when container metadata is encrypted
# actual_etag, actual_meta = base64.b64decode(actual).split(';')
# self.assertEqual(fake_encrypt(md5hex(body)), actual_etag)
# actual_meta = json.loads(actual_meta.lstrip(' meta='))
# actual_meta = json.loads(unqote_plus(actual_meta.lstrip(' meta=')))
# self.assertEqual('test_cipher', actual_meta['cipher'])
# self.assertEqual('test_iv', base64.b64decode(actual_meta['iv']))
@ -79,25 +81,26 @@ class TestEncrypter(unittest.TestCase):
actual_ctype, actual_meta = actual.split(';')
self.assertEqual(base64.b64encode(fake_encrypt('text/plain')),
actual_ctype)
actual_meta = json.loads(actual_meta.lstrip(' meta='))
actual_meta = json.loads(urllib.unquote_plus(
actual_meta.lstrip(' meta=')))
self.assertEqual('test_cipher', actual_meta['cipher'])
self.assertEqual('test_iv', base64.b64decode(actual_meta['iv']))
# encrypted version of content-type for container update
actual = req_hdrs['X-Backend-Container-Update-Override-Content-Type']
self.assertEqual('text/plain', actual)
# TODO: uncomment when container metadata is encrypted
# actual_ctype, actual_meta = base64.b64decode(actual).split(';')
# self.assertEqual(fake_encrypt('text/plain'), actual_ctype)
# actual_meta = json.loads(actual_meta.lstrip(' meta='))
# self.assertEqual('test_cipher', actual_meta['cipher'])
# self.assertEqual('test_iv', base64.b64decode(actual_meta['iv']))
actual_ctype, actual_meta = actual.split(';')
self.assertEqual(fake_encrypt('text/plain'),
base64.b64decode(actual_ctype))
actual_meta = json.loads(urllib.unquote_plus(
actual_meta.lstrip(' meta=')))
self.assertEqual('test_cipher', actual_meta['cipher'])
self.assertEqual('test_iv', base64.b64decode(actual_meta['iv']))
# user meta is encrypted
self.assertEqual(base64.b64encode(fake_encrypt('encrypt me')),
req_hdrs['X-Object-Meta-Test'])
actual = req_hdrs['X-Object-Sysmeta-Crypto-Meta-Test']
actual = json.loads(actual)
actual = json.loads(urllib.unquote_plus(actual))
self.assertEqual('test_cipher', actual['cipher'])
self.assertEqual('test_iv', base64.b64decode(actual['iv']))
@ -133,7 +136,7 @@ class TestEncrypter(unittest.TestCase):
self.assertEqual(base64.b64encode(fake_encrypt('encrypt me')),
req_hdrs['X-Object-Meta-Test'])
actual = req_hdrs['X-Object-Sysmeta-Crypto-Meta-Test']
actual = json.loads(actual)
actual = json.loads(urllib.unquote_plus(actual))
self.assertEqual('test_cipher', actual['cipher'])
self.assertEqual('test_iv', base64.b64decode(actual['iv']))
@ -141,6 +144,42 @@ class TestEncrypter(unittest.TestCase):
self.assertEqual('do not encrypt me',
req_hdrs['X-Object-Sysmeta-Test'])
def test_swift_bytes_not_encrypted(self):
content_type = 'text/plain; foo=bar; swift_bytes=1234567890'
env = {'REQUEST_METHOD': 'PUT',
'swift.crypto.fetch_crypto_keys': fetch_crypto_keys}
hdrs = {'content-type': content_type,
'content-length': '0'}
req = Request.blank(
'/v1/a/c/o', environ=env, body='', headers=hdrs)
app = FakeSwift()
app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(encrypter.Encrypter(app, {}))
self.assertEqual(resp.status, '201 Created')
self.assertEqual(1, len(app.calls), app.calls)
self.assertEqual('PUT', app.calls[0][0])
req_hdrs = app.headers[0]
def _verify_content_type(actual):
parts = actual.split(';')
self.assertEqual(3, len(parts))
expected_ctype = base64.b64encode(
fake_encrypt('text/plain; foo=bar'))
self.assertEqual(expected_ctype, parts[0])
actual_meta = json.loads(urllib.unquote_plus(
parts[1].lstrip(' meta=')))
self.assertEqual('test_cipher', actual_meta['cipher'])
self.assertEqual('test_iv', base64.b64decode(actual_meta['iv']))
self.assertEqual('swift_bytes=1234567890', parts[2].strip())
# encrypted version of content-type for object server
_verify_content_type(req_hdrs['Content-Type'])
# encrypted version of content-type for container update
actual = req_hdrs['X-Backend-Container-Update-Override-Content-Type']
_verify_content_type(actual)
def test_backend_response_etag_is_replaced(self):
body = 'FAKE APP'
env = {'REQUEST_METHOD': 'PUT',
@ -300,7 +339,7 @@ class TestEncrypterContentType(unittest.TestCase):
param = param.strip()
self.assertTrue(param.startswith('meta='))
actual_meta = json.loads(param[5:])
actual_meta = json.loads(urllib.unquote_plus(param[5:]))
self.assertDictEqual(expected_meta, actual_meta)
def test_PUT_detect_content_type(self):

View File

@ -2863,6 +2863,20 @@ cluster_dfw1 = http://dfw1.host/v1/
self.assertEqual(listing_dict['content_type'],
'text/plain;hello="world"')
def test_extract_swift_bytes(self):
scenarios = {
# maps input value -> expected returned tuple
'': ('', None),
'text/plain': ('text/plain', None),
'text/plain; other=thing': ('text/plain; other=thing', None),
'text/plain; swift_bytes=123': ('text/plain', '123'),
'text/plain; other=thing; swift_bytes=123':
('text/plain; other=thing', '123'),
'text/plain; swift_bytes=123; other=thing':
('text/plain; other=thing', '123')}
for test_value, expected in scenarios.items():
self.assertEqual(expected, utils.extract_swift_bytes(test_value))
def test_clean_content_type(self):
subtests = {
'': '', 'text/plain': 'text/plain',
@ -2870,12 +2884,11 @@ cluster_dfw1 = http://dfw1.host/v1/
'text/plain; swift_bytes=123': 'text/plain',
'text/plain; someother=thing; swift_bytes=123':
'text/plain; someother=thing',
# Since Swift always tacks on the swift_bytes, clean_content_type()
# only strips swift_bytes if it's last. The next item simply shows
# that if for some other odd reason it's not last,
# clean_content_type() will not remove it from the header.
'text/plain; swift_bytes=123; someother=thing':
'text/plain; swift_bytes=123; someother=thing'}
# swift_bytes is not necessarily the last param e.g. encrypter
# middleware may append crypto meta as a param after slo middleware
# has appended swift_bytes.
'text/plain; swift_bytes=123; meta=blah':
'text/plain; meta=blah'}
for before, after in subtests.items():
self.assertEqual(utils.clean_content_type(before), after)