diff --git a/swift/common/crypto_utils.py b/swift/common/crypto_utils.py index 5dbb64a2a1..c707c95b09 100644 --- a/swift/common/crypto_utils.py +++ b/swift/common/crypto_utils.py @@ -88,11 +88,16 @@ def dump_crypto_meta(crypto_meta): :param crypto_meta: a dict containing crypto meta items :returns: a string serialization of a crypto meta dict """ + def b64_encode_meta(crypto_meta): + return { + name: (base64.b64encode(value).decode() if name in ('iv', 'key') + else b64_encode_meta(value) if isinstance(value, dict) + else value) + for name, value in crypto_meta.items()} + # use sort_keys=True to make serialized form predictable for testing - return urllib.quote_plus(json.dumps({ - name: (base64.b64encode(value).decode() if name in ('iv', 'key') - else value) - for name, value in crypto_meta.items()}, sort_keys=True)) + return urllib.quote_plus( + json.dumps(b64_encode_meta(crypto_meta), sort_keys=True)) def load_crypto_meta(value): @@ -110,12 +115,16 @@ def load_crypto_meta(value): :raises EncryptionException: if an error occurs while parsing the crypto meta """ + def b64_decode_meta(crypto_meta): + return { + str(name): (base64.b64decode(val) if name in ('iv', 'key') + else b64_decode_meta(val) if isinstance(val, dict) + else str(val)) + for name, val in crypto_meta.items()} + try: value = urllib.unquote_plus(value) - crypto_meta = {str(name): (base64.b64decode(value) - if name in ('iv', 'key') else str(value)) - for name, value in json.loads(value).items()} - return crypto_meta + return b64_decode_meta(json.loads(value)) except (KeyError, ValueError, TypeError) as err: msg = 'Bad crypto meta %s: %s' % (value, err) raise EncryptionException(msg) diff --git a/swift/common/middleware/crypto.py b/swift/common/middleware/crypto.py index a62ddc6855..7131bb7cd1 100644 --- a/swift/common/middleware/crypto.py +++ b/swift/common/middleware/crypto.py @@ -131,21 +131,23 @@ class Crypto(object): # helper method to create random key of correct length return os.urandom(KEY_LENGTH) - def wrap_key(self, wrapping_key, key_to_wrap, iv): + def wrap_key(self, wrapping_key, key_to_wrap): # we don't use an RFC 3394 key wrap algorithm such as cryptography's # aes_wrap_key because it's slower and we have iv material readily # available so don't need a deterministic algorithm + iv = self._get_random_iv() encryptor = Cipher(algorithms.AES(wrapping_key), modes.CTR(iv), backend=default_backend()).encryptor() - return encryptor.update(key_to_wrap) + return {'key': encryptor.update(key_to_wrap), 'iv': iv} - def unwrap_key(self, wrapping_key, wrapped_key, iv): + def unwrap_key(self, wrapping_key, context): + # unwrap a key from dict of form returned by wrap_key # check the key length early - unwrapping won't change the length - self.check_key(wrapped_key) - decryptor = Cipher(algorithms.AES(wrapping_key), modes.CTR(iv), + self.check_key(context['key']) + decryptor = Cipher(algorithms.AES(wrapping_key), + modes.CTR(context['iv']), backend=default_backend()).decryptor() - key = decryptor.update(wrapped_key) - return key + return decryptor.update(context['key']) def check_key(self, key): if len(key) != KEY_LENGTH: diff --git a/swift/common/middleware/decrypter.py b/swift/common/middleware/decrypter.py index c323aaccde..2a150a8478 100644 --- a/swift/common/middleware/decrypter.py +++ b/swift/common/middleware/decrypter.py @@ -70,8 +70,7 @@ class BaseDecrypterContext(CryptoWSGIContext): """ try: return self.crypto.unwrap_key(wrapping_key, - crypto_meta['key'], - crypto_meta['iv']) + crypto_meta['body_key']) except KeyError as err: err = 'Missing %s' % err except ValueError as err: diff --git a/swift/common/middleware/encrypter.py b/swift/common/middleware/encrypter.py index 46e6169597..7c879f2a37 100644 --- a/swift/common/middleware/encrypter.py +++ b/swift/common/middleware/encrypter.py @@ -68,15 +68,12 @@ class EncInputWrapper(object): # do this once when body is first read if self.body_crypto_ctxt is None: self.body_crypto_meta = self.crypto.create_crypto_meta() - self.body_key = self.crypto.create_random_key() - # wrap the body key with object key re-using body iv - self.body_crypto_meta['key'] = self.crypto.wrap_key( - self.keys['object'], - self.body_key, - self.body_crypto_meta['iv'] - ) + body_key = self.crypto.create_random_key() + # wrap the body key with object key + self.body_crypto_meta['body_key'] = self.crypto.wrap_key( + self.keys['object'], body_key) self.body_crypto_ctxt = self.crypto.create_encryption_ctxt( - self.body_key, self.body_crypto_meta.get('iv')) + body_key, self.body_crypto_meta.get('iv')) self.plaintext_md5 = md5() self.ciphertext_md5 = md5() diff --git a/test/unit/common/middleware/test_crypto.py b/test/unit/common/middleware/test_crypto.py index bac61c046d..fe1937e405 100644 --- a/test/unit/common/middleware/test_crypto.py +++ b/test/unit/common/middleware/test_crypto.py @@ -12,8 +12,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. - - +import mock import unittest import os @@ -194,24 +193,26 @@ class TestCrypto(unittest.TestCase): wrapping_key = os.urandom(32) key_to_wrap = os.urandom(32) iv = os.urandom(16) - wrapped = self.crypto.wrap_key(wrapping_key, key_to_wrap, iv) + with mock.patch('swift.common.middleware.crypto.Crypto._get_random_iv', + return_value=iv): + wrapped = self.crypto.wrap_key(wrapping_key, key_to_wrap) cipher = Cipher(algorithms.AES(wrapping_key), modes.CTR(iv), backend=default_backend()) - expected = cipher.encryptor().update(key_to_wrap) + expected = {'key': cipher.encryptor().update(key_to_wrap), + 'iv': iv} self.assertEqual(expected, wrapped) - unwrapped = self.crypto.unwrap_key(wrapping_key, wrapped, iv) + unwrapped = self.crypto.unwrap_key(wrapping_key, wrapped) self.assertEqual(key_to_wrap, unwrapped) def test_unwrap_bad_key(self): # verify that ValueError is raised if unwrapped key is invalid wrapping_key = os.urandom(32) - iv = os.urandom(16) for length in (0, 16, 24, 31, 33): key_to_wrap = os.urandom(length) - wrapped = self.crypto.wrap_key(wrapping_key, key_to_wrap, iv) + wrapped = self.crypto.wrap_key(wrapping_key, key_to_wrap) with self.assertRaises(ValueError) as cm: - self.crypto.unwrap_key(wrapping_key, wrapped, iv) + self.crypto.unwrap_key(wrapping_key, wrapped) self.assertEqual( cm.exception.message, 'Key must be length 32 bytes') diff --git a/test/unit/common/middleware/test_crypto_utils.py b/test/unit/common/middleware/test_crypto_utils.py index a8a1b9207a..0cbfb55ca7 100644 --- a/test/unit/common/middleware/test_crypto_utils.py +++ b/test/unit/common/middleware/test_crypto_utils.py @@ -149,11 +149,14 @@ class TestModuleMethods(unittest.TestCase): 'iv%22%3A+%22MDEyMzQ1Njc4OWFiY2RlZg%3D%3D%22%7D' meta_with_key = {'iv': '0123456789abcdef', 'cipher': 'AES_CTR_256', - 'key': '0123456789abcdef0123456789abcdef'} - serialized_meta_with_key = '%7B%22cipher%22%3A+%22AES_CTR_256%22%2C+%22' \ - 'iv%22%3A+%22MDEyMzQ1Njc4OWFiY2RlZg%3D%3D%22' \ - '%2C+%22key%22%3A+%22MDEyMzQ1Njc4OWFiY2RlZjA' \ - 'xMjM0NTY3ODlhYmNkZWY%3D%22%7D' + 'body_key': {'key': 'fedcba9876543210fedcba9876543210', + 'iv': 'fedcba9876543210'}} + serialized_meta_with_key = '%7B%22body_key%22%3A+%7B%22iv%22%3A+%22ZmVkY' \ + '2JhOTg3NjU0MzIxMA%3D%3D%22%2C+%22key%22%3A+%' \ + '22ZmVkY2JhOTg3NjU0MzIxMGZlZGNiYTk4NzY1NDMyMT' \ + 'A%3D%22%7D%2C+%22cipher%22%3A+%22AES_CTR_256' \ + '%22%2C+%22iv%22%3A+%22MDEyMzQ1Njc4OWFiY2RlZg' \ + '%3D%3D%22%7D' def test_dump_crypto_meta(self): actual = crypto_utils.dump_crypto_meta(self.meta) diff --git a/test/unit/common/middleware/test_decrypter.py b/test/unit/common/middleware/test_decrypter.py index 2cd772c2cb..9cad16becb 100644 --- a/test/unit/common/middleware/test_decrypter.py +++ b/test/unit/common/middleware/test_decrypter.py @@ -61,8 +61,9 @@ class TestDecrypterObjectRequests(unittest.TestCase): object_key = fetch_crypto_keys()['object'] body_key = os.urandom(32) enc_body = encrypt(body, body_key, FAKE_IV) - body_crypto_meta = fake_get_crypto_meta( - key=encrypt(body_key, object_key, FAKE_IV)) + body_key_meta = {'key': encrypt(body_key, object_key, FAKE_IV), + 'iv': FAKE_IV} + body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta) hdrs = { 'Etag': 'hashOfCiphertext', 'content-type': 'text/plain', @@ -313,7 +314,8 @@ class TestDecrypterObjectRequests(unittest.TestCase): self.decrypter.logger.get_lines_for_level('error')[0]) def test_GET_with_bad_body_key_for_object_body(self): - bad_crypto_meta = fake_get_crypto_meta(key='wrapped too short key') + body_key_meta = {'key': 'wrapped too short key', 'iv': FAKE_IV} + bad_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta) self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta) self.assertIn('Key must be length 32', self.decrypter.logger.get_lines_for_level('error')[0]) @@ -321,7 +323,7 @@ class TestDecrypterObjectRequests(unittest.TestCase): def test_GET_with_missing_body_key_for_object_body(self): bad_crypto_meta = fake_get_crypto_meta() # no key by default self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta) - self.assertIn("Missing 'key'", + self.assertIn("Missing 'body_key'", self.decrypter.logger.get_lines_for_level('error')[0]) def test_HEAD_success(self): @@ -368,8 +370,9 @@ class TestDecrypterObjectRequests(unittest.TestCase): object_key = fetch_crypto_keys()['object'] body_key = os.urandom(32) enc_body = encrypt(body, body_key, FAKE_IV) - body_crypto_meta = fake_get_crypto_meta( - key=encrypt(body_key, object_key, FAKE_IV)) + body_key_meta = {'key': encrypt(body_key, object_key, FAKE_IV), + 'iv': FAKE_IV} + body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta) hdrs = { 'Etag': 'hashOfCiphertext', 'etag': 'hashOfCiphertext', @@ -405,8 +408,9 @@ class TestDecrypterObjectRequests(unittest.TestCase): object_key = fetch_crypto_keys()['object'] body_key = os.urandom(32) enc_body = encrypt(body, body_key, FAKE_IV) - body_crypto_meta = fake_get_crypto_meta( - key=encrypt(body_key, object_key, FAKE_IV)) + body_key_meta = {'key': encrypt(body_key, object_key, FAKE_IV), + 'iv': FAKE_IV} + body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta) hdrs = { 'Etag': 'hashOfCiphertext', 'etag': 'hashOfCiphertext', @@ -471,8 +475,9 @@ class TestDecrypterObjectRequests(unittest.TestCase): cont_key = fetch_crypto_keys()['container'] object_key = fetch_crypto_keys()['object'] body_key = os.urandom(32) - body_crypto_meta = fake_get_crypto_meta( - key=encrypt(body_key, object_key, FAKE_IV)) + body_key_meta = {'key': encrypt(body_key, object_key, FAKE_IV), + 'iv': FAKE_IV} + body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta) ctxt = Crypto().create_encryption_ctxt(body_key, FAKE_IV) enc_body = [encrypt(chunk, ctxt=ctxt) for chunk in chunks] hdrs = { @@ -503,8 +508,9 @@ class TestDecrypterObjectRequests(unittest.TestCase): cont_key = fetch_crypto_keys()['container'] object_key = fetch_crypto_keys()['object'] body_key = os.urandom(32) - body_crypto_meta = fake_get_crypto_meta( - key=encrypt(body_key, object_key, FAKE_IV)) + body_key_meta = {'key': encrypt(body_key, object_key, FAKE_IV), + 'iv': FAKE_IV} + body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta) ctxt = Crypto().create_encryption_ctxt(body_key, FAKE_IV) enc_body = [encrypt(chunk, ctxt=ctxt) for chunk in chunks] enc_body = [enc_body[0][3:], enc_body[1], enc_body[2][:2]] @@ -539,8 +545,9 @@ class TestDecrypterObjectRequests(unittest.TestCase): cont_key = fetch_crypto_keys()['container'] object_key = fetch_crypto_keys()['object'] body_key = os.urandom(32) - body_crypto_meta = fake_get_crypto_meta( - key=encrypt(body_key, object_key, FAKE_IV)) + body_key_meta = {'key': encrypt(body_key, object_key, FAKE_IV), + 'iv': FAKE_IV} + body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta) plaintext = 'Cwm fjord veg balks nth pyx quiz' plaintext_etag = md5hex(plaintext) ciphertext = encrypt(plaintext, body_key, FAKE_IV) @@ -605,8 +612,9 @@ class TestDecrypterObjectRequests(unittest.TestCase): cont_key = fetch_crypto_keys()['container'] object_key = fetch_crypto_keys()['object'] body_key = os.urandom(32) - body_crypto_meta = fake_get_crypto_meta( - key=encrypt(body_key, object_key, FAKE_IV)) + body_key_meta = {'key': encrypt(body_key, object_key, FAKE_IV), + 'iv': FAKE_IV} + body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta) plaintext = 'Cwm fjord veg balks nth pyx quiz' plaintext_etag = md5hex(plaintext) ciphertext = encrypt(plaintext, body_key, FAKE_IV) diff --git a/test/unit/common/middleware/test_encrypter.py b/test/unit/common/middleware/test_encrypter.py index 89a920bfb8..5a44aaabea 100644 --- a/test/unit/common/middleware/test_encrypter.py +++ b/test/unit/common/middleware/test_encrypter.py @@ -75,10 +75,15 @@ class TestEncrypter(unittest.TestCase): # verify body crypto meta actual = req_hdrs['X-Object-Sysmeta-Crypto-Meta'] actual = json.loads(urllib.unquote_plus(actual)) - expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV) self.assertEqual(Crypto().get_cipher(), actual['cipher']) self.assertEqual(FAKE_IV, base64.b64decode(actual['iv'])) - self.assertEqual(expected_wrapped_key, base64.b64decode(actual['key'])) + + # verify wrapped body key + expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV) + self.assertEqual(expected_wrapped_key, + base64.b64decode(actual['body_key']['key'])) + self.assertEqual(FAKE_IV, + base64.b64decode(actual['body_key']['iv'])) # verify etag self.assertEqual(ciphertext_etag, req_hdrs['Etag']) @@ -285,10 +290,15 @@ class TestEncrypter(unittest.TestCase): # verify body crypto meta actual = req_hdrs['X-Object-Sysmeta-Crypto-Meta'] actual = json.loads(urllib.unquote_plus(actual)) - expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV) self.assertEqual(Crypto().get_cipher(), actual['cipher']) self.assertEqual(FAKE_IV, base64.b64decode(actual['iv'])) - self.assertEqual(expected_wrapped_key, base64.b64decode(actual['key'])) + + # verify wrapped body key + expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV) + self.assertEqual(expected_wrapped_key, + base64.b64decode(actual['body_key']['key'])) + self.assertEqual(FAKE_IV, + base64.b64decode(actual['body_key']['iv'])) def test_PUT_with_etag_override_in_headers(self): # verify handling of another middleware's diff --git a/test/unit/common/middleware/test_encrypter_decrypter.py b/test/unit/common/middleware/test_encrypter_decrypter.py index 50db9a469c..71dacc61b9 100644 --- a/test/unit/common/middleware/test_encrypter_decrypter.py +++ b/test/unit/common/middleware/test_encrypter_decrypter.py @@ -300,11 +300,10 @@ class TestCryptoPipelineChanges(unittest.TestCase): # verify on disk data - body body_iv = load_crypto_meta( metadata['x-object-sysmeta-crypto-meta'])['iv'] - wrapped_body_key = load_crypto_meta( - metadata['x-object-sysmeta-crypto-meta'])['key'] + body_key_meta = load_crypto_meta( + metadata['x-object-sysmeta-crypto-meta'])['body_key'] obj_key = self.km.create_key('/a/%s/o' % self.container_name) - body_key = crypto.Crypto({}).unwrap_key( - obj_key, wrapped_body_key, body_iv) + body_key = crypto.Crypto({}).unwrap_key(obj_key, body_key_meta) exp_enc_body = encrypt(self.plaintext, body_key, body_iv) self.assertEqual(exp_enc_body, contents) # verify on disk user metadata