openstack
/
heat
Code Issues Proposed changes

Merge "Add batching for heat-manage encrypt/decrypt"

This commit is contained in:
Jenkins 2015-08-05 05:00:16 +00:00 committed by Gerrit Code Review
parent 4262c72811 7c90690acc
commit bf37a4c749
2 changed files with 118 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
heat/db/sqlalchemy/api.py
View File

@ -1145,14 +1145,23 @@ def db_version(engine):
return migration.db_version(engine)
def db_encrypt_parameters_and_properties(ctxt, encryption_key):
def db_encrypt_parameters_and_properties(ctxt, encryption_key, batch_size=50):
"""Encrypt parameters and properties for all templates in db.
:param ctxt: RPC context
:param encryption_key: key that will be used for parameter and property
encryption
:param batch_size: number of templates requested from db in each iteration.
50 means that heat requests 50 templates, encrypt them
and proceed with next 50 items.
"""
from heat.engine import template
session = get_session()
with session.begin():
raw_templates = session.query(models.RawTemplate).all()
for raw_template in raw_templates:
query = session.query(models.RawTemplate)
for raw_template in _get_batch(
session=session, ctxt=ctxt, query=query,
model=models.RawTemplate, batch_size=batch_size):
tmpl = template.Template.load(ctxt, raw_template.id, raw_template)
env = raw_template.environment
@ -1179,10 +1188,12 @@ def db_encrypt_parameters_and_properties(ctxt, encryption_key):
raw_template_update(ctxt, raw_template.id,
{'environment': environment})
resources = session.query(models.Resource).filter(
query = session.query(models.Resource).filter(
~models.Resource.properties_data.is_(None),
~models.Resource.properties_data_encrypted.is_(True)).all()
for resource in resources:
~models.Resource.properties_data_encrypted.is_(True))
for resource in _get_batch(
session=session, ctxt=ctxt, query=query, model=models.Resource,
batch_size=batch_size):
result = {}
for prop_name, prop_value in resource.properties_data.items():
prop_string = jsonutils.dumps(prop_value)
@ -1197,13 +1208,22 @@ def db_encrypt_parameters_and_properties(ctxt, encryption_key):
resource.atomic_key)
def db_decrypt_parameters_and_properties(ctxt, encryption_key):
def db_decrypt_parameters_and_properties(ctxt, encryption_key, batch_size=50):
"""Decrypt parameters and properties for all templates in db.
:param ctxt: RPC context
:param encryption_key: key that will be used for parameter and property
decryption
:param batch_size: number of templates requested from db in each iteration.
50 means that heat requests 50 templates, encrypt them
and proceed with next 50 items.
"""
session = get_session()
with session.begin():
raw_templates = session.query(models.RawTemplate).all()
for raw_template in raw_templates:
query = session.query(models.RawTemplate)
for raw_template in _get_batch(
session=session, ctxt=ctxt, query=query,
model=models.RawTemplate, batch_size=batch_size):
parameters = raw_template.environment['parameters']
encrypted_params = raw_template.environment[
'encrypted_param_names']
@ -1217,10 +1237,12 @@ def db_decrypt_parameters_and_properties(ctxt, encryption_key):
raw_template_update(ctxt, raw_template.id,
{'environment': environment})
resources = session.query(models.Resource).filter(
query = session.query(models.Resource).filter(
~models.Resource.properties_data.is_(None),
models.Resource.properties_data_encrypted.is_(True)).all()
for resource in resources:
models.Resource.properties_data_encrypted.is_(True))
for resource in _get_batch(
session=session, ctxt=ctxt, query=query, model=models.Resource,
batch_size=batch_size):
result = {}
for prop_name, prop_value in resource.properties_data.items():
method, value = prop_value
@ -1234,3 +1256,17 @@ def db_decrypt_parameters_and_properties(ctxt, encryption_key):
{'properties_data': result,
'properties_data_encrypted': False},
resource.atomic_key)
def _get_batch(session, ctxt, query, model, batch_size=50):
last_batch_marker = None
while True:
results = _paginate_query(
context=ctxt, query=query, model=model, limit=batch_size,
marker=last_batch_marker).all()
if not results:
break
else:
for result in results:
yield result
last_batch_marker = results[-1].id

105
heat/tests/db/test_sqlalchemy_api.py
View File

@ -2709,6 +2709,13 @@ class DBAPICryptParamsPropsTest(common.HeatTestCase):
def setUp(self):
super(DBAPICryptParamsPropsTest, self).setUp()
self.ctx = utils.dummy_context()
self.template = self._create_template()
self.user_creds = create_user_creds(self.ctx)
self.stack = create_stack(self.ctx, self.template, self.user_creds)
self.resources = [create_resource(self.ctx, self.stack, name='res1')]
def _create_template(self):
"""Initialize sample template."""
t = template_format.parse('''
heat_template_version: 2013-05-23
parameters:
@ -2728,67 +2735,87 @@ class DBAPICryptParamsPropsTest(common.HeatTestCase):
'files': {'foo': 'bar'},
'environment': {'parameters': {'param1': 'foo',
'param2': 'bar'}}}
self.template = db_api.raw_template_create(self.ctx, template)
self.user_creds = create_user_creds(self.ctx)
self.stack = create_stack(self.ctx, self.template, self.user_creds)
self.resources = [create_resource(self.ctx, self.stack, name='res1')]
return db_api.raw_template_create(self.ctx, template)
def test_db_encrypt_decrypt(self):
def _test_db_encrypt_decrypt(self, batch_size=50):
session = db_api.get_session()
env = session.query(models.RawTemplate).all()[0].environment
self.assertEqual('bar', env['parameters']['param2'])
prop_data = session.query(models.Resource).all()[0].properties_data
self.assertEqual('bar1', prop_data['foo1'])
for r_tmpl in session.query(models.RawTemplate).all():
self.assertEqual('bar', r_tmpl.environment['parameters']['param2'])
for resource in session.query(models.Resource).all():
self.assertEqual('bar1', resource.properties_data['foo1'])
# Test encryption
db_api.db_encrypt_parameters_and_properties(
self.ctx, cfg.CONF.auth_encryption_key)
self.ctx, cfg.CONF.auth_encryption_key, batch_size=batch_size)
session = db_api.get_session()
env = session.query(models.RawTemplate).all()[0].environment
self.assertEqual('cryptography_decrypt_v1',
env['parameters']['param2'][0])
encrypt_value = env['parameters']['param2'][1]
prop_data = session.query(models.Resource).all()[0].properties_data
self.assertEqual('cryptography_decrypt_v1', prop_data['foo1'][0])
for enc_tmpl in session.query(models.RawTemplate).all():
self.assertEqual('cryptography_decrypt_v1',
enc_tmpl.environment['parameters']['param2'][0])
encrypt_value = enc_tmpl.environment['parameters']['param2'][1]
for enc_prop in session.query(models.Resource).all():
self.assertEqual('cryptography_decrypt_v1',
enc_prop.properties_data['foo1'][0])
# Test that encryption is idempotent
db_api.db_encrypt_parameters_and_properties(
self.ctx, cfg.CONF.auth_encryption_key)
self.ctx, cfg.CONF.auth_encryption_key, batch_size=batch_size)
session = db_api.get_session()
env = session.query(models.RawTemplate).all()[0].environment
self.assertEqual('cryptography_decrypt_v1',
env['parameters']['param2'][0])
prop_data = session.query(models.Resource).all()[0].properties_data
self.assertEqual('cryptography_decrypt_v1', prop_data['foo1'][0])
for enc_tmpl in session.query(models.RawTemplate).all():
self.assertEqual('cryptography_decrypt_v1',
enc_tmpl.environment['parameters']['param2'][0])
for enc_prop in session.query(models.Resource).all():
self.assertEqual('cryptography_decrypt_v1',
enc_prop.properties_data['foo1'][0])
# Test decryption
db_api.db_decrypt_parameters_and_properties(
self.ctx, cfg.CONF.auth_encryption_key)
self.ctx, cfg.CONF.auth_encryption_key, batch_size=batch_size)
session = db_api.get_session()
env = session.query(models.RawTemplate).all()[0].environment
self.assertEqual('bar', env['parameters']['param2'])
prop_data = session.query(models.Resource).all()[0].properties_data
self.assertEqual('bar1', prop_data['foo1'])
for dec_tmpl in session.query(models.RawTemplate).all():
self.assertEqual('bar',
dec_tmpl.environment['parameters']['param2'])
for dec_prop in session.query(models.Resource).all():
self.assertEqual('bar1', dec_prop.properties_data['foo1'])
# Test that decryption is idempotent
db_api.db_decrypt_parameters_and_properties(
self.ctx, cfg.CONF.auth_encryption_key)
self.ctx, cfg.CONF.auth_encryption_key, batch_size=batch_size)
session = db_api.get_session()
env = session.query(models.RawTemplate).all()[0].environment
self.assertEqual('bar', env['parameters']['param2'])
prop_data = session.query(models.Resource).all()[0].properties_data
self.assertEqual('bar1', prop_data['foo1'])
for dec_tmpl in session.query(models.RawTemplate).all():
self.assertEqual('bar',
dec_tmpl.environment['parameters']['param2'])
for dec_prop in session.query(models.Resource).all():
self.assertEqual('bar1', dec_prop.properties_data['foo1'])
# Test using a different encryption key to decrypt
db_api.db_encrypt_parameters_and_properties(
self.ctx, '774c15be099ea74123a9b9592ff12680')
self.ctx, '774c15be099ea74123a9b9592ff12680',
batch_size=batch_size)
session = db_api.get_session()
env = session.query(models.RawTemplate).all()[0].environment
self.assertNotEqual(encrypt_value,
env['parameters']['param2'][1])
for r_tmpl in session.query(models.RawTemplate).all():
self.assertNotEqual(encrypt_value,
r_tmpl.environment['parameters']['param2'][1])
db_api.db_decrypt_parameters_and_properties(
self.ctx, '774c15be099ea74123a9b9592ff12680')
self.ctx, '774c15be099ea74123a9b9592ff12680',
batch_size=batch_size)
session = db_api.get_session()
env = session.query(models.RawTemplate).all()[0].environment
self.assertEqual('bar', env['parameters']['param2'])
for r_tmpl in session.query(models.RawTemplate).all():
self.assertEqual('bar',
r_tmpl.environment['parameters']['param2'])
def test_db_encrypt_decrypt(self):
"""Test encryption and decryption for single template"""
self._create_template()
self._test_db_encrypt_decrypt()
def test_db_encrypt_decrypt_in_batches(self):
"""Test encryption and decryption in for several templates.
Test encryption and decryption when heat requests templates in batch:
predefined amount records.
"""
self._create_template()
self._create_template()
self._test_db_encrypt_decrypt(batch_size=1)