From 7c90690accd68c9d9da440eea007345dc3c912c1 Mon Sep 17 00:00:00 2001 From: kairat_kushaev Date: Thu, 2 Jul 2015 16:23:08 +0300 Subject: [PATCH] Add batching for heat-manage encrypt/decrypt When heat-manage executed encrypt/descrypt previously it requested all templates in db. It lead to some issues with memory. So the patch add batching for encrypt/descrypt operations: it allows to request only predefined number of templates, encrypt/decrypt them and request next batch of templates. It allows to reduce memory overhead when doing encrypt/decrypt operations. Co-Authored-By: Pratik Mallya pratik.mallya@gmail.com Change-Id: I6312235ca7de93a95672755c906156034647b177 Closes-bug: #1470652 --- heat/db/sqlalchemy/api.py | 68 +++++++++++++---- heat/tests/db/test_sqlalchemy_api.py | 105 +++++++++++++++++---------- 2 files changed, 118 insertions(+), 55 deletions(-) diff --git a/heat/db/sqlalchemy/api.py b/heat/db/sqlalchemy/api.py index c9a9fd1a9e..0029bdc154 100644 --- a/heat/db/sqlalchemy/api.py +++ b/heat/db/sqlalchemy/api.py @@ -1120,14 +1120,23 @@ def db_version(engine): return migration.db_version(engine) -def db_encrypt_parameters_and_properties(ctxt, encryption_key): +def db_encrypt_parameters_and_properties(ctxt, encryption_key, batch_size=50): + """Encrypt parameters and properties for all templates in db. + + :param ctxt: RPC context + :param encryption_key: key that will be used for parameter and property + encryption + :param batch_size: number of templates requested from db in each iteration. + 50 means that heat requests 50 templates, encrypt them + and proceed with next 50 items. + """ from heat.engine import template session = get_session() with session.begin(): - - raw_templates = session.query(models.RawTemplate).all() - - for raw_template in raw_templates: + query = session.query(models.RawTemplate) + for raw_template in _get_batch( + session=session, ctxt=ctxt, query=query, + model=models.RawTemplate, batch_size=batch_size): tmpl = template.Template.load(ctxt, raw_template.id, raw_template) env = raw_template.environment @@ -1154,10 +1163,12 @@ def db_encrypt_parameters_and_properties(ctxt, encryption_key): raw_template_update(ctxt, raw_template.id, {'environment': environment}) - resources = session.query(models.Resource).filter( + query = session.query(models.Resource).filter( ~models.Resource.properties_data.is_(None), - ~models.Resource.properties_data_encrypted.is_(True)).all() - for resource in resources: + ~models.Resource.properties_data_encrypted.is_(True)) + for resource in _get_batch( + session=session, ctxt=ctxt, query=query, model=models.Resource, + batch_size=batch_size): result = {} for prop_name, prop_value in resource.properties_data.items(): prop_string = jsonutils.dumps(prop_value) @@ -1172,13 +1183,22 @@ def db_encrypt_parameters_and_properties(ctxt, encryption_key): resource.atomic_key) -def db_decrypt_parameters_and_properties(ctxt, encryption_key): +def db_decrypt_parameters_and_properties(ctxt, encryption_key, batch_size=50): + """Decrypt parameters and properties for all templates in db. + + :param ctxt: RPC context + :param encryption_key: key that will be used for parameter and property + decryption + :param batch_size: number of templates requested from db in each iteration. + 50 means that heat requests 50 templates, encrypt them + and proceed with next 50 items. + """ session = get_session() - with session.begin(): - raw_templates = session.query(models.RawTemplate).all() - - for raw_template in raw_templates: + query = session.query(models.RawTemplate) + for raw_template in _get_batch( + session=session, ctxt=ctxt, query=query, + model=models.RawTemplate, batch_size=batch_size): parameters = raw_template.environment['parameters'] encrypted_params = raw_template.environment[ 'encrypted_param_names'] @@ -1192,10 +1212,12 @@ def db_decrypt_parameters_and_properties(ctxt, encryption_key): raw_template_update(ctxt, raw_template.id, {'environment': environment}) - resources = session.query(models.Resource).filter( + query = session.query(models.Resource).filter( ~models.Resource.properties_data.is_(None), - models.Resource.properties_data_encrypted.is_(True)).all() - for resource in resources: + models.Resource.properties_data_encrypted.is_(True)) + for resource in _get_batch( + session=session, ctxt=ctxt, query=query, model=models.Resource, + batch_size=batch_size): result = {} for prop_name, prop_value in resource.properties_data.items(): method, value = prop_value @@ -1209,3 +1231,17 @@ def db_decrypt_parameters_and_properties(ctxt, encryption_key): {'properties_data': result, 'properties_data_encrypted': False}, resource.atomic_key) + + +def _get_batch(session, ctxt, query, model, batch_size=50): + last_batch_marker = None + while True: + results = _paginate_query( + context=ctxt, query=query, model=model, limit=batch_size, + marker=last_batch_marker).all() + if not results: + break + else: + for result in results: + yield result + last_batch_marker = results[-1].id diff --git a/heat/tests/db/test_sqlalchemy_api.py b/heat/tests/db/test_sqlalchemy_api.py index 8563cb7f6c..e72f68487c 100644 --- a/heat/tests/db/test_sqlalchemy_api.py +++ b/heat/tests/db/test_sqlalchemy_api.py @@ -2702,6 +2702,13 @@ class DBAPICryptParamsPropsTest(common.HeatTestCase): def setUp(self): super(DBAPICryptParamsPropsTest, self).setUp() self.ctx = utils.dummy_context() + self.template = self._create_template() + self.user_creds = create_user_creds(self.ctx) + self.stack = create_stack(self.ctx, self.template, self.user_creds) + self.resources = [create_resource(self.ctx, self.stack, name='res1')] + + def _create_template(self): + """Initialize sample template.""" t = template_format.parse(''' heat_template_version: 2013-05-23 parameters: @@ -2721,67 +2728,87 @@ class DBAPICryptParamsPropsTest(common.HeatTestCase): 'files': {'foo': 'bar'}, 'environment': {'parameters': {'param1': 'foo', 'param2': 'bar'}}} - self.template = db_api.raw_template_create(self.ctx, template) - self.user_creds = create_user_creds(self.ctx) - self.stack = create_stack(self.ctx, self.template, self.user_creds) - self.resources = [create_resource(self.ctx, self.stack, name='res1')] + return db_api.raw_template_create(self.ctx, template) - def test_db_encrypt_decrypt(self): + def _test_db_encrypt_decrypt(self, batch_size=50): session = db_api.get_session() - env = session.query(models.RawTemplate).all()[0].environment - self.assertEqual('bar', env['parameters']['param2']) - prop_data = session.query(models.Resource).all()[0].properties_data - self.assertEqual('bar1', prop_data['foo1']) + + for r_tmpl in session.query(models.RawTemplate).all(): + self.assertEqual('bar', r_tmpl.environment['parameters']['param2']) + for resource in session.query(models.Resource).all(): + self.assertEqual('bar1', resource.properties_data['foo1']) # Test encryption db_api.db_encrypt_parameters_and_properties( - self.ctx, cfg.CONF.auth_encryption_key) + self.ctx, cfg.CONF.auth_encryption_key, batch_size=batch_size) session = db_api.get_session() - env = session.query(models.RawTemplate).all()[0].environment - self.assertEqual('cryptography_decrypt_v1', - env['parameters']['param2'][0]) - encrypt_value = env['parameters']['param2'][1] - prop_data = session.query(models.Resource).all()[0].properties_data - self.assertEqual('cryptography_decrypt_v1', prop_data['foo1'][0]) + for enc_tmpl in session.query(models.RawTemplate).all(): + self.assertEqual('cryptography_decrypt_v1', + enc_tmpl.environment['parameters']['param2'][0]) + encrypt_value = enc_tmpl.environment['parameters']['param2'][1] + for enc_prop in session.query(models.Resource).all(): + self.assertEqual('cryptography_decrypt_v1', + enc_prop.properties_data['foo1'][0]) # Test that encryption is idempotent db_api.db_encrypt_parameters_and_properties( - self.ctx, cfg.CONF.auth_encryption_key) + self.ctx, cfg.CONF.auth_encryption_key, batch_size=batch_size) session = db_api.get_session() - env = session.query(models.RawTemplate).all()[0].environment - self.assertEqual('cryptography_decrypt_v1', - env['parameters']['param2'][0]) - prop_data = session.query(models.Resource).all()[0].properties_data - self.assertEqual('cryptography_decrypt_v1', prop_data['foo1'][0]) + for enc_tmpl in session.query(models.RawTemplate).all(): + self.assertEqual('cryptography_decrypt_v1', + enc_tmpl.environment['parameters']['param2'][0]) + for enc_prop in session.query(models.Resource).all(): + self.assertEqual('cryptography_decrypt_v1', + enc_prop.properties_data['foo1'][0]) # Test decryption db_api.db_decrypt_parameters_and_properties( - self.ctx, cfg.CONF.auth_encryption_key) + self.ctx, cfg.CONF.auth_encryption_key, batch_size=batch_size) session = db_api.get_session() - env = session.query(models.RawTemplate).all()[0].environment - self.assertEqual('bar', env['parameters']['param2']) - prop_data = session.query(models.Resource).all()[0].properties_data - self.assertEqual('bar1', prop_data['foo1']) + for dec_tmpl in session.query(models.RawTemplate).all(): + self.assertEqual('bar', + dec_tmpl.environment['parameters']['param2']) + for dec_prop in session.query(models.Resource).all(): + self.assertEqual('bar1', dec_prop.properties_data['foo1']) # Test that decryption is idempotent db_api.db_decrypt_parameters_and_properties( - self.ctx, cfg.CONF.auth_encryption_key) + self.ctx, cfg.CONF.auth_encryption_key, batch_size=batch_size) session = db_api.get_session() - env = session.query(models.RawTemplate).all()[0].environment - self.assertEqual('bar', env['parameters']['param2']) - prop_data = session.query(models.Resource).all()[0].properties_data - self.assertEqual('bar1', prop_data['foo1']) + for dec_tmpl in session.query(models.RawTemplate).all(): + self.assertEqual('bar', + dec_tmpl.environment['parameters']['param2']) + for dec_prop in session.query(models.Resource).all(): + self.assertEqual('bar1', dec_prop.properties_data['foo1']) # Test using a different encryption key to decrypt db_api.db_encrypt_parameters_and_properties( - self.ctx, '774c15be099ea74123a9b9592ff12680') + self.ctx, '774c15be099ea74123a9b9592ff12680', + batch_size=batch_size) session = db_api.get_session() - env = session.query(models.RawTemplate).all()[0].environment - self.assertNotEqual(encrypt_value, - env['parameters']['param2'][1]) + for r_tmpl in session.query(models.RawTemplate).all(): + self.assertNotEqual(encrypt_value, + r_tmpl.environment['parameters']['param2'][1]) db_api.db_decrypt_parameters_and_properties( - self.ctx, '774c15be099ea74123a9b9592ff12680') + self.ctx, '774c15be099ea74123a9b9592ff12680', + batch_size=batch_size) session = db_api.get_session() - env = session.query(models.RawTemplate).all()[0].environment - self.assertEqual('bar', env['parameters']['param2']) + for r_tmpl in session.query(models.RawTemplate).all(): + self.assertEqual('bar', + r_tmpl.environment['parameters']['param2']) + + def test_db_encrypt_decrypt(self): + """Test encryption and decryption for single template""" + self._create_template() + self._test_db_encrypt_decrypt() + + def test_db_encrypt_decrypt_in_batches(self): + """Test encryption and decryption in for several templates. + + Test encryption and decryption when heat requests templates in batch: + predefined amount records. + """ + self._create_template() + self._create_template() + self._test_db_encrypt_decrypt(batch_size=1)