cmd: Deprecate heat-manage migrate_properties_data command
Well, we actually remove it but keep the entry point to avoid breaking scripts. This has been around since 2016 and anyone that hasn't upgraded now ain't upgrading. This is one less set of APIs to worry about when removing the use of autocommit. Signed-off-by: Stephen Finucane <stephenfin@redhat.com> Change-Id: I43051b3ca6e292f1164d750b04d7a440012b2ef5
This commit is contained in:
Stephen Finucane
Takashi Kajinami
parent
d1124ac268
commit
9ba72bbc18
|
|
@ -146,8 +146,10 @@ def do_crypt_parameters_and_properties():
|
|||
|
||||
|
||||
def do_properties_data_migrate():
|
||||
ctxt = context.get_admin_context()
|
||||
db_api.db_properties_data_migrate(ctxt)
|
||||
print(
|
||||
'This command has been deprecated and is now a no-op. '
|
||||
'It will be removed in a future release.'
|
||||
)
|
||||
|
||||
|
||||
def add_command_parsers(subparsers):
|
||||
|
|
|
|||
316
heat/db/api.py
316
heat/db/api.py
|
|
@ -1782,319 +1782,3 @@ def sync_point_update_input_data(context, entity_id,
|
|||
atomic_key=atomic_key
|
||||
).update({"input_data": input_data, "atomic_key": atomic_key + 1})
|
||||
return rows_updated
|
||||
|
||||
|
||||
# data migration utils
|
||||
|
||||
|
||||
def _crypt_action(encrypt):
|
||||
if encrypt:
|
||||
return _('encrypt')
|
||||
return _('decrypt')
|
||||
|
||||
|
||||
def _db_encrypt_or_decrypt_template_params(
|
||||
context, encryption_key, encrypt=False, batch_size=50, verbose=False,
|
||||
):
|
||||
from heat.engine import template
|
||||
excs = []
|
||||
query = context.session.query(models.RawTemplate)
|
||||
template_batches = _get_batch(
|
||||
context.session, context=context, query=query,
|
||||
model=models.RawTemplate,
|
||||
batch_size=batch_size)
|
||||
next_batch = list(itertools.islice(template_batches, batch_size))
|
||||
while next_batch:
|
||||
with context.session.begin():
|
||||
for raw_template in next_batch:
|
||||
try:
|
||||
if verbose:
|
||||
LOG.info("Processing raw_template %s...",
|
||||
raw_template.id)
|
||||
env = raw_template.environment
|
||||
needs_update = False
|
||||
|
||||
# using "in env.keys()" so an exception is raised
|
||||
# if env is something weird like a string.
|
||||
if env is None or 'parameters' not in env.keys():
|
||||
continue
|
||||
if 'encrypted_param_names' in env:
|
||||
encrypted_params = env['encrypted_param_names']
|
||||
else:
|
||||
encrypted_params = []
|
||||
|
||||
if encrypt:
|
||||
tmpl = template.Template.load(
|
||||
context, raw_template.id, raw_template)
|
||||
param_schemata = tmpl.param_schemata()
|
||||
if not param_schemata:
|
||||
continue
|
||||
|
||||
for param_name, param_val in env['parameters'].items():
|
||||
if (param_name in encrypted_params or
|
||||
param_name not in param_schemata or
|
||||
not param_schemata[param_name].hidden):
|
||||
continue
|
||||
encrypted_val = crypt.encrypt(
|
||||
str(param_val), encryption_key)
|
||||
env['parameters'][param_name] = encrypted_val
|
||||
encrypted_params.append(param_name)
|
||||
needs_update = True
|
||||
if needs_update:
|
||||
newenv = env.copy()
|
||||
newenv['encrypted_param_names'] = encrypted_params
|
||||
else: # decrypt
|
||||
for param_name in encrypted_params:
|
||||
method, value = env['parameters'][param_name]
|
||||
decrypted_val = crypt.decrypt(method, value,
|
||||
encryption_key)
|
||||
env['parameters'][param_name] = decrypted_val
|
||||
needs_update = True
|
||||
if needs_update:
|
||||
newenv = env.copy()
|
||||
newenv['encrypted_param_names'] = []
|
||||
|
||||
if needs_update:
|
||||
raw_template_update(context, raw_template.id,
|
||||
{'environment': newenv})
|
||||
except Exception as exc:
|
||||
LOG.exception('Failed to %(crypt_action)s parameters '
|
||||
'of raw template %(id)d',
|
||||
{'id': raw_template.id,
|
||||
'crypt_action': _crypt_action(encrypt)})
|
||||
excs.append(exc)
|
||||
continue
|
||||
finally:
|
||||
if verbose:
|
||||
LOG.info("Finished %(crypt_action)s processing of "
|
||||
"raw_template %(id)d.",
|
||||
{'id': raw_template.id,
|
||||
'crypt_action': _crypt_action(encrypt)})
|
||||
next_batch = list(itertools.islice(template_batches, batch_size))
|
||||
return excs
|
||||
|
||||
|
||||
def _db_encrypt_or_decrypt_resource_prop_data_legacy(
|
||||
context, encryption_key, encrypt=False, batch_size=50, verbose=False,
|
||||
):
|
||||
excs = []
|
||||
|
||||
# Older resources may have properties_data in the legacy column,
|
||||
# so update those as needed
|
||||
query = context.session.query(models.Resource).filter(
|
||||
models.Resource.properties_data_encrypted.isnot(encrypt))
|
||||
resource_batches = _get_batch(
|
||||
session=context.session, context=context, query=query,
|
||||
model=models.Resource,
|
||||
batch_size=batch_size)
|
||||
next_batch = list(itertools.islice(resource_batches, batch_size))
|
||||
while next_batch:
|
||||
with context.session.begin():
|
||||
for resource in next_batch:
|
||||
if not resource.properties_data:
|
||||
continue
|
||||
try:
|
||||
if verbose:
|
||||
LOG.info("Processing resource %s...",
|
||||
resource.id)
|
||||
if encrypt:
|
||||
result = crypt.encrypted_dict(resource.properties_data,
|
||||
encryption_key)
|
||||
else:
|
||||
result = crypt.decrypted_dict(resource.properties_data,
|
||||
encryption_key)
|
||||
_try_resource_update(
|
||||
context, resource.id,
|
||||
{'properties_data': result,
|
||||
'properties_data_encrypted': encrypt},
|
||||
resource.atomic_key)
|
||||
except Exception as exc:
|
||||
LOG.exception('Failed to %(crypt_action)s '
|
||||
'properties_data of resource %(id)d' %
|
||||
{'id': resource.id,
|
||||
'crypt_action': _crypt_action(encrypt)})
|
||||
excs.append(exc)
|
||||
continue
|
||||
finally:
|
||||
if verbose:
|
||||
LOG.info("Finished processing resource %s.",
|
||||
resource.id)
|
||||
next_batch = list(itertools.islice(resource_batches, batch_size))
|
||||
return excs
|
||||
|
||||
|
||||
def _db_encrypt_or_decrypt_resource_prop_data(
|
||||
context, encryption_key, encrypt=False, batch_size=50, verbose=False,
|
||||
):
|
||||
excs = []
|
||||
|
||||
# Older resources may have properties_data in the legacy column,
|
||||
# so update those as needed
|
||||
query = context.session.query(models.ResourcePropertiesData).filter(
|
||||
models.ResourcePropertiesData.encrypted.isnot(encrypt))
|
||||
rpd_batches = _get_batch(
|
||||
session=context.session, context=context, query=query,
|
||||
model=models.ResourcePropertiesData, batch_size=batch_size)
|
||||
next_batch = list(itertools.islice(rpd_batches, batch_size))
|
||||
while next_batch:
|
||||
with context.session.begin():
|
||||
for rpd in next_batch:
|
||||
if not rpd.data:
|
||||
continue
|
||||
try:
|
||||
if verbose:
|
||||
LOG.info("Processing resource_properties_data "
|
||||
"%s...", rpd.id)
|
||||
if encrypt:
|
||||
result = crypt.encrypted_dict(rpd.data,
|
||||
encryption_key)
|
||||
else:
|
||||
result = crypt.decrypted_dict(rpd.data,
|
||||
encryption_key)
|
||||
rpd.update({'data': result,
|
||||
'encrypted': encrypt})
|
||||
except Exception as exc:
|
||||
LOG.exception(
|
||||
"Failed to %(crypt_action)s "
|
||||
"data of resource_properties_data %(id)d" %
|
||||
{'id': rpd.id,
|
||||
'crypt_action': _crypt_action(encrypt)})
|
||||
excs.append(exc)
|
||||
continue
|
||||
finally:
|
||||
if verbose:
|
||||
LOG.info(
|
||||
"Finished processing resource_properties_data"
|
||||
" %s.", rpd.id)
|
||||
next_batch = list(itertools.islice(rpd_batches, batch_size))
|
||||
return excs
|
||||
|
||||
|
||||
def db_encrypt_parameters_and_properties(
|
||||
context, encryption_key, batch_size=50, verbose=False,
|
||||
):
|
||||
"""Encrypt parameters and properties for all templates in db.
|
||||
|
||||
:param context: RPC context
|
||||
:param encryption_key: key that will be used for parameter and property
|
||||
encryption
|
||||
:param batch_size: number of templates requested from DB in each iteration.
|
||||
50 means that heat requests 50 templates, encrypt them
|
||||
and proceed with next 50 items.
|
||||
:param verbose: log an INFO message when processing of each raw_template or
|
||||
resource begins or ends
|
||||
:return: list of exceptions encountered during encryption
|
||||
"""
|
||||
excs = []
|
||||
excs.extend(_db_encrypt_or_decrypt_template_params(
|
||||
context, encryption_key, True, batch_size, verbose))
|
||||
excs.extend(_db_encrypt_or_decrypt_resource_prop_data(
|
||||
context, encryption_key, True, batch_size, verbose))
|
||||
excs.extend(_db_encrypt_or_decrypt_resource_prop_data_legacy(
|
||||
context, encryption_key, True, batch_size, verbose))
|
||||
return excs
|
||||
|
||||
|
||||
def db_decrypt_parameters_and_properties(
|
||||
context, encryption_key, batch_size=50, verbose=False,
|
||||
):
|
||||
"""Decrypt parameters and properties for all templates in db.
|
||||
|
||||
:param context: RPC context
|
||||
:param encryption_key: key that will be used for parameter and property
|
||||
decryption
|
||||
:param batch_size: number of templates requested from DB in each iteration.
|
||||
50 means that heat requests 50 templates, encrypt them
|
||||
and proceed with next 50 items.
|
||||
:param verbose: log an INFO message when processing of each raw_template or
|
||||
resource begins or ends
|
||||
:return: list of exceptions encountered during decryption
|
||||
"""
|
||||
excs = []
|
||||
excs.extend(_db_encrypt_or_decrypt_template_params(
|
||||
context, encryption_key, False, batch_size, verbose))
|
||||
excs.extend(_db_encrypt_or_decrypt_resource_prop_data(
|
||||
context, encryption_key, False, batch_size, verbose))
|
||||
excs.extend(_db_encrypt_or_decrypt_resource_prop_data_legacy(
|
||||
context, encryption_key, False, batch_size, verbose))
|
||||
return excs
|
||||
|
||||
|
||||
def db_properties_data_migrate(context, batch_size=50):
|
||||
"""Migrate properties data from legacy columns to new location in db.
|
||||
|
||||
:param context: RPC context
|
||||
:param batch_size: number of templates requested from DB in each iteration.
|
||||
50 means that heat requests 50 templates, encrypt them
|
||||
and proceed with next 50 items.
|
||||
"""
|
||||
|
||||
query = context.session.query(models.Resource).filter(and_(
|
||||
models.Resource.properties_data.isnot(None),
|
||||
models.Resource.rsrc_prop_data_id.is_(None)))
|
||||
resource_batches = _get_batch(
|
||||
session=context.session, context=context, query=query,
|
||||
model=models.Resource, batch_size=batch_size)
|
||||
next_batch = list(itertools.islice(resource_batches, batch_size))
|
||||
while next_batch:
|
||||
with context.session.begin():
|
||||
for resource in next_batch:
|
||||
try:
|
||||
encrypted = resource.properties_data_encrypted
|
||||
if encrypted is None:
|
||||
LOG.warning(
|
||||
'Unexpected: resource.encrypted is None for '
|
||||
'resource id %s for legacy '
|
||||
'resource.properties_data, assuming False.',
|
||||
resource.id)
|
||||
encrypted = False
|
||||
rsrc_prop_data = resource_prop_data_create(
|
||||
context, {'encrypted': encrypted,
|
||||
'data': resource.properties_data})
|
||||
resource_update(context, resource.id,
|
||||
{'properties_data_encrypted': None,
|
||||
'properties_data': None,
|
||||
'rsrc_prop_data_id': rsrc_prop_data.id},
|
||||
resource.atomic_key)
|
||||
except Exception:
|
||||
LOG.exception('Failed to migrate properties_data for '
|
||||
'resource %d', resource.id)
|
||||
continue
|
||||
next_batch = list(itertools.islice(resource_batches, batch_size))
|
||||
|
||||
query = context.session.query(models.Event).filter(and_(
|
||||
models.Event.resource_properties.isnot(None),
|
||||
models.Event.rsrc_prop_data_id.is_(None)))
|
||||
event_batches = _get_batch(
|
||||
session=context.session, context=context, query=query,
|
||||
model=models.Event, batch_size=batch_size)
|
||||
next_batch = list(itertools.islice(event_batches, batch_size))
|
||||
while next_batch:
|
||||
with context.session.begin():
|
||||
for event in next_batch:
|
||||
try:
|
||||
prop_data = event.resource_properties
|
||||
rsrc_prop_data = resource_prop_data_create(
|
||||
context,
|
||||
{'encrypted': False, 'data': prop_data})
|
||||
event.update({'resource_properties': None,
|
||||
'rsrc_prop_data_id': rsrc_prop_data.id})
|
||||
except Exception:
|
||||
LOG.exception('Failed to migrate resource_properties '
|
||||
'for event %d', event.id)
|
||||
continue
|
||||
next_batch = list(itertools.islice(event_batches, batch_size))
|
||||
|
||||
|
||||
def _get_batch(session, context, query, model, batch_size=50):
|
||||
last_batch_marker = None
|
||||
while True:
|
||||
results = _paginate_query(
|
||||
context=context, query=query, model=model, limit=batch_size,
|
||||
marker=last_batch_marker).all()
|
||||
if not results:
|
||||
break
|
||||
else:
|
||||
for result in results:
|
||||
yield result
|
||||
last_batch_marker = results[-1].id
|
||||
|
|
|
|||
|
|
@ -11,15 +11,12 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import copy
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from unittest import mock
|
||||
import uuid
|
||||
|
||||
import fixtures
|
||||
from oslo_config import cfg
|
||||
from oslo_db import exception as db_exception
|
||||
from oslo_utils import timeutils
|
||||
|
|
@ -3376,536 +3373,6 @@ class DBAPISyncPointTest(common.HeatTestCase):
|
|||
self.assertEqual(len(self.resources) * 21, add.call_count)
|
||||
|
||||
|
||||
class DBAPIMigratePropertiesDataTest(common.HeatTestCase):
|
||||
def setUp(self):
|
||||
super(DBAPIMigratePropertiesDataTest, self).setUp()
|
||||
self.ctx = utils.dummy_context()
|
||||
templ = create_raw_template(self.ctx)
|
||||
user_creds = create_user_creds(self.ctx)
|
||||
stack = create_stack(self.ctx, templ, user_creds)
|
||||
stack2 = create_stack(self.ctx, templ, user_creds)
|
||||
create_resource(self.ctx, stack, True, name='res1')
|
||||
create_resource(self.ctx, stack2, True, name='res2')
|
||||
create_event(self.ctx, True)
|
||||
create_event(self.ctx, True)
|
||||
|
||||
def _test_migrate_resource(self, batch_size=50):
|
||||
resources = self.ctx.session.query(models.Resource).all()
|
||||
self.assertEqual(2, len(resources))
|
||||
for resource in resources:
|
||||
self.assertEqual('bar1', resource.properties_data['foo1'])
|
||||
|
||||
db_api.db_properties_data_migrate(self.ctx, batch_size=batch_size)
|
||||
for resource in resources:
|
||||
self.assertEqual('bar1', resource.rsrc_prop_data.data['foo1'])
|
||||
self.assertFalse(resource.rsrc_prop_data.encrypted)
|
||||
self.assertIsNone(resource.properties_data)
|
||||
self.assertIsNone(resource.properties_data_encrypted)
|
||||
|
||||
def _test_migrate_event(self, batch_size=50):
|
||||
events = self.ctx.session.query(models.Event).all()
|
||||
self.assertEqual(2, len(events))
|
||||
for event in events:
|
||||
self.assertEqual('ev_bar', event.resource_properties['foo2'])
|
||||
|
||||
db_api.db_properties_data_migrate(self.ctx, batch_size=batch_size)
|
||||
self.ctx.session.expire_all()
|
||||
events = self.ctx.session.query(models.Event).all()
|
||||
for event in events:
|
||||
self.assertEqual('ev_bar', event.rsrc_prop_data.data['foo2'])
|
||||
self.assertFalse(event.rsrc_prop_data.encrypted)
|
||||
self.assertIsNone(event.resource_properties)
|
||||
|
||||
def test_migrate_event(self):
|
||||
self._test_migrate_event()
|
||||
|
||||
def test_migrate_event_in_batches(self):
|
||||
self._test_migrate_event(batch_size=1)
|
||||
|
||||
def test_migrate_resource(self):
|
||||
self._test_migrate_resource()
|
||||
|
||||
def test_migrate_resource_in_batches(self):
|
||||
self._test_migrate_resource(batch_size=1)
|
||||
|
||||
def test_migrate_encrypted_resource(self):
|
||||
resources = self.ctx.session.query(models.Resource).all()
|
||||
db_api.db_encrypt_parameters_and_properties(
|
||||
self.ctx, 'i have a key for you if you want')
|
||||
|
||||
encrypted_data_pre_migration = resources[0].properties_data['foo1'][1]
|
||||
db_api.db_properties_data_migrate(self.ctx)
|
||||
resources = self.ctx.session.query(models.Resource).all()
|
||||
|
||||
self.assertTrue(resources[0].rsrc_prop_data.encrypted)
|
||||
self.assertIsNone(resources[0].properties_data)
|
||||
self.assertIsNone(resources[0].properties_data_encrypted)
|
||||
self.assertEqual('cryptography_decrypt_v1',
|
||||
resources[0].rsrc_prop_data.data['foo1'][0])
|
||||
self.assertEqual(encrypted_data_pre_migration,
|
||||
resources[0].rsrc_prop_data.data['foo1'][1])
|
||||
|
||||
db_api.db_decrypt_parameters_and_properties(
|
||||
self.ctx, 'i have a key for you if you want')
|
||||
self.ctx.session.expire_all()
|
||||
resources = self.ctx.session.query(models.Resource).all()
|
||||
|
||||
self.assertEqual('bar1', resources[0].rsrc_prop_data.data['foo1'])
|
||||
self.assertFalse(resources[0].rsrc_prop_data.encrypted)
|
||||
self.assertIsNone(resources[0].properties_data)
|
||||
self.assertIsNone(resources[0].properties_data_encrypted)
|
||||
|
||||
|
||||
class DBAPICryptParamsPropsTest(common.HeatTestCase):
|
||||
def setUp(self):
|
||||
super(DBAPICryptParamsPropsTest, self).setUp()
|
||||
self.ctx = utils.dummy_context()
|
||||
self.template = self._create_template()
|
||||
self.user_creds = create_user_creds(self.ctx)
|
||||
self.stack = create_stack(self.ctx, self.template, self.user_creds)
|
||||
self.resources = [create_resource(self.ctx, self.stack, name='res1')]
|
||||
|
||||
hidden_params_dict = {
|
||||
'param2': 'bar',
|
||||
'param_number': '456',
|
||||
'param_boolean': '1',
|
||||
'param_map': '{\"test\":\"json\"}',
|
||||
'param_comma_list': '[\"Hola\", \"Senor\"]'}
|
||||
|
||||
def _create_template(self):
|
||||
"""Initialize sample template."""
|
||||
self.t = template_format.parse('''
|
||||
heat_template_version: 2013-05-23
|
||||
parameters:
|
||||
param1:
|
||||
type: string
|
||||
description: value1.
|
||||
param2:
|
||||
type: string
|
||||
description: value2.
|
||||
hidden: true
|
||||
param3:
|
||||
type: string
|
||||
description: value3
|
||||
hidden: true
|
||||
default: "don't encrypt me! I'm not sensitive enough"
|
||||
param_string_default_int:
|
||||
type: string
|
||||
description: String parameter with integer default value
|
||||
default: 4353
|
||||
hidden: true
|
||||
param_number:
|
||||
type: number
|
||||
description: Number parameter
|
||||
default: 4353
|
||||
hidden: true
|
||||
param_boolean:
|
||||
type: boolean
|
||||
description: boolean parameter
|
||||
default: true
|
||||
hidden: true
|
||||
param_map:
|
||||
type: json
|
||||
description: json parameter
|
||||
default: {"fee": {"fi":"fo"}}
|
||||
hidden: true
|
||||
param_comma_list:
|
||||
type: comma_delimited_list
|
||||
description: cdl parameter
|
||||
default: ["hola", "senorita"]
|
||||
hidden: true
|
||||
resources:
|
||||
a_resource:
|
||||
type: GenericResourceType
|
||||
''')
|
||||
template = {
|
||||
'template': self.t,
|
||||
'files': {'foo': 'bar'},
|
||||
'environment': {
|
||||
'parameters': {
|
||||
'param1': 'foo',
|
||||
'param2': 'bar',
|
||||
'param_number': '456',
|
||||
'param_boolean': '1',
|
||||
'param_map': '{\"test\":\"json\"}',
|
||||
'param_comma_list': '[\"Hola\", \"Senor\"]'}}}
|
||||
|
||||
return db_api.raw_template_create(self.ctx, template)
|
||||
|
||||
def encrypt(self, enc_key=None, batch_size=50,
|
||||
legacy_prop_data=False):
|
||||
session = self.ctx.session
|
||||
if enc_key is None:
|
||||
enc_key = cfg.CONF.auth_encryption_key
|
||||
self.assertEqual([], db_api.db_encrypt_parameters_and_properties(
|
||||
self.ctx, enc_key, batch_size=batch_size))
|
||||
for enc_tmpl in session.query(models.RawTemplate).all():
|
||||
for param_name in self.hidden_params_dict.keys():
|
||||
self.assertEqual(
|
||||
'cryptography_decrypt_v1',
|
||||
enc_tmpl.environment['parameters'][param_name][0])
|
||||
self.assertEqual(
|
||||
'foo', enc_tmpl.environment['parameters']['param1'])
|
||||
# test that decryption does not store (or encrypt) default
|
||||
# values in template's environment['parameters']
|
||||
self.assertIsNone(
|
||||
enc_tmpl.environment['parameters'].get('param3'))
|
||||
|
||||
enc_resources = session.query(models.Resource).all()
|
||||
self.assertNotEqual([], enc_resources)
|
||||
for enc_resource in enc_resources:
|
||||
if legacy_prop_data:
|
||||
self.assertEqual(
|
||||
'cryptography_decrypt_v1',
|
||||
enc_resource.properties_data['foo1'][0])
|
||||
else:
|
||||
self.assertEqual(
|
||||
'cryptography_decrypt_v1',
|
||||
enc_resource.rsrc_prop_data.data['foo1'][0])
|
||||
|
||||
ev = enc_tmpl.environment['parameters']['param2'][1]
|
||||
return ev
|
||||
|
||||
def decrypt(self, encrypt_value, enc_key=None,
|
||||
batch_size=50, legacy_prop_data=False):
|
||||
session = self.ctx.session
|
||||
if enc_key is None:
|
||||
enc_key = cfg.CONF.auth_encryption_key
|
||||
|
||||
self.assertEqual([], db_api.db_decrypt_parameters_and_properties(
|
||||
self.ctx, enc_key, batch_size=batch_size))
|
||||
|
||||
for dec_tmpl in session.query(models.RawTemplate).all():
|
||||
self.assertNotEqual(
|
||||
encrypt_value,
|
||||
dec_tmpl.environment['parameters']['param2'][1])
|
||||
for param_name, param_value in self.hidden_params_dict.items():
|
||||
self.assertEqual(
|
||||
param_value,
|
||||
dec_tmpl.environment['parameters'][param_name])
|
||||
self.assertEqual(
|
||||
'foo', dec_tmpl.environment['parameters']['param1'])
|
||||
self.assertIsNone(
|
||||
dec_tmpl.environment['parameters'].get('param3'))
|
||||
|
||||
# test that decryption does not store default
|
||||
# values in template's environment['parameters']
|
||||
self.assertIsNone(dec_tmpl.environment['parameters'].get(
|
||||
'param3'))
|
||||
|
||||
decrypt_value = dec_tmpl.environment['parameters']['param2'][1]
|
||||
|
||||
dec_resources = session.query(models.Resource).all()
|
||||
self.assertNotEqual([], dec_resources)
|
||||
for dec_resource in dec_resources:
|
||||
if legacy_prop_data:
|
||||
self.assertEqual(
|
||||
'bar1', dec_resource.properties_data['foo1'])
|
||||
else:
|
||||
self.assertEqual(
|
||||
'bar1', dec_resource.rsrc_prop_data.data['foo1'])
|
||||
|
||||
return decrypt_value
|
||||
|
||||
def _test_db_encrypt_decrypt(self, batch_size=50, legacy_prop_data=False):
|
||||
session = self.ctx.session
|
||||
raw_templates = session.query(models.RawTemplate).all()
|
||||
self.assertNotEqual([], raw_templates)
|
||||
for r_tmpl in raw_templates:
|
||||
for param_name, param_value in self.hidden_params_dict.items():
|
||||
self.assertEqual(param_value,
|
||||
r_tmpl.environment['parameters'][param_name])
|
||||
self.assertEqual('foo',
|
||||
r_tmpl.environment['parameters']['param1'])
|
||||
|
||||
resources = session.query(models.Resource).all()
|
||||
self.assertNotEqual([], resources)
|
||||
self.assertEqual(len(resources), len(raw_templates))
|
||||
for resource in resources:
|
||||
resource = db_api.resource_get(self.ctx, resource.id)
|
||||
if legacy_prop_data:
|
||||
self.assertEqual(
|
||||
'bar1', resource.properties_data['foo1'])
|
||||
else:
|
||||
self.assertEqual(
|
||||
'bar1', resource.rsrc_prop_data.data['foo1'])
|
||||
|
||||
# Test encryption
|
||||
encrypt_value = self.encrypt(batch_size=batch_size,
|
||||
legacy_prop_data=legacy_prop_data)
|
||||
|
||||
# Test that encryption is idempotent
|
||||
encrypt_value2 = self.encrypt(batch_size=batch_size,
|
||||
legacy_prop_data=legacy_prop_data)
|
||||
self.assertEqual(encrypt_value, encrypt_value2)
|
||||
|
||||
# Test decryption
|
||||
decrypt_value = self.decrypt(encrypt_value, batch_size=batch_size,
|
||||
legacy_prop_data=legacy_prop_data)
|
||||
|
||||
# Test that decryption is idempotent
|
||||
decrypt_value2 = self.decrypt(encrypt_value, batch_size=batch_size,
|
||||
legacy_prop_data=legacy_prop_data)
|
||||
self.assertEqual(decrypt_value, decrypt_value2)
|
||||
|
||||
# Test using a different encryption key to encrypt & decrypt
|
||||
encrypt_value3 = self.encrypt(
|
||||
enc_key='774c15be099ea74123a9b9592ff12680',
|
||||
batch_size=batch_size, legacy_prop_data=legacy_prop_data)
|
||||
decrypt_value3 = self.decrypt(
|
||||
encrypt_value3, enc_key='774c15be099ea74123a9b9592ff12680',
|
||||
batch_size=batch_size, legacy_prop_data=legacy_prop_data)
|
||||
self.assertEqual(decrypt_value, decrypt_value3)
|
||||
self.assertNotEqual(encrypt_value, decrypt_value)
|
||||
self.assertNotEqual(encrypt_value3, decrypt_value3)
|
||||
self.assertNotEqual(encrypt_value, encrypt_value3)
|
||||
|
||||
def test_db_encrypt_decrypt(self):
|
||||
"""Test encryption and decryption for single template and resource."""
|
||||
self._test_db_encrypt_decrypt()
|
||||
|
||||
def test_db_encrypt_decrypt_legacy_prop_data(self):
|
||||
"""Test encryption and decryption for res with legacy prop data."""
|
||||
# delete what setUp created
|
||||
[self.ctx.session.delete(r) for r in
|
||||
self.ctx.session.query(models.Resource).all()]
|
||||
[self.ctx.session.delete(s) for s in
|
||||
self.ctx.session.query(models.Stack).all()]
|
||||
[self.ctx.session.delete(t) for t in
|
||||
self.ctx.session.query(models.RawTemplate).all()]
|
||||
|
||||
tmpl = self._create_template()
|
||||
stack = create_stack(self.ctx, tmpl, self.user_creds)
|
||||
create_resource(self.ctx, stack, True, name='res1')
|
||||
self._test_db_encrypt_decrypt(legacy_prop_data=True)
|
||||
|
||||
def test_db_encrypt_decrypt_in_batches(self):
|
||||
"""Test encryption and decryption in for several templates and resources.
|
||||
|
||||
Test encryption and decryption with set batch size of
|
||||
templates and resources.
|
||||
"""
|
||||
tmpl1 = self._create_template()
|
||||
tmpl2 = self._create_template()
|
||||
stack = create_stack(self.ctx, tmpl1, self.user_creds)
|
||||
create_resource(self.ctx, stack, False, name='res1')
|
||||
stack2 = create_stack(self.ctx, tmpl2, self.user_creds)
|
||||
create_resource(self.ctx, stack2, False, name='res2')
|
||||
|
||||
self._test_db_encrypt_decrypt(batch_size=1)
|
||||
|
||||
def test_db_encrypt_decrypt_exception_continue(self):
|
||||
"""Test that encryption and decryption proceed after an exception"""
|
||||
def create_malformed_template():
|
||||
"""Initialize a malformed template which should fail encryption."""
|
||||
t = template_format.parse('''
|
||||
heat_template_version: 2013-05-23
|
||||
parameters:
|
||||
param1:
|
||||
type: string
|
||||
description: value1.
|
||||
param2:
|
||||
type: string
|
||||
description: value2.
|
||||
hidden: true
|
||||
param3:
|
||||
type: string
|
||||
description: value3
|
||||
hidden: true
|
||||
default: "don't encrypt me! I'm not sensitive enough"
|
||||
resources:
|
||||
a_resource:
|
||||
type: GenericResourceType
|
||||
''')
|
||||
template = {
|
||||
'template': t,
|
||||
'files': {'foo': 'bar'},
|
||||
'environment': ''} # <- environment should be a dict
|
||||
|
||||
return db_api.raw_template_create(self.ctx, template)
|
||||
|
||||
create_malformed_template()
|
||||
self._create_template()
|
||||
|
||||
# Test encryption
|
||||
enc_result = db_api.db_encrypt_parameters_and_properties(
|
||||
self.ctx, cfg.CONF.auth_encryption_key, batch_size=50)
|
||||
self.assertEqual(1, len(enc_result))
|
||||
self.assertIs(AttributeError, type(enc_result[0]))
|
||||
enc_tmpls = self.ctx.session.query(models.RawTemplate).all()
|
||||
self.assertEqual('', enc_tmpls[1].environment)
|
||||
self.assertEqual('cryptography_decrypt_v1',
|
||||
enc_tmpls[2].environment['parameters']['param2'][0])
|
||||
|
||||
# Test decryption
|
||||
dec_result = db_api.db_decrypt_parameters_and_properties(
|
||||
self.ctx, cfg.CONF.auth_encryption_key, batch_size=50)
|
||||
self.assertEqual(len(dec_result), 1)
|
||||
self.assertIs(AttributeError, type(dec_result[0]))
|
||||
dec_tmpls = self.ctx.session.query(models.RawTemplate).all()
|
||||
self.assertEqual('', dec_tmpls[1].environment)
|
||||
self.assertEqual('bar',
|
||||
dec_tmpls[2].environment['parameters']['param2'])
|
||||
|
||||
def test_db_encrypt_no_env(self):
|
||||
template = {
|
||||
'template': self.t,
|
||||
'files': {'foo': 'bar'},
|
||||
'environment': None}
|
||||
db_api.raw_template_create(self.ctx, template)
|
||||
self.assertEqual([], db_api.db_encrypt_parameters_and_properties(
|
||||
self.ctx, cfg.CONF.auth_encryption_key))
|
||||
|
||||
def test_db_encrypt_no_env_parameters(self):
|
||||
template = {
|
||||
'template': self.t,
|
||||
'files': {'foo': 'bar'},
|
||||
'environment': {'encrypted_param_names': ['a']}}
|
||||
db_api.raw_template_create(self.ctx, template)
|
||||
self.assertEqual([], db_api.db_encrypt_parameters_and_properties(
|
||||
self.ctx, cfg.CONF.auth_encryption_key))
|
||||
|
||||
def test_db_encrypt_no_properties_data(self):
|
||||
ctx = utils.dummy_context()
|
||||
template = self._create_template()
|
||||
user_creds = create_user_creds(ctx)
|
||||
stack = create_stack(ctx, template, user_creds)
|
||||
resources = [create_resource(ctx, stack, name='res1')]
|
||||
resources[0].properties_data = None
|
||||
self.assertEqual([], db_api.db_encrypt_parameters_and_properties(
|
||||
ctx, cfg.CONF.auth_encryption_key))
|
||||
|
||||
def test_db_encrypt_decrypt_verbose_on(self):
|
||||
info_logger = self.useFixture(
|
||||
fixtures.FakeLogger(level=logging.INFO,
|
||||
format="%(levelname)8s [%(name)s] "
|
||||
"%(message)s"))
|
||||
ctx = utils.dummy_context()
|
||||
template = self._create_template()
|
||||
user_creds = create_user_creds(ctx)
|
||||
stack = create_stack(ctx, template, user_creds)
|
||||
create_resource(ctx, stack, legacy_prop_data=True, name='res2')
|
||||
|
||||
db_api.db_encrypt_parameters_and_properties(
|
||||
ctx, cfg.CONF.auth_encryption_key, verbose=True)
|
||||
self.assertIn("Processing raw_template 1", info_logger.output)
|
||||
self.assertIn("Finished encrypt processing of raw_template 1",
|
||||
info_logger.output)
|
||||
self.assertIn("Processing resource_properties_data 1",
|
||||
info_logger.output)
|
||||
self.assertIn("Finished processing resource_properties_data 1",
|
||||
info_logger.output)
|
||||
# only the resource with legacy properties data is processed
|
||||
self.assertIn("Processing resource 2", info_logger.output)
|
||||
self.assertIn("Finished processing resource 2", info_logger.output)
|
||||
|
||||
info_logger2 = self.useFixture(
|
||||
fixtures.FakeLogger(level=logging.INFO,
|
||||
format="%(levelname)8s [%(name)s] "
|
||||
"%(message)s"))
|
||||
|
||||
db_api.db_decrypt_parameters_and_properties(
|
||||
ctx, cfg.CONF.auth_encryption_key, verbose=True)
|
||||
self.assertIn("Processing raw_template 1", info_logger2.output)
|
||||
self.assertIn("Finished decrypt processing of raw_template 1",
|
||||
info_logger2.output)
|
||||
self.assertIn("Processing resource_properties_data 1",
|
||||
info_logger.output)
|
||||
self.assertIn("Finished processing resource_properties_data 1",
|
||||
info_logger.output)
|
||||
# only the resource with legacy properties data is processed
|
||||
self.assertIn("Processing resource 2", info_logger2.output)
|
||||
self.assertIn("Finished processing resource 2", info_logger2.output)
|
||||
|
||||
def test_db_encrypt_decrypt_verbose_off(self):
|
||||
info_logger = self.useFixture(
|
||||
fixtures.FakeLogger(level=logging.INFO,
|
||||
format="%(levelname)8s [%(name)s] "
|
||||
"%(message)s"))
|
||||
ctx = utils.dummy_context()
|
||||
template = self._create_template()
|
||||
user_creds = create_user_creds(ctx)
|
||||
stack = create_stack(ctx, template, user_creds)
|
||||
create_resource(ctx, stack, name='res1')
|
||||
|
||||
db_api.db_encrypt_parameters_and_properties(
|
||||
ctx, cfg.CONF.auth_encryption_key, verbose=False)
|
||||
self.assertNotIn("Processing raw_template 1", info_logger.output)
|
||||
self.assertNotIn("Processing resource 1", info_logger.output)
|
||||
self.assertNotIn("Successfully processed raw_template 1",
|
||||
info_logger.output)
|
||||
self.assertNotIn("Successfully processed resource 1",
|
||||
info_logger.output)
|
||||
|
||||
info_logger2 = self.useFixture(
|
||||
fixtures.FakeLogger(level=logging.INFO,
|
||||
format="%(levelname)8s [%(name)s] "
|
||||
"%(message)s"))
|
||||
|
||||
db_api.db_decrypt_parameters_and_properties(
|
||||
ctx, cfg.CONF.auth_encryption_key, verbose=False)
|
||||
self.assertNotIn("Processing raw_template 1", info_logger2.output)
|
||||
self.assertNotIn("Processing resource 1", info_logger2.output)
|
||||
self.assertNotIn("Successfully processed raw_template 1",
|
||||
info_logger2.output)
|
||||
self.assertNotIn("Successfully processed resource 1",
|
||||
info_logger2.output)
|
||||
|
||||
def test_db_encrypt_no_param_schema(self):
|
||||
t = copy.deepcopy(self.t)
|
||||
del(t['parameters']['param2'])
|
||||
template = {
|
||||
'template': t,
|
||||
'files': {'foo': 'bar'},
|
||||
'environment': {'encrypted_param_names': [],
|
||||
'parameters': {'param2': 'foo'}}}
|
||||
db_api.raw_template_create(self.ctx, template)
|
||||
self.assertEqual([], db_api.db_encrypt_parameters_and_properties(
|
||||
self.ctx, cfg.CONF.auth_encryption_key))
|
||||
|
||||
def test_db_encrypt_non_string_param_type(self):
|
||||
t = template_format.parse('''
|
||||
heat_template_version: 2013-05-23
|
||||
parameters:
|
||||
param1:
|
||||
type: string
|
||||
description: value1.
|
||||
param2:
|
||||
type: string
|
||||
description: value2.
|
||||
hidden: true
|
||||
param3:
|
||||
type: string
|
||||
description: value3
|
||||
hidden: true
|
||||
default: 1234
|
||||
resources:
|
||||
a_resource:
|
||||
type: GenericResourceType
|
||||
''')
|
||||
template = {
|
||||
'template': t,
|
||||
'files': {},
|
||||
'environment': {'parameters': {
|
||||
'param1': 'foo',
|
||||
'param2': 'bar',
|
||||
'param3': 12345}}}
|
||||
tmpl = db_api.raw_template_create(self.ctx, template)
|
||||
self.assertEqual([], db_api.db_encrypt_parameters_and_properties(
|
||||
self.ctx, cfg.CONF.auth_encryption_key))
|
||||
tmpl = db_api.raw_template_get(self.ctx, tmpl.id)
|
||||
enc_params = copy.copy(tmpl.environment['parameters'])
|
||||
|
||||
self.assertEqual([], db_api.db_decrypt_parameters_and_properties(
|
||||
self.ctx, cfg.CONF.auth_encryption_key, batch_size=50))
|
||||
tmpl = db_api.raw_template_get(self.ctx, tmpl.id)
|
||||
dec_params = tmpl.environment['parameters']
|
||||
|
||||
self.assertNotEqual(enc_params['param3'], dec_params['param3'])
|
||||
self.assertEqual('bar', dec_params['param2'])
|
||||
self.assertEqual('12345', dec_params['param3'])
|
||||
|
||||
|
||||
class ResetStackStatusTests(common.HeatTestCase):
|
||||
|
||||
def setUp(self):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
upgrade:
|
||||
- |
|
||||
The ``heat-manage migrate_properties_data`` command is deprecated and is
|
||||
now a no-op. It will be removed in a future release.
|
||||
Loading…
Reference in New Issue