Merge "Add heat-manage subcommand to migrate legacy prop. data"
This commit is contained in:
commit
467c542097
|
@ -22,8 +22,9 @@ The standard pattern for executing a heat-manage command is:
|
||||||
Run with -h to see a list of available commands:
|
Run with -h to see a list of available commands:
|
||||||
``heat-manage -h``
|
``heat-manage -h``
|
||||||
|
|
||||||
Commands are ``db_version``, ``db_sync``, ``purge_deleted``, ``migrate_covergence_1``
|
Commands are ``db_version``, ``db_sync``, ``purge_deleted``,
|
||||||
and ``service``. Detailed descriptions are below.
|
``migrate_covergence_1``, ``migrate_properties_data``, and
|
||||||
|
``service``. Detailed descriptions are below.
|
||||||
|
|
||||||
``heat-manage db_version``
|
``heat-manage db_version``
|
||||||
|
|
||||||
|
@ -38,6 +39,12 @@ and ``service``. Detailed descriptions are below.
|
||||||
Purge db entries marked as deleted and older than [age]. When project_id
|
Purge db entries marked as deleted and older than [age]. When project_id
|
||||||
argument is provided, only entries belonging to this project will be purged.
|
argument is provided, only entries belonging to this project will be purged.
|
||||||
|
|
||||||
|
``heat-manage migrate_properties_data``
|
||||||
|
|
||||||
|
Migrates properties data from the legacy locations in the db
|
||||||
|
(resource.properties_data and event.resource_properties) to the
|
||||||
|
modern location, the resource_properties_data table.
|
||||||
|
|
||||||
``heat-manage migrate_convergence_1 [stack_id]``
|
``heat-manage migrate_convergence_1 [stack_id]``
|
||||||
|
|
||||||
Migrates [stack_id] from non-convergence to convergence. This requires running
|
Migrates [stack_id] from non-convergence to convergence. This requires running
|
||||||
|
|
|
@ -148,6 +148,11 @@ def do_crypt_parameters_and_properties():
|
||||||
ctxt, prev_encryption_key, CONF.command.verbose_update_params)
|
ctxt, prev_encryption_key, CONF.command.verbose_update_params)
|
||||||
|
|
||||||
|
|
||||||
|
def do_properties_data_migrate():
|
||||||
|
ctxt = context.get_admin_context()
|
||||||
|
db_api.db_properties_data_migrate(ctxt)
|
||||||
|
|
||||||
|
|
||||||
def add_command_parsers(subparsers):
|
def add_command_parsers(subparsers):
|
||||||
# db_version parser
|
# db_version parser
|
||||||
parser = subparsers.add_parser('db_version')
|
parser = subparsers.add_parser('db_version')
|
||||||
|
@ -215,6 +220,10 @@ def add_command_parsers(subparsers):
|
||||||
parser.add_argument('stack_id',
|
parser.add_argument('stack_id',
|
||||||
help=_('Stack id'))
|
help=_('Stack id'))
|
||||||
|
|
||||||
|
# migrate properties_data parser
|
||||||
|
parser = subparsers.add_parser('migrate_properties_data')
|
||||||
|
parser.set_defaults(func=do_properties_data_migrate)
|
||||||
|
|
||||||
ServiceManageCommand.add_service_parsers(subparsers)
|
ServiceManageCommand.add_service_parsers(subparsers)
|
||||||
|
|
||||||
command_opt = cfg.SubCommandOpt('command',
|
command_opt = cfg.SubCommandOpt('command',
|
||||||
|
|
|
@ -37,6 +37,7 @@ from heat.common import exception
|
||||||
from heat.common.i18n import _
|
from heat.common.i18n import _
|
||||||
from heat.common.i18n import _LE
|
from heat.common.i18n import _LE
|
||||||
from heat.common.i18n import _LI
|
from heat.common.i18n import _LI
|
||||||
|
from heat.common.i18n import _LW
|
||||||
from heat.db.sqlalchemy import filters as db_filters
|
from heat.db.sqlalchemy import filters as db_filters
|
||||||
from heat.db.sqlalchemy import migration
|
from heat.db.sqlalchemy import migration
|
||||||
from heat.db.sqlalchemy import models
|
from heat.db.sqlalchemy import models
|
||||||
|
@ -1692,6 +1693,73 @@ def db_decrypt_parameters_and_properties(ctxt, encryption_key, batch_size=50,
|
||||||
return excs
|
return excs
|
||||||
|
|
||||||
|
|
||||||
|
def db_properties_data_migrate(ctxt, batch_size=50):
|
||||||
|
"""Migrate properties data from legacy columns to new location in db.
|
||||||
|
|
||||||
|
:param ctxt: RPC context
|
||||||
|
:param batch_size: number of templates requested from db in each iteration.
|
||||||
|
50 means that heat requests 50 templates, encrypt them
|
||||||
|
and proceed with next 50 items.
|
||||||
|
"""
|
||||||
|
session = ctxt.session
|
||||||
|
|
||||||
|
query = session.query(models.Resource).filter(and_(
|
||||||
|
models.Resource.properties_data.isnot(None),
|
||||||
|
models.Resource.rsrc_prop_data_id.is_(None)))
|
||||||
|
resource_batches = _get_batch(
|
||||||
|
session=session, ctxt=ctxt, query=query,
|
||||||
|
model=models.Resource, batch_size=batch_size)
|
||||||
|
next_batch = list(itertools.islice(resource_batches, batch_size))
|
||||||
|
while next_batch:
|
||||||
|
with session.begin():
|
||||||
|
for resource in next_batch:
|
||||||
|
try:
|
||||||
|
encrypted = resource.properties_data_encrypted
|
||||||
|
if encrypted is None:
|
||||||
|
LOG.warning(
|
||||||
|
_LW('Unexpected: resource.encrypted is None for '
|
||||||
|
'resource id %(id)d for legacy '
|
||||||
|
'resource.properties_data, assuming False.'),
|
||||||
|
{'id': resource.id})
|
||||||
|
encrypted = False
|
||||||
|
rsrc_prop_data = resource_prop_data_create(
|
||||||
|
ctxt, {'encrypted': encrypted,
|
||||||
|
'data': resource.properties_data})
|
||||||
|
resource_update(ctxt, resource.id,
|
||||||
|
{'properties_data_encrypted': None,
|
||||||
|
'properties_data': None,
|
||||||
|
'rsrc_prop_data_id': rsrc_prop_data.id},
|
||||||
|
resource.atomic_key)
|
||||||
|
except Exception:
|
||||||
|
LOG.exception(_LE('Failed to migrate properties_data for '
|
||||||
|
'resource %(id)d'), {'id': resource.id})
|
||||||
|
continue
|
||||||
|
next_batch = list(itertools.islice(resource_batches, batch_size))
|
||||||
|
|
||||||
|
query = session.query(models.Event).filter(and_(
|
||||||
|
models.Event.resource_properties.isnot(None),
|
||||||
|
models.Event.rsrc_prop_data_id.is_(None)))
|
||||||
|
event_batches = _get_batch(
|
||||||
|
session=session, ctxt=ctxt, query=query,
|
||||||
|
model=models.Event, batch_size=batch_size)
|
||||||
|
next_batch = list(itertools.islice(event_batches, batch_size))
|
||||||
|
while next_batch:
|
||||||
|
with session.begin():
|
||||||
|
for event in next_batch:
|
||||||
|
try:
|
||||||
|
prop_data = event.resource_properties
|
||||||
|
rsrc_prop_data = resource_prop_data_create(
|
||||||
|
ctxt, {'encrypted': False,
|
||||||
|
'data': prop_data})
|
||||||
|
event.update({'resource_properties': None,
|
||||||
|
'rsrc_prop_data_id': rsrc_prop_data.id})
|
||||||
|
except Exception:
|
||||||
|
LOG.exception(_LE('Failed to migrate resource_properties '
|
||||||
|
'for event %(id)d'), {'id': event.id})
|
||||||
|
continue
|
||||||
|
next_batch = list(itertools.islice(event_batches, batch_size))
|
||||||
|
|
||||||
|
|
||||||
def _get_batch(session, ctxt, query, model, batch_size=50):
|
def _get_batch(session, ctxt, query, model, batch_size=50):
|
||||||
last_batch_marker = None
|
last_batch_marker = None
|
||||||
while True:
|
while True:
|
||||||
|
|
|
@ -1439,9 +1439,11 @@ def create_resource_prop_data(ctx, **kwargs):
|
||||||
return db_api.resource_prop_data_create(ctx, **values)
|
return db_api.resource_prop_data_create(ctx, **values)
|
||||||
|
|
||||||
|
|
||||||
def create_event(ctx, **kwargs):
|
def create_event(ctx, legacy_prop_data=False, **kwargs):
|
||||||
rpd = db_api.resource_prop_data_create(ctx, {'data': {'name': 'foo'},
|
if not legacy_prop_data:
|
||||||
'encrypted': False})
|
rpd = db_api.resource_prop_data_create(ctx,
|
||||||
|
{'data': {'foo2': 'ev_bar'},
|
||||||
|
'encrypted': False})
|
||||||
values = {
|
values = {
|
||||||
'stack_id': 'test_stack_id',
|
'stack_id': 'test_stack_id',
|
||||||
'resource_action': 'create',
|
'resource_action': 'create',
|
||||||
|
@ -1449,8 +1451,11 @@ def create_event(ctx, **kwargs):
|
||||||
'resource_name': 'res',
|
'resource_name': 'res',
|
||||||
'physical_resource_id': UUID1,
|
'physical_resource_id': UUID1,
|
||||||
'resource_status_reason': "create_complete",
|
'resource_status_reason': "create_complete",
|
||||||
'rsrc_prop_data': rpd,
|
|
||||||
}
|
}
|
||||||
|
if not legacy_prop_data:
|
||||||
|
values['rsrc_prop_data'] = rpd
|
||||||
|
else:
|
||||||
|
values['resource_properties'] = {'foo2': 'ev_bar'}
|
||||||
values.update(kwargs)
|
values.update(kwargs)
|
||||||
return db_api.event_create(ctx, values)
|
return db_api.event_create(ctx, values)
|
||||||
|
|
||||||
|
@ -2721,7 +2726,7 @@ class DBAPIEventTest(common.HeatTestCase):
|
||||||
self.assertEqual('res', ret_event.resource_name)
|
self.assertEqual('res', ret_event.resource_name)
|
||||||
self.assertEqual(UUID1, ret_event.physical_resource_id)
|
self.assertEqual(UUID1, ret_event.physical_resource_id)
|
||||||
self.assertEqual('create_complete', ret_event.resource_status_reason)
|
self.assertEqual('create_complete', ret_event.resource_status_reason)
|
||||||
self.assertEqual({'name': 'foo'}, ret_event.rsrc_prop_data.data)
|
self.assertEqual({'foo2': 'ev_bar'}, ret_event.rsrc_prop_data.data)
|
||||||
|
|
||||||
def test_event_get_all(self):
|
def test_event_get_all(self):
|
||||||
self.stack1 = create_stack(self.ctx, self.template, self.user_creds,
|
self.stack1 = create_stack(self.ctx, self.template, self.user_creds,
|
||||||
|
@ -3294,6 +3299,86 @@ class DBAPISyncPointTest(common.HeatTestCase):
|
||||||
self.assertEqual(len(self.resources) * 4, add.call_count)
|
self.assertEqual(len(self.resources) * 4, add.call_count)
|
||||||
|
|
||||||
|
|
||||||
|
class DBAPIMigratePropertiesDataTest(common.HeatTestCase):
|
||||||
|
def setUp(self):
|
||||||
|
super(DBAPIMigratePropertiesDataTest, self).setUp()
|
||||||
|
self.ctx = utils.dummy_context()
|
||||||
|
templ = create_raw_template(self.ctx)
|
||||||
|
user_creds = create_user_creds(self.ctx)
|
||||||
|
stack = create_stack(self.ctx, templ, user_creds)
|
||||||
|
stack2 = create_stack(self.ctx, templ, user_creds)
|
||||||
|
create_resource(self.ctx, stack, True, name='res1')
|
||||||
|
create_resource(self.ctx, stack2, True, name='res2')
|
||||||
|
create_event(self.ctx, True)
|
||||||
|
create_event(self.ctx, True)
|
||||||
|
|
||||||
|
def _test_migrate_resource(self, batch_size=50):
|
||||||
|
resources = self.ctx.session.query(models.Resource).all()
|
||||||
|
self.assertEqual(2, len(resources))
|
||||||
|
for resource in resources:
|
||||||
|
self.assertEqual('bar1', resource.properties_data['foo1'])
|
||||||
|
|
||||||
|
db_api.db_properties_data_migrate(self.ctx, batch_size=batch_size)
|
||||||
|
for resource in resources:
|
||||||
|
self.assertEqual('bar1', resource.rsrc_prop_data.data['foo1'])
|
||||||
|
self.assertFalse(resource.rsrc_prop_data.encrypted)
|
||||||
|
self.assertIsNone(resource.properties_data)
|
||||||
|
self.assertIsNone(resource.properties_data_encrypted)
|
||||||
|
|
||||||
|
def _test_migrate_event(self, batch_size=50):
|
||||||
|
events = self.ctx.session.query(models.Event).all()
|
||||||
|
self.assertEqual(2, len(events))
|
||||||
|
for event in events:
|
||||||
|
self.assertEqual('ev_bar', event.resource_properties['foo2'])
|
||||||
|
|
||||||
|
db_api.db_properties_data_migrate(self.ctx, batch_size=batch_size)
|
||||||
|
self.ctx.session.expire_all()
|
||||||
|
events = self.ctx.session.query(models.Event).all()
|
||||||
|
for event in events:
|
||||||
|
self.assertEqual('ev_bar', event.rsrc_prop_data.data['foo2'])
|
||||||
|
self.assertFalse(event.rsrc_prop_data.encrypted)
|
||||||
|
self.assertIsNone(event.resource_properties)
|
||||||
|
|
||||||
|
def test_migrate_event(self):
|
||||||
|
self._test_migrate_event()
|
||||||
|
|
||||||
|
def test_migrate_event_in_batches(self):
|
||||||
|
self._test_migrate_event(batch_size=1)
|
||||||
|
|
||||||
|
def test_migrate_resource(self):
|
||||||
|
self._test_migrate_resource()
|
||||||
|
|
||||||
|
def test_migrate_resource_in_batches(self):
|
||||||
|
self._test_migrate_resource(batch_size=1)
|
||||||
|
|
||||||
|
def test_migrate_encrypted_resource(self):
|
||||||
|
resources = self.ctx.session.query(models.Resource).all()
|
||||||
|
db_api.db_encrypt_parameters_and_properties(
|
||||||
|
self.ctx, 'i have a key for you if you want')
|
||||||
|
|
||||||
|
encrypted_data_pre_migration = resources[0].properties_data['foo1'][1]
|
||||||
|
db_api.db_properties_data_migrate(self.ctx)
|
||||||
|
resources = self.ctx.session.query(models.Resource).all()
|
||||||
|
|
||||||
|
self.assertTrue(resources[0].rsrc_prop_data.encrypted)
|
||||||
|
self.assertIsNone(resources[0].properties_data)
|
||||||
|
self.assertIsNone(resources[0].properties_data_encrypted)
|
||||||
|
self.assertEqual('cryptography_decrypt_v1',
|
||||||
|
resources[0].rsrc_prop_data.data['foo1'][0])
|
||||||
|
self.assertEqual(encrypted_data_pre_migration,
|
||||||
|
resources[0].rsrc_prop_data.data['foo1'][1])
|
||||||
|
|
||||||
|
db_api.db_decrypt_parameters_and_properties(
|
||||||
|
self.ctx, 'i have a key for you if you want')
|
||||||
|
self.ctx.session.expire_all()
|
||||||
|
resources = self.ctx.session.query(models.Resource).all()
|
||||||
|
|
||||||
|
self.assertEqual('bar1', resources[0].rsrc_prop_data.data['foo1'])
|
||||||
|
self.assertFalse(resources[0].rsrc_prop_data.encrypted)
|
||||||
|
self.assertIsNone(resources[0].properties_data)
|
||||||
|
self.assertIsNone(resources[0].properties_data_encrypted)
|
||||||
|
|
||||||
|
|
||||||
class DBAPICryptParamsPropsTest(common.HeatTestCase):
|
class DBAPICryptParamsPropsTest(common.HeatTestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
super(DBAPICryptParamsPropsTest, self).setUp()
|
super(DBAPICryptParamsPropsTest, self).setUp()
|
||||||
|
|
Loading…
Reference in New Issue