openstack/nova
Code Issues Proposed changes

Add online migration to move instance groups to API database

Browse Source 
This moves instance groups from the main database to the API database
in an online migration.

Part of blueprint cells-instance-groups-api-db

Change-Id: I946767509903c76e9089bda02e24d964e383b91c
This commit is contained in:
melanie witt 2016-06-23 03:46:50 +00:00 committed by Dan Smith
parent b292bf3441
commit d35e1577c9
4 changed files with 112 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
nova/cmd/manage.py
View File

@ -83,6 +83,7 @@ from nova import objects
from nova.objects import aggregate as aggregate_obj
from nova.objects import flavor as flavor_obj
from nova.objects import instance as instance_obj
from nova.objects import instance_group as instance_group_obj
from nova.objects import keypair as keypair_obj
from nova.objects import request_spec
from nova import quota
@ -791,6 +792,7 @@ class DbCommands(object):
keypair_obj.migrate_keypairs_to_api_db,
aggregate_obj.migrate_aggregates,
aggregate_obj.migrate_aggregate_reset_autoincrement,
instance_group_obj.migrate_instance_groups_to_api_db,
)
def __init__(self):

58
nova/objects/instance_group.py
View File

@ -24,6 +24,7 @@ from nova.compute import utils as compute_utils
from nova import db
from nova.db.sqlalchemy import api as db_api
from nova.db.sqlalchemy import api_models
from nova.db.sqlalchemy import models as main_models
from nova import exception
from nova import objects
from nova.objects import base
@ -372,8 +373,10 @@ class InstanceGroup(base.NovaPersistentObject, base.NovaObject,
self[field] = current[field]
self.obj_reset_changes()
@base.remotable
def create(self):
def _create(self, skipcheck=False):
# NOTE(danms): This is just for the migration routine, and
# can be removed once we're no longer supporting the migration
# of instance groups from the main to api database.
if self.obj_attr_is_set('id'):
raise exception.ObjectActionError(action='create',
reason='already created')
@ -387,13 +390,14 @@ class InstanceGroup(base.NovaPersistentObject, base.NovaObject,
self.uuid = uuidutils.generate_uuid()
updates['uuid'] = self.uuid
try:
db.instance_group_get(self._context, self.uuid)
raise exception.ObjectActionError(
action='create',
reason='already created in main')
except exception.InstanceGroupNotFound:
pass
if not skipcheck:
try:
db.instance_group_get(self._context, self.uuid)
raise exception.ObjectActionError(
action='create',
reason='already created in main')
except exception.InstanceGroupNotFound:
pass
db_group = self._create_in_db(self._context, updates,
policies=policies,
members=members)
@ -402,6 +406,10 @@ class InstanceGroup(base.NovaPersistentObject, base.NovaObject,
compute_utils.notify_about_server_group_update(self._context,
"create", payload)
@base.remotable
def create(self):
self._create()
@base.remotable
def destroy(self):
payload = {'server_group_id': self.uuid}
@ -502,3 +510,35 @@ class InstanceGroupList(base.ObjectListBase, base.NovaObject):
main_db_groups = db.instance_group_get_all(context)
return base.obj_make_list(context, cls(context), objects.InstanceGroup,
api_db_groups + main_db_groups)
@db_api.main_context_manager.reader
def _get_main_instance_groups(context, limit):
return context.session.query(main_models.InstanceGroup).\
options(joinedload('_policies')).\
options(joinedload('_members')).\
filter_by(deleted=0).\
limit(limit).\
all()
def migrate_instance_groups_to_api_db(context, count):
main_groups = _get_main_instance_groups(context, count)
done = 0
for db_group in main_groups:
group = objects.InstanceGroup(context=context,
user_id=db_group.user_id,
project_id=db_group.project_id,
uuid=db_group.uuid,
name=db_group.name,
policies=db_group.policies,
members=db_group.members)
try:
group._create(skipcheck=True)
except exception.InstanceGroupIdExists:
# NOTE(melwitt): This might happen if there's a failure right after
# the InstanceGroup was created and the migration is re-run.
pass
db_api.instance_group_delete(context, db_group.uuid)
done += 1
return len(main_groups), done

55
nova/tests/functional/db/test_instance_group.py
View File

@ -18,6 +18,7 @@ from nova.db.sqlalchemy import api as db_api
from nova import exception
from nova import objects
from nova.objects import base
from nova.objects import instance_group
from nova import test
from nova.tests import uuidsentinel as uuids
@ -183,3 +184,57 @@ class InstanceGroupObjectTestCase(test.TestCase):
self.assertEqual(2, len(get_groups))
self.assertTrue(base.obj_equal_prims(create_group, get_groups[0]))
ovo_fixture.compare_obj(self, get_groups[1], db_group)
def test_migrate_instance_groups(self):
self._api_group(name='apigroup')
orig_main_models = []
orig_main_models.append(self._main_group(name='maingroup1'))
orig_main_models.append(self._main_group(name='maingroup2'))
orig_main_models.append(self._main_group(name='maingroup3'))
total, done = instance_group.migrate_instance_groups_to_api_db(
self.context, 2)
self.assertEqual(2, total)
self.assertEqual(2, done)
# This only fetches from the api db
api_groups = objects.InstanceGroupList._get_from_db(self.context)
self.assertEqual(3, len(api_groups))
# This only fetches from the main db
main_groups = db_api.instance_group_get_all(self.context)
self.assertEqual(1, len(main_groups))
self.assertEqual((1, 1),
instance_group.migrate_instance_groups_to_api_db(
self.context, 100))
self.assertEqual((0, 0),
instance_group.migrate_instance_groups_to_api_db(
self.context, 100))
# Verify the api_models have all their attributes set properly
api_models = objects.InstanceGroupList._get_from_db(self.context)
# Filter out the group that was created in the api db originally
api_models = [x for x in api_models if x.name != 'apigroup']
key_func = lambda model: model.uuid
api_models = sorted(api_models, key=key_func)
orig_main_models = sorted(orig_main_models, key=key_func)
ignore_fields = ('id', 'hosts', 'deleted', 'deleted_at', 'created_at',
'updated_at')
for i in range(len(api_models)):
for field in instance_group.InstanceGroup.fields:
if field not in ignore_fields:
self.assertEqual(orig_main_models[i][field],
api_models[i][field])
def test_migrate_instance_groups_skips_existing(self):
self._api_group(uuid=uuids.group)
self._main_group(uuid=uuids.group)
total, done = instance_group.migrate_instance_groups_to_api_db(
self.context, 100)
self.assertEqual(1, total)
self.assertEqual(1, done)
total, done = instance_group.migrate_instance_groups_to_api_db(
self.context, 100)
self.assertEqual(0, total)
self.assertEqual(0, done)

6
releasenotes/notes/bp-cells-instance-groups-api-db-910a44ef5f2f7769.yaml Normal file
View File

@ -0,0 +1,6 @@
---
upgrade:
- The nova-manage db online_data_migrations command will now migrate server
groups to the API database. New server groups will be automatically created
in the API database but existing server groups must be manually migrated
using the nova-manage command.