openstack/nova
Code Issues Proposed changes

Add online migration to move keypairs from main to API database

Browse Source 
This moves keypairs from the main database to the API database in
an online migration. Note that it refuses to run until all of the
instances have keypair information, so that we don't strand instances
that might need to lazy-load those things.

Related to blueprint cells-keypairs-api-db

Change-Id: I97b72ae3e7e8ea3d6b596870d8da3aaa689fd6b5
This commit is contained in:
Dan Smith 2016-05-06 10:59:10 -07:00
parent 54d66ee894
commit b8aac794d4
4 changed files with 104 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
nova/cmd/manage.py
View File

@ -82,6 +82,7 @@ from nova.i18n import _
from nova import objects
from nova.objects import flavor as flavor_obj
from nova.objects import instance as instance_obj
from nova.objects import keypair as keypair_obj
from nova.objects import request_spec
from nova import quota
from nova import rpc
@ -728,6 +729,7 @@ class DbCommands(object):
flavor_obj.migrate_flavor_reset_autoincrement,
instance_obj.migrate_instance_keypairs,
request_spec.migrate_instances_add_request_spec,
keypair_obj.migrate_keypairs_to_api_db,
)
def __init__(self):

53
nova/objects/keypair.py
View File

@ -13,18 +13,22 @@
# under the License.
from oslo_db import exception as db_exc
from oslo_log import log as logging
from oslo_utils import versionutils
from nova import db
from nova.db.sqlalchemy import api as db_api
from nova.db.sqlalchemy import api_models
from nova.db.sqlalchemy import models as main_models
from nova import exception
from nova.i18n import _LE
from nova import objects
from nova.objects import base
from nova.objects import fields
KEYPAIR_TYPE_SSH = 'ssh'
KEYPAIR_TYPE_X509 = 'x509'
LOG = logging.getLogger(__name__)
@db_api.api_context_manager.reader
@ -155,6 +159,9 @@ class KeyPair(base.NovaPersistentObject, base.NovaObject,
except exception.KeypairNotFound:
pass
self._create()
def _create(self):
updates = self.obj_get_changes()
db_keypair = self._create_in_db(self._context, updates)
self._from_db_object(self._context, self, db_keypair)
@ -198,3 +205,49 @@ class KeyPairList(base.ObjectListBase, base.NovaObject):
def get_count_by_user(cls, context, user_id):
return (cls._get_count_from_db(context, user_id) +
db.key_pair_count_by_user(context, user_id))
@db_api.main_context_manager.reader
def _count_unmigrated_instances(context):
return context.session.query(main_models.InstanceExtra).\
filter_by(keypairs=None).\
filter_by(deleted=0).\
count()
@db_api.main_context_manager.reader
def _get_main_keypairs(context, limit):
return context.session.query(main_models.KeyPair).\
filter_by(deleted=0).\
limit(limit).\
all()
def migrate_keypairs_to_api_db(context, count):
bad_instances = _count_unmigrated_instances(context)
if bad_instances:
LOG.error(_LE('Some instances are still missing keypair '
'information. Unable to run keypair migration '
'at this time.'))
return 0, 0
main_keypairs = _get_main_keypairs(context, count)
done = 0
for db_keypair in main_keypairs:
kp = objects.KeyPair(context=context,
user_id=db_keypair.user_id,
name=db_keypair.name,
fingerprint=db_keypair.fingerprint,
public_key=db_keypair.public_key,
type=db_keypair.type)
try:
kp._create()
except exception.KeyPairExists:
# NOTE(danms): If this got created somehow in the API DB,
# then it's newer and we just continue on to destroy the
# old one in the cell DB.
pass
db_api.key_pair_destroy(context, db_keypair.user_id, db_keypair.name)
done += 1
return len(main_keypairs), done

43
nova/tests/functional/db/test_keypair.py
View File

@ -134,3 +134,46 @@ class KeyPairObjectTestCase(test.NoDBTestCase):
count = objects.KeyPairList.get_count_by_user(self.context,
self.context.user_id)
self.assertEqual(2, count)
def test_migrate_keypairs(self):
self._api_kp(name='apikey')
self._main_kp(name='mainkey1')
self._main_kp(name='mainkey2')
self._main_kp(name='mainkey3')
total, done = keypair.migrate_keypairs_to_api_db(self.context, 2)
self.assertEqual(2, total)
self.assertEqual(2, done)
# NOTE(danms): This only fetches from the API DB
api_keys = objects.KeyPairList._get_from_db(self.context,
self.context.user_id)
self.assertEqual(3, len(api_keys))
# NOTE(danms): This only fetches from the main DB
main_keys = db_api.key_pair_get_all_by_user(self.context,
self.context.user_id)
self.assertEqual(1, len(main_keys))
self.assertEqual((1, 1),
keypair.migrate_keypairs_to_api_db(self.context, 100))
self.assertEqual((0, 0),
keypair.migrate_keypairs_to_api_db(self.context, 100))
def test_migrate_keypairs_bails_on_unmigrated_instances(self):
objects.Instance(context=self.context, user_id=self.context.user_id,
project_id=self.context.project_id).create()
self._api_kp(name='apikey')
self._main_kp(name='mainkey1')
total, done = keypair.migrate_keypairs_to_api_db(self.context, 100)
self.assertEqual(0, total)
self.assertEqual(0, done)
def test_migrate_keypairs_skips_existing(self):
self._api_kp(name='mykey')
self._main_kp(name='mykey')
total, done = keypair.migrate_keypairs_to_api_db(self.context, 100)
self.assertEqual(1, total)
self.assertEqual(1, done)
total, done = keypair.migrate_keypairs_to_api_db(self.context, 100)
self.assertEqual(0, total)
self.assertEqual(0, done)

6
releasenotes/notes/keypairs-moved-to-api-9cde30acac6f76b6.yaml Normal file
View File

@ -0,0 +1,6 @@
---
upgrade:
- Keypairs have been moved to the API database, using an online
data migration. During the first phase of the migration, instances
will be given local storage of their key, after which keypairs will
be moved to the API database.