From e05acd2005d22556918a60a7621e463b593c34e6 Mon Sep 17 00:00:00 2001 From: Dan Smith Date: Fri, 18 Mar 2016 12:13:35 -0700 Subject: [PATCH] Add flavor migration routine This adds a function and nova-manage command that moves flavors from the main DB to the API DB. Note that it also includes an online migration that doesn't actually change any data, but only resets the autoincrement value for postgres, if necessary. This is not put into the main migration simply because that one may run small chunks of flavors and won't know what the real maximum is. Related to blueprint flavor-cell-api Depends-On: Ibc02acc60b1023039f2613f8949b95d55169fd30 Change-Id: I4ac31d6ff96595ea31d96d8604f690aa964d5709 --- nova/cmd/manage.py | 3 ++ nova/objects/flavor.py | 63 +++++++++++++++++++++++++ nova/tests/functional/db/test_flavor.py | 26 ++++++++++ 3 files changed, 92 insertions(+) diff --git a/nova/cmd/manage.py b/nova/cmd/manage.py index 71d9292279..77359e7538 100644 --- a/nova/cmd/manage.py +++ b/nova/cmd/manage.py @@ -79,6 +79,7 @@ from nova.db import migration from nova import exception from nova.i18n import _ from nova import objects +from nova.objects import flavor as flavor_obj from nova.openstack.common import cliutils from nova import quota from nova import rpc @@ -924,6 +925,8 @@ class DbCommands(object): db.pcidevice_online_data_migration, db.computenode_uuids_online_data_migration, db.aggregate_uuids_online_data_migration, + flavor_obj.migrate_flavors, + flavor_obj.migrate_flavor_reset_autoincrement, ) def __init__(self): diff --git a/nova/objects/flavor.py b/nova/objects/flavor.py index 9e648dc8ed..77f69b624e 100644 --- a/nova/objects/flavor.py +++ b/nova/objects/flavor.py @@ -14,9 +14,12 @@ from oslo_db import exception as db_exc from oslo_db.sqlalchemy import utils as sqlalchemyutils +from oslo_log import log as logging from sqlalchemy import or_ from sqlalchemy.orm import joinedload from sqlalchemy.sql.expression import asc +from sqlalchemy.sql import func +from sqlalchemy.sql import text from sqlalchemy.sql import true from nova import db @@ -24,11 +27,13 @@ from nova.db.sqlalchemy import api as db_api from nova.db.sqlalchemy.api import require_context from nova.db.sqlalchemy import api_models from nova import exception +from nova.i18n import _LW from nova import objects from nova.objects import base from nova.objects import fields +LOG = logging.getLogger(__name__) OPTIONAL_FIELDS = ['extra_specs', 'projects'] DEPRECATED_FIELDS = ['deleted', 'deleted_at'] @@ -654,3 +659,61 @@ class FlavorList(base.ObjectListBase, base.NovaObject): return base.obj_make_list(context, cls(context), objects.Flavor, api_db_flavors + db_flavors, expected_attrs=['extra_specs']) + + +@db_api.main_context_manager.reader +def _get_main_db_flavor_ids(context, limit): + # NOTE(danms): We don't need this imported at runtime, so + # keep it separate here + from nova.db.sqlalchemy import models + return [x[0] for x in context.session.query(models.InstanceTypes.id). + filter_by(deleted=0). + limit(limit)] + + +def migrate_flavors(ctxt, count): + main_db_ids = _get_main_db_flavor_ids(ctxt, count) + if not main_db_ids: + return 0, 0 + + count_all = len(main_db_ids) + count_hit = 0 + + for flavor_id in main_db_ids: + try: + flavor = Flavor.get_by_id(ctxt, flavor_id) + flavor_values = {field: getattr(flavor, field) + for field in flavor.fields} + flavor._flavor_create(ctxt, flavor_values) + count_hit += 1 + db.flavor_destroy(ctxt, flavor.name) + except exception.FlavorNotFound: + LOG.warning(_LW('Flavor id %(id)i disappeared during migration'), + {'id': flavor_id}) + except (exception.FlavorExists, exception.FlavorIdExists) as e: + LOG.error(str(e)) + + return count_all, count_hit + + +def _adjust_autoincrement(context, value): + engine = db_api.get_api_engine() + if engine.name == 'postgresql': + # NOTE(danms): If we migrated some flavors in the above function, + # then we will have confused postgres' sequence for the autoincrement + # primary key. MySQL does not care about this, but since postgres does, + # we need to reset this to avoid a failure on the next flavor creation. + engine.execute( + text('ALTER SEQUENCE flavors_id_seq RESTART WITH %i;' % ( + value))) + + +@db_api.api_context_manager.reader +def _get_max_flavor_id(context): + return context.session.query(func.max(api_models.Flavors.id)).one()[0] + + +def migrate_flavor_reset_autoincrement(ctxt, count): + max_id = _get_max_flavor_id(ctxt) + _adjust_autoincrement(ctxt, max_id + 1) + return 0, 0 diff --git a/nova/tests/functional/db/test_flavor.py b/nova/tests/functional/db/test_flavor.py index e409fed247..00c8a1e0f5 100644 --- a/nova/tests/functional/db/test_flavor.py +++ b/nova/tests/functional/db/test_flavor.py @@ -16,6 +16,7 @@ from nova.db.sqlalchemy import api as db_api from nova.db.sqlalchemy import api_models from nova import exception from nova import objects +from nova.objects import flavor as flavor_obj from nova import test from nova.tests import fixtures @@ -229,3 +230,28 @@ class FlavorObjectTestCase(test.NoDBTestCase): flavor.create() self.assertRaises(exception.MarkerNotFound, self._test_get_all, 2, marker='noflavoratall') + + +class FlavorMigrationTestCase(test.NoDBTestCase): + def setUp(self): + super(FlavorMigrationTestCase, self).setUp() + self.useFixture(fixtures.Database()) + self.useFixture(fixtures.Database(database='api')) + self.context = context.get_admin_context() + + def test_migration(self): + main_flavors = len(db.flavor_get_all(self.context)) + match, done = flavor_obj.migrate_flavors(self.context, 50) + self.assertEqual(main_flavors, match) + self.assertEqual(main_flavors, done) + self.assertEqual(0, len(db.flavor_get_all(self.context))) + self.assertEqual(main_flavors, + len(objects.FlavorList.get_all(self.context))) + + def test_migrate_flavor_reset_autoincrement(self): + # NOTE(danms): Not much we can do here other than just make + # sure that the non-postgres case does not explode. + match, done = flavor_obj.migrate_flavor_reset_autoincrement( + self.context, 0) + self.assertEqual(0, match) + self.assertEqual(0, done)