Add flavor migration routine

This adds a function and nova-manage command that moves flavors from
the main DB to the API DB. Note that it also includes an online
migration that doesn't actually change any data, but only resets the
autoincrement value for postgres, if necessary. This is not put
into the main migration simply because that one may run small chunks
of flavors and won't know what the real maximum is.

Related to blueprint flavor-cell-api

Depends-On: Ibc02acc60b1023039f2613f8949b95d55169fd30
Change-Id: I4ac31d6ff96595ea31d96d8604f690aa964d5709

nova/cmd/manage.py

@@ -79,6 +79,7 @@ from nova.db import migration
 from nova import exception
 from nova.i18n import _
 from nova import objects
+from nova.objects import flavor as flavor_obj
 from nova.openstack.common import cliutils
 from nova import quota
 from nova import rpc
@@ -924,6 +925,8 @@ class DbCommands(object):
         db.pcidevice_online_data_migration,
         db.computenode_uuids_online_data_migration,
         db.aggregate_uuids_online_data_migration,
+        flavor_obj.migrate_flavors,
+        flavor_obj.migrate_flavor_reset_autoincrement,
     )
 
     def __init__(self):

nova/objects/flavor.py

@@ -14,9 +14,12 @@
 
 from oslo_db import exception as db_exc
 from oslo_db.sqlalchemy import utils as sqlalchemyutils
+from oslo_log import log as logging
 from sqlalchemy import or_
 from sqlalchemy.orm import joinedload
 from sqlalchemy.sql.expression import asc
+from sqlalchemy.sql import func
+from sqlalchemy.sql import text
 from sqlalchemy.sql import true
 
 from nova import db
@@ -24,11 +27,13 @@ from nova.db.sqlalchemy import api as db_api
 from nova.db.sqlalchemy.api import require_context
 from nova.db.sqlalchemy import api_models
 from nova import exception
+from nova.i18n import _LW
 from nova import objects
 from nova.objects import base
 from nova.objects import fields
 
 
+LOG = logging.getLogger(__name__)
 OPTIONAL_FIELDS = ['extra_specs', 'projects']
 DEPRECATED_FIELDS = ['deleted', 'deleted_at']
 
@@ -654,3 +659,61 @@ class FlavorList(base.ObjectListBase, base.NovaObject):
         return base.obj_make_list(context, cls(context), objects.Flavor,
                                   api_db_flavors + db_flavors,
                                   expected_attrs=['extra_specs'])
+
+
+@db_api.main_context_manager.reader
+def _get_main_db_flavor_ids(context, limit):
+    # NOTE(danms): We don't need this imported at runtime, so
+    # keep it separate here
+    from nova.db.sqlalchemy import models
+    return [x[0] for x in context.session.query(models.InstanceTypes.id).
+            filter_by(deleted=0).
+            limit(limit)]
+
+
+def migrate_flavors(ctxt, count):
+    main_db_ids = _get_main_db_flavor_ids(ctxt, count)
+    if not main_db_ids:
+        return 0, 0
+
+    count_all = len(main_db_ids)
+    count_hit = 0
+
+    for flavor_id in main_db_ids:
+        try:
+            flavor = Flavor.get_by_id(ctxt, flavor_id)
+            flavor_values = {field: getattr(flavor, field)
+                             for field in flavor.fields}
+            flavor._flavor_create(ctxt, flavor_values)
+            count_hit += 1
+            db.flavor_destroy(ctxt, flavor.name)
+        except exception.FlavorNotFound:
+            LOG.warning(_LW('Flavor id %(id)i disappeared during migration'),
+                        {'id': flavor_id})
+        except (exception.FlavorExists, exception.FlavorIdExists) as e:
+            LOG.error(str(e))
+
+    return count_all, count_hit
+
+
+def _adjust_autoincrement(context, value):
+    engine = db_api.get_api_engine()
+    if engine.name == 'postgresql':
+        # NOTE(danms): If we migrated some flavors in the above function,
+        # then we will have confused postgres' sequence for the autoincrement
+        # primary key. MySQL does not care about this, but since postgres does,
+        # we need to reset this to avoid a failure on the next flavor creation.
+        engine.execute(
+            text('ALTER SEQUENCE flavors_id_seq RESTART WITH %i;' % (
+                value)))
+
+
+@db_api.api_context_manager.reader
+def _get_max_flavor_id(context):
+    return context.session.query(func.max(api_models.Flavors.id)).one()[0]
+
+
+def migrate_flavor_reset_autoincrement(ctxt, count):
+    max_id = _get_max_flavor_id(ctxt)
+    _adjust_autoincrement(ctxt, max_id + 1)
+    return 0, 0

nova/tests/functional/db/test_flavor.py

@@ -16,6 +16,7 @@ from nova.db.sqlalchemy import api as db_api
 from nova.db.sqlalchemy import api_models
 from nova import exception
 from nova import objects
+from nova.objects import flavor as flavor_obj
 from nova import test
 from nova.tests import fixtures
 
@@ -229,3 +230,28 @@ class FlavorObjectTestCase(test.NoDBTestCase):
         flavor.create()
         self.assertRaises(exception.MarkerNotFound,
                           self._test_get_all, 2, marker='noflavoratall')
+
+
+class FlavorMigrationTestCase(test.NoDBTestCase):
+    def setUp(self):
+        super(FlavorMigrationTestCase, self).setUp()
+        self.useFixture(fixtures.Database())
+        self.useFixture(fixtures.Database(database='api'))
+        self.context = context.get_admin_context()
+
+    def test_migration(self):
+        main_flavors = len(db.flavor_get_all(self.context))
+        match, done = flavor_obj.migrate_flavors(self.context, 50)
+        self.assertEqual(main_flavors, match)
+        self.assertEqual(main_flavors, done)
+        self.assertEqual(0, len(db.flavor_get_all(self.context)))
+        self.assertEqual(main_flavors,
+                         len(objects.FlavorList.get_all(self.context)))
+
+    def test_migrate_flavor_reset_autoincrement(self):
+        # NOTE(danms): Not much we can do here other than just make
+        # sure that the non-postgres case does not explode.
+        match, done = flavor_obj.migrate_flavor_reset_autoincrement(
+            self.context, 0)
+        self.assertEqual(0, match)
+        self.assertEqual(0, done)