Add generic code for online data migrations

To limit downtime during upgrade we've decided to ban data migrations
done in migrations scripts and do them online during the runtime of the
cloud. This commit implements generic infrastructure for that.

This was part of a commit adding a mapping table between volume types
and consistency groups, but I've separated it because it's needed
independently. Also the aforementioned commit will probably be obsolete
when we finish generic volume groups efforts.

UpgradeImpact
Related: blueprint online-schema-upgrades
Change-Id: I655d47c39e44251cbbad6e4b3fbc8e8d73995d30

cinder/cmd/manage.py

@@ -208,6 +208,8 @@ class HostCommands(object):
 class DbCommands(object):
     """Class for managing the database."""
 
+    online_migrations = ()
+
     def __init__(self):
         pass
 
@@ -243,6 +245,55 @@ class DbCommands(object):
                     "logs for more details."))
             sys.exit(1)
 
+    def _run_migration(self, ctxt, max_count, ignore_state):
+        ran = 0
+        for migration_meth in self.online_migrations:
+            count = max_count - ran
+            try:
+                found, done = migration_meth(ctxt, count, ignore_state)
+            except Exception:
+                print(_("Error attempting to run %(method)s") %
+                      {'method': migration_meth.__name__})
+                found = done = 0
+
+            if found:
+                print(_('%(total)i rows matched query %(meth)s, %(done)i '
+                        'migrated') % {'total': found,
+                                       'meth': migration_meth.__name__,
+                                       'done': done})
+            if max_count is not None:
+                ran += done
+                if ran >= max_count:
+                    break
+        return ran
+
+    @args('--max_count', metavar='<number>', dest='max_count', type=int,
+          help='Maximum number of objects to consider.')
+    @args('--ignore_state', action='store_true', dest='ignore_state',
+          help='Force records to migrate even if another operation is '
+               'performed on them. This may be dangerous, please refer to '
+               'release notes for more information.')
+    def online_data_migrations(self, max_count=None, ignore_state=False):
+        """Perform online data migrations for the release in batches."""
+        ctxt = context.get_admin_context()
+        if max_count is not None:
+            unlimited = False
+            if max_count < 1:
+                print(_('Must supply a positive value for max_number.'))
+                sys.exit(127)
+        else:
+            unlimited = True
+            max_count = 50
+            print(_('Running batches of %i until complete.') % max_count)
+
+        ran = None
+        while ran is None or ran != 0:
+            ran = self._run_migration(ctxt, max_count, ignore_state)
+            if not unlimited:
+                break
+
+        sys.exit(1 if ran else 0)
+
 
 class VersionCommands(object):
     """Class for exposing the codebase version."""

cinder/tests/unit/test_cmd.py

@@ -234,6 +234,34 @@ class TestCinderManageCmd(test.TestCase):
         with mock.patch('sys.stdout', new=six.StringIO()):
             self.assertRaises(exception.InvalidInput, db_cmds.sync, 1)
 
+    @mock.patch('cinder.cmd.manage.DbCommands.online_migrations',
+                (mock.Mock(side_effect=((2, 2), (0, 0)), __name__='foo'),))
+    def test_db_commands_online_data_migrations(self):
+        db_cmds = cinder_manage.DbCommands()
+        exit = self.assertRaises(SystemExit, db_cmds.online_data_migrations)
+        self.assertEqual(0, exit.code)
+        cinder_manage.DbCommands.online_migrations[0].assert_has_calls(
+            (mock.call(mock.ANY, 50, False),) * 2)
+
+    @mock.patch('cinder.cmd.manage.DbCommands.online_migrations',
+                (mock.Mock(side_effect=((2, 2), (0, 0)), __name__='foo'),))
+    def test_db_commands_online_data_migrations_ignore_state_and_max(self):
+        db_cmds = cinder_manage.DbCommands()
+        exit = self.assertRaises(SystemExit, db_cmds.online_data_migrations,
+                                 2, True)
+        self.assertEqual(1, exit.code)
+        cinder_manage.DbCommands.online_migrations[0].assert_called_once_with(
+            mock.ANY, 2, True)
+
+    @mock.patch('cinder.cmd.manage.DbCommands.online_migrations',
+                (mock.Mock(side_effect=((2, 2), (0, 0)), __name__='foo'),))
+    def test_db_commands_online_data_migrations_max_negative(self):
+        db_cmds = cinder_manage.DbCommands()
+        exit = self.assertRaises(SystemExit, db_cmds.online_data_migrations,
+                                 -1)
+        self.assertEqual(127, exit.code)
+        cinder_manage.DbCommands.online_migrations[0].assert_not_called()
+
     @mock.patch('cinder.version.version_string')
     def test_versions_commands_list(self, version_string):
         version_cmds = cinder_manage.VersionCommands()

releasenotes/notes/cinder-manage-db-online-schema-migrations-d1c0d40f26d0f033.yaml

@@ -0,0 +1,13 @@
+---
+upgrade:
+  - To get rid of long running DB data migrations that must be run offline,
+    Cinder will now be able to execute them online, on a live cloud. Before
+    upgrading from Newton to Ocata operator needs to perform all the Newton
+    data migrations. To achieve that he needs to perform `cinder-manage db
+    online-data-migrations` until there are no records to be updated. To limit
+    DB performance impact migrations can be performed in chunks limited by
+    `--max_number` option. If your intent is to upgrade Cinder in a non-live
+    manner, you can use `--ignore-state` option safely. Please note that
+    finishing all the Newton data migrations will be enforced by the first
+    schema migration in Ocata, so you won't be able to upgrade to Ocata without
+    that.