From 6f00d3be95edcbcd6252af5ee42fb8f3b1dfd7d0 Mon Sep 17 00:00:00 2001 From: Dan Smith Date: Wed, 28 Sep 2016 08:26:22 -0700 Subject: [PATCH] Allow running db archiving continuously This adds a --until_complete flag to the db archive command, which will attempt to run batches of max_rows continuously until archiving has completed (or stops being able to archive anything). We also take care to catch KeyboardInterrupt so that it will, if stopped, still print the verbose report at the end. Instead of printing the table on each iteration, we print some progress information and then provide a completed summary table at the end of the run. This also defaults max_rows to a sane value. The previous meaning of not providing a value was "archive a whole table in a single operation", which was useful only in the most simple of cases. Now that we have a more reasonable "do it until you are done" mechanism, we default this to 1000 rows for sanity, which also serves as the batch size for the continuous mode. Implements blueprint archive-deleted-rows-all Change-Id: I2929c2f5b3116b63a4666b258bf078bc51d4e45a --- nova/cmd/manage.py | 53 +++++++++++---- nova/tests/unit/test_nova_manage.py | 64 +++++++++++++++++++ .../archive-all-db-aadf2ce0394c24fa.yaml | 7 ++ 3 files changed, 110 insertions(+), 14 deletions(-) create mode 100644 releasenotes/notes/archive-all-db-aadf2ce0394c24fa.yaml diff --git a/nova/cmd/manage.py b/nova/cmd/manage.py index 965d581acd31..d16e7eab63a8 100644 --- a/nova/cmd/manage.py +++ b/nova/cmd/manage.py @@ -825,28 +825,53 @@ class DbCommands(object): """Print the current database version.""" print(migration.db_version()) - @args('--max_rows', metavar='', + @args('--max_rows', metavar='', default=1000, help='Maximum number of deleted rows to archive') @args('--verbose', action='store_true', dest='verbose', default=False, help='Print how many rows were archived per table.') - def archive_deleted_rows(self, max_rows, verbose=False): - """Move up to max_rows deleted rows from production tables to shadow - tables. + @args('--until-complete', action='store_true', dest='until_complete', + default=False, + help=('Run continuously until all deleted rows are archived. Use ' + 'max_rows as a batch size for each iteration.')) + def archive_deleted_rows(self, max_rows, verbose=False, + until_complete=False): + """Move deleted rows from production tables to shadow tables. Returns 0 if nothing was archived, 1 if some number of rows were archived, 2 if max_rows is invalid. If automating, this should be run continuously while the result is 1, stopping at 0. """ - if max_rows is not None: - max_rows = int(max_rows) - if max_rows < 0: - print(_("Must supply a positive value for max_rows")) - return(2) - if max_rows > db.MAX_INT: - print(_('max rows must be <= %(max_value)d') % - {'max_value': db.MAX_INT}) - return(2) - table_to_rows_archived = db.archive_deleted_rows(max_rows) + max_rows = int(max_rows) + if max_rows < 0: + print(_("Must supply a positive value for max_rows")) + return(2) + if max_rows > db.MAX_INT: + print(_('max rows must be <= %(max_value)d') % + {'max_value': db.MAX_INT}) + return(2) + + table_to_rows_archived = {} + if until_complete and verbose: + sys.stdout.write(_('Archiving') + '..') # noqa + while True: + try: + run = db.archive_deleted_rows(max_rows) + except KeyboardInterrupt: + run = {} + if until_complete and verbose: + print('.' + _('stopped')) # noqa + break + for k, v in run.items(): + table_to_rows_archived.setdefault(k, 0) + table_to_rows_archived[k] += v + if not until_complete: + break + elif not run: + if verbose: + print('.' + _('complete')) # noqa + break + if verbose: + sys.stdout.write('.') if verbose: if table_to_rows_archived: utils.print_dict(table_to_rows_archived, _('Table'), diff --git a/nova/tests/unit/test_nova_manage.py b/nova/tests/unit/test_nova_manage.py index 7dcafd581f26..f6ae8efbe409 100644 --- a/nova/tests/unit/test_nova_manage.py +++ b/nova/tests/unit/test_nova_manage.py @@ -503,6 +503,70 @@ class DBCommandsTestCase(test.NoDBTestCase): # Tests that we get table output. self._test_archive_deleted_rows(verbose=True) + @mock.patch.object(db, 'archive_deleted_rows') + def test_archive_deleted_rows_until_complete(self, mock_db_archive, + verbose=False): + mock_db_archive.side_effect = [ + {'instances': 10, 'instance_extra': 5}, + {'instances': 5, 'instance_faults': 1}, + {}] + result = self.commands.archive_deleted_rows(20, verbose=verbose, + until_complete=True) + self.assertEqual(1, result) + if verbose: + expected = """\ +Archiving.....complete ++-----------------+-------------------------+ +| Table | Number of Rows Archived | ++-----------------+-------------------------+ +| instance_extra | 5 | +| instance_faults | 1 | +| instances | 15 | ++-----------------+-------------------------+ +""" + else: + expected = '' + + self.assertEqual(expected, self.output.getvalue()) + mock_db_archive.assert_has_calls([mock.call(20), + mock.call(20), + mock.call(20)]) + + def test_archive_deleted_rows_until_complete_quiet(self): + self.test_archive_deleted_rows_until_complete(verbose=False) + + @mock.patch.object(db, 'archive_deleted_rows') + def test_archive_deleted_rows_until_stopped(self, mock_db_archive, + verbose=True): + mock_db_archive.side_effect = [ + {'instances': 10, 'instance_extra': 5}, + {'instances': 5, 'instance_faults': 1}, + KeyboardInterrupt] + result = self.commands.archive_deleted_rows(20, verbose=verbose, + until_complete=True) + self.assertEqual(1, result) + if verbose: + expected = """\ +Archiving.....stopped ++-----------------+-------------------------+ +| Table | Number of Rows Archived | ++-----------------+-------------------------+ +| instance_extra | 5 | +| instance_faults | 1 | +| instances | 15 | ++-----------------+-------------------------+ +""" + else: + expected = '' + + self.assertEqual(expected, self.output.getvalue()) + mock_db_archive.assert_has_calls([mock.call(20), + mock.call(20), + mock.call(20)]) + + def test_archive_deleted_rows_until_stopped_quiet(self): + self.test_archive_deleted_rows_until_stopped(verbose=False) + @mock.patch.object(db, 'archive_deleted_rows', return_value={}) def test_archive_deleted_rows_verbose_no_results(self, mock_db_archive): result = self.commands.archive_deleted_rows(20, verbose=True) diff --git a/releasenotes/notes/archive-all-db-aadf2ce0394c24fa.yaml b/releasenotes/notes/archive-all-db-aadf2ce0394c24fa.yaml new file mode 100644 index 000000000000..f2a53a585320 --- /dev/null +++ b/releasenotes/notes/archive-all-db-aadf2ce0394c24fa.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Support for archiving all deleted rows from the database has + been added to the ``nova-manage db archive_deleted_rows`` + command. The ``--until-complete`` option will continuously + run the process until no more rows are available for archiving.