openstack/nova
Code Issues Proposed changes

Allow running db archiving continuously

Browse Source 
This adds a --until_complete flag to the db archive command, which will
attempt to run batches of max_rows continuously until archiving has
completed (or stops being able to archive anything). We also take care
to catch KeyboardInterrupt so that it will, if stopped, still print
the verbose report at the end. Instead of printing the table on each
iteration, we print some progress information and then provide a completed
summary table at the end of the run.

This also defaults max_rows to a sane value. The previous meaning of not
providing a value was "archive a whole table in a single operation",
which was useful only in the most simple of cases. Now that we have
a more reasonable "do it until you are done" mechanism, we default this
to 1000 rows for sanity, which also serves as the batch size for the
continuous mode.

Implements blueprint archive-deleted-rows-all
Change-Id: I2929c2f5b3116b63a4666b258bf078bc51d4e45a
This commit is contained in:
Dan Smith 2016-09-28 08:26:22 -07:00
parent 85bf71b2ae
commit 6f00d3be95
3 changed files with 110 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
nova/cmd/manage.py
View File

@ -825,28 +825,53 @@ class DbCommands(object):
"""Print the current database version."""
print(migration.db_version())
@args('--max_rows', metavar='<number>',
@args('--max_rows', metavar='<number>', default=1000,
help='Maximum number of deleted rows to archive')
@args('--verbose', action='store_true', dest='verbose', default=False,
help='Print how many rows were archived per table.')
def archive_deleted_rows(self, max_rows, verbose=False):
"""Move up to max_rows deleted rows from production tables to shadow
tables.
@args('--until-complete', action='store_true', dest='until_complete',
default=False,
help=('Run continuously until all deleted rows are archived. Use '
'max_rows as a batch size for each iteration.'))
def archive_deleted_rows(self, max_rows, verbose=False,
until_complete=False):
"""Move deleted rows from production tables to shadow tables.
Returns 0 if nothing was archived, 1 if some number of rows were
archived, 2 if max_rows is invalid. If automating, this should be
run continuously while the result is 1, stopping at 0.
"""
if max_rows is not None:
max_rows = int(max_rows)
if max_rows < 0:
print(_("Must supply a positive value for max_rows"))
return(2)
if max_rows > db.MAX_INT:
print(_('max rows must be <= %(max_value)d') %
{'max_value': db.MAX_INT})
return(2)
table_to_rows_archived = db.archive_deleted_rows(max_rows)
max_rows = int(max_rows)
if max_rows < 0:
print(_("Must supply a positive value for max_rows"))
return(2)
if max_rows > db.MAX_INT:
print(_('max rows must be <= %(max_value)d') %
{'max_value': db.MAX_INT})
return(2)
table_to_rows_archived = {}
if until_complete and verbose:
sys.stdout.write(_('Archiving') + '..') # noqa
while True:
try:
run = db.archive_deleted_rows(max_rows)
except KeyboardInterrupt:
run = {}
if until_complete and verbose:
print('.' + _('stopped')) # noqa
break
for k, v in run.items():
table_to_rows_archived.setdefault(k, 0)
table_to_rows_archived[k] += v
if not until_complete:
break
elif not run:
if verbose:
print('.' + _('complete')) # noqa
break
if verbose:
sys.stdout.write('.')
if verbose:
if table_to_rows_archived:
utils.print_dict(table_to_rows_archived, _('Table'),

64
nova/tests/unit/test_nova_manage.py
View File

@ -503,6 +503,70 @@ class DBCommandsTestCase(test.NoDBTestCase):
# Tests that we get table output.
self._test_archive_deleted_rows(verbose=True)
@mock.patch.object(db, 'archive_deleted_rows')
def test_archive_deleted_rows_until_complete(self, mock_db_archive,
verbose=False):
mock_db_archive.side_effect = [
{'instances': 10, 'instance_extra': 5},
{'instances': 5, 'instance_faults': 1},
{}]
result = self.commands.archive_deleted_rows(20, verbose=verbose,
until_complete=True)
self.assertEqual(1, result)
if verbose:
expected = """\
Archiving.....complete
+-----------------+-------------------------+
| Table | Number of Rows Archived |
+-----------------+-------------------------+
| instance_extra | 5 |
| instance_faults | 1 |
| instances | 15 |
+-----------------+-------------------------+
"""
else:
expected = ''
self.assertEqual(expected, self.output.getvalue())
mock_db_archive.assert_has_calls([mock.call(20),
mock.call(20),
mock.call(20)])
def test_archive_deleted_rows_until_complete_quiet(self):
self.test_archive_deleted_rows_until_complete(verbose=False)
@mock.patch.object(db, 'archive_deleted_rows')
def test_archive_deleted_rows_until_stopped(self, mock_db_archive,
verbose=True):
mock_db_archive.side_effect = [
{'instances': 10, 'instance_extra': 5},
{'instances': 5, 'instance_faults': 1},
KeyboardInterrupt]
result = self.commands.archive_deleted_rows(20, verbose=verbose,
until_complete=True)
self.assertEqual(1, result)
if verbose:
expected = """\
Archiving.....stopped
+-----------------+-------------------------+
| Table | Number of Rows Archived |
+-----------------+-------------------------+
| instance_extra | 5 |
| instance_faults | 1 |
| instances | 15 |
+-----------------+-------------------------+
"""
else:
expected = ''
self.assertEqual(expected, self.output.getvalue())
mock_db_archive.assert_has_calls([mock.call(20),
mock.call(20),
mock.call(20)])
def test_archive_deleted_rows_until_stopped_quiet(self):
self.test_archive_deleted_rows_until_stopped(verbose=False)
@mock.patch.object(db, 'archive_deleted_rows', return_value={})
def test_archive_deleted_rows_verbose_no_results(self, mock_db_archive):
result = self.commands.archive_deleted_rows(20, verbose=True)

7
releasenotes/notes/archive-all-db-aadf2ce0394c24fa.yaml Normal file
View File

@ -0,0 +1,7 @@
---
features:
- |
Support for archiving all deleted rows from the database has
been added to the ``nova-manage db archive_deleted_rows``
command. The ``--until-complete`` option will continuously
run the process until no more rows are available for archiving.