openstack/nova
Code Issues Proposed changes

Make nova-manage db purge take --all-cells

Browse Source 
This makes purge iterate over all cells if requested. This also makes our
post_test_hook.sh use the --all-cells variant with just the base config
file.

Related to blueprint purge-db

Change-Id: I7eb5ed05224838cdba18e96724162cc930f4422e
This commit is contained in:
Dan Smith 2018-03-07 07:08:14 -08:00
parent ff47787e11
commit fd59fbd4d1
6 changed files with 88 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
doc/source/cli/nova-manage.rst
View File

@ -73,15 +73,17 @@ Nova Database
is desired for the purge, then run ``nova-manage db purge --before
<date>`` manually after archiving is complete.
``nova-manage db purge [--all] [--before <date>] [--verbose]``
``nova-manage db purge [--all] [--before <date>] [--verbose] [--all-cells]``
Delete rows from shadow tables. Specifying --all will delete all data from
all shadow tables. Specifying --before will delete data from all shadow tables
that is older than the date provided. Date strings may be fuzzy, such as
``Oct 21 2015``. Returns exit code 0 if rows were deleted, 1 if required
arguments are not provided, 2 if an invalid date is provided, 3 if no data
was deleted. Specifying --verbose will cause information to be printed about
purged records.
``Oct 21 2015``. Specifying --verbose will cause information to be printed about
purged records. Specifying --all-cells will cause the purge to be applied against
all cell databases. For --all-cells to work, the api database connection
information must be configured. Returns exit code 0 if rows were deleted, 1 if
required arguments are not provided, 2 if an invalid date is provided, 3 if no
data was deleted, 4 if the list of cells cannot be obtained.
``nova-manage db null_instance_uuid_scan [--delete]``

4
gate/post_test_hook.sh
View File

@ -19,7 +19,7 @@ function archive_deleted_rows {
}
function purge_db {
$MANAGE $* db purge --all --verbose
$MANAGE db purge --all --verbose --all-cells
RET=$?
if [[ $RET -eq 0 ]]; then
echo Purge successful
@ -40,7 +40,7 @@ cell_conf=$(conductor_conf 1)
conf="--config-file $NOVA_CONF --config-file $cell_conf"
archive_deleted_rows $conf
purge_db $conf
purge_db
set -e
# We need to get the admin credentials to run the OSC CLIs for Placement.

28
nova/cmd/manage.py
View File

@ -562,7 +562,10 @@ Error: %s""") % six.text_type(e))
help='Purge all rows in the shadow tables')
@args('--verbose', dest='verbose', action='store_true', default=False,
help='Print information about purged records')
def purge(self, before=None, purge_all=False, verbose=False):
@args('--all-cells', dest='all_cells', action='store_true', default=False,
help='Run against all cell databases')
def purge(self, before=None, purge_all=False, verbose=False,
all_cells=False):
if before is None and purge_all is False:
print(_('Either --before or --all is required'))
return 1
@ -577,9 +580,28 @@ Error: %s""") % six.text_type(e))
def status(msg):
if verbose:
print(msg)
print('%s: %s' % (identity, msg))
deleted = sa_db.purge_shadow_tables(before_date, status_fn=status)
deleted = 0
admin_ctxt = context.get_admin_context()
if all_cells:
try:
cells = objects.CellMappingList.get_all(admin_ctxt)
except db_exc.DBError:
print(_('Unable to get cell list from API DB. '
'Is it configured?'))
return 4
for cell in cells:
identity = _('Cell %s') % cell.identity
with context.target_cell(admin_ctxt, cell) as cctxt:
deleted += sa_db.purge_shadow_tables(cctxt,
before_date,
status_fn=status)
else:
identity = _('DB')
deleted = sa_db.purge_shadow_tables(admin_ctxt,
before_date, status_fn=status)
if deleted:
return 0
else:

4
nova/db/sqlalchemy/api.py
View File

@ -5926,8 +5926,8 @@ def _purgeable_tables(metadata):
t.name.endswith('migrate_version'))]
def purge_shadow_tables(before_date, status_fn=None):
engine = get_engine()
def purge_shadow_tables(context, before_date, status_fn=None):
engine = get_engine(context=context)
conn = engine.connect()
metadata = MetaData()
metadata.bind = engine

13
nova/tests/functional/db/test_archive.py
View File

@ -178,7 +178,9 @@ class TestDatabaseArchive(test_servers.ServersTestBase):
def status(msg):
lines.append(msg)
deleted = sqlalchemy_api.purge_shadow_tables(None, status_fn=status)
admin_context = context.get_admin_context()
deleted = sqlalchemy_api.purge_shadow_tables(admin_context,
None, status_fn=status)
self.assertNotEqual(0, deleted)
self.assertNotEqual(0, len(lines))
for line in lines:
@ -199,7 +201,9 @@ class TestDatabaseArchive(test_servers.ServersTestBase):
pre_purge_results = self._get_table_counts()
past = timeutils.utcnow() - datetime.timedelta(hours=1)
deleted = sqlalchemy_api.purge_shadow_tables(past)
admin_context = context.get_admin_context()
deleted = sqlalchemy_api.purge_shadow_tables(admin_context,
past)
# Make sure we didn't delete anything if the marker is before
# we started
self.assertEqual(0, deleted)
@ -209,7 +213,7 @@ class TestDatabaseArchive(test_servers.ServersTestBase):
self.assertEqual(pre_purge_results, results)
future = timeutils.utcnow() + datetime.timedelta(hours=1)
deleted = sqlalchemy_api.purge_shadow_tables(future)
deleted = sqlalchemy_api.purge_shadow_tables(admin_context, future)
# Make sure we deleted things when the marker is after
# we started
self.assertNotEqual(0, deleted)
@ -228,5 +232,6 @@ class TestDatabaseArchive(test_servers.ServersTestBase):
results, deleted_ids = db.archive_deleted_rows(max_rows=1000)
self.assertEqual([server_id], deleted_ids)
date = dateutil_parser.parse('oct 21 2015', fuzzy=True)
deleted = sqlalchemy_api.purge_shadow_tables(date)
admin_context = context.get_admin_context()
deleted = sqlalchemy_api.purge_shadow_tables(admin_context, date)
self.assertEqual(0, deleted)

46
nova/tests/unit/test_nova_manage.py
View File

@ -479,7 +479,8 @@ Rows were archived, running purge...
mock_db_archive.assert_has_calls([mock.call(20),
mock.call(20),
mock.call(20)])
mock_db_purge.assert_called_once_with(None, status_fn=mock.ANY)
mock_db_purge.assert_called_once_with(mock.ANY, None,
status_fn=mock.ANY)
def test_archive_deleted_rows_until_stopped_quiet(self):
self.test_archive_deleted_rows_until_stopped(verbose=False)
@ -547,14 +548,15 @@ Rows were archived, running purge...
mock_purge.return_value = 1
ret = self.commands.purge(purge_all=True)
self.assertEqual(0, ret)
mock_purge.assert_called_once_with(None, status_fn=mock.ANY)
mock_purge.assert_called_once_with(mock.ANY, None, status_fn=mock.ANY)
@mock.patch('nova.db.sqlalchemy.api.purge_shadow_tables')
def test_purge_date(self, mock_purge):
mock_purge.return_value = 1
ret = self.commands.purge(before='oct 21 2015')
self.assertEqual(0, ret)
mock_purge.assert_called_once_with(datetime.datetime(2015, 10, 21),
mock_purge.assert_called_once_with(mock.ANY,
datetime.datetime(2015, 10, 21),
status_fn=mock.ANY)
@mock.patch('nova.db.sqlalchemy.api.purge_shadow_tables')
@ -575,6 +577,44 @@ Rows were archived, running purge...
ret = self.commands.purge(purge_all=True)
self.assertEqual(3, ret)
@mock.patch('nova.db.sqlalchemy.api.purge_shadow_tables')
@mock.patch('nova.objects.CellMappingList.get_all')
def test_purge_all_cells(self, mock_get_cells, mock_purge):
cell1 = objects.CellMapping(uuid=uuidsentinel.cell1, name='cell1',
database_connection='foo1',
transport_url='bar1')
cell2 = objects.CellMapping(uuid=uuidsentinel.cell2, name='cell2',
database_connection='foo2',
transport_url='bar2')
mock_get_cells.return_value = [cell1, cell2]
values = [123, 456]
def fake_purge(*args, **kwargs):
val = values.pop(0)
kwargs['status_fn'](val)
return val
mock_purge.side_effect = fake_purge
ret = self.commands.purge(purge_all=True, all_cells=True, verbose=True)
self.assertEqual(0, ret)
mock_get_cells.assert_called_once_with(mock.ANY)
output = self.output.getvalue()
expected = """\
Cell %s: 123
Cell %s: 456
""" % (cell1.identity, cell2.identity)
self.assertEqual(expected, output)
@mock.patch('nova.objects.CellMappingList.get_all')
def test_purge_all_cells_no_api_config(self, mock_get_cells):
mock_get_cells.side_effect = db_exc.DBError
ret = self.commands.purge(purge_all=True, all_cells=True)
self.assertEqual(4, ret)
self.assertIn('Unable to get cell list', self.output.getvalue())
@mock.patch.object(migration, 'db_null_instance_uuid_scan',
return_value={'foo': 0})
def test_null_instance_uuid_scan_no_records_found(self, mock_scan):