diff --git a/doc/source/cli/nova-manage.rst b/doc/source/cli/nova-manage.rst index ab33e7c4b619..36f6fe189d60 100644 --- a/doc/source/cli/nova-manage.rst +++ b/doc/source/cli/nova-manage.rst @@ -73,15 +73,17 @@ Nova Database is desired for the purge, then run ``nova-manage db purge --before `` manually after archiving is complete. -``nova-manage db purge [--all] [--before ] [--verbose]`` +``nova-manage db purge [--all] [--before ] [--verbose] [--all-cells]`` Delete rows from shadow tables. Specifying --all will delete all data from all shadow tables. Specifying --before will delete data from all shadow tables that is older than the date provided. Date strings may be fuzzy, such as - ``Oct 21 2015``. Returns exit code 0 if rows were deleted, 1 if required - arguments are not provided, 2 if an invalid date is provided, 3 if no data - was deleted. Specifying --verbose will cause information to be printed about - purged records. + ``Oct 21 2015``. Specifying --verbose will cause information to be printed about + purged records. Specifying --all-cells will cause the purge to be applied against + all cell databases. For --all-cells to work, the api database connection + information must be configured. Returns exit code 0 if rows were deleted, 1 if + required arguments are not provided, 2 if an invalid date is provided, 3 if no + data was deleted, 4 if the list of cells cannot be obtained. ``nova-manage db null_instance_uuid_scan [--delete]`` diff --git a/gate/post_test_hook.sh b/gate/post_test_hook.sh index c56e15569f4c..b27274b049a4 100755 --- a/gate/post_test_hook.sh +++ b/gate/post_test_hook.sh @@ -19,7 +19,7 @@ function archive_deleted_rows { } function purge_db { - $MANAGE $* db purge --all --verbose + $MANAGE db purge --all --verbose --all-cells RET=$? if [[ $RET -eq 0 ]]; then echo Purge successful @@ -40,7 +40,7 @@ cell_conf=$(conductor_conf 1) conf="--config-file $NOVA_CONF --config-file $cell_conf" archive_deleted_rows $conf -purge_db $conf +purge_db set -e # We need to get the admin credentials to run the OSC CLIs for Placement. diff --git a/nova/cmd/manage.py b/nova/cmd/manage.py index 656dfa8d853a..9a5975541d17 100644 --- a/nova/cmd/manage.py +++ b/nova/cmd/manage.py @@ -562,7 +562,10 @@ Error: %s""") % six.text_type(e)) help='Purge all rows in the shadow tables') @args('--verbose', dest='verbose', action='store_true', default=False, help='Print information about purged records') - def purge(self, before=None, purge_all=False, verbose=False): + @args('--all-cells', dest='all_cells', action='store_true', default=False, + help='Run against all cell databases') + def purge(self, before=None, purge_all=False, verbose=False, + all_cells=False): if before is None and purge_all is False: print(_('Either --before or --all is required')) return 1 @@ -577,9 +580,28 @@ Error: %s""") % six.text_type(e)) def status(msg): if verbose: - print(msg) + print('%s: %s' % (identity, msg)) - deleted = sa_db.purge_shadow_tables(before_date, status_fn=status) + deleted = 0 + admin_ctxt = context.get_admin_context() + + if all_cells: + try: + cells = objects.CellMappingList.get_all(admin_ctxt) + except db_exc.DBError: + print(_('Unable to get cell list from API DB. ' + 'Is it configured?')) + return 4 + for cell in cells: + identity = _('Cell %s') % cell.identity + with context.target_cell(admin_ctxt, cell) as cctxt: + deleted += sa_db.purge_shadow_tables(cctxt, + before_date, + status_fn=status) + else: + identity = _('DB') + deleted = sa_db.purge_shadow_tables(admin_ctxt, + before_date, status_fn=status) if deleted: return 0 else: diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 54b28013cc5d..5c9388da8060 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -5926,8 +5926,8 @@ def _purgeable_tables(metadata): t.name.endswith('migrate_version'))] -def purge_shadow_tables(before_date, status_fn=None): - engine = get_engine() +def purge_shadow_tables(context, before_date, status_fn=None): + engine = get_engine(context=context) conn = engine.connect() metadata = MetaData() metadata.bind = engine diff --git a/nova/tests/functional/db/test_archive.py b/nova/tests/functional/db/test_archive.py index f51c0b14921e..c6bd59556bd7 100644 --- a/nova/tests/functional/db/test_archive.py +++ b/nova/tests/functional/db/test_archive.py @@ -178,7 +178,9 @@ class TestDatabaseArchive(test_servers.ServersTestBase): def status(msg): lines.append(msg) - deleted = sqlalchemy_api.purge_shadow_tables(None, status_fn=status) + admin_context = context.get_admin_context() + deleted = sqlalchemy_api.purge_shadow_tables(admin_context, + None, status_fn=status) self.assertNotEqual(0, deleted) self.assertNotEqual(0, len(lines)) for line in lines: @@ -199,7 +201,9 @@ class TestDatabaseArchive(test_servers.ServersTestBase): pre_purge_results = self._get_table_counts() past = timeutils.utcnow() - datetime.timedelta(hours=1) - deleted = sqlalchemy_api.purge_shadow_tables(past) + admin_context = context.get_admin_context() + deleted = sqlalchemy_api.purge_shadow_tables(admin_context, + past) # Make sure we didn't delete anything if the marker is before # we started self.assertEqual(0, deleted) @@ -209,7 +213,7 @@ class TestDatabaseArchive(test_servers.ServersTestBase): self.assertEqual(pre_purge_results, results) future = timeutils.utcnow() + datetime.timedelta(hours=1) - deleted = sqlalchemy_api.purge_shadow_tables(future) + deleted = sqlalchemy_api.purge_shadow_tables(admin_context, future) # Make sure we deleted things when the marker is after # we started self.assertNotEqual(0, deleted) @@ -228,5 +232,6 @@ class TestDatabaseArchive(test_servers.ServersTestBase): results, deleted_ids = db.archive_deleted_rows(max_rows=1000) self.assertEqual([server_id], deleted_ids) date = dateutil_parser.parse('oct 21 2015', fuzzy=True) - deleted = sqlalchemy_api.purge_shadow_tables(date) + admin_context = context.get_admin_context() + deleted = sqlalchemy_api.purge_shadow_tables(admin_context, date) self.assertEqual(0, deleted) diff --git a/nova/tests/unit/test_nova_manage.py b/nova/tests/unit/test_nova_manage.py index 24fe11676d34..e4e14c7cf4b1 100644 --- a/nova/tests/unit/test_nova_manage.py +++ b/nova/tests/unit/test_nova_manage.py @@ -479,7 +479,8 @@ Rows were archived, running purge... mock_db_archive.assert_has_calls([mock.call(20), mock.call(20), mock.call(20)]) - mock_db_purge.assert_called_once_with(None, status_fn=mock.ANY) + mock_db_purge.assert_called_once_with(mock.ANY, None, + status_fn=mock.ANY) def test_archive_deleted_rows_until_stopped_quiet(self): self.test_archive_deleted_rows_until_stopped(verbose=False) @@ -547,14 +548,15 @@ Rows were archived, running purge... mock_purge.return_value = 1 ret = self.commands.purge(purge_all=True) self.assertEqual(0, ret) - mock_purge.assert_called_once_with(None, status_fn=mock.ANY) + mock_purge.assert_called_once_with(mock.ANY, None, status_fn=mock.ANY) @mock.patch('nova.db.sqlalchemy.api.purge_shadow_tables') def test_purge_date(self, mock_purge): mock_purge.return_value = 1 ret = self.commands.purge(before='oct 21 2015') self.assertEqual(0, ret) - mock_purge.assert_called_once_with(datetime.datetime(2015, 10, 21), + mock_purge.assert_called_once_with(mock.ANY, + datetime.datetime(2015, 10, 21), status_fn=mock.ANY) @mock.patch('nova.db.sqlalchemy.api.purge_shadow_tables') @@ -575,6 +577,44 @@ Rows were archived, running purge... ret = self.commands.purge(purge_all=True) self.assertEqual(3, ret) + @mock.patch('nova.db.sqlalchemy.api.purge_shadow_tables') + @mock.patch('nova.objects.CellMappingList.get_all') + def test_purge_all_cells(self, mock_get_cells, mock_purge): + cell1 = objects.CellMapping(uuid=uuidsentinel.cell1, name='cell1', + database_connection='foo1', + transport_url='bar1') + cell2 = objects.CellMapping(uuid=uuidsentinel.cell2, name='cell2', + database_connection='foo2', + transport_url='bar2') + + mock_get_cells.return_value = [cell1, cell2] + + values = [123, 456] + + def fake_purge(*args, **kwargs): + val = values.pop(0) + kwargs['status_fn'](val) + return val + mock_purge.side_effect = fake_purge + + ret = self.commands.purge(purge_all=True, all_cells=True, verbose=True) + self.assertEqual(0, ret) + mock_get_cells.assert_called_once_with(mock.ANY) + output = self.output.getvalue() + expected = """\ +Cell %s: 123 +Cell %s: 456 +""" % (cell1.identity, cell2.identity) + + self.assertEqual(expected, output) + + @mock.patch('nova.objects.CellMappingList.get_all') + def test_purge_all_cells_no_api_config(self, mock_get_cells): + mock_get_cells.side_effect = db_exc.DBError + ret = self.commands.purge(purge_all=True, all_cells=True) + self.assertEqual(4, ret) + self.assertIn('Unable to get cell list', self.output.getvalue()) + @mock.patch.object(migration, 'db_null_instance_uuid_scan', return_value={'foo': 0}) def test_null_instance_uuid_scan_no_records_found(self, mock_scan):