From fc77ce191fe9393be815bc32e6da867808dab054 Mon Sep 17 00:00:00 2001 From: melanie witt Date: Thu, 21 Jan 2021 04:14:51 +0000 Subject: [PATCH] Add --sleep option for archive_deleted_rows --until-complete Currently, when 'nova-manage db archive_deleted_rows' is run with the --until-complete option, the process will archive rows in batches in a tight loop, which can cause problems in busy environments where the aggressive archiving interferes with other requests trying to write to the database. This adds an option for users to specify an amount of time in seconds to sleep between batches of rows while archiving with --until-complete, allowing the process to be throttled. Closes-Bug: #1912579 Change-Id: I638b2fa78b81919373e607458e6f68a7983a79aa --- doc/source/cli/nova-manage.rst | 10 ++++++-- nova/cmd/manage.py | 25 ++++++++++++++----- nova/tests/unit/cmd/test_manage.py | 14 ++++++++--- .../notes/archive-sleep-a0cc3d3e7784e5df.yaml | 11 ++++++++ 4 files changed, 49 insertions(+), 11 deletions(-) create mode 100644 releasenotes/notes/archive-sleep-a0cc3d3e7784e5df.yaml diff --git a/doc/source/cli/nova-manage.rst b/doc/source/cli/nova-manage.rst index 8354e9bc2b46..4dc614db2aa9 100644 --- a/doc/source/cli/nova-manage.rst +++ b/doc/source/cli/nova-manage.rst @@ -227,6 +227,7 @@ db archive_deleted_rows nova-manage db archive_deleted_rows [--max_rows ] [--verbose] [--until-complete] [--before ] [--purge] [--all-cells] [--task-log] + [--sleep] Move deleted rows from production tables to shadow tables. Note that the corresponding rows in the ``instance_mappings``, ``request_specs`` and @@ -240,7 +241,7 @@ stopping at 0, or use the :option:`--until-complete` option. .. versionchanged:: 24.0.0 (Xena) - Added :option:`--task-log` option. + Added :option:`--task-log`, :option:`--sleep` options. .. rubric:: Options @@ -295,7 +296,12 @@ stopping at 0, or use the :option:`--until-complete` option. record data via the `/os-instance_usage_audit_log`__ API (example: Telemetry). -.. __: https://docs.openstack.org/api-ref/compute/#server-usage-audit-log-os-instance-usage-audit-log + .. __: https://docs.openstack.org/api-ref/compute/#server-usage-audit-log-os-instance-usage-audit-log + +.. option:: --sleep + + The amount of time in seconds to sleep between batches when + :option:`--until-complete` is used. Defaults to 0. .. rubric:: Return codes diff --git a/nova/cmd/manage.py b/nova/cmd/manage.py index 221fed443724..95c455d94861 100644 --- a/nova/cmd/manage.py +++ b/nova/cmd/manage.py @@ -26,6 +26,7 @@ import functools import os import re import sys +import time import traceback from urllib import parse as urlparse @@ -249,9 +250,14 @@ class DbCommands(object): '``--before`` option to avoid races for those consuming ' '``task_log`` record data via the ' '``/os-instance_usage_audit_log`` API (example: Telemetry).')) - def archive_deleted_rows(self, max_rows=1000, verbose=False, - until_complete=False, purge=False, - before=None, all_cells=False, task_log=False): + @args('--sleep', type=int, metavar='', dest='sleep', + help='The amount of time in seconds to sleep between batches when ' + '``--until-complete`` is used. Defaults to 0.') + def archive_deleted_rows( + self, max_rows=1000, verbose=False, + until_complete=False, purge=False, + before=None, all_cells=False, task_log=False, sleep=0, + ): """Move deleted rows from production tables to shadow tables. Returns 0 if nothing was archived, 1 if some number of rows were @@ -344,7 +350,8 @@ class DbCommands(object): verbose, before_date, cell_name, - task_log) + task_log, + sleep) except KeyboardInterrupt: interrupt = True break @@ -377,8 +384,10 @@ class DbCommands(object): # NOTE(danms): Return nonzero if we archived something return int(bool(table_to_rows_archived)) - def _do_archive(self, table_to_rows_archived, cctxt, max_rows, - until_complete, verbose, before_date, cell_name, task_log): + def _do_archive( + self, table_to_rows_archived, cctxt, max_rows, + until_complete, verbose, before_date, cell_name, task_log, sleep, + ): """Helper function for archiving deleted rows for a cell. This will archive deleted rows for a cell database and remove the @@ -398,6 +407,8 @@ class DbCommands(object): :param cell_name: Name of the cell or None if not archiving across all cells :param task_log: Whether to archive task_log table rows + :param sleep: The amount of time in seconds to sleep between batches + when ``until_complete`` is True. """ ctxt = context.get_admin_context() while True: @@ -437,6 +448,8 @@ class DbCommands(object): break if verbose: sys.stdout.write('.') + # Optionally sleep between batches to throttle the archiving. + time.sleep(sleep) return total_rows_archived @args('--before', metavar='', dest='before', diff --git a/nova/tests/unit/cmd/test_manage.py b/nova/tests/unit/cmd/test_manage.py index fdf0a6d3aafa..4f9c732c9443 100644 --- a/nova/tests/unit/cmd/test_manage.py +++ b/nova/tests/unit/cmd/test_manage.py @@ -315,17 +315,20 @@ Archiving.....complete # Tests that we get table output. self._test_archive_deleted_rows(verbose=True) + @mock.patch('time.sleep') @mock.patch.object(db, 'archive_deleted_rows') @mock.patch.object(objects.CellMappingList, 'get_all') def test_archive_deleted_rows_until_complete(self, mock_get_all, - mock_db_archive, - verbose=False): + mock_db_archive, mock_sleep, + verbose=False, + sleep=0): mock_db_archive.side_effect = [ ({'instances': 10, 'instance_extra': 5}, list(), 15), ({'instances': 5, 'instance_faults': 1}, list(), 6), ({}, list(), 0)] result = self.commands.archive_deleted_rows(20, verbose=verbose, - until_complete=True) + until_complete=True, + sleep=sleep) self.assertEqual(1, result) if verbose: expected = """\ @@ -353,10 +356,15 @@ Archiving.....complete test.MatchType(context.RequestContext), 20, before=None, task_log=False), ]) + self.assertEqual(2, mock_sleep.call_count) + mock_sleep.assert_has_calls([mock.call(sleep), mock.call(sleep)]) def test_archive_deleted_rows_until_complete_quiet(self): self.test_archive_deleted_rows_until_complete(verbose=False) + def test_archive_deleted_rows_until_complete_sleep(self): + self.test_archive_deleted_rows_until_complete(sleep=30) + @mock.patch('nova.db.main.api.purge_shadow_tables') @mock.patch.object(db, 'archive_deleted_rows') @mock.patch.object(objects.CellMappingList, 'get_all') diff --git a/releasenotes/notes/archive-sleep-a0cc3d3e7784e5df.yaml b/releasenotes/notes/archive-sleep-a0cc3d3e7784e5df.yaml new file mode 100644 index 000000000000..9715567fd0a8 --- /dev/null +++ b/releasenotes/notes/archive-sleep-a0cc3d3e7784e5df.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + A ``--sleep`` option has been added to the ``nova-manage db + archive_deleted_rows`` CLI. When this command is run with the + ``--until-complete`` option, the process will archive rows in batches + in a tight loop, which can cause problems in busy environments where + the aggressive archiving interferes with other requests trying to write + to the database. The ``--sleep`` option can be used to specify a time to + sleep between batches of rows while archiving with ``--until-complete``, + allowing the process to be throttled.