Add --sleep option for archive_deleted_rows --until-complete

Currently, when 'nova-manage db archive_deleted_rows' is run with
the --until-complete option, the process will archive rows in batches
in a tight loop, which can cause problems in busy environments where
the aggressive archiving interferes with other requests trying to write
to the database.

This adds an option for users to specify an amount of time in seconds
to sleep between batches of rows while archiving with --until-complete,
allowing the process to be throttled.

Closes-Bug: #1912579

Change-Id: I638b2fa78b81919373e607458e6f68a7983a79aa

doc/source/cli/nova-manage.rst

@@ -227,6 +227,7 @@ db archive_deleted_rows
 
     nova-manage db archive_deleted_rows [--max_rows <rows>] [--verbose]
       [--until-complete] [--before <date>] [--purge] [--all-cells] [--task-log]
+      [--sleep]
 
 Move deleted rows from production tables to shadow tables. Note that the
 corresponding rows in the ``instance_mappings``, ``request_specs`` and
@@ -240,7 +241,7 @@ stopping at 0, or use the :option:`--until-complete` option.
 
 .. versionchanged:: 24.0.0 (Xena)
 
-    Added :option:`--task-log` option.
+    Added :option:`--task-log`, :option:`--sleep` options.
 
 .. rubric:: Options
 
@@ -295,7 +296,12 @@ stopping at 0, or use the :option:`--until-complete` option.
     record data via the `/os-instance_usage_audit_log`__ API (example:
     Telemetry).
 
-.. __: https://docs.openstack.org/api-ref/compute/#server-usage-audit-log-os-instance-usage-audit-log
+    .. __: https://docs.openstack.org/api-ref/compute/#server-usage-audit-log-os-instance-usage-audit-log
+
+.. option:: --sleep
+
+    The amount of time in seconds to sleep between batches when
+    :option:`--until-complete` is used. Defaults to 0.
 
 .. rubric:: Return codes
 

nova/cmd/manage.py

@@ -26,6 +26,7 @@ import functools
 import os
 import re
 import sys
+import time
 import traceback
 from urllib import parse as urlparse
 
@@ -249,9 +250,14 @@ class DbCommands(object):
                 '``--before`` option to avoid races for those consuming '
                 '``task_log`` record data via the '
                 '``/os-instance_usage_audit_log`` API (example: Telemetry).'))
-    def archive_deleted_rows(self, max_rows=1000, verbose=False,
-                             until_complete=False, purge=False,
-                             before=None, all_cells=False, task_log=False):
+    @args('--sleep', type=int, metavar='<seconds>', dest='sleep',
+          help='The amount of time in seconds to sleep between batches when '
+               '``--until-complete`` is used. Defaults to 0.')
+    def archive_deleted_rows(
+        self, max_rows=1000, verbose=False,
+        until_complete=False, purge=False,
+        before=None, all_cells=False, task_log=False, sleep=0,
+    ):
         """Move deleted rows from production tables to shadow tables.
 
         Returns 0 if nothing was archived, 1 if some number of rows were
@@ -344,7 +350,8 @@ class DbCommands(object):
                         verbose,
                         before_date,
                         cell_name,
-                        task_log)
+                        task_log,
+                        sleep)
                 except KeyboardInterrupt:
                     interrupt = True
                     break
@@ -377,8 +384,10 @@ class DbCommands(object):
         # NOTE(danms): Return nonzero if we archived something
         return int(bool(table_to_rows_archived))
 
-    def _do_archive(self, table_to_rows_archived, cctxt, max_rows,
-                    until_complete, verbose, before_date, cell_name, task_log):
+    def _do_archive(
+        self, table_to_rows_archived, cctxt, max_rows,
+        until_complete, verbose, before_date, cell_name, task_log, sleep,
+    ):
         """Helper function for archiving deleted rows for a cell.
 
         This will archive deleted rows for a cell database and remove the
@@ -398,6 +407,8 @@ class DbCommands(object):
         :param cell_name: Name of the cell or None if not archiving across all
             cells
         :param task_log: Whether to archive task_log table rows
+        :param sleep: The amount of time in seconds to sleep between batches
+            when ``until_complete`` is True.
         """
         ctxt = context.get_admin_context()
         while True:
@@ -437,6 +448,8 @@ class DbCommands(object):
                 break
             if verbose:
                 sys.stdout.write('.')
+            # Optionally sleep between batches to throttle the archiving.
+            time.sleep(sleep)
         return total_rows_archived
 
     @args('--before', metavar='<before>', dest='before',

nova/tests/unit/cmd/test_manage.py

@@ -315,17 +315,20 @@ Archiving.....complete
         # Tests that we get table output.
         self._test_archive_deleted_rows(verbose=True)
 
+    @mock.patch('time.sleep')
     @mock.patch.object(db, 'archive_deleted_rows')
     @mock.patch.object(objects.CellMappingList, 'get_all')
     def test_archive_deleted_rows_until_complete(self, mock_get_all,
-                                                 mock_db_archive,
-                                                 verbose=False):
+                                                 mock_db_archive, mock_sleep,
+                                                 verbose=False,
+                                                 sleep=0):
         mock_db_archive.side_effect = [
             ({'instances': 10, 'instance_extra': 5}, list(), 15),
             ({'instances': 5, 'instance_faults': 1}, list(), 6),
             ({}, list(), 0)]
         result = self.commands.archive_deleted_rows(20, verbose=verbose,
-                                                    until_complete=True)
+                                                    until_complete=True,
+                                                    sleep=sleep)
         self.assertEqual(1, result)
         if verbose:
             expected = """\
@@ -353,10 +356,15 @@ Archiving.....complete
                 test.MatchType(context.RequestContext), 20, before=None,
                 task_log=False),
         ])
+        self.assertEqual(2, mock_sleep.call_count)
+        mock_sleep.assert_has_calls([mock.call(sleep), mock.call(sleep)])
 
     def test_archive_deleted_rows_until_complete_quiet(self):
         self.test_archive_deleted_rows_until_complete(verbose=False)
 
+    def test_archive_deleted_rows_until_complete_sleep(self):
+        self.test_archive_deleted_rows_until_complete(sleep=30)
+
     @mock.patch('nova.db.main.api.purge_shadow_tables')
     @mock.patch.object(db, 'archive_deleted_rows')
     @mock.patch.object(objects.CellMappingList, 'get_all')

releasenotes/notes/archive-sleep-a0cc3d3e7784e5df.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    A ``--sleep`` option has been added to the ``nova-manage db
+    archive_deleted_rows`` CLI. When this command is run with the
+    ``--until-complete`` option, the process will archive rows in batches
+    in a tight loop, which can cause problems in busy environments where
+    the aggressive archiving interferes with other requests trying to write
+    to the database. The ``--sleep`` option can be used to specify a time to
+    sleep between batches of rows while archiving with ``--until-complete``,
+    allowing the process to be throttled.