From 02adbab96924b5f4f12ccb148df1ef0242acb0df Mon Sep 17 00:00:00 2001 From: james lin Date: Wed, 5 Jun 2024 17:10:07 +0800 Subject: [PATCH] Add new action purge-data The action will run `nova-manage db purge --verbose --all` to delete rows from shadow tables. Related-Bug: 2066940 Change-Id: I4602a9cf38126b50bfe188b68a75cff1d2597342 --- actions.yaml | 9 +++- actions/actions.py | 11 +++++ actions/purge-data | 1 + hooks/nova_cc_utils.py | 28 ++++++++++++ unit_tests/test_nova_cc_utils.py | 74 ++++++++++++++++++++++++++++++++ 5 files changed, 122 insertions(+), 1 deletion(-) create mode 120000 actions/purge-data diff --git a/actions.yaml b/actions.yaml index 99f2d3b6..d6f5bf85 100644 --- a/actions.yaml +++ b/actions.yaml @@ -16,7 +16,14 @@ archive-data: batch-size: type: integer default: 10000 - description: Archive old data to shadow tables +purge-data: + description: Run job to purge stale soft deleted rows in database + params: + before: + type: string + description: | + Delete data from all shadow tables that is older than the date provided. + Date strings may be fuzzy, such as `Oct 21 2015`. security-checklist: description: | Validate the running configuration against the OpenStack security guides diff --git a/actions/actions.py b/actions/actions.py index 116999b0..3d2f9ead 100755 --- a/actions/actions.py +++ b/actions/actions.py @@ -48,6 +48,16 @@ def resume(args): utils.resume_unit_helper(utils.register_configs()) +def purge_data(args): + """Run data purge process + @raises Exception should the purge fail""" + hookenv.action_set({ + 'output': utils.purge_stale_soft_deleted_rows( + before=hookenv.action_get('before'), + ) + }) + + def archive_data(args): """Run data archival process @raises Exception should the archival fail""" @@ -198,6 +208,7 @@ ACTIONS = { "archive-data": archive_data, "clear-unit-knownhost-cache": clear_unit_knownhost_cache, "sync-compute-availability-zones": sync_compute_availability_zones, + "purge-data": purge_data, } diff --git a/actions/purge-data b/actions/purge-data new file mode 120000 index 00000000..405a394e --- /dev/null +++ b/actions/purge-data @@ -0,0 +1 @@ +actions.py \ No newline at end of file diff --git a/hooks/nova_cc_utils.py b/hooks/nova_cc_utils.py index 4dcc52ab..8c577902 100644 --- a/hooks/nova_cc_utils.py +++ b/hooks/nova_cc_utils.py @@ -946,6 +946,33 @@ def map_instances(): hookenv.log(msg, level=hookenv.INFO) +def purge_stale_soft_deleted_rows(before=""): + '''Purge all stale soft-deleted rows.''' + hookenv.log('Purging stale soft-deleted rows', level=hookenv.INFO) + cmd = ['nova-manage', 'db', 'purge', '--verbose'] + if before: + cmd.extend(['--before', str(before)]) + else: + cmd.extend(['--all']) + process = subprocess.Popen(cmd, stdout=subprocess.PIPE) + stdout, stderr = process.communicate() + exit_code = process.wait() + + # exit_code 3 means no data was deleted. + if exit_code not in [0, 3]: + msg = ( + 'Purging stale soft-deleted rows failed\nsstdout: {}\nsstderr: {}' + .format(stdout, stderr) + ) + hookenv.log(msg, level=hookenv.ERROR) + raise Exception(msg) + if exit_code == 3: + msg = 'Purging stale soft-deleted rows and no data was deleted' + hookenv.log(msg, level=hookenv.INFO) + return msg + return stdout + + def archive_deleted_rows(max_rows=None): hookenv.log('Archiving deleted rows', level=hookenv.INFO) cmd = ['nova-manage', 'db', 'archive_deleted_rows', '--verbose'] @@ -954,6 +981,7 @@ def archive_deleted_rows(max_rows=None): process = subprocess.Popen(cmd, stdout=subprocess.PIPE) stdout, stderr = process.communicate() exit_code = process.wait() + if exit_code not in [0, 1]: msg = 'Archiving deleted rows failed\nstdout: {}\nstderr: {}'.format( stdout, diff --git a/unit_tests/test_nova_cc_utils.py b/unit_tests/test_nova_cc_utils.py index 981f3644..8a9d091e 100644 --- a/unit_tests/test_nova_cc_utils.py +++ b/unit_tests/test_nova_cc_utils.py @@ -1622,6 +1622,80 @@ class NovaCCUtilsTests(CharmTestCase): with self.assertRaises(Exception): utils.archive_deleted_rows() + @patch('subprocess.Popen') + def test_purge_stale_soft_deleted_rows(self, mock_popen): + process_mock = MagicMock() + attrs = { + 'communicate.return_value': ('output', 'error'), + 'wait.return_value': 0} + process_mock.configure_mock(**attrs) + mock_popen.return_value = process_mock + expectd_calls = [ + call([ + 'nova-manage', + 'db', + 'purge', + '--verbose', + '--all'], stdout=-1), + call().communicate(), + call().wait()] + utils.purge_stale_soft_deleted_rows() + self.assertEqual(mock_popen.mock_calls, expectd_calls) + + @patch('subprocess.Popen') + def test_purge_stale_soft_deleted_rows_no_data(self, mock_popen): + process_mock = MagicMock() + attrs = { + 'communicate.return_value': ('output', 'error'), + 'wait.return_value': 3} + process_mock.configure_mock(**attrs) + mock_popen.return_value = process_mock + expectd_calls = [ + call([ + 'nova-manage', + 'db', + 'purge', + '--verbose', + '--all'], stdout=-1), + call().communicate(), + call().wait()] + msg = utils.purge_stale_soft_deleted_rows() + self.assertEqual(mock_popen.mock_calls, expectd_calls) + self.assertEqual( + msg, 'Purging stale soft-deleted rows and no data was deleted') + + @patch('subprocess.Popen') + def test_purge_stale_soft_deleted_rows_with_before(self, mock_popen): + process_mock = MagicMock() + attrs = { + 'communicate.return_value': ('output', 'error'), + 'wait.return_value': 0} + process_mock.configure_mock(**attrs) + mock_popen.return_value = process_mock + expectd_calls = [ + call([ + 'nova-manage', + 'db', + 'purge', + '--verbose', + '--before', + '2024-06-06'], stdout=-1), + call().communicate(), + call().wait()] + utils.purge_stale_soft_deleted_rows(before='2024-06-06') + self.assertEqual(mock_popen.mock_calls, expectd_calls) + + @patch('subprocess.Popen') + def test_purge_stale_soft_deleted_rows_exception(self, mock_popen): + process_mock = MagicMock() + attrs = { + 'communicate.return_value': ('output', 'error'), + 'wait.return_value': 123} + process_mock.configure_mock(**attrs) + mock_popen.return_value = process_mock + with self.assertRaises(Exception): + utils.purge_stale_soft_deleted_rows() + def test_is_serial_console_enabled_on_juno(self): self.os_release.return_value = 'juno' self.test_config.set('enable-serial-console', True)