From a86390aeabca27167e26f38afb202e7daf39b185 Mon Sep 17 00:00:00 2001 From: Rafael Lopez Date: Thu, 12 Jan 2023 05:01:59 +0000 Subject: [PATCH] Additional check to replace missing seeded file The additional check is based on cluster being bootstrapped and the last backup being a SST. The change includes new function for checking the last backup was SST and unittests to verify said function as well as the main charm_check_func where the check is used and seeded file is replaced. Closes-Bug: #2000107 Signed-off-by: Rafael Lopez Change-Id: I8e516059da5299cc0e0ce8ef0802d3a46abb1a54 --- hooks/percona_utils.py | 36 ++++++++++++++++++++- unit_tests/test_percona_utils.py | 54 ++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/hooks/percona_utils.py b/hooks/percona_utils.py index 1ff6a55..28294aa 100644 --- a/hooks/percona_utils.py +++ b/hooks/percona_utils.py @@ -87,6 +87,7 @@ KEY = "keys/repo.percona.com" REPO = """deb http://repo.percona.com/apt {release} main deb-src http://repo.percona.com/apt {release} main""" SEEDED_MARKER = "{data_dir}/seeded" +BACKUP_INFO = "{data_dir}/xtrabackup_info" HOSTS_FILE = '/etc/hosts' DEFAULT_MYSQL_PORT = 3306 INITIAL_CLUSTERED_KEY = 'initial-cluster-complete' @@ -166,6 +167,36 @@ def mark_seeded(): seeded.write('done') +def last_backup_sst(): + """ Check if the last backup was an SST + The percona xtrabackup_info file (BACKUP_INFO) contains information about + the last backup/sync to this node, including the type of backup, either + incremental (IST) or full (SST). + + We return True if we can successfully determine the last backup was SST + from the BACKUP_INFO file contents, otherwise we assume the last backup + was an incremental and return False. + + @returns boolean + """ + result = False + try: + with open(BACKUP_INFO.format(data_dir=resolve_data_dir()), 'r') as f: + lines = f.readlines() + for line in lines: + if re.match('^incremental = N($|\n)', line): + result = True + except FileNotFoundError: + log("""Backup info file not found: %s, assuming last backup was + incremental""" % + BACKUP_INFO.format(data_dir=resolve_data_dir()), level=DEBUG) + except Exception: + log("""Unable to read backup info file: %s, assuming last backup was + incremental""" % + BACKUP_INFO.format(data_dir=resolve_data_dir()), level=DEBUG) + return result + + def setup_percona_repo(): ''' Configure service unit to use percona repositories ''' with open('/etc/apt/sources.list.d/percona.list', 'w') as sources: @@ -702,7 +733,10 @@ def charm_check_func(ensure_seeded=False): :returns: (status, message) :rtype: Tuple[str, str] """ - if ensure_seeded and not seeded(): + # Ensure seeded file is replaced if told or after any SST event post + # bootstrap. resolves bug #2000107 + if (not seeded() and + (ensure_seeded or (is_bootstrapped() and last_backup_sst()))): log("'seeded' file is missing but should exists; putting it back.") mark_seeded() if is_unit_upgrading_set(): diff --git a/unit_tests/test_percona_utils.py b/unit_tests/test_percona_utils.py index e029910..072a520 100644 --- a/unit_tests/test_percona_utils.py +++ b/unit_tests/test_percona_utils.py @@ -643,6 +643,40 @@ class UtilsTests(CharmTestCase): thresholds = percona_utils.get_nrpe_threads_connected_thresholds() self.assertEqual(thresholds, (80, 90)) + def test_last_backup_sst(self): + # test backup info file when backup was SST + mock_read_data = 'incremental = N\n' + mock_open = mock.mock_open(read_data=mock_read_data) + with mock.patch('percona_utils.open', mock_open): + result = percona_utils.last_backup_sst() + self.assertEqual(result, True) + + # test backup info file when backup was IST + mock_read_data = 'incremental = Y\n' + mock_open = mock.mock_open(read_data=mock_read_data) + with mock.patch('percona_utils.open', mock_open): + result = percona_utils.last_backup_sst() + self.assertEqual(result, False) + + # test backup info file with other 'incremental' string + mock_read_data = 'something incremental = N\n' + mock_open = mock.mock_open(read_data=mock_read_data) + with mock.patch('percona_utils.open', mock_open): + result = percona_utils.last_backup_sst() + self.assertEqual(result, False) + + # test backup info file with two lines incremental + mock_read_data = 'incremental incremental = Y\nincremental = N\n' + mock_open = mock.mock_open(read_data=mock_read_data) + with mock.patch('percona_utils.open', mock_open): + result = percona_utils.last_backup_sst() + self.assertEqual(result, True) + + # test non existant backup info file + percona_utils.BACKUP_INFO = '/some/non/existant/file' + result = percona_utils.last_backup_sst() + self.assertEqual(result, False) + class UtilsTestsStatus(CharmTestCase): @@ -722,6 +756,26 @@ class UtilsTestsStatus(CharmTestCase): stat, _ = percona_utils.charm_check_func() assert stat == 'active' + @mock.patch.object(percona_utils, 'last_backup_sst') + def test_bootstrapped_seeded_missing_sst(self, mock_last_backup_sst): + self.is_bootstrapped.return_value = True + self.seeded.side_effect = [False, True] + self.config.return_value = None + percona_utils.SEEDED_MARKER = '/tmp/seeded' + mock_last_backup_sst.return_value = True + stat, _ = percona_utils.charm_check_func() + assert stat == 'active' + + @mock.patch.object(percona_utils, 'last_backup_sst') + def test_not_bootstrapped_seeded_missing_sst(self, mock_last_backup_sst): + self.is_bootstrapped.return_value = False + self.seeded.side_effect = [False, False] + self.config.return_value = None + percona_utils.SEEDED_MARKER = '/tmp/seeded' + mock_last_backup_sst.return_value = True + stat, _ = percona_utils.charm_check_func() + assert stat == 'waiting' + class UtilsTestsCTC(CharmTestCase): TO_PATCH = [