Additional check to replace missing seeded file

The additional check is based on cluster being bootstrapped and the last
backup being a SST.

The change includes new function for checking the last backup was SST and unittests to verify said function as well as the main charm_check_func where the check is used and seeded file is replaced.

Closes-Bug: #2000107
Signed-off-by: Rafael Lopez <rafael.lopez@canonical.com>
Change-Id: I8e516059da5299cc0e0ce8ef0802d3a46abb1a54

hooks/percona_utils.py

@@ -87,6 +87,7 @@ KEY = "keys/repo.percona.com"
 REPO = """deb http://repo.percona.com/apt {release} main
 deb-src http://repo.percona.com/apt {release} main"""
 SEEDED_MARKER = "{data_dir}/seeded"
+BACKUP_INFO = "{data_dir}/xtrabackup_info"
 HOSTS_FILE = '/etc/hosts'
 DEFAULT_MYSQL_PORT = 3306
 INITIAL_CLUSTERED_KEY = 'initial-cluster-complete'
@@ -166,6 +167,36 @@ def mark_seeded():
         seeded.write('done')
 
 
+def last_backup_sst():
+    """ Check if the last backup was an SST
+    The percona xtrabackup_info file (BACKUP_INFO) contains information about
+    the last backup/sync to this node, including the type of backup, either
+    incremental (IST) or full (SST).
+
+    We return True if we can successfully determine the last backup was SST
+    from the BACKUP_INFO file contents, otherwise we assume the last backup
+    was an incremental and return False.
+
+    @returns boolean
+    """
+    result = False
+    try:
+        with open(BACKUP_INFO.format(data_dir=resolve_data_dir()), 'r') as f:
+            lines = f.readlines()
+            for line in lines:
+                if re.match('^incremental = N($|\n)', line):
+                    result = True
+    except FileNotFoundError:
+        log("""Backup info file not found: %s, assuming last backup was
+            incremental""" %
+            BACKUP_INFO.format(data_dir=resolve_data_dir()), level=DEBUG)
+    except Exception:
+        log("""Unable to read backup info file: %s, assuming last backup was
+            incremental""" %
+            BACKUP_INFO.format(data_dir=resolve_data_dir()), level=DEBUG)
+    return result
+
+
 def setup_percona_repo():
     ''' Configure service unit to use percona repositories '''
     with open('/etc/apt/sources.list.d/percona.list', 'w') as sources:
@@ -702,7 +733,10 @@ def charm_check_func(ensure_seeded=False):
     :returns: (status, message)
     :rtype: Tuple[str, str]
     """
-    if ensure_seeded and not seeded():
+    # Ensure seeded file is replaced if told or after any SST event post
+    # bootstrap. resolves bug #2000107
+    if (not seeded() and
+            (ensure_seeded or (is_bootstrapped() and last_backup_sst()))):
         log("'seeded' file is missing but should exists; putting it back.")
         mark_seeded()
     if is_unit_upgrading_set():

unit_tests/test_percona_utils.py

@@ -643,6 +643,40 @@ class UtilsTests(CharmTestCase):
         thresholds = percona_utils.get_nrpe_threads_connected_thresholds()
         self.assertEqual(thresholds, (80, 90))
 
+    def test_last_backup_sst(self):
+        # test backup info file when backup was SST
+        mock_read_data = 'incremental = N\n'
+        mock_open = mock.mock_open(read_data=mock_read_data)
+        with mock.patch('percona_utils.open', mock_open):
+            result = percona_utils.last_backup_sst()
+        self.assertEqual(result, True)
+
+        # test backup info file when backup was IST
+        mock_read_data = 'incremental = Y\n'
+        mock_open = mock.mock_open(read_data=mock_read_data)
+        with mock.patch('percona_utils.open', mock_open):
+            result = percona_utils.last_backup_sst()
+        self.assertEqual(result, False)
+
+        # test backup info file with other 'incremental' string
+        mock_read_data = 'something incremental = N\n'
+        mock_open = mock.mock_open(read_data=mock_read_data)
+        with mock.patch('percona_utils.open', mock_open):
+            result = percona_utils.last_backup_sst()
+        self.assertEqual(result, False)
+
+        # test backup info file with two lines incremental
+        mock_read_data = 'incremental incremental = Y\nincremental = N\n'
+        mock_open = mock.mock_open(read_data=mock_read_data)
+        with mock.patch('percona_utils.open', mock_open):
+            result = percona_utils.last_backup_sst()
+        self.assertEqual(result, True)
+
+        # test non existant backup info file
+        percona_utils.BACKUP_INFO = '/some/non/existant/file'
+        result = percona_utils.last_backup_sst()
+        self.assertEqual(result, False)
+
 
 class UtilsTestsStatus(CharmTestCase):
 
@@ -722,6 +756,26 @@ class UtilsTestsStatus(CharmTestCase):
         stat, _ = percona_utils.charm_check_func()
         assert stat == 'active'
 
+    @mock.patch.object(percona_utils, 'last_backup_sst')
+    def test_bootstrapped_seeded_missing_sst(self, mock_last_backup_sst):
+        self.is_bootstrapped.return_value = True
+        self.seeded.side_effect = [False, True]
+        self.config.return_value = None
+        percona_utils.SEEDED_MARKER = '/tmp/seeded'
+        mock_last_backup_sst.return_value = True
+        stat, _ = percona_utils.charm_check_func()
+        assert stat == 'active'
+
+    @mock.patch.object(percona_utils, 'last_backup_sst')
+    def test_not_bootstrapped_seeded_missing_sst(self, mock_last_backup_sst):
+        self.is_bootstrapped.return_value = False
+        self.seeded.side_effect = [False, False]
+        self.config.return_value = None
+        percona_utils.SEEDED_MARKER = '/tmp/seeded'
+        mock_last_backup_sst.return_value = True
+        stat, _ = percona_utils.charm_check_func()
+        assert stat == 'waiting'
+
 
 class UtilsTestsCTC(CharmTestCase):
     TO_PATCH = [