Display bootstrap grstate after cold boot

To assist in DB cluster recovery after a cold boot, display the
grstate sequence number and the safe to boot status of the instance
in workload status.

Change-Id: Ia4e0e86e7d10b2b22148237688ff77cac4ebee7d

hooks/percona_hooks.py

@@ -110,6 +110,7 @@ from percona_utils import (
     is_bootstrapped,
     clustered_once,
     INITIAL_CLUSTERED_KEY,
+    INITIAL_CLIENT_UPDATE_KEY,
     is_leader_bootstrapped,
     get_wsrep_value,
     assess_status,
@@ -158,8 +159,6 @@ RES_MONITOR_PARAMS = ('params user="sstuser" password="%(sstpass)s" '
                       'op monitor interval="1s" timeout="30s" '
                       'OCF_CHECK_LEVEL="1"')
 
-INITIAL_CLIENT_UPDATE_KEY = 'initial_client_update_done'
-
 MYSQL_SOCKET = "/var/run/mysqld/mysqld.sock"
 
 

hooks/percona_utils.py

@@ -12,6 +12,7 @@ import six
 import uuid
 from functools import partial
 import time
+import yaml
 
 from charmhelpers.core.decorators import retry_on_exception
 from charmhelpers.core.host import (
@@ -84,6 +85,7 @@ SEEDED_MARKER = "{data_dir}/seeded"
 HOSTS_FILE = '/etc/hosts'
 DEFAULT_MYSQL_PORT = 3306
 INITIAL_CLUSTERED_KEY = 'initial-cluster-complete'
+INITIAL_CLIENT_UPDATE_KEY = 'initial_client_update_done'
 
 # NOTE(ajkavanagh) - this is 'required' for the pause/resume code for
 # maintenance mode, but is currently not populated as the
@@ -111,6 +113,11 @@ class DesyncedException(Exception):
     pass
 
 
+class GRStateFileNotFound(Exception):
+    """Raised when the grstate file does not exist"""
+    pass
+
+
 class FakeOSConfigRenderer(object):
     """This class is to provide to register_configs() as a 'fake'
     OSConfigRenderer object that has a complete_contexts method that returns
@@ -679,6 +686,15 @@ def charm_check_func():
         # Avoid looping through attempting to determine cluster_in_sync
         return ("blocked", "Unit upgrading.")
 
+    kvstore = kv()
+    # Using INITIAL_CLIENT_UPDATE_KEY as this is a step beyond merely
+    # clustered, but rather clustered and clients were previously notified.
+    if (kvstore.get(INITIAL_CLIENT_UPDATE_KEY, False) and
+            not check_mysql_connection()):
+        return ('blocked',
+                'MySQL is down. Sequence Number: {}. Safe To Bootstrap: {}'
+                .format(get_grstate_seqno(), get_grstate_safe_to_bootstrap()))
+
     @retry_on_exception(num_retries=10,
                         base_delay=2,
                         exc_type=DesyncedException)
@@ -1452,3 +1468,79 @@ def list_replication_users():
                                   "User='replication';"):
         replication_users.append(result[0])
     return replication_users
+
+
+def check_mysql_connection():
+    """Check if local instance of mysql is accessible.
+
+    Attempt a connection to the local instance of mysql to determine if it is
+    running and accessible.
+
+    :side effect: Uses get_db_helper to execute a connection to the DB.
+    :returns: boolean
+    """
+
+    m_helper = get_db_helper()
+    try:
+        m_helper.connect(password=m_helper.get_mysql_root_password())
+        return True
+    except OperationalError:
+        log("Could not connect to db", DEBUG)
+        return False
+
+
+def get_grstate_seqno():
+    """Get GR State safe sequence number.
+
+    Read the grstate yaml file to determine the sequence number for this
+    instance.
+
+    :returns: int Sequence Number
+    """
+
+    grstate_file = os.path.join(resolve_data_dir(), "grastate.dat")
+    if os.path.exists(grstate_file):
+        with open(grstate_file, 'r') as f:
+            grstate = yaml.safe_load(f)
+        return grstate.get("seqno")
+
+
+def get_grstate_safe_to_bootstrap():
+    """Get GR State safe to bootstrap.
+
+    Read the grstate yaml file to determine if it is safe to bootstrap from
+    this instance.
+
+    :returns: int Safe to bootstrap 0 or 1
+    """
+
+    grstate_file = os.path.join(resolve_data_dir(), "grastate.dat")
+    if os.path.exists(grstate_file):
+        with open(grstate_file, 'r') as f:
+            grstate = yaml.safe_load(f)
+        return grstate.get("safe_to_bootstrap")
+
+
+def set_grstate_safe_to_bootstrap():
+    """Set GR State safe to bootstrap.
+
+    Update the grstate yaml file to indicate it is safe to bootstrap from
+    this instance.
+
+    :side effect: Writes the grstate.dat file.
+    :raises GRStateFileNotFound: If grstate.dat file does not exist.
+    :returns: None
+    """
+
+    grstate_file = os.path.join(resolve_data_dir(), "grastate.dat")
+    if not os.path.exists(grstate_file):
+        raise GRStateFileNotFound("{} file does not exist"
+                                  .format(grstate_file))
+    with open(grstate_file, 'r') as f:
+        grstate = yaml.safe_load(f)
+
+    # Force safe to bootstrap
+    grstate["safe_to_bootstrap"] = 1
+
+    with open(grstate_file, 'w') as f:
+        f.write(yaml.dump(grstate))

unit_tests/test_percona_utils.py

@@ -6,7 +6,7 @@ import mock
 
 import percona_utils
 
-from test_utils import CharmTestCase
+from test_utils import CharmTestCase, patch_open
 
 os.environ['JUJU_UNIT_NAME'] = 'percona-cluster/2'
 
@@ -19,6 +19,8 @@ class UtilsTests(CharmTestCase):
         'related_units',
         'relation_get',
         'relation_set',
+        'get_db_helper',
+        'yaml',
     ]
 
     def setUp(self):
@@ -420,6 +422,57 @@ class UtilsTests(CharmTestCase):
             percona_utils.check_for_socket("filename", exists=False)
         _time.sleep.assert_called_with(10)
 
+    def test_check_mysql_connection(self):
+        _db_helper = mock.MagicMock()
+        _db_helper.get_mysql_root_password.return_value = "password"
+        self.get_db_helper.return_value = _db_helper
+
+        _db_helper.connect.return_value = mock.MagicMock()
+        self.assertTrue(percona_utils.check_mysql_connection())
+
+        # The MySQLdb module is fully mocked out, including the
+        # OperationalError. Make OperationalError behave like an exception.
+        percona_utils.OperationalError = Exception
+        _db_helper.connect.side_effect = percona_utils.OperationalError
+        self.assertFalse(percona_utils.check_mysql_connection())
+
+    @mock.patch("percona_utils.resolve_data_dir")
+    @mock.patch("percona_utils.os")
+    def test_get_grstate_seqno(self, _os, _resolve_dd):
+        _resolve_dd.return_value = "/tmp"
+        _seqno = "25"
+        _os.path.exists.return_value = True
+        self.yaml.safe_load.return_value = {"seqno": _seqno}
+        with patch_open() as (_open, _file):
+            _open.return_value = _file
+            self.assertEqual(_seqno, percona_utils.get_grstate_seqno())
+
+    @mock.patch("percona_utils.resolve_data_dir")
+    @mock.patch("percona_utils.os")
+    def test_get_grstate_safe_to_bootstrap(self, _os, _resolve_dd):
+        _resolve_dd.return_value = "/tmp"
+        _bootstrap = "0"
+        _os.path.exists.return_value = True
+        self.yaml.safe_load.return_value = {"safe_to_bootstrap": _bootstrap}
+        with patch_open() as (_open, _file):
+            _open.return_value = _file
+            self.assertEqual(
+                _bootstrap, percona_utils.get_grstate_safe_to_bootstrap())
+
+    @mock.patch("percona_utils.resolve_data_dir")
+    @mock.patch("percona_utils.os")
+    def test_set_grstate_safe_to_bootstrap(self, _os, _resolve_dd):
+        _resolve_dd.return_value = "/tmp"
+        _bootstrap = "0"
+        _os.path.exists.return_value = True
+        self.yaml.safe_load.return_value = {"safe_to_bootstrap": _bootstrap}
+        with patch_open() as (_open, _file):
+            _open.return_value = _file
+            _file.write = mock.MagicMock()
+            percona_utils.set_grstate_safe_to_bootstrap()
+            self.yaml.dump.assert_called_once_with({"safe_to_bootstrap": 1})
+            _file.write.assert_called_once()
+
 
 class UtilsTestsStatus(CharmTestCase):
 

unit_tests/test_utils.py

@@ -1,3 +1,4 @@
+import io
 import os
 import logging
 import unittest
@@ -125,17 +126,20 @@ class TestRelation(object):
 
 @contextmanager
 def patch_open():
-    '''Patch open() to allow mocking both open() itself and the file that is
+    """Patch open().
+
+    Patch open() to allow mocking both open() itself and the file that is
     yielded.
 
-    Yields the mock for "open" and "file", respectively.'''
+    Yields the mock for "open" and "file", respectively.
+    """
     mock_open = MagicMock(spec=open)
-    mock_file = MagicMock(spec=__file__)
+    mock_file = MagicMock(spec=io.FileIO)
 
     @contextmanager
     def stub_open(*args, **kwargs):
         mock_open(*args, **kwargs)
         yield mock_file
 
-    with patch('__builtin__.open', stub_open):
+    with patch('builtins.open', stub_open):
         yield mock_open, mock_file