Fixing broken mysql-router configuration

A customer faced an issue when they face full disk. Mysql router configuration file was broken. As this charm doesn't handle whole configuration file, charm wrote just part of them based on template. so re-bootstrapping is needed when this symptom happens.

To do that, I put code to check configuration file size, and if it is under 1000bytes, run bootstrap.

func-test-pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/1196
Closes-Bug: #2004088
Change-Id: Ic28f0406eb8d5dbd9131ae83090498297ac48c31

src/lib/charm/openstack/mysql_router.py

@@ -479,7 +479,20 @@ class MySQLRouterCharm(charms_openstack.charm.OpenStackCharm):
 
         return None, None
 
-    def bootstrap_mysqlrouter(self):
+    def validate_configuration(self):
+        """Validate Configuration
+
+        Check if mysql router configuration file is less than 1024 bytes.
+        If so, then the configuration file is probably damaged, and then
+        re-run the `bootstrap_mysqlrouter()` function with True to force
+        it.
+        """
+
+        conf_size = os.path.getsize(self.mysqlrouter_conf)
+        if conf_size <= 1024:
+            self.bootstrap_mysqlrouter(True)
+
+    def bootstrap_mysqlrouter(self, force=False):
         """Bootstrap MySQL Router.
 
         Execute the mysqlrouter bootstrap command. MySQL Router bootstraps into
@@ -494,7 +507,7 @@ class MySQLRouterCharm(charms_openstack.charm.OpenStackCharm):
         :rtype: None
         """
 
-        if reactive.flags.is_flag_set(MYSQL_ROUTER_BOOTSTRAPPED):
+        if not force and reactive.flags.is_flag_set(MYSQL_ROUTER_BOOTSTRAPPED):
             ch_core.hookenv.log(
                 "Bootstrap mysqlrouter is being called after we set the "
                 "bootstrapped flag: {}. This may require manual intervention,"
@@ -522,8 +535,19 @@ class MySQLRouterCharm(charms_openstack.charm.OpenStackCharm):
 
         # If we have attempted to bootstrap before but unsuccessfully,
         # use the force option to avoid LP Bug#1919560
-        if reactive.flags.is_flag_set(MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED):
+        is_bootstrap_attempted = reactive.flags.is_flag_set(
+            MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED)
+        if is_bootstrap_attempted or force:
             cmd.append("--force")
+            # clear configuration before force bootstrap
+            # because if there are some regex string, it fails
+            try:
+                with open(self.mysqlrouter_conf, "wt") as f:
+                    f.write("[DEFAULT]\n")
+            except Exception:
+                ch_core.hookenv.log(
+                    "ignored, because the bootstrap will overwrite the file.")
+                pass
 
         # Set and attempt the bootstrap
         reactive.flags.set_flag(MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED)

src/reactive/mysql_router_handlers.py

@@ -11,7 +11,6 @@ charms_openstack.bus.discover()
 charm.use_defaults(
     'charm.installed',
     'config.changed',
-    'update-status',
     'upgrade-charm')
 
 
@@ -99,6 +98,7 @@ def proxy_shared_db_responses(shared_db, db_router):
     :type db_router_interface: MySQLRouterRequires object
     """
     with charm.provide_charm_instance() as instance:
+        instance.validate_configuration()
         instance.config_changed()
         instance.proxy_db_and_user_responses(db_router, shared_db)
         instance.assess_status()
@@ -112,3 +112,10 @@ def stop_charm():
     with charm.provide_charm_instance() as instance:
         instance.stop_mysqlrouter()
         instance.config_cleanup()
+
+
+@reactive.hook('update-status')
+def update_status():
+    with charm.provide_charm_instance() as instance:
+        instance.validate_configuration()
+        instance.assess_status()

unit_tests/test_lib_charm_openstack_mysql_router.py

@@ -484,6 +484,62 @@ class TestMySQLRouterCharm(test_utils.PatchHelper):
         self.clear_flag.assert_called_once_with(
             mysql_router.MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED)
 
+    def test_bootstrap_mysqlrouter_force(self):
+        _json_addr = '"10.10.10.60"'
+        _json_pass = '"clusterpass"'
+        _pass = json.loads(_json_pass)
+        _addr = json.loads(_json_addr)
+        _user = "mysql"
+        _port = "3006"
+        self.patch_object(mysql_router.reactive.flags, "is_flag_set")
+        self.endpoint_from_flag.return_value = self.db_router
+        self.db_router.password.return_value = _json_pass
+        self.db_router.db_host.return_value = _json_addr
+        self.is_flag_set.return_value = False
+
+        mrc = mysql_router.MySQLRouterCharm()
+        mrc.options.system_user = _user
+        mrc.options.base_port = _port
+
+        _relations = ["relid"]
+
+        self.patch_object(mysql_router.ch_core.hookenv, "relation_ids")
+        self.relation_ids.return_value = _relations
+
+        _related_units = ["relunits"]
+
+        self.patch_object(mysql_router.ch_core.hookenv, "related_units")
+        self.related_units.return_value = _related_units
+
+        _config_data = {
+            "mysqlrouter_password": json.dumps(_pass),
+            "db_host": json.dumps(_addr),
+        }
+
+        self.patch_object(mysql_router.ch_core.hookenv, "relation_get")
+        self.relation_get.return_value = _config_data
+
+        self.cmp_pkgrevno.return_value = 1
+        self.is_flag_set.side_effect = [False, True]
+        self.subprocess.check_output.side_effect = None
+        mrc.bootstrap_mysqlrouter(True)
+        self.subprocess.check_output.assert_called_once_with(
+            [mrc.mysqlrouter_bin, "--user", _user, "--name", mrc.name,
+             "--bootstrap", "{}:{}@{}"
+             .format(mrc.db_router_user, _pass, _addr),
+             "--directory", mrc.mysqlrouter_working_dir,
+             "--conf-use-sockets",
+             "--conf-bind-address", mrc.shared_db_address,
+             "--report-host", mrc.db_router_address,
+             "--conf-base-port", _port,
+             "--disable-rest", "--force"],
+            stderr=self.stdout)
+        self.set_flag.assert_has_calls([
+            mock.call(mysql_router.MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED),
+            mock.call(mysql_router.MYSQL_ROUTER_BOOTSTRAPPED)])
+        self.clear_flag.assert_called_once_with(
+            mysql_router.MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED)
+
     def test_start_mysqlrouter(self):
         self.patch_object(mysql_router.ch_core.host, "service_start")
         _name = "keystone-mysql-router"

unit_tests/test_mysql_router_handlers.py

@@ -25,7 +25,6 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks):
     def test_hooks(self):
         defaults = [
             "config.changed",
-            "update-status",
             "upgrade-charm",
             "charm.installed",
         ]
@@ -56,7 +55,10 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks):
             "hook": {
                 "stop_charm": (
                     "stop",
-                )
+                ),
+                "update_status": (
+                    "update-status",
+                ),
             }
         }
         # test that the hooks were registered via the