Restart router if connections fail with 2003 code

At the moment if a connection through the router fails after a
configuration update the router is only restarted if the connection
error has a code of 2013 but often the error thrown is 2003 (see
*1) . This patch alters the charms
behaviour to also restart the router on a 2013 error.

While testing this patch it became apparent that a connection
attempt through the router immediatly after the router has been
restarted very often fail. So, the connection attempt has been
moved into its own method with its own tenacity retry logic.
A side effect of this is that the total possible wait time
has increased from 5 * 10 (outer tenacity loop) to 5 * 10 * 5
(outer tenacity loop and inner tenacity loop).

*1 https://dev.mysql.com/doc/mysql-errors/8.0/en/client-error-reference.html

Closes-Bug: #1973177
Change-Id: I9c2846bf4f21d2dcb1958bee4c9fa72dd4464b6c
(cherry picked from commit 5942b035f6)

src/lib/charm/openstack/mysql_router.py

@@ -96,6 +96,10 @@ class MySQLRouterCharm(charms_openstack.charm.OpenStackCharm):
     # LP Bug #1915842
     _waiting_for_initial_communication_packet_error = 2013
 
+    # mysql.MySQLdb._exceptions.OperationalError error 2003
+    # LP Bug #1973177
+    _cannot_connect_via_ip = 2003
+
     @property
     def mysqlrouter_pid_file(self):
         """Determine the path for the mysqlrouter PID file.
@@ -771,11 +775,28 @@ class MySQLRouterCharm(charms_openstack.charm.OpenStackCharm):
             ch_core.hookenv.log("Updating configuration parameters", "DEBUG")
             self.update_config_parameters(_parameters)
 
-    @tenacity.retry(wait=tenacity.wait_fixed(10),
-                    retry=tenacity.retry_if_exception_type(
-                        mysql.MySQLdb._exceptions.OperationalError),
-                    reraise=True,
-                    stop=tenacity.stop_after_attempt(5))
+    @tenacity.retry(
+        wait=tenacity.wait_fixed(10),
+        retry=tenacity.retry_if_exception_type(
+            mysql.MySQLdb._exceptions.OperationalError),
+        reraise=True,
+        stop=tenacity.stop_after_attempt(5))
+    def retry_conection_check(self):
+        """Retry database connection check."""
+        ch_core.hookenv.log("Checking connection through router", "DEBUG")
+        # Only raise an exception if it matches
+        # mysql.MySQLdb._exceptions.OperationalError error 2003 or 2013
+        # LP Bug #1915842 & #1973177
+        self.check_mysql_connection(
+            reraise_on=[
+                self._waiting_for_initial_communication_packet_error,
+                self._cannot_connect_via_ip])
+
+    @tenacity.retry(
+        retry=tenacity.retry_if_exception_type(
+            mysql.MySQLdb._exceptions.OperationalError),
+        reraise=True,
+        stop=tenacity.stop_after_attempt(5))
     def custom_restart_function(self, service_name):
         """Tenacity retry custom restart function for restart_on_change
 
@@ -791,8 +812,7 @@ class MySQLRouterCharm(charms_openstack.charm.OpenStackCharm):
             "Custom restart of {}".format(service_name), "DEBUG")
         self.service_stop(service_name)
         self.service_start(service_name)
-        # Only raise an exception if it matches
-        # mysql.MySQLdb._exceptions.OperationalError error 2013
-        # LP Bug #1915842
-        self.check_mysql_connection(
-            reraise_on=[self._waiting_for_initial_communication_packet_error])
+        # In the case of the db-router service it reports itself as having
+        # started prior to being fully initialised. So when checking the
+        # connection retry a few times.
+        self.retry_conection_check()

src/tests/bundles/focal-full.yaml

@@ -8,16 +8,28 @@ series: &series focal
 applications:
   keystone-mysql-router:
     charm: ../../../mysql-router.charm
+    options:
+      debug: True
   nova-mysql-router:
     charm: ../../../mysql-router.charm
+    options:
+      debug: True
   glance-mysql-router:
     charm: ../../../mysql-router.charm
+    options:
+      debug: True
   neutron-mysql-router:
     charm: ../../../mysql-router.charm
+    options:
+      debug: True
   placement-mysql-router:
     charm: ../../../mysql-router.charm
+    options:
+      debug: True
   vault-mysql-router:
     charm: ../../../mysql-router.charm
+    options:
+      debug: True
   mysql-innodb-cluster:
     charm: ch:mysql-innodb-cluster
     num_units: 3

src/tests/bundles/focal.yaml

@@ -5,9 +5,13 @@ applications:
   keystone-mysql-router:
     series: focal
     charm: ../../../mysql-router.charm
+    options:
+      debug: True
   vault-mysql-router:
     series: focal
     charm: ../../../mysql-router.charm
+    options:
+      debug: True
   mysql-innodb-cluster:
     charm: ch:mysql-innodb-cluster
     num_units: 3

src/tests/bundles/jammy-full.yaml

@@ -8,16 +8,28 @@ series: &series jammy
 applications:
   keystone-mysql-router:
     charm: ../../../mysql-router.charm
+    options:
+      debug: True
   nova-mysql-router:
     charm: ../../../mysql-router.charm
+    options:
+      debug: True
   glance-mysql-router:
     charm: ../../../mysql-router.charm
+    options:
+      debug: True
   neutron-mysql-router:
     charm: ../../../mysql-router.charm
+    options:
+      debug: True
   placement-mysql-router:
     charm: ../../../mysql-router.charm
+    options:
+      debug: True
   vault-mysql-router:
     charm: ../../../mysql-router.charm
+    options:
+      debug: True
   mysql-innodb-cluster:
     charm: ch:mysql-innodb-cluster
     num_units: 3

src/tests/bundles/jammy.yaml

@@ -4,8 +4,12 @@ series: jammy
 applications:
   keystone-mysql-router:
     charm: ../../../mysql-router.charm
+    options:
+      debug: True
   vault-mysql-router:
     charm: ../../../mysql-router.charm
+    options:
+      debug: True
   mysql-innodb-cluster:
     charm: ch:mysql-innodb-cluster
     num_units: 3