Do not fail on Cloned recoveryMethod

When the recoveryMethod clone actually needs to overwrite the remote
node the mysql-shell unfortunately returns with returncode 1. Both
"Clone process has finished" and "Group Replication is running"
actually indicate successful states.

Handle these two edge cassess as succesful.

func-test-pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/565

Closes-Bug: #1912688
Change-Id: Ia0e99feee76f403ba5ed6e631bd0671c017c9c2c

src/lib/charm/openstack/mysql_innodb_cluster.py

@@ -717,7 +717,8 @@ class MySQLInnoDBClusterCharm(charms_openstack.charm.OpenStackCharm):
             "cluster.add_instance("
             "{{'user': '{user}', 'host': '{addr}', 'password': '{pw}', "
             "'port': '3306'}},"
-            "{{'recoveryMethod': 'clone'}})"
+            "{{'recoveryMethod': 'clone', 'waitRecovery': '2', "
+            "'interactive': False}})"
             .format(
                 user=self.cluster_user, pw=self.cluster_password,
                 caddr=_primary or self.cluster_address,
@@ -725,10 +726,38 @@ class MySQLInnoDBClusterCharm(charms_openstack.charm.OpenStackCharm):
         try:
             output = self.run_mysqlsh_script(_script)
         except subprocess.CalledProcessError as e:
-            ch_core.hookenv.log(
-                "Failed adding instance {} to cluster: {}"
-                .format(address, e.stderr.decode("UTF-8")), "ERROR")
-            return
+            # LP Bug#1912688
+            # When the recoveryMethod clone actually needs to overwrite the
+            # remote node the mysql-shell unfortunately returns with returncode
+            # 1. Both "Clone process has finished" and "Group Replication is
+            # running" actually indicate successful states.
+            # Creating separate checks in order to get good logging on each
+            # outcome.
+            output = None
+            _stderr = e.stderr.decode("UTF-8")
+            if "Clone process has finished" in _stderr:
+                output = e.stderr
+                ch_core.hookenv.log(
+                    "Add instance {} raised CalledProcessError with "
+                    "returncode 1, however, the output contains 'Clone "
+                    "process has finished' an indication of successfully "
+                    "adding the instance to the cluster."
+                    .format(address), "WARNING")
+            if "Group Replication is running" in _stderr:
+                output = e.stderr
+                ch_core.hookenv.log(
+                    "Add instance {} raised CalledProcessError with "
+                    "returncode 1, however, the output contains 'Group "
+                    "Replication is running' an indication of previously "
+                    "successfully adding the instance to the cluster."
+                    .format(address), "WARNING")
+            # Some failure has occured, return without setting instance
+            # clustered flag.
+            if not output:
+                ch_core.hookenv.log(
+                    "Failed adding instance {} to cluster: {}"
+                    .format(address, _stderr), "ERROR")
+                return
         ch_core.hookenv.log("Instance Clustered {}: {}"
                             .format(address, output.decode("UTF-8")),
                             level="DEBUG")

unit_tests/test_lib_charm_openstack_mysql_innodb_cluster.py

@@ -634,7 +634,8 @@ class TestMySQLInnoDBClusterCharm(test_utils.PatchHelper):
             "cluster = dba.get_cluster('{}')\n"
             "cluster.add_instance("
             "{{'user': '{}', 'host': '{}', 'password': '{}', 'port': '3306'}},"
-            "{{'recoveryMethod': 'clone'}})"
+            "{{'recoveryMethod': 'clone', 'waitRecovery': '2', "
+            "'interactive': False}})"
             .format(
                 midbc.cluster_user, midbc.cluster_password,
                 midbc.cluster_address, midbc.cluster_name,