Add retries to MaaS BMC user configuration

It has been observed that MaaS will fail to enlist/commission/deploy
nodes if it fails to set up its own user in the BMC during cloud
init. This patch set adds a git patch file to update the MaaS source
code in order to retry setting up the MaaS BMC user if it fails.

This patch set also adds to the exception message sent when MaaS
fails to set up a BMC user.

Change-Id: I475988875acffac620302fae3eed8d236a5a46f7

images/maas-region-controller/2.3_configure_ipmi_user.patch

@@ -0,0 +1,57 @@
+diff --git a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py
+index e2c3ce5..7370963 100644
+--- a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py
++++ b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py
+@@ -229,17 +229,40 @@ def make_ipmi_user_settings(username, password):
+     return user_settings
+ 
+ 
++def configure_ipmi_user_with_backoff(username):
++    """Create/configure an IPMI user, but with several tries"""
++    attempt = 1
++    max_attempts = 5
++    backoff_amount = 30
++    while attempt <= max_attempts:
++        password = None
++        try:
++            password = configure_ipmi_user(username)
++        except:
++            if (attempt + 1) > max_attempts:
++                # This is our last attempt, don't catch anything:
++                raise
++
++        if password is None:
++            time.sleep(attempt * backoff_amount)
++        else:
++            return password
++        attempt += 1
++
++
+ def configure_ipmi_user(username):
+     """Create or configure an IPMI user for remote use."""
++    exceptions_caught = []
+     for password in [generate_random_password(),
+                      generate_random_password(with_special_chars=True)]:
+         user_settings = make_ipmi_user_settings(username, password)
+         try:
+             apply_ipmi_user_settings(user_settings)
+             return password
+-        except subprocess.CalledProcessError:
+-            pass
+-    raise IPMIError("Unable to set BMC password.")
++        except subprocess.CalledProcessError as e:
++            exceptions_caught.append(e)
++    raise IPMIError(
++        "Unable to set BMC password:\n{}".format(exceptions_caught))
+ 
+ 
+ def set_ipmi_lan_channel_settings():
+@@ -357,7 +380,7 @@ def main():
+     IPMI_MAAS_USER = "maas"
+     IPMI_MAAS_PASSWORD = None
+ 
+-    IPMI_MAAS_PASSWORD = configure_ipmi_user(IPMI_MAAS_USER)
++    IPMI_MAAS_PASSWORD = configure_ipmi_user_with_backoff(IPMI_MAAS_USER)
+ 
+     # Attempt to enable IPMI Over Lan. If it is disabled, MAAS won't
+     # be able to remotely communicate to the BMC.

images/maas-region-controller/Dockerfile

@@ -72,11 +72,15 @@ COPY 2.3_mac_address.patch /tmp/2.3_mac_address.patch
 # sh8121att: allow all requests via the proxy to allow it to work
 # behind ingress
 COPY 2.3_proxy_acl.patch /tmp/2.3_proxy_acl.patch
+# Patch to add retrying to MaaS BMC user setup, and improve exception handling
+copy 2.3_configure_ipmi_user.patch /tmp/2.3_configure_ipmi_user.patch
+
 RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_network.py < /tmp/2.3_route.patch
 RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed.py < /tmp/2.3_kernel_package.patch
 RUN cd /usr/lib/python3/dist-packages/maasserver/models && patch partition.py < /tmp/2.3_bios_grub_partition.patch
 RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_storage.py < /tmp/2.3_bios_grub_preseed.patch
 RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets && patch maas_enlist.sh < /tmp/2.3_maas_enlist.patch
+RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets && patch maas_ipmi_autodetect.py < /tmp/2.3_configure_ipmi_user.patch
 RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch ipaddr.py < /tmp/2.3_mac_address.patch
 RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch ipaddr.py < /tmp/2.3_mac_address.patch
 RUN cd /usr/lib/python3/dist-packages/provisioningserver/templates/proxy && patch maas-proxy.conf.template < /tmp/2.3_proxy_acl.patch