Add kayobe overcloud host upgrade

This command performs necessary changes on the host to prepare the control
plane for an upgrade.

Currently this performs a workaround for issue #14, RabbitMQ upgrade failure.

We clear stale entries from /etc/hosts on the overcloud hosts and from the
rabbitmq containers, which allows the upgrade to complete successfully. The
source of the stale entries is currently unknown.

ansible/overcloud-etc-hosts-fixup.yml

@@ -0,0 +1,51 @@
+---
+# For some currently unknown reason, overcloud hosts end up with multiple
+# entries in /etc/hosts that map their own hostname to their provisioning
+# network IP address, in addition to one that maps their own hostname to their
+# internal network IP address. This causes RabbitMQ upgrades to fail, as
+# RabbitMQ expects the system's hostname to resolve to the IP address on
+# which it is listening. As a workaround, we remove the stale entries from
+# /etc/hosts.  See https://github.com/stackhpc/kayobe/issues/14.
+
+- name: Ensure overcloud hosts' /etc/hosts does not contain provisioning network IP
+  hosts: overcloud
+  tasks:
+    - name: Ensure overcloud hosts' /etc/hosts does not contain provisioning network IP
+      lineinfile:
+        dest: /etc/hosts
+        regexp: "^{{ provision_oc_net_name | net_ip }}[ \t]*{{ inventory_hostname }}"
+        state: absent
+      when: provision_oc_net_name | net_ip != None
+      become: True
+
+- name: Ensure rabbitmq containers' /etc/hosts does not contain provisioning network IP
+  hosts: controllers
+  vars:
+    rabbitmq_containers:
+      - rabbitmq
+      - outward_rabbitmq
+  tasks:
+    - block:
+        - name: Check whether rabbitmq container is running
+          command: docker inspect -f {{ '{{.Id}}' }} {{ item }}
+          changed_when: False
+          failed_when: False
+          with_items: "{{ rabbitmq_containers }}"
+          register: ps_result
+
+        - name: Ensure rabbitmq containers' /etc/hosts does not contain provisioning network IP
+          command: >
+            docker exec -u root {{ item.item }}
+            bash -c
+            'cp /etc/hosts /tmp/hosts &&
+             sed -i -e "/^{{ provision_oc_net_name | net_ip }}[ \t]*{{ inventory_hostname }}/d" /tmp/hosts &&
+             if ! diff -q /tmp/hosts /etc/hosts >/dev/null; then
+               cp /tmp/hosts /etc/hosts &&
+               echo changed
+             fi &&
+             rm /tmp/hosts'
+          changed_when: "'changed' in sed_result.stdout"
+          with_items: "{{ ps_result.results }}"
+          when: item.rc == 0
+          register: sed_result
+      when: provision_oc_net_name | net_ip != None

doc/source/upgrading.rst

@@ -87,6 +87,17 @@ should be obtained either by building them locally or pulling them from an
 image registry.  Second, the overcloud services should be replaced with new
 containers created from the new container images.
 
+Upgrading Host Services
+-----------------------
+
+Prior to upgrading the OpenStack control plane, the overcloud host services
+should be upgraded::
+
+    (kayobe-venv) $ kayobe overcloud host upgrade
+
+Note that this will not perform full configuration of the host, and will
+instead perform a targeted upgrade of specific services where necessary.
+
 Upgrading the Ironic Deployment Images
 --------------------------------------
 

kayobe/cli/commands.py

@@ -442,7 +442,7 @@ class OvercloudDeprovision(KayobeAnsibleMixin, VaultMixin, Command):
 
 class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
                              Command):
-    """Configure the overcloud host OS."""
+    """Configure the overcloud host services."""
 
     def get_parser(self, prog_name):
         parser = super(OvercloudHostConfigure, self).get_parser(prog_name)
@@ -479,6 +479,21 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
         self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud")
 
 
+class OvercloudHostUpgrade(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
+                           Command):
+    """Upgrade the overcloud host services.
+
+    Performs the changes necessary to make the host services suitable for the
+    configured OpenStack release.
+    """
+
+    def take_action(self, parsed_args):
+        self.app.LOG.debug("Upgrading overcloud host services")
+        playbooks = _build_playbook_list(
+            "overcloud-etc-hosts-fixup")
+        self.run_kayobe_playbooks(parsed_args, playbooks)
+
+
 class OvercloudServiceDeploy(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
                              Command):
     """Deploy the overcloud services."""

setup.py

@@ -62,6 +62,7 @@ setup(
             'overcloud_deprovision = kayobe.cli.commands:OvercloudDeprovision',
             'overcloud_hardware_inspect = kayobe.cli.commands:OvercloudHardwareInspect',
             'overcloud_host_configure = kayobe.cli.commands:OvercloudHostConfigure',
+            'overcloud_host_upgrade = kayobe.cli.commands:OvercloudHostUpgrade',
             'overcloud_introspection_data_save = kayobe.cli.commands:OvercloudIntrospectionDataSave',
             'overcloud_inventory_discover = kayobe.cli.commands:OvercloudInventoryDiscover',
             'overcloud_post_configure = kayobe.cli.commands:OvercloudPostConfigure',