Allow combining system_upgrade_prepare and system_upgrade_run into system_upgrade

Previously we required the operator to run two separate commands for
the "prepare" and "run" phases of operating system upgrade. This
commit refactors the upgrade_tasks with these tags so that running the
whole system upgrade is possible via a single command with `--tags
system_upgrade`.

Allowing to run in a single command requires being more careful about
what can happen in which step number in the upgrade tasks. The upgrade
steps for system upgrade are now explicitly documented in composable
services readme.

The existing system_upgrade_run and system_upgrade_prepare tasks were
checked and moved into the appropriate steps. In the case of
pacemaker, it required moving the cluster stop/destroy action into a
single file with removing all containers, to guarantee that the
cluster is stopped before the container removal, otherwise pacemaker
would try to spawn new containers.

Change-Id: I3cd78de8d07be46ee01006dd7e039c285991d14a
Partial-Bug: #1831690

deployment/README.rst

@@ -291,6 +291,24 @@ Note that the services are not started in the upgrade tasks - we instead re-run
 puppet which does any reconfiguration required for the new version, then starts
 the services.
 
+When running an OS upgrade via the tags `system_upgrade_prepare` and
+`system_upgrade_run`, or the combined tag `system_upgrade`, the steps
+corellate to the following:
+
+   1) Any pre-service-stop actions. (`system_upgrade_prepare`)
+
+   2) Stop all services. (`system_upgrade_prepare`)
+
+   3) Post-service-stop actions like removing packages before the
+      upgrade. (`system_upgrade_prepare`)
+
+   4) Step reserved for the `tripleo-packages` service. Only package
+      download for upgrade (under `system_upgrade_prepare` tag), and
+      reboot for performing the offline upgrade (under
+      `system_upgrade_run` tag) happens here.
+
+   5) Any post-upgrade tasks (`system_upgrade_run`).
+
 Nova Server Metadata Settings
 -----------------------------
 

deployment/database/mysql-pacemaker-puppet.yaml

@@ -393,14 +393,13 @@ outputs:
         - vars:
             mysql_upgrade_persist: {get_param: MysqlUpgradePersist}
           when:
-            - step|int == 1
+            - step|int == 3
             - mysql_upgrade_persist
           tags:
             - never
+            - system_upgrade
             - system_upgrade_prepare
           block:
-            - name: Ban galera from local node
-              command: /usr/sbin/pcs resource ban galera-bundle {{ansible_hostname}} --wait
             - name: Persist mysql data
               include_role:
                 name: tripleo-persist
@@ -410,10 +409,11 @@ outputs:
         - vars:
             mysql_upgrade_persist: {get_param: MysqlUpgradePersist}
           when:
-            - step|int == 1
+            - step|int == 5
             - mysql_upgrade_persist
           tags:
             - never
+            - system_upgrade
             - system_upgrade_run
           block:
             - name: Restore mysql data

deployment/pacemaker/pacemaker-baremetal-puppet.yaml

@@ -161,22 +161,6 @@ outputs:
       step_config: |
         include ::tripleo::profile::base::pacemaker
       upgrade_tasks:
-        - name: system_upgrade_prepare step 2
-          tags:
-            - never
-            - system_upgrade_prepare
-          when: step|int == 2
-          block:
-            - name: Check if pcs is present
-              stat:
-                path: /usr/sbin/pcs
-              register: pcs_stat
-            - name: Stop pacemaker cluster
-              pacemaker_cluster: state=offline
-              when: pcs_stat.stat.exists
-            - name: destroy pacemaker cluster
-              command: /usr/sbin/pcs cluster destroy
-              when: pcs_stat.stat.exists
         - name: upgrade step 0
           when: step|int == 0
           vars:

deployment/podman/podman-baremetal-ansible.yaml

@@ -127,15 +127,32 @@ outputs:
       service_config_settings: {}
 
       upgrade_tasks:
-        - name: Stop all services by stopping all docker containers
+        - name: system_upgrade_prepare step 2
-          when:
-            - (step | int) == 3
-          include_role:
-            name: tripleo-podman
-            tasks_from: tripleo_docker_stop.yml
           tags:
             - never
+            - system_upgrade
             - system_upgrade_prepare
+          when:
+            - (step | int) == 2
+          block:
+            - name: Check if pcs is present
+              stat:
+                path: /usr/sbin/pcs
+              register: pcs_stat
+            - name: Stop pacemaker cluster before stopping all docker containers
+              pacemaker_cluster: state=offline
+              when: pcs_stat.stat.exists
+            - name: Destroy pacemaker cluster
+              command: /usr/sbin/pcs cluster destroy
+              when: pcs_stat.stat.exists
+            - name: Stop all services by stopping all docker containers
+              include_role:
+                name: tripleo-podman
+                tasks_from: tripleo_docker_stop.yml
+              tags:
+                - never
+                - system_upgrade
+                - system_upgrade_prepare
 
         - name: Run podman install
           when:

deployment/tripleo-packages/tripleo-packages-baremetal-puppet.yaml

@@ -99,9 +99,11 @@ outputs:
             upgrade_leapp_debug: {get_param: UpgradeLeappDebug}
             upgrade_leapp_devel_skip_rhsm: {get_param: UpgradeLeappDevelSkipRhsm}
             upgrade_leapp_reboot_timeout: {get_param: UpgradeLeappRebootTimeout}
+
         - name: system_upgrade_prepare step 3
           tags:
             - never
+            - system_upgrade
             - system_upgrade_prepare
           when: step|int == 3
           block:
@@ -121,22 +123,37 @@ outputs:
                 name: leapp
                 state: latest
               when: upgrade_leapp_enabled
+
+        - name: system_upgrade_prepare step 4
+          tags:
+            - never
+            - system_upgrade
+            - system_upgrade_prepare
+          when: step|int == 4
+          block:
             - name: run leapp upgrade (download packages)
               shell: >
                 {% if upgrade_leapp_devel_skip_rhsm|default(false) %}LEAPP_DEVEL_SKIP_RHSM=1{% endif %}
                 leapp upgrade
                 {% if upgrade_leapp_debug|default(true) %}--debug{% endif %}
               when: upgrade_leapp_enabled
-        - name: system_upgrade_run step 5
+
+        - name: system_upgrade_run step 4
           tags:
             - never
+            - system_upgrade
             - system_upgrade_run
-          when: step|int == 5
+            # In case someone needs to re-run system_upgrade_run post-tasks
+            # but doesn't want to reboot, they can run with
+            # `--skip-tags system_upgrade_reboot`.
+            - system_upgrade_reboot
+          when: step|int == 4
           block:
             - name: reboot to perform the upgrade
               reboot:
                 reboot_timeout: "{{upgrade_leapp_reboot_timeout}}"
               when: upgrade_leapp_enabled
+
         - name: Package and repo update tasks
           when: step|int == 0
           block: