From 7f7960a5367e29647cd79a6325f7ca857a2d0084 Mon Sep 17 00:00:00 2001 From: Jiri Stransky Date: Tue, 13 Aug 2019 15:01:44 +0200 Subject: [PATCH] Allow combining system_upgrade_prepare and system_upgrade_run into system_upgrade Previously we required the operator to run two separate commands for the "prepare" and "run" phases of operating system upgrade. This commit refactors the upgrade_tasks with these tags so that running the whole system upgrade is possible via a single command with `--tags system_upgrade`. Allowing to run in a single command requires being more careful about what can happen in which step number in the upgrade tasks. The upgrade steps for system upgrade are now explicitly documented in composable services readme. The existing system_upgrade_run and system_upgrade_prepare tasks were checked and moved into the appropriate steps. In the case of pacemaker, it required moving the cluster stop/destroy action into a single file with removing all containers, to guarantee that the cluster is stopped before the container removal, otherwise pacemaker would try to spawn new containers. Change-Id: I3cd78de8d07be46ee01006dd7e039c285991d14a Partial-Bug: #1831690 (cherry picked from commit 206625d4f5bd9a643e76d7f52e13cc171ee4edad) --- .../database/mysql-pacemaker-puppet.yaml | 8 +++---- .../podman/podman-baremetal-ansible.yaml | 21 +++++++++++++++---- .../tripleo-packages-baremetal-puppet.yaml | 21 +++++++++++++++++-- puppet/services/README.rst | 18 ++++++++++++++++ puppet/services/pacemaker.yaml | 16 -------------- 5 files changed, 58 insertions(+), 26 deletions(-) diff --git a/deployment/database/mysql-pacemaker-puppet.yaml b/deployment/database/mysql-pacemaker-puppet.yaml index f91fe3432b..40fb0b9ea5 100644 --- a/deployment/database/mysql-pacemaker-puppet.yaml +++ b/deployment/database/mysql-pacemaker-puppet.yaml @@ -393,14 +393,13 @@ outputs: - vars: mysql_upgrade_persist: {get_param: MysqlUpgradePersist} when: - - step|int == 1 + - step|int == 3 - mysql_upgrade_persist tags: - never + - system_upgrade - system_upgrade_prepare block: - - name: Ban galera from local node - command: /usr/sbin/pcs resource ban galera-bundle {{ansible_hostname}} --wait - name: Persist mysql data include_role: name: tripleo-persist @@ -410,10 +409,11 @@ outputs: - vars: mysql_upgrade_persist: {get_param: MysqlUpgradePersist} when: - - step|int == 1 + - step|int == 5 - mysql_upgrade_persist tags: - never + - system_upgrade - system_upgrade_run block: - name: Restore mysql data diff --git a/deployment/podman/podman-baremetal-ansible.yaml b/deployment/podman/podman-baremetal-ansible.yaml index 8ec6646652..ecb67f3e72 100644 --- a/deployment/podman/podman-baremetal-ansible.yaml +++ b/deployment/podman/podman-baremetal-ansible.yaml @@ -139,13 +139,26 @@ outputs: service_config_settings: {} upgrade_tasks: - - block: - - name: Stop all services by stopping all Docker containers - when: step|int == 3 - shell: docker ps -q | xargs --no-run-if-empty -n1 docker stop + - name: system_upgrade_prepare step 2 tags: - never + - system_upgrade - system_upgrade_prepare + when: + - (step | int) == 2 + block: + - name: Check if pcs is present + stat: + path: /usr/sbin/pcs + register: pcs_stat + - name: Stop pacemaker cluster before stopping all docker containers + pacemaker_cluster: state=offline + when: pcs_stat.stat.exists + - name: Destroy pacemaker cluster + command: /usr/sbin/pcs cluster destroy + when: pcs_stat.stat.exists + - name: Stop all services by stopping all Docker containers + shell: docker ps -q | xargs --no-run-if-empty -n1 docker stop # Upgrade tasks for Pacemaker-managed services tasks pull # container images in step 2, we need insecure registries diff --git a/deployment/tripleo-packages/tripleo-packages-baremetal-puppet.yaml b/deployment/tripleo-packages/tripleo-packages-baremetal-puppet.yaml index 28367d4d82..2fba7af394 100644 --- a/deployment/tripleo-packages/tripleo-packages-baremetal-puppet.yaml +++ b/deployment/tripleo-packages/tripleo-packages-baremetal-puppet.yaml @@ -99,9 +99,11 @@ outputs: upgrade_leapp_debug: {get_param: UpgradeLeappDebug} upgrade_leapp_devel_skip_rhsm: {get_param: UpgradeLeappDevelSkipRhsm} upgrade_leapp_reboot_timeout: {get_param: UpgradeLeappRebootTimeout} + - name: system_upgrade_prepare step 3 tags: - never + - system_upgrade - system_upgrade_prepare when: step|int == 3 block: @@ -121,22 +123,37 @@ outputs: name: leapp state: latest when: upgrade_leapp_enabled + + - name: system_upgrade_prepare step 4 + tags: + - never + - system_upgrade + - system_upgrade_prepare + when: step|int == 4 + block: - name: run leapp upgrade (download packages) shell: > {% if upgrade_leapp_devel_skip_rhsm|default(false) %}LEAPP_DEVEL_SKIP_RHSM=1{% endif %} leapp upgrade {% if upgrade_leapp_debug|default(true) %}--debug{% endif %} when: upgrade_leapp_enabled - - name: system_upgrade_run step 5 + + - name: system_upgrade_run step 4 tags: - never + - system_upgrade - system_upgrade_run - when: step|int == 5 + # In case someone needs to re-run system_upgrade_run post-tasks + # but doesn't want to reboot, they can run with + # `--skip-tags system_upgrade_reboot`. + - system_upgrade_reboot + when: step|int == 4 block: - name: reboot to perform the upgrade reboot: reboot_timeout: "{{upgrade_leapp_reboot_timeout}}" when: upgrade_leapp_enabled + - name: Package and repo update tasks when: step|int == 0 block: diff --git a/puppet/services/README.rst b/puppet/services/README.rst index 519fa1f7ee..709bfad1f4 100644 --- a/puppet/services/README.rst +++ b/puppet/services/README.rst @@ -179,6 +179,24 @@ Note that the services are not started in the upgrade tasks - we instead re-run puppet which does any reconfiguration required for the new version, then starts the services. +When running an OS upgrade via the tags `system_upgrade_prepare` and +`system_upgrade_run`, or the combined tag `system_upgrade`, the steps +corellate to the following: + + 1) Any pre-service-stop actions. (`system_upgrade_prepare`) + + 2) Stop all services. (`system_upgrade_prepare`) + + 3) Post-service-stop actions like removing packages before the + upgrade. (`system_upgrade_prepare`) + + 4) Step reserved for the `tripleo-packages` service. Only package + download for upgrade (under `system_upgrade_prepare` tag), and + reboot for performing the offline upgrade (under + `system_upgrade_run` tag) happens here. + + 5) Any post-upgrade tasks (`system_upgrade_run`). + Nova Server Metadata Settings ----------------------------- diff --git a/puppet/services/pacemaker.yaml b/puppet/services/pacemaker.yaml index 072676309f..631a16f75d 100644 --- a/puppet/services/pacemaker.yaml +++ b/puppet/services/pacemaker.yaml @@ -148,22 +148,6 @@ outputs: step_config: | include ::tripleo::profile::base::pacemaker upgrade_tasks: - - name: system_upgrade_prepare step 2 - tags: - - never - - system_upgrade_prepare - when: step|int == 2 - block: - - name: Check if pcs is present - stat: - path: /usr/sbin/pcs - register: pcs_stat - - name: Stop pacemaker cluster - pacemaker_cluster: state=offline - when: pcs_stat.stat.exists - - name: destroy pacemaker cluster - command: /usr/sbin/pcs cluster destroy - when: pcs_stat.stat.exists - name: upgrade step 0 when: step|int == 0 vars: