Enable system upgrade and upgrade run per host.

Allow the execution of the overcloud upgrade steps by hosts
withing the Overcloud roles. After this patch is merged, the hosts
contained in the ovecloud roles (Controllers, Computes, Storage, etc...)
will be taken serially to apply the upgrade host by host.

For each of the hosts within these roles we'll execute the following
steps:

1- Pre overcloud system upgrade workarounds (Workaround executed in the
   host via Ansible with --limit)
2a- Overcloud system upgrade prepare + overcloud system upgrade run (if
   system_upgrade_compact == false)
2b- Overcloud system upgrade (if system_upgrade_compact == true)[Default]
3- Overcloud system upgrade transfer (if role == Controller and
   host == first)
4- Post overcloud system upgrade workaronuds (Workaround executed in the
   host via Ansible with --limit)
5- Pre overcloud upgrade run workarounds (Workaround executed in the
   host via Ansible with --limit)
6- Overcloud upgrade run
7- Post overcloud upgrade run workarounds (Workaround executed in the
   host via Ansible with --limit)

These sets of steps will be executed for all the hosts belonging to the
Overcloud Ansible group.

Change-Id: Ia13f479d58979561fca1c274ac7dde5dc8c2343a

defaults/main.yml

@@ -68,6 +68,7 @@ workload_swap: 512
 undercloud_upgrade_template: undercloud_upgrade.sh.j2
 undercloud_update_template: undercloud_upgrade.sh.j2
 overcloud_upgrade_prepare_template: overcloud_upgrade_prepare.sh.j2
+overcloud_system_upgrade_template: overcloud_system_upgrade.sh.j2
 overcloud_upgrade_run_template: overcloud_upgrade_run.sh.j2
 overcloud_upgrade_converge_template: overcloud_upgrade_converge.sh.j2
 workload_launch_template: workload_launch.sh.j2
@@ -77,6 +78,7 @@ undercloud_upgrade_script: "{{ working_dir }}/undercloud_upgrade.sh"
 undercloud_update_script: "{{ working_dir }}/undercloud_update.sh"
 overcloud_deploy_script: "{{ working_dir }}/overcloud_deploy.sh"
 overcloud_upgrade_prepare_script: "{{ working_dir }}/overcloud_upgrade_prepare.sh"
+overcloud_system_upgrade_script_base: "{{ working_dir }}/overcloud_system_upgrade"
 overcloud_upgrade_run_script_base: "{{ working_dir }}/overcloud_upgrade_run"
 overcloud_upgrade_converge_script: "{{ working_dir }}/overcloud_upgrade_converge.sh"
 workload_launch_script: "{{ working_dir }}/workload_launch.sh"
@@ -260,3 +262,7 @@ leapp_unsubscribed: false
 
 # Leapp OS upgrade variables
 leapp_unregister: true
+
+# Compact system upgrade prepare and
+# system upgrade run into a single step
+system_upgrade_compact: true

tasks/upgrade/create-overcloud-upgrade-hosts-scripts.yaml

@@ -0,0 +1,18 @@
+---
+- name: create overcloud system upgrade script for {{ role }}
+  vars:
+    role_hosts: "{{ oc_roles_hosts[role] }}"
+  template:
+    src: "{{ overcloud_system_upgrade_template }}"
+    dest: "{{ overcloud_system_upgrade_script_base }}-{{ item }}.sh"
+    mode: 0775
+    force: true
+  loop: "{{ oc_roles_hosts[role] }}"
+
+- name: create overcloud upgrade script for {{ role }}
+  template:
+    src: "{{ overcloud_upgrade_run_template }}"
+    dest: "{{ overcloud_upgrade_run_script_base }}-{{ item }}.sh"
+    mode: 0775
+    force: true
+  loop: "{{ oc_roles_hosts[role] }}"

tasks/upgrade/create-overcloud-upgrade-scripts.yaml

@@ -42,6 +42,8 @@
   loop:
     - 'pre_overcloud_upgrade_prepare_workarounds'
     - 'post_overcloud_upgrade_prepare_workarounds'
+    - 'pre_overcloud_system_upgrade_workarounds'
+    - 'post_overcloud_system_upgrade_workarounds'
     - 'pre_overcloud_upgrade_workarounds'
     - 'post_overcloud_upgrade_workarounds'
     - 'pre_overcloud_upgrade_converge_workarounds'
@@ -67,13 +69,10 @@
     mode: 0755
     force: true
 
-- name: create overcloud upgrade script
-  template:
-    src: "{{ overcloud_upgrade_run_template }}"
-    dest: "{{ overcloud_upgrade_run_script_base }}-{{ item }}.sh"
-    mode: 0775
-    force: true
-  loop: "{{ oc_roles|default(['all']) }}"
+- include_tasks: create-overcloud-upgrade-hosts-scripts.yaml
+  loop: "{{ oc_roles }}"
+  loop_control:
+    loop_var: role
 
 - name: create online data migration script
   template:

tasks/upgrade/main.yml

@@ -4,7 +4,7 @@
   args:
     apply:
       tags: create_undercloud_upgrade_scripts
-  tags: create_undercloud_upgrade_scripts
+  tags: always
   when: >
       upgrade_noop|bool or
       undercloud_upgrade|bool or
@@ -121,35 +121,15 @@
       tags: overcloud_upgrade_run
       become: true
 
-    - name: apply pre overcloud upgrade workarounds
-      shell: |
-        set -o pipefail
-        bash {{ working_dir }}/pre_overcloud_upgrade_workarounds.sh 2>&1 {{ timestamper_cmd }} > pre_overcloud_upgrade_workarounds.log
-      args:
-        chdir: "{{ working_dir }}"
-      when: upgrade_workarounds
-      tags: overcloud_upgrade_run
-
-    - name: launch workload
-      shell: |
-        set -o pipefail
-        bash {{ workload_launch_script }} 2>&1 {{ timestamper_cmd }} > workload_launch.log
-      args:
-        chdir: "{{ working_dir }}"
-      when: workload_launch|bool
-
-    - import_tasks: overcloud_upgrade_run.yml
-      tags: overcloud_upgrade_run
-
-    - name: apply post overcloud upgrade workarounds
-      shell: |
-        set -o pipefail
-        bash {{ working_dir }}/post_overcloud_upgrade_workarounds.sh 2>&1 {{ timestamper_cmd }} > post_overcloud_upgrade_workarounds.log
-      args:
-        chdir: "{{ working_dir }}"
-      when: upgrade_workarounds
-      tags: overcloud_upgrade_run
+    ######## OS upgrade + Overcloud node upgrade run #######
+    - include_tasks: overcloud_upgrade_roles.yaml
+      loop: "{{ oc_roles }}"
+      loop_control:
+        loop_var: role
+      tags:
+        - always
 
+    ######## CEPH upgrade ########
     - name: apply pre Ceph upgrade workarounds
       shell: |
         set -o pipefail
@@ -177,6 +157,7 @@
         set -o pipefail
         bash {{ working_dir }}/overcloud_online_data_migration.sh 2>&1 {{ timestamper_cmd }} > overcloud_upgrade_data_migration.log
 
+    ######## Overcloud upgrade converge ########
     - name: apply pre overcloud upgrade converge workarounds
       shell: |
         set -o pipefail

tasks/upgrade/overcloud_upgrade_hosts.yaml

@@ -0,0 +1,74 @@
+---
+######## RHEL system upgrade ########
+- name: apply pre overcloud system upgrade workarounds in {{ host }}
+  shell: |
+    set -o pipefail
+    bash {{ working_dir }}/pre_overcloud_system_upgrade_workarounds.sh --limit {{ host}} 2>&1 {{ timestamper_cmd }} > pre_overcloud_system_upgrade_workarounds_{{ host }}.log
+  args:
+    chdir: "{{ working_dir }}"
+  when: upgrade_workarounds
+  tags: overcloud_system_upgrade
+
+- name: run the RHEL upgrade steps for the host {{ host }}
+  shell: |
+    set -o pipefail
+    bash {{ overcloud_system_upgrade_script_base }}-{{ host }}.sh 2>&1 {{ timestamper_cmd }} > \
+        {{ working_dir }}/overcloud_system_upgrade_{{ host }}.log
+  register: overcloud_system_upgrade_res
+  ignore_errors: true
+  tags: overcloud_system_upgrade
+
+- name: was the RHEL upgrade step successful.
+  fail: msg="Overcloud Operating System upgrade failed in {{ host }}."
+  when: overcloud_system_upgrade_res.rc != 0
+  tags: overcloud_system_upgrade
+
+- name: apply post overcloud system upgrade workarounds in {{ host }}
+  shell: |
+    set -o pipefail
+    bash {{ working_dir }}/post_overcloud_system_upgrade_workarounds.sh --limit {{ host}} 2>&1 {{ timestamper_cmd }} > post_overcloud_system_upgrade_workarounds_{{ host }}.log
+  args:
+    chdir: "{{ working_dir }}"
+  when: upgrade_workarounds
+  tags: overcloud_system_upgrade
+
+  ######## overcloud upgrade run ########
+- name: apply pre overcloud upgrade workarounds in {{ host }}
+  shell: |
+    set -o pipefail
+    bash {{ working_dir }}/pre_overcloud_upgrade_workarounds.sh --limit {{ host}} 2>&1 {{ timestamper_cmd }} > pre_overcloud_upgrade_workarounds_{{ host }}.log
+  args:
+    chdir: "{{ working_dir }}"
+  when: upgrade_workarounds
+  tags: overcloud_upgrade_run
+
+- name: launch workload
+  shell: |
+    set -o pipefail
+    bash {{ workload_launch_script }} 2>&1 {{ timestamper_cmd }} > workload_launch.log
+  args:
+    chdir: "{{ working_dir }}"
+  when: workload_launch|bool
+
+- name: run overcloud major upgrade for the host {{ host }}
+  shell: |
+    set -o pipefail
+    bash {{ overcloud_upgrade_run_script_base }}-{{ host }}.sh 2>&1 {{ timestamper_cmd }} > \
+        {{ working_dir }}/overcloud_upgrade_run_{{ host }}.log
+  register: overcloud_upgrade_res
+  ignore_errors: true
+  tags: overcloud_upgrade_run
+
+- name: was the overcloud upgrade composable step successful.
+  fail: msg="Overcloud upgrade composable step failed for {{ host }}... :("
+  when: overcloud_upgrade_res.rc != 0
+  tags: overcloud_upgrade_run
+
+- name: apply post overcloud upgrade workarounds in {{ host }}
+  shell: |
+    set -o pipefail
+    bash {{ working_dir }}/post_overcloud_upgrade_workarounds.sh --limit {{ host}} 2>&1 {{ timestamper_cmd }} > post_overcloud_upgrade_workarounds_{{ host }}.log
+  args:
+    chdir: "{{ working_dir }}"
+  when: upgrade_workarounds
+  tags: overcloud_upgrade_run

tasks/upgrade/overcloud_upgrade_roles.yaml

@@ -0,0 +1,8 @@
+---
+- include_tasks: overcloud_upgrade_hosts.yaml
+  loop: "{{ oc_roles_hosts[role] }}"
+  loop_control:
+    loop_var: host
+  tags:
+    - always
+  when: "{{ role != 'all' }}"

tasks/upgrade/overcloud_upgrade_run.yml

@@ -1,14 +1,12 @@
 ---
-- name: run overcloud major upgrade in each of the roles/hostgroups
+- name: run overcloud major upgrade for the host {{ host }}
   shell: |
     set -o pipefail
-    bash {{ overcloud_upgrade_run_script_base }}-{{ item }}.sh 2>&1 {{ timestamper_cmd }} > \
-        {{ working_dir }}/overcloud_upgrade_run_{{ item }}.log
-  loop: "{{ oc_roles|default(['all']) }}"
-  register: overcloud_upgrade_nodes
+    bash {{ overcloud_upgrade_run_script_base }}-{{ host }}.sh 2>&1 {{ timestamper_cmd }} > \
+        {{ working_dir }}/overcloud_upgrade_run_{{ host }}.log
+  register: overcloud_upgrade_res
   ignore_errors: true
 
 - name: was the overcloud upgrade composable step successful.
-  fail: msg="Overcloud upgrade composable step failed... :("
-  loop: "{{ overcloud_upgrade_nodes.results }}"
-  when: item.rc != 0
+  fail: msg="Overcloud upgrade composable step failed for {{ host }}... :("
+  when: overcloud_upgrade_res.rc != 0

templates/overcloud_system_upgrade.sh.j2

@@ -0,0 +1,53 @@
+#!/bin/env bash
+#
+# Run overcloud Operating System upgrade on overcloud node {{ item }}
+#
+
+set -euo pipefail
+
+source {{ undercloud_rc }}
+
+{% if system_upgrade_compact|bool -%}
+
+echo "[$(date)] Started system upgrade step for {{ item }} role"
+
+openstack overcloud upgrade run \
+        --playbook upgrade_steps_playbook.yaml \
+        --tags system_upgrade \
+        --limit {{ item }} 2>&1
+
+echo "[$(date)] Finished system upgrade step for {{ item }} role"
+
+{% else -%}
+
+echo "[$(date)] Started system upgrade prepare for {{ item }} role"
+
+openstack overcloud upgrade run \
+        --playbook upgrade_steps_playbook.yaml \
+        --tags system_upgrade_prepare \
+        --limit {{ item }} 2>&1
+
+echo "[$(date)] Finished system upgrade prepare for {{ item }} role"
+
+echo "[$(date)] Started system upgrade run for {{ item }} role"
+
+openstack overcloud upgrade run \
+        --playbook upgrade_steps_playbook.yaml \
+        --tags system_upgrade_run \
+        --limit {{ item }} 2>&1
+
+echo "[$(date)] Finished system upgrade run for {{ item }} role"
+
+{% endif -%}
+
+{% if role == 'Controller' and role_hosts|length > 1 and role_hosts.index(item) == 0 -%}
+
+echo "[$(date)] Started system upgrade transfer data for {{ item }} role"
+
+openstack overcloud external-upgrade run \
+        --tags system_upgrade_transfer_data 2>&1
+
+echo "[$(date)] Finished system upgrade transfer data for {{ item }} role"
+{% endif -%}
+
+

templates/overcloud_upgrade_run.sh.j2

@@ -4,6 +4,17 @@
 #
 set -euo pipefail
 
+{# Elements will contain all the hosts in oc_roles_hosts for the
+role under loop from the first element of the list to the element in the item
+variable. Example:
+oc_roles_hosts = { 'Controller': ['controller-0', 'controller-1', 'controller-2',
+                   'Compute' : ['compute-0'] }
+If role = 'Controller' and item = 'controller-1'
+   elements = ['controller-0', 'controller-1']
+#}
+
+{% set elements = oc_roles_hosts[role][0:oc_roles_hosts[role].index(item)+1] %}
+
 {% if overcloud_upgrade_multibooks|bool %}
 
     {%- for playbook in overcloud_upgrade_playbooks -%}
@@ -30,15 +41,15 @@ bash {{ working_dir }}/fip_http_check_start.sh &
 
 source {{ undercloud_rc }}
 
-echo "[$(date)] Runing major upgrade {{ playbook }} playbook for {{ item }} role"
+echo "[$(date)] Running major upgrade {{ playbook }} playbook for {{ elements }} role"
 openstack overcloud upgrade run \
         --stack {{ overcloud_stack_name }} \
         {% if tripleo_upgrade_debug|bool -%}
         --debug \
         {% endif -%}
-        --limit {{ item }} --playbook {{ playbook }} 2>&1
+        --limit {{ elements | join(',') }} --playbook {{ playbook }} 2>&1
 
-echo "[$(date)] Finished major upgrade {{ playbook }} playbook for {{ item }} role"
+echo "[$(date)] Finished major upgrade {{ playbook }} playbook for {{ elements }} role"
 
     {%- endfor %}
 {# validation scripts should run after all playbooks are finished #}
@@ -90,16 +101,16 @@ bash {{ working_dir }}/fip_http_check_start.sh &
 
 source {{ undercloud_rc }}
 
-echo "[$(date)] Runing major upgrade for {{ item }} role"
+echo "[$(date)] Running major upgrade for {{ elements }} hosts"
 
 openstack overcloud upgrade run \
         --stack {{ overcloud_stack_name }} \
         {% if tripleo_upgrade_debug|bool %}
         --debug \
         {% endif -%}
-        --limit {{ item }} --playbook all 2>&1
+        --limit {{ elements | join(',') }} --playbook all 2>&1
 
-echo "[$(date)] Finished major upgrade for {{ item }} role"
+echo "[$(date)] Finished major upgrade for {{ elements }} hosts"
 
 {% if l3_agent_connectivity_check|bool %}
 if [[ -e {{ working_dir }}/l3_agent_stop_ping.sh ]]; then

templates/workarounds.sh.j2

@@ -138,6 +138,10 @@ ansible_patch {{ key }}
 {# finish pre_overcloud_upgrade_prepare_workarounds #}
 {{ render_workarounds(post_overcloud_upgrade_prepare_workarounds|default([]),'post_overcloud_upgrade_prepare_workarounds') }}
 {# finish post_overcloud_upgrade_prepare_workarounds #}
+{{ render_workarounds(pre_overcloud_system_upgrade_workarounds|default([]),'pre_overcloud_system_upgrade_workarounds') }}
+{# finish pre_overcloud_system_upgrade_workarounds #}
+{{ render_workarounds(post_overcloud_system_upgrade_workarounds|default([]),'post_overcloud_system_upgrade_workarounds') }}
+{# finish post_overcloud_system_upgrade_workarounds #}
 {{ render_workarounds(pre_overcloud_upgrade_workarounds|default([]),'pre_overcloud_upgrade_workarounds') }}
 {# finish pre_overcloud_upgrade_workarounds #}
 {{ render_workarounds(post_overcloud_upgrade_workarounds|default([]),'post_overcloud_upgrade_workarounds') }}