Merge "Enable system upgrade and upgrade run per host."

defaults/main.yml

@@ -68,6 +68,7 @@ workload_swap: 512
 undercloud_upgrade_template: undercloud_upgrade.sh.j2
 undercloud_update_template: undercloud_upgrade.sh.j2
 overcloud_upgrade_prepare_template: overcloud_upgrade_prepare.sh.j2
+overcloud_system_upgrade_template: overcloud_system_upgrade.sh.j2
 overcloud_upgrade_run_template: overcloud_upgrade_run.sh.j2
 overcloud_upgrade_converge_template: overcloud_upgrade_converge.sh.j2
 workload_launch_template: workload_launch.sh.j2
@@ -77,6 +78,7 @@ undercloud_upgrade_script: "{{ working_dir }}/undercloud_upgrade.sh"
 undercloud_update_script: "{{ working_dir }}/undercloud_update.sh"
 overcloud_deploy_script: "{{ working_dir }}/overcloud_deploy.sh"
 overcloud_upgrade_prepare_script: "{{ working_dir }}/overcloud_upgrade_prepare.sh"
+overcloud_system_upgrade_script_base: "{{ working_dir }}/overcloud_system_upgrade"
 overcloud_upgrade_run_script_base: "{{ working_dir }}/overcloud_upgrade_run"
 overcloud_upgrade_converge_script: "{{ working_dir }}/overcloud_upgrade_converge.sh"
 workload_launch_script: "{{ working_dir }}/workload_launch.sh"
@@ -260,3 +262,7 @@ leapp_unsubscribed: false
 
 # Leapp OS upgrade variables
 leapp_unregister: true
+
+# Compact system upgrade prepare and
+# system upgrade run into a single step
+system_upgrade_compact: true

tasks/upgrade/create-overcloud-upgrade-hosts-scripts.yaml

@@ -0,0 +1,18 @@
+---
+- name: create overcloud system upgrade script for {{ role }}
+  vars:
+    role_hosts: "{{ oc_roles_hosts[role] }}"
+  template:
+    src: "{{ overcloud_system_upgrade_template }}"
+    dest: "{{ overcloud_system_upgrade_script_base }}-{{ item }}.sh"
+    mode: 0775
+    force: true
+  loop: "{{ oc_roles_hosts[role] }}"
+
+- name: create overcloud upgrade script for {{ role }}
+  template:
+    src: "{{ overcloud_upgrade_run_template }}"
+    dest: "{{ overcloud_upgrade_run_script_base }}-{{ item }}.sh"
+    mode: 0775
+    force: true
+  loop: "{{ oc_roles_hosts[role] }}"

tasks/upgrade/create-overcloud-upgrade-scripts.yaml

@@ -42,6 +42,8 @@
   loop:
     - 'pre_overcloud_upgrade_prepare_workarounds'
     - 'post_overcloud_upgrade_prepare_workarounds'
+    - 'pre_overcloud_system_upgrade_workarounds'
+    - 'post_overcloud_system_upgrade_workarounds'
     - 'pre_overcloud_upgrade_workarounds'
     - 'post_overcloud_upgrade_workarounds'
     - 'pre_overcloud_upgrade_converge_workarounds'
@@ -67,13 +69,10 @@
     mode: 0755
     force: true
 
-- name: create overcloud upgrade script
-  template:
-    src: "{{ overcloud_upgrade_run_template }}"
-    dest: "{{ overcloud_upgrade_run_script_base }}-{{ item }}.sh"
-    mode: 0775
-    force: true
-  loop: "{{ oc_roles|default(['all']) }}"
+- include_tasks: create-overcloud-upgrade-hosts-scripts.yaml
+  loop: "{{ oc_roles }}"
+  loop_control:
+    loop_var: role
 
 - name: create online data migration script
   template:

tasks/upgrade/main.yml

@@ -4,7 +4,7 @@
   args:
     apply:
       tags: create_undercloud_upgrade_scripts
-  tags: create_undercloud_upgrade_scripts
+  tags: always
   when: >
       upgrade_noop|bool or
       undercloud_upgrade|bool or
@@ -121,35 +121,15 @@
       tags: overcloud_upgrade_run
       become: true
 
-    - name: apply pre overcloud upgrade workarounds
-      shell: |
-        set -o pipefail
-        bash {{ working_dir }}/pre_overcloud_upgrade_workarounds.sh 2>&1 {{ timestamper_cmd }} > pre_overcloud_upgrade_workarounds.log
-      args:
-        chdir: "{{ working_dir }}"
-      when: upgrade_workarounds
-      tags: overcloud_upgrade_run
-
-    - name: launch workload
-      shell: |
-        set -o pipefail
-        bash {{ workload_launch_script }} 2>&1 {{ timestamper_cmd }} > workload_launch.log
-      args:
-        chdir: "{{ working_dir }}"
-      when: workload_launch|bool
-
-    - import_tasks: overcloud_upgrade_run.yml
-      tags: overcloud_upgrade_run
-
-    - name: apply post overcloud upgrade workarounds
-      shell: |
-        set -o pipefail
-        bash {{ working_dir }}/post_overcloud_upgrade_workarounds.sh 2>&1 {{ timestamper_cmd }} > post_overcloud_upgrade_workarounds.log
-      args:
-        chdir: "{{ working_dir }}"
-      when: upgrade_workarounds
-      tags: overcloud_upgrade_run
+    ######## OS upgrade + Overcloud node upgrade run #######
+    - include_tasks: overcloud_upgrade_roles.yaml
+      loop: "{{ oc_roles }}"
+      loop_control:
+        loop_var: role
+      tags:
+        - always
 
+    ######## CEPH upgrade ########
     - name: apply pre Ceph upgrade workarounds
       shell: |
         set -o pipefail
@@ -177,6 +157,7 @@
         set -o pipefail
         bash {{ working_dir }}/overcloud_online_data_migration.sh 2>&1 {{ timestamper_cmd }} > overcloud_upgrade_data_migration.log
 
+    ######## Overcloud upgrade converge ########
     - name: apply pre overcloud upgrade converge workarounds
       shell: |
         set -o pipefail

tasks/upgrade/overcloud_upgrade_hosts.yaml

@@ -0,0 +1,74 @@
+---
+######## RHEL system upgrade ########
+- name: apply pre overcloud system upgrade workarounds in {{ host }}
+  shell: |
+    set -o pipefail
+    bash {{ working_dir }}/pre_overcloud_system_upgrade_workarounds.sh --limit {{ host}} 2>&1 {{ timestamper_cmd }} > pre_overcloud_system_upgrade_workarounds_{{ host }}.log
+  args:
+    chdir: "{{ working_dir }}"
+  when: upgrade_workarounds
+  tags: overcloud_system_upgrade
+
+- name: run the RHEL upgrade steps for the host {{ host }}
+  shell: |
+    set -o pipefail
+    bash {{ overcloud_system_upgrade_script_base }}-{{ host }}.sh 2>&1 {{ timestamper_cmd }} > \
+        {{ working_dir }}/overcloud_system_upgrade_{{ host }}.log
+  register: overcloud_system_upgrade_res
+  ignore_errors: true
+  tags: overcloud_system_upgrade
+
+- name: was the RHEL upgrade step successful.
+  fail: msg="Overcloud Operating System upgrade failed in {{ host }}."
+  when: overcloud_system_upgrade_res.rc != 0
+  tags: overcloud_system_upgrade
+
+- name: apply post overcloud system upgrade workarounds in {{ host }}
+  shell: |
+    set -o pipefail
+    bash {{ working_dir }}/post_overcloud_system_upgrade_workarounds.sh --limit {{ host}} 2>&1 {{ timestamper_cmd }} > post_overcloud_system_upgrade_workarounds_{{ host }}.log
+  args:
+    chdir: "{{ working_dir }}"
+  when: upgrade_workarounds
+  tags: overcloud_system_upgrade
+
+  ######## overcloud upgrade run ########
+- name: apply pre overcloud upgrade workarounds in {{ host }}
+  shell: |
+    set -o pipefail
+    bash {{ working_dir }}/pre_overcloud_upgrade_workarounds.sh --limit {{ host}} 2>&1 {{ timestamper_cmd }} > pre_overcloud_upgrade_workarounds_{{ host }}.log
+  args:
+    chdir: "{{ working_dir }}"
+  when: upgrade_workarounds
+  tags: overcloud_upgrade_run
+
+- name: launch workload
+  shell: |
+    set -o pipefail
+    bash {{ workload_launch_script }} 2>&1 {{ timestamper_cmd }} > workload_launch.log
+  args:
+    chdir: "{{ working_dir }}"
+  when: workload_launch|bool
+
+- name: run overcloud major upgrade for the host {{ host }}
+  shell: |
+    set -o pipefail
+    bash {{ overcloud_upgrade_run_script_base }}-{{ host }}.sh 2>&1 {{ timestamper_cmd }} > \
+        {{ working_dir }}/overcloud_upgrade_run_{{ host }}.log
+  register: overcloud_upgrade_res
+  ignore_errors: true
+  tags: overcloud_upgrade_run
+
+- name: was the overcloud upgrade composable step successful.
+  fail: msg="Overcloud upgrade composable step failed for {{ host }}... :("
+  when: overcloud_upgrade_res.rc != 0
+  tags: overcloud_upgrade_run
+
+- name: apply post overcloud upgrade workarounds in {{ host }}
+  shell: |
+    set -o pipefail
+    bash {{ working_dir }}/post_overcloud_upgrade_workarounds.sh --limit {{ host}} 2>&1 {{ timestamper_cmd }} > post_overcloud_upgrade_workarounds_{{ host }}.log
+  args:
+    chdir: "{{ working_dir }}"
+  when: upgrade_workarounds
+  tags: overcloud_upgrade_run

tasks/upgrade/overcloud_upgrade_roles.yaml

@@ -0,0 +1,8 @@
+---
+- include_tasks: overcloud_upgrade_hosts.yaml
+  loop: "{{ oc_roles_hosts[role] }}"
+  loop_control:
+    loop_var: host
+  tags:
+    - always
+  when: "{{ role != 'all' }}"

tasks/upgrade/overcloud_upgrade_run.yml

@@ -1,14 +1,12 @@
 ---
-- name: run overcloud major upgrade in each of the roles/hostgroups
+- name: run overcloud major upgrade for the host {{ host }}
   shell: |
     set -o pipefail
-    bash {{ overcloud_upgrade_run_script_base }}-{{ item }}.sh 2>&1 {{ timestamper_cmd }} > \
-        {{ working_dir }}/overcloud_upgrade_run_{{ item }}.log
-  loop: "{{ oc_roles|default(['all']) }}"
-  register: overcloud_upgrade_nodes
+    bash {{ overcloud_upgrade_run_script_base }}-{{ host }}.sh 2>&1 {{ timestamper_cmd }} > \
+        {{ working_dir }}/overcloud_upgrade_run_{{ host }}.log
+  register: overcloud_upgrade_res
   ignore_errors: true
 
 - name: was the overcloud upgrade composable step successful.
-  fail: msg="Overcloud upgrade composable step failed... :("
-  loop: "{{ overcloud_upgrade_nodes.results }}"
-  when: item.rc != 0
+  fail: msg="Overcloud upgrade composable step failed for {{ host }}... :("
+  when: overcloud_upgrade_res.rc != 0

templates/overcloud_system_upgrade.sh.j2

@@ -0,0 +1,53 @@
+#!/bin/env bash
+#
+# Run overcloud Operating System upgrade on overcloud node {{ item }}
+#
+
+set -euo pipefail
+
+source {{ undercloud_rc }}
+
+{% if system_upgrade_compact|bool -%}
+
+echo "[$(date)] Started system upgrade step for {{ item }} role"
+
+openstack overcloud upgrade run \
+        --playbook upgrade_steps_playbook.yaml \
+        --tags system_upgrade \
+        --limit {{ item }} 2>&1
+
+echo "[$(date)] Finished system upgrade step for {{ item }} role"
+
+{% else -%}
+
+echo "[$(date)] Started system upgrade prepare for {{ item }} role"
+
+openstack overcloud upgrade run \
+        --playbook upgrade_steps_playbook.yaml \
+        --tags system_upgrade_prepare \
+        --limit {{ item }} 2>&1
+
+echo "[$(date)] Finished system upgrade prepare for {{ item }} role"
+
+echo "[$(date)] Started system upgrade run for {{ item }} role"
+
+openstack overcloud upgrade run \
+        --playbook upgrade_steps_playbook.yaml \
+        --tags system_upgrade_run \
+        --limit {{ item }} 2>&1
+
+echo "[$(date)] Finished system upgrade run for {{ item }} role"
+
+{% endif -%}
+
+{% if role == 'Controller' and role_hosts|length > 1 and role_hosts.index(item) == 0 -%}
+
+echo "[$(date)] Started system upgrade transfer data for {{ item }} role"
+
+openstack overcloud external-upgrade run \
+        --tags system_upgrade_transfer_data 2>&1
+
+echo "[$(date)] Finished system upgrade transfer data for {{ item }} role"
+{% endif -%}
+
+

templates/overcloud_upgrade_run.sh.j2

@@ -4,6 +4,17 @@
 #
 set -euo pipefail
 
+{# Elements will contain all the hosts in oc_roles_hosts for the
+role under loop from the first element of the list to the element in the item
+variable. Example:
+oc_roles_hosts = { 'Controller': ['controller-0', 'controller-1', 'controller-2',
+                   'Compute' : ['compute-0'] }
+If role = 'Controller' and item = 'controller-1'
+   elements = ['controller-0', 'controller-1']
+#}
+
+{% set elements = oc_roles_hosts[role][0:oc_roles_hosts[role].index(item)+1] %}
+
 {% if overcloud_upgrade_multibooks|bool %}
 
     {%- for playbook in overcloud_upgrade_playbooks -%}
@@ -30,15 +41,15 @@ bash {{ working_dir }}/fip_http_check_start.sh &
 
 source {{ undercloud_rc }}
 
-echo "[$(date)] Runing major upgrade {{ playbook }} playbook for {{ item }} role"
+echo "[$(date)] Running major upgrade {{ playbook }} playbook for {{ elements }} role"
 openstack overcloud upgrade run \
         --stack {{ overcloud_stack_name }} \
         {% if tripleo_upgrade_debug|bool -%}
         --debug \
         {% endif -%}
-        --limit {{ item }} --playbook {{ playbook }} 2>&1
+        --limit {{ elements | join(',') }} --playbook {{ playbook }} 2>&1
 
-echo "[$(date)] Finished major upgrade {{ playbook }} playbook for {{ item }} role"
+echo "[$(date)] Finished major upgrade {{ playbook }} playbook for {{ elements }} role"
 
     {%- endfor %}
 {# validation scripts should run after all playbooks are finished #}
@@ -90,16 +101,16 @@ bash {{ working_dir }}/fip_http_check_start.sh &
 
 source {{ undercloud_rc }}
 
-echo "[$(date)] Runing major upgrade for {{ item }} role"
+echo "[$(date)] Running major upgrade for {{ elements }} hosts"
 
 openstack overcloud upgrade run \
         --stack {{ overcloud_stack_name }} \
         {% if tripleo_upgrade_debug|bool %}
         --debug \
         {% endif -%}
-        --limit {{ item }} --playbook all 2>&1
+        --limit {{ elements | join(',') }} --playbook all 2>&1
 
-echo "[$(date)] Finished major upgrade for {{ item }} role"
+echo "[$(date)] Finished major upgrade for {{ elements }} hosts"
 
 {% if l3_agent_connectivity_check|bool %}
 if [[ -e {{ working_dir }}/l3_agent_stop_ping.sh ]]; then

templates/workarounds.sh.j2

@@ -138,6 +138,10 @@ ansible_patch {{ key }}
 {# finish pre_overcloud_upgrade_prepare_workarounds #}
 {{ render_workarounds(post_overcloud_upgrade_prepare_workarounds|default([]),'post_overcloud_upgrade_prepare_workarounds') }}
 {# finish post_overcloud_upgrade_prepare_workarounds #}
+{{ render_workarounds(pre_overcloud_system_upgrade_workarounds|default([]),'pre_overcloud_system_upgrade_workarounds') }}
+{# finish pre_overcloud_system_upgrade_workarounds #}
+{{ render_workarounds(post_overcloud_system_upgrade_workarounds|default([]),'post_overcloud_system_upgrade_workarounds') }}
+{# finish post_overcloud_system_upgrade_workarounds #}
 {{ render_workarounds(pre_overcloud_upgrade_workarounds|default([]),'pre_overcloud_upgrade_workarounds') }}
 {# finish pre_overcloud_upgrade_workarounds #}
 {{ render_workarounds(post_overcloud_upgrade_workarounds|default([]),'post_overcloud_upgrade_workarounds') }}