ffu: Introduce ffu upgrade tasks

This change introduces the steps used for upgrading a Newton env
to Queens by following the fast forward upgrade steps.

Co-Authored-By: Marius Cornea <mcornea@redhat.com>
Change-Id: I7a1478edc4a55a64960c136490019f202439b8dc

defaults/main.yml

@@ -14,12 +14,20 @@ overcloud_upgrade: false
 undercloud_update: false
 overcloud_update: false
 
+# enable ffu upgrade
+ffu_undercloud_upgrade: false
+ffu_overcloud_upgrade: false
+ffu_noop: false
+
 # enable upgrade workarounds
 upgrade_workarounds: false
 
 # enable update workarounds
 updates_workarounds: false
 
+# enable ffu workarounds
+ffu_upgrade_workarounds: false
+
 # use oooq
 use_oooq: false
 
@@ -136,3 +144,15 @@ upgrade_hci: false
 upgrade_init_command_template: upgrade_init_command.yaml.j2
 upgrade_init_command_dest: "{{working_dir}}/UpgradeInitCommand.yaml"
 upgrade_init_command: false
+
+# ffu undercloud upgrade
+ffu_undercloud_releases: ['ocata', 'pike', 'queens']
+ffu_undercloud_repo_type: tripleo-repos
+ffu_undercloud_repo_args:
+  tripleo_repos:
+    - release: ocata
+      args: -b ocata current
+    - release: pike
+      args: -b pike current
+    - release: queens
+      args: current

infrared_plugin/main.yml

@@ -122,5 +122,28 @@
       set_fact:
          overcloud_rc: "{{ working_dir }}/{{ install.overcloud.stack }}rc"
 
+    - name: Set undercloud FFU upgrade
+      set_fact:
+         ffu_undercloud_upgrade: true
+      when: install.undercloud.ffu.upgrade
+
+    - name: Set undercloud FFU upgrade releases
+      set_fact:
+         ffu_undercloud_releases: "{{ install.undercloud.ffu.releases }}"
+
+    - name: Set undercloud FFU upgrade repo
+      set_fact:
+         ffu_undercloud_repo_type: "{{ install.undercloud.ffu.repo }}"
+
+    - name: Set FFU upgrade workarounds
+      set_fact:
+         ffu_upgrade_workarounds: true
+      when: install.upgrade.ffu.workarounds
+
+    - name: Set overcloud FFU upgrade
+      set_fact:
+         ffu_overcloud_upgrade: true
+      when: install.overcloud.ffu.upgrade
+
   roles:
     - tripleo-upgrade

infrared_plugin/plugin.spec

@@ -126,3 +126,32 @@ subparsers:
                       type: Value
                       help: Overcloud stack name
                       default: "overcloud"
+            - title: TripleO Fast Forward Upgrade
+              options:
+                  overcloud-ffu-upgrade:
+                      type: Bool
+                      help: |
+                          Fast Forward Upgrade Overcloud
+                          NOTE: Upgrade require overcloud deployment script to be available in home directory of undercloud
+                          user at undercloud node
+                      default: false
+                  undercloud-ffu-upgrade:
+                      type: Bool
+                      help: |
+                          Fast Forward Upgrade Undercloud
+                      default: false
+                  undercloud-ffu-releases:
+                      type: ListValue
+                      help: |
+                          Undercloud FFU upgrade releases
+                      default: 11,12,13
+                  undercloud-ffu-repo:
+                      type: Value
+                      help: |
+                          Undercloud FFU upgrade repository method
+                      default: 'rhos-release'
+                  upgrade-ffu-workarounds:
+                      type: Bool
+                      help: |
+                          Apply FFU upgrade workarounds
+                      default: false

tasks/fast-forward-upgrade/create-ffu-scripts.yaml

@@ -0,0 +1,8 @@
+- name: create {{ release }} undercloud upgrade script
+  template:
+    src: templates/fast-forward-upgrade/undercloud_upgrade.sh.j2
+    dest: "{{ working_dir }}/undercloud_upgrade_{{ release }}.sh"
+    mode: 0775
+  vars:
+    query: "tripleo_repos[?release=='{{release}}'].args"
+    cmd_args: "{{ ffu_undercloud_repo_args | json_query(query) }}"

tasks/fast-forward-upgrade/create-prepare-scripts.yaml

@@ -0,0 +1,81 @@
+- name: create a comptatible deployment scripts from oooq
+  include: ../upgrade/use_oooq.yaml
+  tags: use_oooq
+  when: use_oooq|bool
+
+- name: make a copy of the initial overcloud deploy script
+  copy:
+    remote_src: yes
+    src: "{{ overcloud_deploy_script }}"
+    dest: "{{ overcloud_deploy_script }}.orig"
+
+- name: create ffu upgrade workaround scripts
+  template:
+    src: fast-forward-upgrade/workarounds.sh.j2
+    dest: "{{working_dir}}/{{ item }}.sh"
+    mode: 0775
+  with_items:
+     - 'post_ffu_undercloud_upgrade_workarounds'
+     - 'pre_ffu_update_stack_outputs_workarounds'
+     - 'pre_ffu_download_config_workarounds'
+     - 'pre_ffu_upgrade_playbook_workarounds'
+     - 'pre_ffu_upgrade_steps_playbook_workarounds'
+     - 'pre_ffu_deploy_steps_playbook_workarounds'
+  when: ffu_upgrade_workarounds
+
+- name: register is_upstream variable
+  shell: |
+    rpm -q openstack-tripleo-heat-templates | grep '\.el[[:digit:]]\.'
+  failed_when: false
+  register: is_upstream
+
+- name: Check if overcoud deployed with pacemaker
+  shell: |
+    source {{ undercloud_rc }}
+    openstack stack output show overcloud EnabledServices | grep '"pacemaker"'
+  failed_when: false
+  register: pacemaker_present
+
+- name: set deployment_with_ha fact
+  set_fact:
+    deployment_with_ha: "{{ is_upstream.rc == 0 and pacemaker_present.rc == 0 }}"
+
+- name: register used service environment files
+  # Find all environment files matching which include one of the
+  # listed patterns. Supports even when there's many of them on one
+  # line, which is how OOOQ passes env files. The lookahead expression
+  # needs to have fixed width, so remove any extra leading spaces from
+  # results via sed rather than in the grep expression itself.
+  shell: |
+    grep -oP '(^| )(?<=-e |--environment-file ) *[^ ]*(environments/docker|environments/services|environments/storage|environments/ceph-ansible|ci/environments)[^ ]*' {{ overcloud_deploy_script }} \
+        | sed -E 's/^ *//'
+  register: services
+  ignore_errors: true
+
+- name: create container images download script
+  template:
+    src: "{{ container_images_download_template }}"
+    dest: "{{ container_images_download_script }}"
+    mode: 0775
+  when: use_local_docker_registry
+
+- name: create registry environment file script
+  template:
+    src: "{{ local_docker_registry_env_template }}"
+    dest: "{{ local_docker_registry_env_script }}"
+    mode: 0775
+  when: create_docker_registry
+
+- name: create ffu overcloud upgrade scripts
+  include: ../upgrade/step_upgrade.yml
+  loop_control:
+    loop_var: ugstage
+  with_items:
+    - step: "FFU update stack outputs"
+      script: "{{ working_dir }}/ffu_update_stack_outputs.sh"
+      environment_file:
+        - "{% if upstream_container_images %}{{ tht_directory }}/environments/docker.yaml{% endif %}"
+        - "{% if upstream_container_images and (deployment_with_ha.rc == 0)  %}{{ tht_directory }}/environments/docker-ha.yaml{% endif %}"
+        - "{{ containers_default_parameters }}"
+        - "{{ tht_directory }}/environments/fast-forward-upgrade.yaml"
+        - "{{ tht_directory }}/environments/noop-deploy-steps.yaml"

tasks/fast-forward-upgrade/ffu_deploy_steps_playbook.yaml

@@ -0,0 +1,5 @@
+---
+- name: run deploy_steps_playbook playbook
+  shell: |
+    source {{ undercloud_rc }}
+    ansible-playbook -i /usr/bin/tripleo-ansible-inventory {{ overcloud_config_dir }}/deploy_steps_playbook.yaml

tasks/fast-forward-upgrade/ffu_download_config.yaml

@@ -0,0 +1,8 @@
+- name: overcloud config download
+  shell: |
+    source {{ undercloud_rc }}
+    openstack overcloud config download
+  register: config_download
+
+- set_fact:
+    overcloud_config_dir: "{{ config_download.stdout | regex_search('(/.*tripleo.*)')}}"

tasks/fast-forward-upgrade/ffu_update_stack_outputs.yaml

@@ -0,0 +1,22 @@
+---
+- name: update stack outputs
+  shell: |
+    source {{ undercloud_rc }}
+    {{ working_dir }}/ffu_update_stack_outputs.sh &> {{ working_dir }}/ffu_update_stack_outputs.log
+  register: ffu_stack_output
+  ignore_errors: yes
+
+- name: print stack failures
+  shell: |
+    source {{ undercloud_rc }}
+    openstack stack failures list --long overcloud | tee {{ working_dir}}/ffu_update_stack_outputs_failed.log
+  when: ffu_stack_output|failed
+
+- name: print resource list
+  shell: |
+    source {{ undercloud_rc }}
+    openstack stack resource list --filter status=FAILED --nested-depth 5 overcloud | tee {{ working_dir}}/ffu_update_stack_outputs_failed_resources.log
+  when: ffu_stack_output|failed
+
+- fail: msg="FFU update stack outputs step failed... :("
+  when: ffu_stack_output|failed

tasks/fast-forward-upgrade/ffu_upgrade_playbook.yaml

@@ -0,0 +1,5 @@
+---
+- name: run fast_forward_upgrade_playbook playbook
+  shell: |
+    source {{ undercloud_rc }}
+    ansible-playbook -i /usr/bin/tripleo-ansible-inventory {{ overcloud_config_dir }}/fast_forward_upgrade_playbook.yaml

tasks/fast-forward-upgrade/ffu_upgrade_steps_playbook.yaml

@@ -0,0 +1,5 @@
+---
+- name: run upgrade_steps_playbook playbook
+  shell: |
+    source {{ undercloud_rc }}
+    ansible-playbook -i /usr/bin/tripleo-ansible-inventory -M /usr/share/ansible-modules/ {{ overcloud_config_dir }}/upgrade_steps_playbook.yaml

tasks/fast-forward-upgrade/main.yml

@@ -0,0 +1,76 @@
+- name: create upgrade prepare scripts
+  include: create-prepare-scripts.yaml
+
+- name: create fast forward upgrade scripts
+  include: create-ffu-scripts.yaml
+  tags: create_ffu_scripts
+  loop_control:
+    loop_var: release
+  with_items: "{{ ffu_undercloud_releases }}"
+  when: ffu_noop|bool or ffu_undercloud_upgrade|bool or ffu_overcloud_upgrade|bool
+
+- block:
+    - name: ffu undercloud upgrade
+      include: undercloud-upgrade.yml
+      loop_control:
+        loop_var: release
+      with_items: "{{ ffu_undercloud_releases }}"
+
+    - name: apply post ffu undercloud upgrade workarounds
+      command: "{{working_dir}}/post_ffu_undercloud_upgrade_workarounds.sh"
+      when: ffu_upgrade_workarounds|bool
+  when: ffu_undercloud_upgrade|bool
+
+- block:
+    # TODO: move this out of upgrade
+    - name: download container images
+      include: ../upgrade/container_images.yaml
+      when: use_local_docker_registry|bool
+      tags: container_images
+
+    - name: create local docker registry environment file
+      include: ../upgrade/docker_registry_images_env.yaml
+      when: create_docker_registry|bool
+      tags: docker_registry_images_env
+
+    - name: apply pre ffu update stack outputs workarounds
+      command: "{{working_dir}}/pre_ffu_update_stack_outputs_workarounds.sh"
+      when: ffu_upgrade_workarounds|bool
+      tags: ffu_update_stack_outputs
+
+    - include: ffu_update_stack_outputs.yaml
+      tags: ffu_update_stack_outputs
+
+    - name: apply pre ffu download config workarounds
+      command: "{{working_dir}}/pre_ffu_download_config_workarounds.sh"
+      when: ffu_upgrade_workarounds|bool
+      tags: ffu_download_config
+
+    - include: ffu_download_config.yaml
+      tags: ffu_download_config
+
+    - name: apply pre fast forward upgrade playbook workarounds
+      command: "{{working_dir}}/pre_ffu_upgrade_playbook_workarounds.sh"
+      when: ffu_upgrade_workarounds|bool
+      tags: ffu_upgrade_playbook
+
+    - include: ffu_upgrade_playbook.yaml
+      tags: ffu_upgrade_playbook
+
+    - name: apply pre ffu upgrade steps playbook workarounds
+      command: "{{working_dir}}/pre_ffu_upgrade_steps_playbook_workarounds.sh"
+      when: ffu_upgrade_workarounds|bool
+      tags: ffu_upgrade_steps_playbook
+
+    - include: ffu_upgrade_steps_playbook.yaml
+      tags: ffu_upgrade_steps_playbook
+
+    - name: apply pre ffu deploy steps playbook workarounds
+      command: "{{working_dir}}/pre_ffu_deploy_steps_playbook_workarounds.sh"
+      when: ffu_upgrade_workarounds|bool
+      tags: ffu_deploy_steps_playbook
+
+    - include: ffu_deploy_steps_playbook.yaml
+      tags: ffu_deploy_steps_playbook
+
+  when: ffu_overcloud_upgrade|bool

tasks/fast-forward-upgrade/undercloud-upgrade.yml

@@ -0,0 +1,14 @@
+- include_role:
+    name: rhos-release
+  vars:
+    director_build: "{{ install.get('director', {}).build|default(omit) }}"
+    build: "{{ install.build | default(omit) }}"
+    mirror: "{{ install.mirror | default(omit) }}"
+    rr_distro_version: "{{ install.osrelease|default(omit) }}"
+    rhos_release_rpm: "https://url.corp.redhat.com/rhos-release-latest-rpm"
+  become: true
+  become_user: root
+  when: ffu_undercloud_repo_type == 'rhos-release'
+
+- name: Upgrade undercloud to {{ release }}
+  command: "{{ working_dir }}/undercloud_upgrade_{{ release }}.sh"

tasks/main.yml

@@ -15,3 +15,6 @@
 
 - include: update/main.yml
   when: update_noop|bool or undercloud_update|bool or overcloud_update|bool
+
+- include: fast-forward-upgrade/main.yml
+  when: ffu_noop|bool or ffu_undercloud_upgrade|bool or ffu_overcloud_upgrade|bool

templates/fast-forward-upgrade/undercloud_upgrade.sh.j2

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Perform undercloud upgrade and related steps
+set -euo pipefail
+
+{% if ffu_undercloud_repo_type == 'tripleo-repos' %}
+TRIPLEO_REPOS_RPM=$(curl -L --silent https://trunk.rdoproject.org/centos7/current/ | grep python2-tripleo-repos | awk -F "href" {'print $2'} | awk -F '"' {'print $2'})
+sudo yum localinstall -y https://trunk.rdoproject.org/centos7/current/${TRIPLEO_REPOS_RPM}
+
+sudo tripleo-repos {{ cmd_args | join(' ') }} ceph
+{% endif %}
+
+{% if release == 'ocata' or release == '11' %}
+sudo systemctl stop openstack-* neutron-* httpd
+sudo yum update -y instack-undercloud openstack-puppet-modules openstack-tripleo-common python-tripleoclient
+{% else %}
+sudo yum -y install ceph-ansible
+sudo yum -y update python-tripleoclient
+{% endif %}
+
+openstack undercloud upgrade &> {{ working_dir }}/undercloud_upgrade_{{ release }}.log

templates/fast-forward-upgrade/workarounds.sh.j2

@@ -0,0 +1,86 @@
+#!/bin/bash
+#
+# Apply upgrade workarounds
+set -euo pipefail
+
+{% if 'post_ffu_undercloud_upgrade_workarounds' in item %}
+{% for bugs in post_ffu_undercloud_upgrade_workarounds|default([]) %}
+{% for key, value in bugs.items() %}
+echo {{ key }}
+{% if value.patch %}
+curl -4 https://review.openstack.org/changes/{{ value.id }}/revisions/current/patch?download | \
+    base64 -d | \
+    sudo patch  -d {{ value.basedir }} -p1
+
+{% else %}
+{{ value.command }}
+
+{% endif %}
+{% endfor %}
+{% endfor %}
+
+{% elif 'pre_ffu_update_stack_outputs_workarounds' in item %}
+{% for bugs in pre_ffu_update_stack_outputs_workarounds|default([]) %}
+{% for key, value in bugs.items() %}
+echo {{ key }}
+{% if value.patch %}
+curl -4 https://review.openstack.org/changes/{{ value.id }}/revisions/current/patch?download | \
+    base64 -d | \
+    sudo patch  -d {{ value.basedir }} -p1
+
+{% else %}
+{{ value.command }}
+
+{% endif %}
+{% endfor %}
+{% endfor %}
+
+{% elif 'pre_ffu_download_config_workarounds' in item %}
+{% for bugs in pre_ffu_download_config_workarounds|default([]) %}
+{% for key, value in bugs.items() %}
+echo {{ key }}
+{% if value.patch %}
+curl -4 https://review.openstack.org/changes/{{ value.id }}/revisions/current/patch?download | \
+    base64 -d | \
+    sudo patch  -d {{ value.basedir }} -p1
+
+{% else %}
+{{ value.command }}
+
+{% endif %}
+{% endfor %}
+{% endfor %}
+
+{% elif 'pre_ffu_upgrade_playbook_workarounds' in item %}
+{% for bugs in pre_ffu_upgrade_playbook_workarounds|default([]) %}
+{% for key, value in bugs.items() %}
+echo {{ key }}
+{% if value.patch %}
+curl -4 https://review.openstack.org/changes/{{ value.id }}/revisions/current/patch?download | \
+     base64 -d | \
+     sudo patch  -d {{ value.basedir }} -p1
+
+{% else %}
+{{ value.command }}
+
+{% endif %}
+{% endfor %}
+{% endfor %}
+
+{% elif 'pre_ffu_deploy_steps_playbook_workarounds' in item %}
+{% for bugs in pre_ffu_deploy_steps_playbook_workarounds|default([]) %}
+{% for key, value in bugs.items() %}
+echo {{ key }}
+{% if value.patch %}
+curl -4 https://review.openstack.org/changes/{{ value.id }}/revisions/current/patch?download | \
+    base64 -d | \
+    sudo patch  -d {{ value.basedir }} -p1
+
+{% else %}
+{{ value.command }}
+
+{% endif %}
+{% endfor %}
+{% endfor %}
+
+{% endif %}