From 6d4d27c7434e6c79bffb52b2e4e988fe6a169480 Mon Sep 17 00:00:00 2001 From: Lee Yarwood Date: Thu, 23 Nov 2017 10:37:16 +0000 Subject: [PATCH] ffu: Introduce ffu upgrade tasks This change introduces the steps used for upgrading a Newton env to Queens by following the fast forward upgrade steps. Co-Authored-By: Marius Cornea Change-Id: I7a1478edc4a55a64960c136490019f202439b8dc --- defaults/main.yml | 20 +++++ infrared_plugin/main.yml | 23 +++++ infrared_plugin/plugin.spec | 29 +++++++ .../create-ffu-scripts.yaml | 8 ++ .../create-prepare-scripts.yaml | 81 +++++++++++++++++ .../ffu_deploy_steps_playbook.yaml | 5 ++ .../ffu_download_config.yaml | 8 ++ .../ffu_update_stack_outputs.yaml | 22 +++++ .../ffu_upgrade_playbook.yaml | 5 ++ .../ffu_upgrade_steps_playbook.yaml | 5 ++ tasks/fast-forward-upgrade/main.yml | 76 ++++++++++++++++ .../undercloud-upgrade.yml | 14 +++ tasks/main.yml | 3 + .../undercloud_upgrade.sh.j2 | 21 +++++ .../fast-forward-upgrade/workarounds.sh.j2 | 86 +++++++++++++++++++ 15 files changed, 406 insertions(+) create mode 100644 tasks/fast-forward-upgrade/create-ffu-scripts.yaml create mode 100644 tasks/fast-forward-upgrade/create-prepare-scripts.yaml create mode 100644 tasks/fast-forward-upgrade/ffu_deploy_steps_playbook.yaml create mode 100644 tasks/fast-forward-upgrade/ffu_download_config.yaml create mode 100644 tasks/fast-forward-upgrade/ffu_update_stack_outputs.yaml create mode 100644 tasks/fast-forward-upgrade/ffu_upgrade_playbook.yaml create mode 100644 tasks/fast-forward-upgrade/ffu_upgrade_steps_playbook.yaml create mode 100644 tasks/fast-forward-upgrade/main.yml create mode 100644 tasks/fast-forward-upgrade/undercloud-upgrade.yml create mode 100644 templates/fast-forward-upgrade/undercloud_upgrade.sh.j2 create mode 100644 templates/fast-forward-upgrade/workarounds.sh.j2 diff --git a/defaults/main.yml b/defaults/main.yml index bcfb0322..922bf632 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -14,12 +14,20 @@ overcloud_upgrade: false undercloud_update: false overcloud_update: false +# enable ffu upgrade +ffu_undercloud_upgrade: false +ffu_overcloud_upgrade: false +ffu_noop: false + # enable upgrade workarounds upgrade_workarounds: false # enable update workarounds updates_workarounds: false +# enable ffu workarounds +ffu_upgrade_workarounds: false + # use oooq use_oooq: false @@ -136,3 +144,15 @@ upgrade_hci: false upgrade_init_command_template: upgrade_init_command.yaml.j2 upgrade_init_command_dest: "{{working_dir}}/UpgradeInitCommand.yaml" upgrade_init_command: false + +# ffu undercloud upgrade +ffu_undercloud_releases: ['ocata', 'pike', 'queens'] +ffu_undercloud_repo_type: tripleo-repos +ffu_undercloud_repo_args: + tripleo_repos: + - release: ocata + args: -b ocata current + - release: pike + args: -b pike current + - release: queens + args: current diff --git a/infrared_plugin/main.yml b/infrared_plugin/main.yml index d5597e76..cbe022ad 100644 --- a/infrared_plugin/main.yml +++ b/infrared_plugin/main.yml @@ -122,5 +122,28 @@ set_fact: overcloud_rc: "{{ working_dir }}/{{ install.overcloud.stack }}rc" + - name: Set undercloud FFU upgrade + set_fact: + ffu_undercloud_upgrade: true + when: install.undercloud.ffu.upgrade + + - name: Set undercloud FFU upgrade releases + set_fact: + ffu_undercloud_releases: "{{ install.undercloud.ffu.releases }}" + + - name: Set undercloud FFU upgrade repo + set_fact: + ffu_undercloud_repo_type: "{{ install.undercloud.ffu.repo }}" + + - name: Set FFU upgrade workarounds + set_fact: + ffu_upgrade_workarounds: true + when: install.upgrade.ffu.workarounds + + - name: Set overcloud FFU upgrade + set_fact: + ffu_overcloud_upgrade: true + when: install.overcloud.ffu.upgrade + roles: - tripleo-upgrade diff --git a/infrared_plugin/plugin.spec b/infrared_plugin/plugin.spec index f9771d7b..1cb803e1 100644 --- a/infrared_plugin/plugin.spec +++ b/infrared_plugin/plugin.spec @@ -126,3 +126,32 @@ subparsers: type: Value help: Overcloud stack name default: "overcloud" + - title: TripleO Fast Forward Upgrade + options: + overcloud-ffu-upgrade: + type: Bool + help: | + Fast Forward Upgrade Overcloud + NOTE: Upgrade require overcloud deployment script to be available in home directory of undercloud + user at undercloud node + default: false + undercloud-ffu-upgrade: + type: Bool + help: | + Fast Forward Upgrade Undercloud + default: false + undercloud-ffu-releases: + type: ListValue + help: | + Undercloud FFU upgrade releases + default: 11,12,13 + undercloud-ffu-repo: + type: Value + help: | + Undercloud FFU upgrade repository method + default: 'rhos-release' + upgrade-ffu-workarounds: + type: Bool + help: | + Apply FFU upgrade workarounds + default: false diff --git a/tasks/fast-forward-upgrade/create-ffu-scripts.yaml b/tasks/fast-forward-upgrade/create-ffu-scripts.yaml new file mode 100644 index 00000000..c78daf3d --- /dev/null +++ b/tasks/fast-forward-upgrade/create-ffu-scripts.yaml @@ -0,0 +1,8 @@ +- name: create {{ release }} undercloud upgrade script + template: + src: templates/fast-forward-upgrade/undercloud_upgrade.sh.j2 + dest: "{{ working_dir }}/undercloud_upgrade_{{ release }}.sh" + mode: 0775 + vars: + query: "tripleo_repos[?release=='{{release}}'].args" + cmd_args: "{{ ffu_undercloud_repo_args | json_query(query) }}" diff --git a/tasks/fast-forward-upgrade/create-prepare-scripts.yaml b/tasks/fast-forward-upgrade/create-prepare-scripts.yaml new file mode 100644 index 00000000..7aeb1e7a --- /dev/null +++ b/tasks/fast-forward-upgrade/create-prepare-scripts.yaml @@ -0,0 +1,81 @@ +- name: create a comptatible deployment scripts from oooq + include: ../upgrade/use_oooq.yaml + tags: use_oooq + when: use_oooq|bool + +- name: make a copy of the initial overcloud deploy script + copy: + remote_src: yes + src: "{{ overcloud_deploy_script }}" + dest: "{{ overcloud_deploy_script }}.orig" + +- name: create ffu upgrade workaround scripts + template: + src: fast-forward-upgrade/workarounds.sh.j2 + dest: "{{working_dir}}/{{ item }}.sh" + mode: 0775 + with_items: + - 'post_ffu_undercloud_upgrade_workarounds' + - 'pre_ffu_update_stack_outputs_workarounds' + - 'pre_ffu_download_config_workarounds' + - 'pre_ffu_upgrade_playbook_workarounds' + - 'pre_ffu_upgrade_steps_playbook_workarounds' + - 'pre_ffu_deploy_steps_playbook_workarounds' + when: ffu_upgrade_workarounds + +- name: register is_upstream variable + shell: | + rpm -q openstack-tripleo-heat-templates | grep '\.el[[:digit:]]\.' + failed_when: false + register: is_upstream + +- name: Check if overcoud deployed with pacemaker + shell: | + source {{ undercloud_rc }} + openstack stack output show overcloud EnabledServices | grep '"pacemaker"' + failed_when: false + register: pacemaker_present + +- name: set deployment_with_ha fact + set_fact: + deployment_with_ha: "{{ is_upstream.rc == 0 and pacemaker_present.rc == 0 }}" + +- name: register used service environment files + # Find all environment files matching which include one of the + # listed patterns. Supports even when there's many of them on one + # line, which is how OOOQ passes env files. The lookahead expression + # needs to have fixed width, so remove any extra leading spaces from + # results via sed rather than in the grep expression itself. + shell: | + grep -oP '(^| )(?<=-e |--environment-file ) *[^ ]*(environments/docker|environments/services|environments/storage|environments/ceph-ansible|ci/environments)[^ ]*' {{ overcloud_deploy_script }} \ + | sed -E 's/^ *//' + register: services + ignore_errors: true + +- name: create container images download script + template: + src: "{{ container_images_download_template }}" + dest: "{{ container_images_download_script }}" + mode: 0775 + when: use_local_docker_registry + +- name: create registry environment file script + template: + src: "{{ local_docker_registry_env_template }}" + dest: "{{ local_docker_registry_env_script }}" + mode: 0775 + when: create_docker_registry + +- name: create ffu overcloud upgrade scripts + include: ../upgrade/step_upgrade.yml + loop_control: + loop_var: ugstage + with_items: + - step: "FFU update stack outputs" + script: "{{ working_dir }}/ffu_update_stack_outputs.sh" + environment_file: + - "{% if upstream_container_images %}{{ tht_directory }}/environments/docker.yaml{% endif %}" + - "{% if upstream_container_images and (deployment_with_ha.rc == 0) %}{{ tht_directory }}/environments/docker-ha.yaml{% endif %}" + - "{{ containers_default_parameters }}" + - "{{ tht_directory }}/environments/fast-forward-upgrade.yaml" + - "{{ tht_directory }}/environments/noop-deploy-steps.yaml" diff --git a/tasks/fast-forward-upgrade/ffu_deploy_steps_playbook.yaml b/tasks/fast-forward-upgrade/ffu_deploy_steps_playbook.yaml new file mode 100644 index 00000000..f1a9a456 --- /dev/null +++ b/tasks/fast-forward-upgrade/ffu_deploy_steps_playbook.yaml @@ -0,0 +1,5 @@ +--- +- name: run deploy_steps_playbook playbook + shell: | + source {{ undercloud_rc }} + ansible-playbook -i /usr/bin/tripleo-ansible-inventory {{ overcloud_config_dir }}/deploy_steps_playbook.yaml diff --git a/tasks/fast-forward-upgrade/ffu_download_config.yaml b/tasks/fast-forward-upgrade/ffu_download_config.yaml new file mode 100644 index 00000000..103e307f --- /dev/null +++ b/tasks/fast-forward-upgrade/ffu_download_config.yaml @@ -0,0 +1,8 @@ +- name: overcloud config download + shell: | + source {{ undercloud_rc }} + openstack overcloud config download + register: config_download + +- set_fact: + overcloud_config_dir: "{{ config_download.stdout | regex_search('(/.*tripleo.*)')}}" diff --git a/tasks/fast-forward-upgrade/ffu_update_stack_outputs.yaml b/tasks/fast-forward-upgrade/ffu_update_stack_outputs.yaml new file mode 100644 index 00000000..cef26e82 --- /dev/null +++ b/tasks/fast-forward-upgrade/ffu_update_stack_outputs.yaml @@ -0,0 +1,22 @@ +--- +- name: update stack outputs + shell: | + source {{ undercloud_rc }} + {{ working_dir }}/ffu_update_stack_outputs.sh &> {{ working_dir }}/ffu_update_stack_outputs.log + register: ffu_stack_output + ignore_errors: yes + +- name: print stack failures + shell: | + source {{ undercloud_rc }} + openstack stack failures list --long overcloud | tee {{ working_dir}}/ffu_update_stack_outputs_failed.log + when: ffu_stack_output|failed + +- name: print resource list + shell: | + source {{ undercloud_rc }} + openstack stack resource list --filter status=FAILED --nested-depth 5 overcloud | tee {{ working_dir}}/ffu_update_stack_outputs_failed_resources.log + when: ffu_stack_output|failed + +- fail: msg="FFU update stack outputs step failed... :(" + when: ffu_stack_output|failed diff --git a/tasks/fast-forward-upgrade/ffu_upgrade_playbook.yaml b/tasks/fast-forward-upgrade/ffu_upgrade_playbook.yaml new file mode 100644 index 00000000..b2bdfc3d --- /dev/null +++ b/tasks/fast-forward-upgrade/ffu_upgrade_playbook.yaml @@ -0,0 +1,5 @@ +--- +- name: run fast_forward_upgrade_playbook playbook + shell: | + source {{ undercloud_rc }} + ansible-playbook -i /usr/bin/tripleo-ansible-inventory {{ overcloud_config_dir }}/fast_forward_upgrade_playbook.yaml diff --git a/tasks/fast-forward-upgrade/ffu_upgrade_steps_playbook.yaml b/tasks/fast-forward-upgrade/ffu_upgrade_steps_playbook.yaml new file mode 100644 index 00000000..69e095f7 --- /dev/null +++ b/tasks/fast-forward-upgrade/ffu_upgrade_steps_playbook.yaml @@ -0,0 +1,5 @@ +--- +- name: run upgrade_steps_playbook playbook + shell: | + source {{ undercloud_rc }} + ansible-playbook -i /usr/bin/tripleo-ansible-inventory -M /usr/share/ansible-modules/ {{ overcloud_config_dir }}/upgrade_steps_playbook.yaml diff --git a/tasks/fast-forward-upgrade/main.yml b/tasks/fast-forward-upgrade/main.yml new file mode 100644 index 00000000..071ce02f --- /dev/null +++ b/tasks/fast-forward-upgrade/main.yml @@ -0,0 +1,76 @@ +- name: create upgrade prepare scripts + include: create-prepare-scripts.yaml + +- name: create fast forward upgrade scripts + include: create-ffu-scripts.yaml + tags: create_ffu_scripts + loop_control: + loop_var: release + with_items: "{{ ffu_undercloud_releases }}" + when: ffu_noop|bool or ffu_undercloud_upgrade|bool or ffu_overcloud_upgrade|bool + +- block: + - name: ffu undercloud upgrade + include: undercloud-upgrade.yml + loop_control: + loop_var: release + with_items: "{{ ffu_undercloud_releases }}" + + - name: apply post ffu undercloud upgrade workarounds + command: "{{working_dir}}/post_ffu_undercloud_upgrade_workarounds.sh" + when: ffu_upgrade_workarounds|bool + when: ffu_undercloud_upgrade|bool + +- block: + # TODO: move this out of upgrade + - name: download container images + include: ../upgrade/container_images.yaml + when: use_local_docker_registry|bool + tags: container_images + + - name: create local docker registry environment file + include: ../upgrade/docker_registry_images_env.yaml + when: create_docker_registry|bool + tags: docker_registry_images_env + + - name: apply pre ffu update stack outputs workarounds + command: "{{working_dir}}/pre_ffu_update_stack_outputs_workarounds.sh" + when: ffu_upgrade_workarounds|bool + tags: ffu_update_stack_outputs + + - include: ffu_update_stack_outputs.yaml + tags: ffu_update_stack_outputs + + - name: apply pre ffu download config workarounds + command: "{{working_dir}}/pre_ffu_download_config_workarounds.sh" + when: ffu_upgrade_workarounds|bool + tags: ffu_download_config + + - include: ffu_download_config.yaml + tags: ffu_download_config + + - name: apply pre fast forward upgrade playbook workarounds + command: "{{working_dir}}/pre_ffu_upgrade_playbook_workarounds.sh" + when: ffu_upgrade_workarounds|bool + tags: ffu_upgrade_playbook + + - include: ffu_upgrade_playbook.yaml + tags: ffu_upgrade_playbook + + - name: apply pre ffu upgrade steps playbook workarounds + command: "{{working_dir}}/pre_ffu_upgrade_steps_playbook_workarounds.sh" + when: ffu_upgrade_workarounds|bool + tags: ffu_upgrade_steps_playbook + + - include: ffu_upgrade_steps_playbook.yaml + tags: ffu_upgrade_steps_playbook + + - name: apply pre ffu deploy steps playbook workarounds + command: "{{working_dir}}/pre_ffu_deploy_steps_playbook_workarounds.sh" + when: ffu_upgrade_workarounds|bool + tags: ffu_deploy_steps_playbook + + - include: ffu_deploy_steps_playbook.yaml + tags: ffu_deploy_steps_playbook + + when: ffu_overcloud_upgrade|bool diff --git a/tasks/fast-forward-upgrade/undercloud-upgrade.yml b/tasks/fast-forward-upgrade/undercloud-upgrade.yml new file mode 100644 index 00000000..538518fb --- /dev/null +++ b/tasks/fast-forward-upgrade/undercloud-upgrade.yml @@ -0,0 +1,14 @@ +- include_role: + name: rhos-release + vars: + director_build: "{{ install.get('director', {}).build|default(omit) }}" + build: "{{ install.build | default(omit) }}" + mirror: "{{ install.mirror | default(omit) }}" + rr_distro_version: "{{ install.osrelease|default(omit) }}" + rhos_release_rpm: "https://url.corp.redhat.com/rhos-release-latest-rpm" + become: true + become_user: root + when: ffu_undercloud_repo_type == 'rhos-release' + +- name: Upgrade undercloud to {{ release }} + command: "{{ working_dir }}/undercloud_upgrade_{{ release }}.sh" diff --git a/tasks/main.yml b/tasks/main.yml index 1784225e..305fb367 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -15,3 +15,6 @@ - include: update/main.yml when: update_noop|bool or undercloud_update|bool or overcloud_update|bool + +- include: fast-forward-upgrade/main.yml + when: ffu_noop|bool or ffu_undercloud_upgrade|bool or ffu_overcloud_upgrade|bool diff --git a/templates/fast-forward-upgrade/undercloud_upgrade.sh.j2 b/templates/fast-forward-upgrade/undercloud_upgrade.sh.j2 new file mode 100644 index 00000000..71487da9 --- /dev/null +++ b/templates/fast-forward-upgrade/undercloud_upgrade.sh.j2 @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Perform undercloud upgrade and related steps +set -euo pipefail + +{% if ffu_undercloud_repo_type == 'tripleo-repos' %} +TRIPLEO_REPOS_RPM=$(curl -L --silent https://trunk.rdoproject.org/centos7/current/ | grep python2-tripleo-repos | awk -F "href" {'print $2'} | awk -F '"' {'print $2'}) +sudo yum localinstall -y https://trunk.rdoproject.org/centos7/current/${TRIPLEO_REPOS_RPM} + +sudo tripleo-repos {{ cmd_args | join(' ') }} ceph +{% endif %} + +{% if release == 'ocata' or release == '11' %} +sudo systemctl stop openstack-* neutron-* httpd +sudo yum update -y instack-undercloud openstack-puppet-modules openstack-tripleo-common python-tripleoclient +{% else %} +sudo yum -y install ceph-ansible +sudo yum -y update python-tripleoclient +{% endif %} + +openstack undercloud upgrade &> {{ working_dir }}/undercloud_upgrade_{{ release }}.log diff --git a/templates/fast-forward-upgrade/workarounds.sh.j2 b/templates/fast-forward-upgrade/workarounds.sh.j2 new file mode 100644 index 00000000..8d390fd6 --- /dev/null +++ b/templates/fast-forward-upgrade/workarounds.sh.j2 @@ -0,0 +1,86 @@ +#!/bin/bash +# +# Apply upgrade workarounds +set -euo pipefail + +{% if 'post_ffu_undercloud_upgrade_workarounds' in item %} +{% for bugs in post_ffu_undercloud_upgrade_workarounds|default([]) %} +{% for key, value in bugs.items() %} +echo {{ key }} +{% if value.patch %} +curl -4 https://review.openstack.org/changes/{{ value.id }}/revisions/current/patch?download | \ + base64 -d | \ + sudo patch -d {{ value.basedir }} -p1 + +{% else %} +{{ value.command }} + +{% endif %} +{% endfor %} +{% endfor %} + +{% elif 'pre_ffu_update_stack_outputs_workarounds' in item %} +{% for bugs in pre_ffu_update_stack_outputs_workarounds|default([]) %} +{% for key, value in bugs.items() %} +echo {{ key }} +{% if value.patch %} +curl -4 https://review.openstack.org/changes/{{ value.id }}/revisions/current/patch?download | \ + base64 -d | \ + sudo patch -d {{ value.basedir }} -p1 + +{% else %} +{{ value.command }} + +{% endif %} +{% endfor %} +{% endfor %} + +{% elif 'pre_ffu_download_config_workarounds' in item %} +{% for bugs in pre_ffu_download_config_workarounds|default([]) %} +{% for key, value in bugs.items() %} +echo {{ key }} +{% if value.patch %} +curl -4 https://review.openstack.org/changes/{{ value.id }}/revisions/current/patch?download | \ + base64 -d | \ + sudo patch -d {{ value.basedir }} -p1 + +{% else %} +{{ value.command }} + +{% endif %} +{% endfor %} +{% endfor %} + +{% elif 'pre_ffu_upgrade_playbook_workarounds' in item %} +{% for bugs in pre_ffu_upgrade_playbook_workarounds|default([]) %} +{% for key, value in bugs.items() %} +echo {{ key }} +{% if value.patch %} +curl -4 https://review.openstack.org/changes/{{ value.id }}/revisions/current/patch?download | \ + base64 -d | \ + sudo patch -d {{ value.basedir }} -p1 + +{% else %} +{{ value.command }} + +{% endif %} +{% endfor %} +{% endfor %} + +{% elif 'pre_ffu_deploy_steps_playbook_workarounds' in item %} +{% for bugs in pre_ffu_deploy_steps_playbook_workarounds|default([]) %} +{% for key, value in bugs.items() %} +echo {{ key }} +{% if value.patch %} +curl -4 https://review.openstack.org/changes/{{ value.id }}/revisions/current/patch?download | \ + base64 -d | \ + sudo patch -d {{ value.basedir }} -p1 + +{% else %} +{{ value.command }} + +{% endif %} +{% endfor %} +{% endfor %} + +{% endif %}