Re-enable stx-openstack restore
This commit re-enables stx-openstack restore after recent code churn
in Ansible playbooks.
For a partial restore where ceph data remain intact during restore,
run restore_openstack playbook once to restore stx-openstack.
For a full restore where ceph data is wiped during restore, following
these steps to restore stx-openstack:
- Run restore_openstack playbook with restore_ceph_data set to true
- Follow the procedure to restore glance images and cinder volumes
- Run restore_openstack playbook with restore_openstack_continue set
to true
Change-Id: I96838392e4fb06c1d84e3f884bda4cd9be4f9d08
Story: 2004761
Task: 36813
Signed-off-by: Wei Zhou <wei.zhou@windriver.com>
This commit is contained in:
Wei Zhou
parent
0bd13cd69e
commit
567e89f682
|
|
@ -80,3 +80,8 @@ host_backup_dir: "{{ lookup('env', 'HOME') }}"
|
|||
# Flag file to indicate if platform restore is in progress
|
||||
#
|
||||
restore_in_progress_flag: /etc/platform/.restore_in_progress
|
||||
|
||||
# This variable is set to true when restore_openstack playbook is
|
||||
# played again to bring up remaining OpenStack services after Ceph
|
||||
# data is restored.
|
||||
restore_openstack_continue: false
|
||||
|
|
|
|||
|
|
@ -10,26 +10,12 @@
|
|||
gather_facts: no
|
||||
|
||||
vars_files:
|
||||
- host_vars/default.yml
|
||||
|
||||
pre_tasks:
|
||||
- stat:
|
||||
path: "{{ item }}"
|
||||
register: files_to_import
|
||||
with_items:
|
||||
- "{{ override_files_dir }}/secrets.yml"
|
||||
- "{{ override_files_dir }}/{{ inventory_hostname }}_secrets.yml"
|
||||
- "{{ override_files_dir }}/site.yml"
|
||||
- "{{ override_files_dir }}/{{ inventory_hostname }}.yml"
|
||||
delegate_to: localhost
|
||||
|
||||
- include_vars: "{{ item.item }}"
|
||||
when: item.stat.exists
|
||||
with_items: "{{ files_to_import.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item }}"
|
||||
- host_vars/backup-restore/default.yml
|
||||
|
||||
# Main play
|
||||
roles:
|
||||
- common
|
||||
- { role: common/prepare-env }
|
||||
- { role: backup-restore/prepare-env }
|
||||
- { role: restore-openstack/prepare-env, when: not restore_openstack_continue|bool }
|
||||
- { role: backup-restore/transfer-file, when: not restore_openstack_continue|bool }
|
||||
- { role: restore-openstack/restore, become: yes }
|
||||
|
|
|
|||
|
|
@ -11,6 +11,10 @@
|
|||
# - reboot the controller if it is required by the patching
|
||||
#
|
||||
- block:
|
||||
- name: Set fact for restore_in_progress flag file
|
||||
set_fact:
|
||||
restore_in_progress_flag: /etc/platform/.restore_in_progress
|
||||
|
||||
- name: Set fact for patching staging dir
|
||||
set_fact:
|
||||
patching_staging_dir: /scratch/patching
|
||||
|
|
|
|||
|
|
@ -0,0 +1,82 @@
|
|||
---
|
||||
#
|
||||
# Copyright (c) 2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# ROLE DESCRIPTION:
|
||||
# This role verifies if platform restore should proceed
|
||||
#
|
||||
- name: Fail if backup_filename is not defined or set
|
||||
fail:
|
||||
msg: "Mandatory configuration parameter backup_filename is not defined or set."
|
||||
when: backup_filename is not defined or backup_filename is none
|
||||
|
||||
- name: Fail if initial_backup_dir is not defined or set
|
||||
fail:
|
||||
msg: "Mandatory configuration parameter initial_backup_dir is not defined or set."
|
||||
when: initial_backup_dir is not defined or initial_backup_dir is none
|
||||
|
||||
- name: Check if restoring StarlingX Openstack is already in progress
|
||||
stat:
|
||||
path: "{{ restore_openstack_in_progress_flag }}"
|
||||
register: restore_openstack_in_progress
|
||||
|
||||
- name: Fail if restoring StarlingX Openstack is already in progress
|
||||
fail:
|
||||
msg: "Restoring StarlingX Openstack is already in progress!"
|
||||
when: restore_openstack_in_progress.stat.exists
|
||||
|
||||
- name: Check if it is the active controller
|
||||
shell: source /etc/platform/openrc; system host-show $(cat /etc/hostname) | grep -i "Controller-Active"
|
||||
register: active_ctlr
|
||||
|
||||
- name: Fail if it is not the active controller
|
||||
fail:
|
||||
msg: "Restore StarlingX Openstack application can only be done on the active controller."
|
||||
when: active_ctlr.rc != 0
|
||||
|
||||
- name: Search for stx-openstack application
|
||||
shell: source /etc/platform/openrc; system application-list | grep 'stx-openstack'
|
||||
register: check_openstack
|
||||
failed_when: false
|
||||
|
||||
# If stx-application doesn't exist or is in applied state, the restore is complete.
|
||||
- block:
|
||||
- debug:
|
||||
msg: >-
|
||||
Either StarlingX OpenStack application had not been applied when the system was
|
||||
backed up or it has already been restored successfully.
|
||||
- meta: end_play
|
||||
|
||||
when: check_openstack.rc != 0 or check_openstack.stdout is search('applied')
|
||||
|
||||
# If stx-application is not in uploaded state, it means restore_openstack playbook
|
||||
# has been run and was not successful. Re-run the playbook is not supported.
|
||||
- name: Fail if StarlingX Openstack is not in the right state for restore
|
||||
fail:
|
||||
msg: >-
|
||||
stx-openstack application must be in ‘uploaded’ state for the restore to proceed.
|
||||
The restore must have failed already.
|
||||
when: check_openstack.stdout is not search('uploaded')
|
||||
|
||||
- name: Check if platform-integ-apps was applied successfully during controller-0 unlock
|
||||
shell: >-
|
||||
source /etc/platform/openrc; system application-list |
|
||||
grep 'platform-integ-apps.*applied.*completed'
|
||||
register: check_platform_integ
|
||||
|
||||
- name: Fail if platform-integ-apps was not applied successfully
|
||||
fail:
|
||||
msg: Application platform-integ-apps was not applied successfully during controller-0 unlock
|
||||
when: check_platform_integ.rc != 0
|
||||
|
||||
- name: For remote play set target_backup_dir to {{ backup_dir }}
|
||||
set_fact:
|
||||
target_backup_dir: "{{ backup_dir }}"
|
||||
when: inventory_hostname != "localhost"
|
||||
|
||||
- name: For local play set target_backup_dir to initial_backup_dir
|
||||
set_fact:
|
||||
target_backup_dir: "{{ initial_backup_dir }}"
|
||||
when: inventory_hostname == "localhost"
|
||||
|
|
@ -7,80 +7,51 @@
|
|||
# ROLE DESCRIPTION:
|
||||
# This role is to restore StarlingX Openstack application.
|
||||
#
|
||||
|
||||
- name: Fail if backup_filename is not defined or set
|
||||
fail:
|
||||
msg: "Mandatory configuration parameter backup_filename is not defined or set."
|
||||
when: backup_filename is not defined or backup_filename is none
|
||||
|
||||
# Check if the backup tarball exists in {{ backup_dir }}. This check can
|
||||
# serve as a guard to ensure restore StarlingX Openstack is run on the
|
||||
# right system because after the platform is restored the backup tarball is
|
||||
# stored in {{ backup_dir }} on controller-0.
|
||||
- name: Check if the backup tarball exists in {{ backup_dir }}
|
||||
stat:
|
||||
path: "{{ backup_dir }}/{{ backup_filename }}"
|
||||
register: backup_file_result
|
||||
|
||||
- name: Fail if the backup tarball doesn't exist
|
||||
fail:
|
||||
msg: Cannot find backup tarball {{ backup_dir }}/{{ backup_filename }}!
|
||||
when: not backup_file_result.stat.exists
|
||||
|
||||
- name: Check if it is the active controller
|
||||
shell: source /etc/platform/openrc; system host-show $(cat /etc/hostname) | grep -i "Controller-Active"
|
||||
register: active_ctlr
|
||||
|
||||
- name: Fail if it is not the active controller
|
||||
fail:
|
||||
msg: "Restore StarlingX Openstack application can only be done on the active controller."
|
||||
when: active_ctlr.rc != 0
|
||||
|
||||
- name: Check if restoring StarlingX Openstack is already in progress
|
||||
stat:
|
||||
path: "{{ restore_openstack_in_progress_flag }}"
|
||||
register: restore_openstack_in_progress
|
||||
|
||||
- name: Fail if restoring StarlingX Openstack is already in progress
|
||||
fail:
|
||||
msg: "Restoring StarlingX Openstack is already in progress!"
|
||||
when: restore_openstack_in_progress.stat.exists
|
||||
|
||||
- name: Search for stx-openstack application
|
||||
shell: source /etc/platform/openrc; system application-list | grep 'stx-openstack'
|
||||
register: check_openstack
|
||||
failed_when: false
|
||||
|
||||
# If stx-application doesn't exist or is in applied state, the restore is complete.
|
||||
# To bring up remaining StarlingX Openstack services after storage data is restored
|
||||
- block:
|
||||
- debug:
|
||||
- name: Check if restore OpenStack continue flag file exists
|
||||
stat:
|
||||
path: "{{ restore_openstack_continue_flag }}"
|
||||
register: check_restore_openstack_continue
|
||||
|
||||
- name: Fail if restore OpenStack continue flag file does not exist
|
||||
fail:
|
||||
msg: >-
|
||||
Either StarlingX OpenStack application had not been applied when the system was
|
||||
backed up or it has already been restored successfully.
|
||||
"{{ restore_openstack_continue_flag }} does not exist. Cannot continue
|
||||
to restore OpenStack application."
|
||||
when: not check_restore_openstack_continue.stat.exists
|
||||
|
||||
- name: Continue to bring up the remaining StarlingX Openstack services
|
||||
shell: source /etc/platform/openrc; system application-apply --mode normal stx-openstack
|
||||
|
||||
- name: Checking stx-openstack apply progress for the remaining Openstack services... (max 50 mins)
|
||||
shell: source /etc/platform/openrc; system application-list | grep 'stx-openstack'
|
||||
register: check_app_state
|
||||
until: check_app_state.stdout is search('applied') or
|
||||
check_app_state.stdout is search('apply-failed')
|
||||
retries: 50
|
||||
delay: 60
|
||||
|
||||
- name: Fail if the remaining Openstack services cannot be restored successfully
|
||||
fail:
|
||||
msg: stx-openstack application is not applied successfully.
|
||||
when: check_app_state.stdout is not search('applied')
|
||||
|
||||
- name: Remove restore_openstack_in_progress flag file
|
||||
file:
|
||||
path: "{{ restore_openstack_in_progress_flag }}"
|
||||
state: absent
|
||||
|
||||
- name: Remove restore_openstack_continue flag file
|
||||
file:
|
||||
path: "{{ restore_openstack_continue_flag }}"
|
||||
state: absent
|
||||
|
||||
- debug:
|
||||
msg: Restore stx-openstack application is successful!
|
||||
|
||||
- meta: end_play
|
||||
|
||||
when: check_openstack.rc != 0 or check_openstack.stdout is search('applied')
|
||||
|
||||
# If stx-application is not in uploaded state, it means restore_openstack playbook
|
||||
# has been run and was not successfully. Re-run the playbook is not supported.
|
||||
- name: Fail if StarlingX Openstack is not in the right state for restore
|
||||
fail:
|
||||
msg: >-
|
||||
stx-openstack application must be in ‘uploaded’ state for the restore to proceed.
|
||||
The restore must have failed already.
|
||||
|
||||
when: check_openstack.stdout is not search('uploaded')
|
||||
|
||||
- name: Check if platform-integ-apps was applied successfully during controller-0 unlock
|
||||
shell: >-
|
||||
source /etc/platform/openrc; system application-list |
|
||||
grep 'platform-integ-apps.*applied.*completed'
|
||||
register: check_platform_integ
|
||||
|
||||
- name: Fail if platform-integ-apps was not applied successfully
|
||||
fail:
|
||||
msg: Application platform-integ-apps was not applied successfully during controller-0 unlock
|
||||
when: check_platform_integ.rc != 0
|
||||
when: restore_openstack_continue
|
||||
|
||||
- name: Create restore StarlingX Openstack in progress flag file in /opt/platform
|
||||
file:
|
||||
|
|
@ -88,6 +59,20 @@
|
|||
state: touch
|
||||
|
||||
- block:
|
||||
- name: Set config path facts
|
||||
set_fact:
|
||||
armada_permdir: "{{ platform_path }}/armada/{{ software_version }}"
|
||||
helm_charts_permdir: "{{ platform_path }}/helm_charts"
|
||||
|
||||
- name: Restore Helm charts, armada manifests and extension filesystem
|
||||
command: tar -C / --overwrite -xpf {{ target_backup_dir }}/{{ backup_filename }} {{ item }}
|
||||
args:
|
||||
warn: false
|
||||
become_user: root
|
||||
with_items:
|
||||
- "{{ helm_charts_permdir | regex_replace('^\\/', '') }}"
|
||||
- "{{ armada_permdir | regex_replace('^\\/', '') }}"
|
||||
|
||||
- name: Bring up MariaDB and its prerequisite services
|
||||
shell: >-
|
||||
source /etc/platform/openrc; system application-apply --mode restore_db stx-openstack
|
||||
|
|
@ -125,7 +110,7 @@
|
|||
|
||||
- name: Fail if openstack-garbd pod is not running
|
||||
fail:
|
||||
msg: Openstack-garbd pod is not running. Restore StarlingX Openstack application failed.
|
||||
msg: Openstack-garbd pod is not running. Restore stx-openstack application failed.
|
||||
when: check_garbd_pod.rc != 0
|
||||
|
||||
when: system_type == "Standard"
|
||||
|
|
@ -141,7 +126,7 @@
|
|||
|
||||
- name: Extract MariaDB data to backup directory
|
||||
command: >-
|
||||
tar -C {{ backup_dir }}/mariadb -xpf {{ backup_dir }}/{{ backup_filename }}
|
||||
tar -C {{ backup_dir }}/mariadb -xpf {{ target_backup_dir }}/{{ backup_filename }}
|
||||
--transform='s,.*/,,' '*/*\.mariadb\.*'
|
||||
args:
|
||||
warn: false
|
||||
|
|
@ -189,7 +174,7 @@
|
|||
fail:
|
||||
msg: >-
|
||||
Cinder and Glance services cannot be restored successfully.
|
||||
Restore StarlingX Openstack application failed.
|
||||
Restore stx-openstack application failed.
|
||||
when: check_app_state.stdout is not search('applied')
|
||||
|
||||
- block:
|
||||
|
|
@ -199,6 +184,11 @@
|
|||
Please refer to the system administration guide to restore Glance images
|
||||
and Cinder volumes.
|
||||
|
||||
- name: Create restore StarlingX Openstack continue flag file
|
||||
file:
|
||||
path: "{{ restore_openstack_continue_flag }}"
|
||||
state: touch
|
||||
|
||||
- meta: end_play
|
||||
when: restore_ceph_data
|
||||
|
||||
|
|
@ -215,11 +205,11 @@
|
|||
|
||||
- name: Fail if the remaining Openstack services cannot be restored successfully
|
||||
fail:
|
||||
msg: StarlingX Openstack application is not applied successfully.
|
||||
msg: stx-openstack application is not applied successfully.
|
||||
when: check_app_state.stdout is not search('applied')
|
||||
|
||||
- debug:
|
||||
msg: Restore StarlingX Openstack application is successful!
|
||||
msg: Restore stx-openstack application is successful!
|
||||
|
||||
always:
|
||||
- name: Remove restore_openstack_in_progress flag file
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@
|
|||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
platform_permdir: /opt/platform
|
||||
restore_openstack_in_progress_flag: "{{ platform_permdir }}/.restore_openstack_in_progress"
|
||||
platform_path: /opt/platform
|
||||
restore_openstack_in_progress_flag: "{{ platform_path }}/.restore_openstack_in_progress"
|
||||
restore_openstack_continue_flag: "{{ platform_path }}/.restore_openstack_continue"
|
||||
kube_config_dir: /etc/kubernetes/admin.conf
|
||||
mariadb_pod: mariadb-server-0
|
||||
|
|
|
|||
Loading…
Reference in New Issue