567e89f682
This commit re-enables stx-openstack restore after recent code churn in Ansible playbooks. For a partial restore where ceph data remain intact during restore, run restore_openstack playbook once to restore stx-openstack. For a full restore where ceph data is wiped during restore, following these steps to restore stx-openstack: - Run restore_openstack playbook with restore_ceph_data set to true - Follow the procedure to restore glance images and cinder volumes - Run restore_openstack playbook with restore_openstack_continue set to true Change-Id: I96838392e4fb06c1d84e3f884bda4cd9be4f9d08 Story: 2004761 Task: 36813 Signed-off-by: Wei Zhou <wei.zhou@windriver.com>
219 lines
7.5 KiB
YAML
219 lines
7.5 KiB
YAML
---
|
|
#
|
|
# Copyright (c) 2019 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
# ROLE DESCRIPTION:
|
|
# This role is to restore StarlingX Openstack application.
|
|
#
|
|
# To bring up remaining StarlingX Openstack services after storage data is restored
|
|
- block:
|
|
- name: Check if restore OpenStack continue flag file exists
|
|
stat:
|
|
path: "{{ restore_openstack_continue_flag }}"
|
|
register: check_restore_openstack_continue
|
|
|
|
- name: Fail if restore OpenStack continue flag file does not exist
|
|
fail:
|
|
msg: >-
|
|
"{{ restore_openstack_continue_flag }} does not exist. Cannot continue
|
|
to restore OpenStack application."
|
|
when: not check_restore_openstack_continue.stat.exists
|
|
|
|
- name: Continue to bring up the remaining StarlingX Openstack services
|
|
shell: source /etc/platform/openrc; system application-apply --mode normal stx-openstack
|
|
|
|
- name: Checking stx-openstack apply progress for the remaining Openstack services... (max 50 mins)
|
|
shell: source /etc/platform/openrc; system application-list | grep 'stx-openstack'
|
|
register: check_app_state
|
|
until: check_app_state.stdout is search('applied') or
|
|
check_app_state.stdout is search('apply-failed')
|
|
retries: 50
|
|
delay: 60
|
|
|
|
- name: Fail if the remaining Openstack services cannot be restored successfully
|
|
fail:
|
|
msg: stx-openstack application is not applied successfully.
|
|
when: check_app_state.stdout is not search('applied')
|
|
|
|
- name: Remove restore_openstack_in_progress flag file
|
|
file:
|
|
path: "{{ restore_openstack_in_progress_flag }}"
|
|
state: absent
|
|
|
|
- name: Remove restore_openstack_continue flag file
|
|
file:
|
|
path: "{{ restore_openstack_continue_flag }}"
|
|
state: absent
|
|
|
|
- debug:
|
|
msg: Restore stx-openstack application is successful!
|
|
|
|
- meta: end_play
|
|
when: restore_openstack_continue
|
|
|
|
- name: Create restore StarlingX Openstack in progress flag file in /opt/platform
|
|
file:
|
|
path: "{{ restore_openstack_in_progress_flag }}"
|
|
state: touch
|
|
|
|
- block:
|
|
- name: Set config path facts
|
|
set_fact:
|
|
armada_permdir: "{{ platform_path }}/armada/{{ software_version }}"
|
|
helm_charts_permdir: "{{ platform_path }}/helm_charts"
|
|
|
|
- name: Restore Helm charts, armada manifests and extension filesystem
|
|
command: tar -C / --overwrite -xpf {{ target_backup_dir }}/{{ backup_filename }} {{ item }}
|
|
args:
|
|
warn: false
|
|
become_user: root
|
|
with_items:
|
|
- "{{ helm_charts_permdir | regex_replace('^\\/', '') }}"
|
|
- "{{ armada_permdir | regex_replace('^\\/', '') }}"
|
|
|
|
- name: Bring up MariaDB and its prerequisite services
|
|
shell: >-
|
|
source /etc/platform/openrc; system application-apply --mode restore_db stx-openstack
|
|
|
|
- name: Checking MariaDB apply progress ... (max 30 mins)
|
|
shell: source /etc/platform/openrc; system application-list | grep 'stx-openstack'
|
|
register: check_app_state
|
|
until: check_app_state.stdout is search('applied') or
|
|
check_app_state.stdout is search('apply-failed')
|
|
retries: 30
|
|
delay: 60
|
|
|
|
- name: Check if MariaDB pod is running
|
|
shell: >-
|
|
kubectl --kubeconfig={{ kube_config_dir }} get pods -n openstack |
|
|
grep {{ mariadb_pod }} | grep -i 'Running'
|
|
failed_when: false
|
|
register: check_mariadb_pod
|
|
|
|
- name: Fail if MariaDB service cannot be restored successfully
|
|
fail:
|
|
msg: >-
|
|
MariaDB service cannot be restored successfully.
|
|
Restore StarlingX Openstack application failed.
|
|
when: check_app_state.stdout is not search('applied') or
|
|
check_mariadb_pod.rc != 0
|
|
|
|
- block:
|
|
- name: Check if openstack-garbd pod is running if it is not an all-in-one system
|
|
shell: >-
|
|
kubectl --kubeconfig={{ kube_config_dir }} get pods -n openstack |
|
|
grep openstack-garbd | grep -i 'Running'
|
|
failed_when: false
|
|
register: check_garbd_pod
|
|
|
|
- name: Fail if openstack-garbd pod is not running
|
|
fail:
|
|
msg: Openstack-garbd pod is not running. Restore stx-openstack application failed.
|
|
when: check_garbd_pod.rc != 0
|
|
|
|
when: system_type == "Standard"
|
|
|
|
- name: Create MariaDB backup directory
|
|
file:
|
|
path: "{{ backup_dir }}/mariadb"
|
|
state: directory
|
|
recurse: yes
|
|
owner: root
|
|
group: root
|
|
mode: 0755
|
|
|
|
- name: Extract MariaDB data to backup directory
|
|
command: >-
|
|
tar -C {{ backup_dir }}/mariadb -xpf {{ target_backup_dir }}/{{ backup_filename }}
|
|
--transform='s,.*/,,' '*/*\.mariadb\.*'
|
|
args:
|
|
warn: false
|
|
|
|
- name: Set k8s cmd prefix
|
|
set_fact:
|
|
kube_cmd_prefix: "kubectl --kubeconfig={{ kube_config_dir }} exec -i {{ mariadb_pod }} -n openstack -- bash -c "
|
|
|
|
- name: List MariaDB backup files
|
|
shell: ls {{ backup_dir }}/mariadb/*
|
|
register: mariadb_files
|
|
|
|
- name: Create MariaDB database
|
|
shell: >-
|
|
{{ kube_cmd_prefix }}
|
|
'exec mysql -uroot -p"$MYSQL_DBADMIN_PASSWORD" -e"create database {{ (item|basename).split('.')[0] }}"'
|
|
with_items: "{{ mariadb_files.stdout_lines }}"
|
|
|
|
- name: Restore MariaDB data
|
|
shell: >-
|
|
{{ kube_cmd_prefix }}
|
|
'exec mysql -uroot -p"$MYSQL_DBADMIN_PASSWORD" {{ (item|basename).split('.')[0] }}'
|
|
< {{ item }}
|
|
with_items: "{{ mariadb_files.stdout_lines }}"
|
|
|
|
- name: Remove MariaDB backup directory
|
|
file:
|
|
path: "{{ backup_dir }}/mariadb"
|
|
state: absent
|
|
|
|
- name: Bring up Cinder and Glance services and the prerequisites
|
|
shell: >-
|
|
source /etc/platform/openrc; system application-apply --mode restore_storage stx-openstack
|
|
|
|
- name: Checking Cinder and Glance apply progress... (max 30 mins)
|
|
shell: >-
|
|
source /etc/platform/openrc; system application-list | grep 'stx-openstack'
|
|
register: check_app_state
|
|
until: check_app_state.stdout is search('applied') or
|
|
check_app_state.stdout is search('apply-failed')
|
|
retries: 30
|
|
delay: 60
|
|
|
|
- name: Fail if Cinder and Glance services cannot be restored successfully
|
|
fail:
|
|
msg: >-
|
|
Cinder and Glance services cannot be restored successfully.
|
|
Restore stx-openstack application failed.
|
|
when: check_app_state.stdout is not search('applied')
|
|
|
|
- block:
|
|
- name: Inform user to restore Glance images and Cinder volumes
|
|
debug:
|
|
msg: >-
|
|
Please refer to the system administration guide to restore Glance images
|
|
and Cinder volumes.
|
|
|
|
- name: Create restore StarlingX Openstack continue flag file
|
|
file:
|
|
path: "{{ restore_openstack_continue_flag }}"
|
|
state: touch
|
|
|
|
- meta: end_play
|
|
when: restore_ceph_data
|
|
|
|
- name: Continue to bring up the remaining StarlingX Openstack services
|
|
shell: source /etc/platform/openrc; system application-apply --mode normal stx-openstack
|
|
|
|
- name: Checking stx-openstack apply progress for the remaining Openstack services... (max 50 mins)
|
|
shell: source /etc/platform/openrc; system application-list | grep 'stx-openstack'
|
|
register: check_app_state
|
|
until: check_app_state.stdout is search('applied') or
|
|
check_app_state.stdout is search('apply-failed')
|
|
retries: 50
|
|
delay: 60
|
|
|
|
- name: Fail if the remaining Openstack services cannot be restored successfully
|
|
fail:
|
|
msg: stx-openstack application is not applied successfully.
|
|
when: check_app_state.stdout is not search('applied')
|
|
|
|
- debug:
|
|
msg: Restore stx-openstack application is successful!
|
|
|
|
always:
|
|
- name: Remove restore_openstack_in_progress flag file
|
|
file:
|
|
path: "{{ restore_openstack_in_progress_flag }}"
|
|
state: absent
|