diff --git a/playbookconfig/src/playbooks/host_vars/backup-restore/default.yml b/playbookconfig/src/playbooks/host_vars/backup-restore/default.yml index 5b56bbec3..827983cd5 100644 --- a/playbookconfig/src/playbooks/host_vars/backup-restore/default.yml +++ b/playbookconfig/src/playbooks/host_vars/backup-restore/default.yml @@ -80,3 +80,8 @@ host_backup_dir: "{{ lookup('env', 'HOME') }}" # Flag file to indicate if platform restore is in progress # restore_in_progress_flag: /etc/platform/.restore_in_progress + +# This variable is set to true when restore_openstack playbook is +# played again to bring up remaining OpenStack services after Ceph +# data is restored. +restore_openstack_continue: false diff --git a/playbookconfig/src/playbooks/restore_openstack.yml b/playbookconfig/src/playbooks/restore_openstack.yml index 7e3c3f078..c5edf5b86 100644 --- a/playbookconfig/src/playbooks/restore_openstack.yml +++ b/playbookconfig/src/playbooks/restore_openstack.yml @@ -10,26 +10,12 @@ gather_facts: no vars_files: - - host_vars/default.yml - - pre_tasks: - - stat: - path: "{{ item }}" - register: files_to_import - with_items: - - "{{ override_files_dir }}/secrets.yml" - - "{{ override_files_dir }}/{{ inventory_hostname }}_secrets.yml" - - "{{ override_files_dir }}/site.yml" - - "{{ override_files_dir }}/{{ inventory_hostname }}.yml" - delegate_to: localhost - - - include_vars: "{{ item.item }}" - when: item.stat.exists - with_items: "{{ files_to_import.results }}" - loop_control: - label: "{{ item.item }}" + - host_vars/backup-restore/default.yml # Main play roles: - - common + - { role: common/prepare-env } + - { role: backup-restore/prepare-env } + - { role: restore-openstack/prepare-env, when: not restore_openstack_continue|bool } + - { role: backup-restore/transfer-file, when: not restore_openstack_continue|bool } - { role: restore-openstack/restore, become: yes } diff --git a/playbookconfig/src/playbooks/roles/bootstrap/prepare-env/tasks/load_patching_tasks.yml b/playbookconfig/src/playbooks/roles/bootstrap/prepare-env/tasks/load_patching_tasks.yml index 06b04ae7f..762413020 100644 --- a/playbookconfig/src/playbooks/roles/bootstrap/prepare-env/tasks/load_patching_tasks.yml +++ b/playbookconfig/src/playbooks/roles/bootstrap/prepare-env/tasks/load_patching_tasks.yml @@ -11,6 +11,10 @@ # - reboot the controller if it is required by the patching # - block: + - name: Set fact for restore_in_progress flag file + set_fact: + restore_in_progress_flag: /etc/platform/.restore_in_progress + - name: Set fact for patching staging dir set_fact: patching_staging_dir: /scratch/patching diff --git a/playbookconfig/src/playbooks/roles/restore-openstack/prepare-env/tasks/main.yml b/playbookconfig/src/playbooks/roles/restore-openstack/prepare-env/tasks/main.yml new file mode 100644 index 000000000..b30d51989 --- /dev/null +++ b/playbookconfig/src/playbooks/roles/restore-openstack/prepare-env/tasks/main.yml @@ -0,0 +1,82 @@ +--- +# +# Copyright (c) 2019 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# ROLE DESCRIPTION: +# This role verifies if platform restore should proceed +# +- name: Fail if backup_filename is not defined or set + fail: + msg: "Mandatory configuration parameter backup_filename is not defined or set." + when: backup_filename is not defined or backup_filename is none + +- name: Fail if initial_backup_dir is not defined or set + fail: + msg: "Mandatory configuration parameter initial_backup_dir is not defined or set." + when: initial_backup_dir is not defined or initial_backup_dir is none + +- name: Check if restoring StarlingX Openstack is already in progress + stat: + path: "{{ restore_openstack_in_progress_flag }}" + register: restore_openstack_in_progress + +- name: Fail if restoring StarlingX Openstack is already in progress + fail: + msg: "Restoring StarlingX Openstack is already in progress!" + when: restore_openstack_in_progress.stat.exists + +- name: Check if it is the active controller + shell: source /etc/platform/openrc; system host-show $(cat /etc/hostname) | grep -i "Controller-Active" + register: active_ctlr + +- name: Fail if it is not the active controller + fail: + msg: "Restore StarlingX Openstack application can only be done on the active controller." + when: active_ctlr.rc != 0 + +- name: Search for stx-openstack application + shell: source /etc/platform/openrc; system application-list | grep 'stx-openstack' + register: check_openstack + failed_when: false + +# If stx-application doesn't exist or is in applied state, the restore is complete. +- block: + - debug: + msg: >- + Either StarlingX OpenStack application had not been applied when the system was + backed up or it has already been restored successfully. + - meta: end_play + + when: check_openstack.rc != 0 or check_openstack.stdout is search('applied') + +# If stx-application is not in uploaded state, it means restore_openstack playbook +# has been run and was not successful. Re-run the playbook is not supported. +- name: Fail if StarlingX Openstack is not in the right state for restore + fail: + msg: >- + stx-openstack application must be in ‘uploaded’ state for the restore to proceed. + The restore must have failed already. + when: check_openstack.stdout is not search('uploaded') + +- name: Check if platform-integ-apps was applied successfully during controller-0 unlock + shell: >- + source /etc/platform/openrc; system application-list | + grep 'platform-integ-apps.*applied.*completed' + register: check_platform_integ + +- name: Fail if platform-integ-apps was not applied successfully + fail: + msg: Application platform-integ-apps was not applied successfully during controller-0 unlock + when: check_platform_integ.rc != 0 + +- name: For remote play set target_backup_dir to {{ backup_dir }} + set_fact: + target_backup_dir: "{{ backup_dir }}" + when: inventory_hostname != "localhost" + +- name: For local play set target_backup_dir to initial_backup_dir + set_fact: + target_backup_dir: "{{ initial_backup_dir }}" + when: inventory_hostname == "localhost" diff --git a/playbookconfig/src/playbooks/roles/restore-openstack/restore/tasks/main.yml b/playbookconfig/src/playbooks/roles/restore-openstack/restore/tasks/main.yml index b3d324584..9f049c4f7 100644 --- a/playbookconfig/src/playbooks/roles/restore-openstack/restore/tasks/main.yml +++ b/playbookconfig/src/playbooks/roles/restore-openstack/restore/tasks/main.yml @@ -7,80 +7,51 @@ # ROLE DESCRIPTION: # This role is to restore StarlingX Openstack application. # - -- name: Fail if backup_filename is not defined or set - fail: - msg: "Mandatory configuration parameter backup_filename is not defined or set." - when: backup_filename is not defined or backup_filename is none - -# Check if the backup tarball exists in {{ backup_dir }}. This check can -# serve as a guard to ensure restore StarlingX Openstack is run on the -# right system because after the platform is restored the backup tarball is -# stored in {{ backup_dir }} on controller-0. -- name: Check if the backup tarball exists in {{ backup_dir }} - stat: - path: "{{ backup_dir }}/{{ backup_filename }}" - register: backup_file_result - -- name: Fail if the backup tarball doesn't exist - fail: - msg: Cannot find backup tarball {{ backup_dir }}/{{ backup_filename }}! - when: not backup_file_result.stat.exists - -- name: Check if it is the active controller - shell: source /etc/platform/openrc; system host-show $(cat /etc/hostname) | grep -i "Controller-Active" - register: active_ctlr - -- name: Fail if it is not the active controller - fail: - msg: "Restore StarlingX Openstack application can only be done on the active controller." - when: active_ctlr.rc != 0 - -- name: Check if restoring StarlingX Openstack is already in progress - stat: - path: "{{ restore_openstack_in_progress_flag }}" - register: restore_openstack_in_progress - -- name: Fail if restoring StarlingX Openstack is already in progress - fail: - msg: "Restoring StarlingX Openstack is already in progress!" - when: restore_openstack_in_progress.stat.exists - -- name: Search for stx-openstack application - shell: source /etc/platform/openrc; system application-list | grep 'stx-openstack' - register: check_openstack - failed_when: false - -# If stx-application doesn't exist or is in applied state, the restore is complete. +# To bring up remaining StarlingX Openstack services after storage data is restored - block: - - debug: + - name: Check if restore OpenStack continue flag file exists + stat: + path: "{{ restore_openstack_continue_flag }}" + register: check_restore_openstack_continue + + - name: Fail if restore OpenStack continue flag file does not exist + fail: msg: >- - Either StarlingX OpenStack application had not been applied when the system was - backed up or it has already been restored successfully. + "{{ restore_openstack_continue_flag }} does not exist. Cannot continue + to restore OpenStack application." + when: not check_restore_openstack_continue.stat.exists + + - name: Continue to bring up the remaining StarlingX Openstack services + shell: source /etc/platform/openrc; system application-apply --mode normal stx-openstack + + - name: Checking stx-openstack apply progress for the remaining Openstack services... (max 50 mins) + shell: source /etc/platform/openrc; system application-list | grep 'stx-openstack' + register: check_app_state + until: check_app_state.stdout is search('applied') or + check_app_state.stdout is search('apply-failed') + retries: 50 + delay: 60 + + - name: Fail if the remaining Openstack services cannot be restored successfully + fail: + msg: stx-openstack application is not applied successfully. + when: check_app_state.stdout is not search('applied') + + - name: Remove restore_openstack_in_progress flag file + file: + path: "{{ restore_openstack_in_progress_flag }}" + state: absent + + - name: Remove restore_openstack_continue flag file + file: + path: "{{ restore_openstack_continue_flag }}" + state: absent + + - debug: + msg: Restore stx-openstack application is successful! + - meta: end_play - - when: check_openstack.rc != 0 or check_openstack.stdout is search('applied') - -# If stx-application is not in uploaded state, it means restore_openstack playbook -# has been run and was not successfully. Re-run the playbook is not supported. -- name: Fail if StarlingX Openstack is not in the right state for restore - fail: - msg: >- - stx-openstack application must be in ‘uploaded’ state for the restore to proceed. - The restore must have failed already. - - when: check_openstack.stdout is not search('uploaded') - -- name: Check if platform-integ-apps was applied successfully during controller-0 unlock - shell: >- - source /etc/platform/openrc; system application-list | - grep 'platform-integ-apps.*applied.*completed' - register: check_platform_integ - -- name: Fail if platform-integ-apps was not applied successfully - fail: - msg: Application platform-integ-apps was not applied successfully during controller-0 unlock - when: check_platform_integ.rc != 0 + when: restore_openstack_continue - name: Create restore StarlingX Openstack in progress flag file in /opt/platform file: @@ -88,6 +59,20 @@ state: touch - block: + - name: Set config path facts + set_fact: + armada_permdir: "{{ platform_path }}/armada/{{ software_version }}" + helm_charts_permdir: "{{ platform_path }}/helm_charts" + + - name: Restore Helm charts, armada manifests and extension filesystem + command: tar -C / --overwrite -xpf {{ target_backup_dir }}/{{ backup_filename }} {{ item }} + args: + warn: false + become_user: root + with_items: + - "{{ helm_charts_permdir | regex_replace('^\\/', '') }}" + - "{{ armada_permdir | regex_replace('^\\/', '') }}" + - name: Bring up MariaDB and its prerequisite services shell: >- source /etc/platform/openrc; system application-apply --mode restore_db stx-openstack @@ -125,7 +110,7 @@ - name: Fail if openstack-garbd pod is not running fail: - msg: Openstack-garbd pod is not running. Restore StarlingX Openstack application failed. + msg: Openstack-garbd pod is not running. Restore stx-openstack application failed. when: check_garbd_pod.rc != 0 when: system_type == "Standard" @@ -141,7 +126,7 @@ - name: Extract MariaDB data to backup directory command: >- - tar -C {{ backup_dir }}/mariadb -xpf {{ backup_dir }}/{{ backup_filename }} + tar -C {{ backup_dir }}/mariadb -xpf {{ target_backup_dir }}/{{ backup_filename }} --transform='s,.*/,,' '*/*\.mariadb\.*' args: warn: false @@ -189,7 +174,7 @@ fail: msg: >- Cinder and Glance services cannot be restored successfully. - Restore StarlingX Openstack application failed. + Restore stx-openstack application failed. when: check_app_state.stdout is not search('applied') - block: @@ -199,6 +184,11 @@ Please refer to the system administration guide to restore Glance images and Cinder volumes. + - name: Create restore StarlingX Openstack continue flag file + file: + path: "{{ restore_openstack_continue_flag }}" + state: touch + - meta: end_play when: restore_ceph_data @@ -215,11 +205,11 @@ - name: Fail if the remaining Openstack services cannot be restored successfully fail: - msg: StarlingX Openstack application is not applied successfully. + msg: stx-openstack application is not applied successfully. when: check_app_state.stdout is not search('applied') - debug: - msg: Restore StarlingX Openstack application is successful! + msg: Restore stx-openstack application is successful! always: - name: Remove restore_openstack_in_progress flag file diff --git a/playbookconfig/src/playbooks/roles/restore-openstack/restore/vars/main.yml b/playbookconfig/src/playbooks/roles/restore-openstack/restore/vars/main.yml index adac0e543..093c0c933 100644 --- a/playbookconfig/src/playbooks/roles/restore-openstack/restore/vars/main.yml +++ b/playbookconfig/src/playbooks/roles/restore-openstack/restore/vars/main.yml @@ -4,8 +4,8 @@ # # SPDX-License-Identifier: Apache-2.0 # - -platform_permdir: /opt/platform -restore_openstack_in_progress_flag: "{{ platform_permdir }}/.restore_openstack_in_progress" +platform_path: /opt/platform +restore_openstack_in_progress_flag: "{{ platform_path }}/.restore_openstack_in_progress" +restore_openstack_continue_flag: "{{ platform_path }}/.restore_openstack_continue" kube_config_dir: /etc/kubernetes/admin.conf mariadb_pod: mariadb-server-0