Make execution of the backup idempotent
Currently, the playbook stops the services and the pacemaker to perform the backup. If the backup fails for whatever reason, the playbook cannot run again and fails due to the services being down. This patch fixes the issue in two directions. First, by using the block-always ansible feature, we ensure that whenever an error happens, the playbook will restart the pacemaker and the services before exiting.
The second direction is on the execution itself. By controlling certain actions, we ensure that the backup will be succesful not matter what state the services and the pacemaker are when the playbook is ran. This ensures success even in the case an operator stops the ansible execution manually (which will not trigger the 'always' condition).
BZ: #1954818
Change-Id: Id2aff61f219b0c4992f6f0045f1aba2c7d129758
(cherry picked from commit 9a865d1769
)
This commit is contained in:
parent
0aef9a7faa
commit
414c2dae80
|
@ -32,6 +32,12 @@
|
|||
tags:
|
||||
- always
|
||||
|
||||
- name: Unpause ceph mon container if paused
|
||||
command: "{{ tripleo_container_cli }} unpause ceph-mon-{{ ansible_facts['hostname'] }}"
|
||||
failed_when: false
|
||||
tags:
|
||||
- bar_create_recover_image
|
||||
|
||||
- name: Export ceph authentication
|
||||
shell: |
|
||||
set -o pipefail
|
||||
|
|
|
@ -48,6 +48,29 @@
|
|||
tags:
|
||||
- bar_create_recover_image
|
||||
|
||||
- name: Enable pacemaker if it is stopped
|
||||
command: pcs cluster start --all
|
||||
when:
|
||||
- enabled_galera
|
||||
- tripleo_backup_and_restore_service_manager|bool
|
||||
run_once: true
|
||||
tags:
|
||||
- bar_create_recover_image
|
||||
|
||||
- name: Wait until pacemaker has Galera up&running
|
||||
shell: |
|
||||
set -o pipefail
|
||||
ss -tunlp | grep ":3306 " | sed -e 's/.*\///'
|
||||
register: mysql_result
|
||||
retries: 300
|
||||
until: mysql_result is search('mysqld')
|
||||
delay: 5
|
||||
when:
|
||||
- enabled_galera
|
||||
- tripleo_backup_and_restore_service_manager|bool
|
||||
tags:
|
||||
- bar_create_recover_image
|
||||
|
||||
- name: Get the mysql container id when galera is enabled
|
||||
shell: |
|
||||
set -o pipefail
|
||||
|
@ -65,6 +88,17 @@
|
|||
tags:
|
||||
- bar_create_recover_image
|
||||
|
||||
- name: Unpause mysql for backup if it is paused
|
||||
command: "{{ tripleo_container_cli }} unpause {{ tripleo_backup_and_restore_mysql_container }}"
|
||||
when:
|
||||
- mysql_password.stderr is defined
|
||||
- tripleo_backup_and_restore_mysql_container == "mysql"
|
||||
- not enabled_galera
|
||||
- tripleo_backup_and_restore_service_manager|bool
|
||||
failed_when: false
|
||||
tags:
|
||||
- bar_create_recover_image
|
||||
|
||||
- name: MySQL Grants backup
|
||||
shell: |
|
||||
set -o pipefail
|
||||
|
|
|
@ -49,8 +49,15 @@
|
|||
- name: Backup pacemaker configuration
|
||||
import_tasks: pacemaker_backup.yml
|
||||
|
||||
- name: Create recovery images with ReaR
|
||||
import_tasks: run_backup.yml
|
||||
- name: Perform backup
|
||||
block:
|
||||
- name: Create recovery images with ReaR
|
||||
import_tasks: run_backup.yml
|
||||
always:
|
||||
- name: Service management
|
||||
import_tasks: service_manager_unpause.yml
|
||||
when:
|
||||
- tripleo_backup_and_restore_service_manager|bool
|
||||
|
||||
- name: Service management
|
||||
import_tasks: service_manager_unpause.yml
|
||||
|
|
|
@ -39,6 +39,7 @@
|
|||
/usr/bin/{{ tripleo_container_cli }} ps --format '{{ '{{' }}.Names {{ '}}' }} ' | /usr/bin/egrep -v 'galera|mysql|bundle'
|
||||
register: container_services
|
||||
changed_when: container_services.stdout is undefined
|
||||
failed_when: false
|
||||
tags:
|
||||
- bar_create_recover_image
|
||||
|
||||
|
|
Loading…
Reference in New Issue