Make execution of the backup idempotent

Currently, the playbook stops the services and the pacemaker to perform the backup. If the backup fails for whatever reason, the playbook cannot run again and fails due to the services being down. This patch fixes the issue in two directions. First, by using the block-always ansible feature, we ensure that whenever an error happens, the playbook will restart the pacemaker and the services before exiting.

The second direction is on the execution itself. By controlling certain actions, we ensure that the backup will be succesful not matter what state the services and the pacemaker are when the playbook is ran. This ensures success even in the case an operator stops the ansible execution manually (which will not trigger the 'always' condition).

BZ: #1954818
Change-Id: Id2aff61f219b0c4992f6f0045f1aba2c7d129758
(cherry picked from commit 9a865d1769)
This commit is contained in:
Juan Larriba 2021-05-14 14:31:46 +02:00
parent 0aef9a7faa
commit 414c2dae80
4 changed files with 50 additions and 2 deletions

View File

@ -32,6 +32,12 @@
tags:
- always
- name: Unpause ceph mon container if paused
command: "{{ tripleo_container_cli }} unpause ceph-mon-{{ ansible_facts['hostname'] }}"
failed_when: false
tags:
- bar_create_recover_image
- name: Export ceph authentication
shell: |
set -o pipefail

View File

@ -48,6 +48,29 @@
tags:
- bar_create_recover_image
- name: Enable pacemaker if it is stopped
command: pcs cluster start --all
when:
- enabled_galera
- tripleo_backup_and_restore_service_manager|bool
run_once: true
tags:
- bar_create_recover_image
- name: Wait until pacemaker has Galera up&running
shell: |
set -o pipefail
ss -tunlp | grep ":3306 " | sed -e 's/.*\///'
register: mysql_result
retries: 300
until: mysql_result is search('mysqld')
delay: 5
when:
- enabled_galera
- tripleo_backup_and_restore_service_manager|bool
tags:
- bar_create_recover_image
- name: Get the mysql container id when galera is enabled
shell: |
set -o pipefail
@ -65,6 +88,17 @@
tags:
- bar_create_recover_image
- name: Unpause mysql for backup if it is paused
command: "{{ tripleo_container_cli }} unpause {{ tripleo_backup_and_restore_mysql_container }}"
when:
- mysql_password.stderr is defined
- tripleo_backup_and_restore_mysql_container == "mysql"
- not enabled_galera
- tripleo_backup_and_restore_service_manager|bool
failed_when: false
tags:
- bar_create_recover_image
- name: MySQL Grants backup
shell: |
set -o pipefail

View File

@ -49,8 +49,15 @@
- name: Backup pacemaker configuration
import_tasks: pacemaker_backup.yml
- name: Create recovery images with ReaR
import_tasks: run_backup.yml
- name: Perform backup
block:
- name: Create recovery images with ReaR
import_tasks: run_backup.yml
always:
- name: Service management
import_tasks: service_manager_unpause.yml
when:
- tripleo_backup_and_restore_service_manager|bool
- name: Service management
import_tasks: service_manager_unpause.yml

View File

@ -39,6 +39,7 @@
/usr/bin/{{ tripleo_container_cli }} ps --format '{{ '{{' }}.Names {{ '}}' }} ' | /usr/bin/egrep -v 'galera|mysql|bundle'
register: container_services
changed_when: container_services.stdout is undefined
failed_when: false
tags:
- bar_create_recover_image