From 9b26f440d3ca163ed97d2ec3dc87ca96b2745d9c Mon Sep 17 00:00:00 2001 From: toure Date: Mon, 9 Sep 2019 09:13:35 -0400 Subject: [PATCH] Add service management support. The current role for backup and restore will need service management to insure that all running services on the nodes are stopped and restarted in order to capture a consistent state. This change also includes database dump functionality which will allow the operator to create a clean backup of the database. Change-Id: Ic75f40a750880be4ea00ed79de4575cdbbe0f190 --- .../backup/tasks/db_backup.yml | 104 ++++++++++++++++++ .../backup_and_restore/backup/tasks/main.yml | 9 +- .../backup/tasks/service_manager_pause.yml | 50 +++++++++ .../backup/tasks/service_manager_unpause.yml | 51 +++++++++ .../backup_and_restore/defaults/main.yml | 8 ++ .../molecule/default/playbook.yml | 1 + .../setup_rear/tasks/main.yml | 22 ++-- .../roles/backup_and_restore/tasks/main.yml | 15 +++ 8 files changed, 240 insertions(+), 20 deletions(-) create mode 100644 tripleo_ansible/roles/backup_and_restore/backup/tasks/db_backup.yml create mode 100644 tripleo_ansible/roles/backup_and_restore/backup/tasks/service_manager_pause.yml create mode 100644 tripleo_ansible/roles/backup_and_restore/backup/tasks/service_manager_unpause.yml diff --git a/tripleo_ansible/roles/backup_and_restore/backup/tasks/db_backup.yml b/tripleo_ansible/roles/backup_and_restore/backup/tasks/db_backup.yml new file mode 100644 index 000000000..cf30175b3 --- /dev/null +++ b/tripleo_ansible/roles/backup_and_restore/backup/tasks/db_backup.yml @@ -0,0 +1,104 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Create a backup for each database into seperate files. +- name: Get database root password + command: | + /bin/hiera -c /etc/puppet/hiera.yaml mysql::server::root_password + when: mysql_password is undefined + register: mysql_password + become: true + tags: + - bar_create_recover_image + +- name: Get galera bind_address + command: | + /bin/hiera -c /etc/puppet/hiera.yaml 'tripleo::profile::pacemaker::database::mysql::bind_address' + when: tripleo_backup_and_restore_pacemaker_galera_bind_address is undefined + register: tripleo_backup_and_restore_pacemaker_galera_bind_address + become: true + tags: + - bar_create_recover_image + +- name: Disable galera when there is no pacemaker mysql bind address + set_fact: + enabled_galera: false + when: tripleo_backup_and_restore_pacemaker_galera_bind_address.stdout == 'nil' + tags: + - bar_create_recover_image + +- name: Enable galera when there is pacemaker mysql bind address + set_fact: + enabled_galera: true + when: tripleo_backup_and_restore_pacemaker_galera_bind_address.stdout != 'nil' + tags: + - bar_create_recover_image + +- name: Get the mysql container id when galera is enabled + command: | + {{ tripleo_container_cli }} ps -a | grep galera | awk '{print $1}' + when: enabled_galera + register: galera_container_id + become: true + tags: + - bar_create_recover_image + +- name: Set the tripleo_backup_and_restore_mysql_container id + set_fact: + tripleo_backup_and_restore_mysql_container: "{{ galera_container_id.stdout }}" + when: enabled_galera + tags: + - bar_create_recover_image + +- name: MySQL Grants backup + shell: | + set -o pipefail + {{ tripleo_container_cli }} exec {{ tripleo_backup_and_restore_mysql_container }} bash -c "mysql -uroot \ + -p{{ mysql_password.stdout }} -s -N \ + -e \"SELECT CONCAT('\\\"SHOW GRANTS FOR ''',user,'''@''',host,''';\\\"') \ + FROM mysql.user where (length(user) > 0 and user NOT LIKE 'root')\" | xargs -n1 mysql \ + -uroot -p{{ mysql_password.stdout }} -s -N -e | sed 's/$/;/' " > openstack-backup-mysql-grants.sql + when: mysql_password.stderr is defined + tags: + - bar_create_recover_image + +- name: MySQL BBDDs backup + shell: | + set -o pipefail + {{ tripleo_container_cli }} exec {{ tripleo_backup_and_restore_mysql_container }} bash -c "mysql -uroot \ + -p{{ mysql_password.stdout }} -s -N \ + -e \"select distinct table_schema from information_schema.tables \ + where engine='innodb' and table_schema != 'mysql';\" | xargs mysqldump -uroot \ + -p{{ mysql_password.stdout }} --single-transaction --databases" > openstack-backup-mysql.sql + when: mysql_password.stderr is defined + tags: + - bar_create_recover_image + +- name: Pause mysql. + command: "{{ tripleo_container_cli }} pause {{ tripleo_backup_and_restore_mysql_container }}" + when: + - mysql_password.stderr is defined + - tripleo_backup_and_restore_mysql_container == "mysql" + - not enabled_galera + tags: + - bar_create_recover_image + +- name: Stop pacemaker + command: pcs cluster stop --all + when: enabled_galera + run_once: true + tags: + - bar_create_recover_image diff --git a/tripleo_ansible/roles/backup_and_restore/backup/tasks/main.yml b/tripleo_ansible/roles/backup_and_restore/backup/tasks/main.yml index b4bed318c..705492d18 100644 --- a/tripleo_ansible/roles/backup_and_restore/backup/tasks/main.yml +++ b/tripleo_ansible/roles/backup_and_restore/backup/tasks/main.yml @@ -32,16 +32,11 @@ tags: - always -- name: Create backup command - set_fact: - tripleo_backup_and_restore_rear_command: rear {{ '-s ' if tripleo_backup_and_restore_rear_simulate else '' }}-d -v mkbackup - tags: - - bar_create_recover_image - - name: Create the node backup become: true - command: '{{ tripleo_backup_and_restore_rear_command }}' + command: rear {{ '-s ' if tripleo_backup_and_restore_rear_simulate else '' }}-d -v mkbackup register: tripleo_backup_and_restore_rear_output + when: tripleo_backup_and_restore_rear_output is undefined tags: - bar_create_recover_image diff --git a/tripleo_ansible/roles/backup_and_restore/backup/tasks/service_manager_pause.yml b/tripleo_ansible/roles/backup_and_restore/backup/tasks/service_manager_pause.yml new file mode 100644 index 000000000..c39084f90 --- /dev/null +++ b/tripleo_ansible/roles/backup_and_restore/backup/tasks/service_manager_pause.yml @@ -0,0 +1,50 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Call to podman to list running containers then commit all state to +# disk. Once services state has been flushed dump the database then allow +# the backup to start. + +- name: Get Container cli + command: /bin/hiera -c /etc/puppet/hiera.yaml container_cli + register: tripleo_backup_and_restore_container_cli + changed_when: tripleo_backup_and_restore_container_cli.stdout is undefined + tags: + - bar_create_recover_image + +- name: set tripleo_container_cli + set_fact: + tripleo_container_cli: "{{ tripleo_backup_and_restore_container_cli.stdout }}" + when: + - tripleo_backup_and_restore_container_cli.stdout != 'nil' + tags: + - bar_create_recover_image + +- name: Gather Container Service Name + shell: | + set -o pipefail + /usr/bin/{{ tripleo_container_cli }} ps --format '{{ '{{' }}.Names {{ '}}' }} ' | /usr/bin/egrep -v 'galera|mysql|bundle' + register: container_services + changed_when: container_services.stdout is undefined + tags: + - bar_create_recover_image + +- name: Pause containers for database backup. + command: "{{ tripleo_container_cli }} pause {{ item }}" + with_items: "{{ container_services.stdout_lines }}" + when: container_services is defined + tags: + - bar_create_recover_image diff --git a/tripleo_ansible/roles/backup_and_restore/backup/tasks/service_manager_unpause.yml b/tripleo_ansible/roles/backup_and_restore/backup/tasks/service_manager_unpause.yml new file mode 100644 index 000000000..10b917f4b --- /dev/null +++ b/tripleo_ansible/roles/backup_and_restore/backup/tasks/service_manager_unpause.yml @@ -0,0 +1,51 @@ +--- +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Call to podman to list running containers then commit all state to +# disk. Once services state has been flushed dump the database then allow +# the backup to start. + +- name: Enable pacemaker + command: pcs cluster start --all + when: enabled_galera + run_once: true + tags: + - bar_create_recover_image + +- name: unPause database container + command: "{{ tripleo_container_cli }} unpause {{ tripleo_backup_and_restore_mysql_container }}" + when: + - tripleo_container_cli is defined + - not enabled_galera + - tripleo_backup_and_restore_mysql_container is defined + tags: + - bar_create_recover_image + +- name: Gather Container Service Name + shell: | + set -o pipefail + /usr/bin/{{ tripleo_container_cli }} ps -a --filter='status=paused' --format '{{ '{{' }}.Names {{ '}}' }} ' + register: container_services + changed_when: container_services.stdout is defined + tags: + - bar_create_recover_image + +- name: Pause containers for database backup. + command: "{{ tripleo_container_cli }} unpause {{ item }}" + with_items: "{{ container_services.stdout_lines }}" + when: tripleo_container_cli is defined + tags: + - bar_create_recover_image diff --git a/tripleo_ansible/roles/backup_and_restore/defaults/main.yml b/tripleo_ansible/roles/backup_and_restore/defaults/main.yml index 125b7d8f0..93b633795 100644 --- a/tripleo_ansible/roles/backup_and_restore/defaults/main.yml +++ b/tripleo_ansible/roles/backup_and_restore/defaults/main.yml @@ -17,6 +17,14 @@ # All variables intended for modification should be placed in this file. +# Set the container command line entry-point +tripleo_container_cli: "{{ container_cli | default('docker') }}" +# Stop and start all running services before backup is ran. +tripleo_backup_and_restore_service_manager: true + +# Set the name of the mysql container +tripleo_backup_and_restore_mysql_container: mysql + # All variables within this role should have a prefix of "tripleo_backup_and_restore" tripleo_backup_and_restore_debug: false # By default this should be the Undercloud node diff --git a/tripleo_ansible/roles/backup_and_restore/molecule/default/playbook.yml b/tripleo_ansible/roles/backup_and_restore/molecule/default/playbook.yml index 048ad31c9..e70c54291 100644 --- a/tripleo_ansible/roles/backup_and_restore/molecule/default/playbook.yml +++ b/tripleo_ansible/roles/backup_and_restore/molecule/default/playbook.yml @@ -22,6 +22,7 @@ - role: backup_and_restore tripleo_backup_and_restore_nfs_server: undercloud tripleo_backup_and_restore_rear_simulate: true + tripleo_backup_and_restore_service_manager: false tripleo_backup_and_restore_hiera_config_file: "{{ ansible_user_dir }}/hiera.yaml" vars: ansible_python_interpreter: "{{ ansible_user_dir }}/test-python/bin/python" diff --git a/tripleo_ansible/roles/backup_and_restore/setup_rear/tasks/main.yml b/tripleo_ansible/roles/backup_and_restore/setup_rear/tasks/main.yml index e02dd2fd6..55d31d6af 100644 --- a/tripleo_ansible/roles/backup_and_restore/setup_rear/tasks/main.yml +++ b/tripleo_ansible/roles/backup_and_restore/setup_rear/tasks/main.yml @@ -43,20 +43,21 @@ - name: Get local hostname command: hostname register: tripleo_backup_and_restore_hostname + when: tripleo_backup_and_restore_hostname is undefined tags: - bar_setup_rear -- name: Get bootstrap_nodeid +- name: Set bootstrap nodeid become: true - command: "hiera -c {{ tripleo_backup_and_restore_hiera_config_file }} bootstrap_nodeid" - register: tripleo_backup_and_restore_bootstrap_nodeid + set_fact: + tripleo_backup_and_restore_bootstrap_nodeid: "{{ lookup('hiera', 'bootstrap_nodeid _hierarchy_file=/etc/puppet/hiera.yml') }}" tags: - bar_setup_rear -- name: Get enabled services by node +- name: List enabled services by node become: true - command: "hiera -c {{ tripleo_backup_and_restore_hiera_config_file }} service_names" - register: tripleo_backup_and_restore_enabled_services + set_fact: + tripleo_backup_and_restore_restore_enabled_services: "{{ lookup('hiera', 'service_names _hierarchy_file=/etc/puppet/hiera.yml') }}" tags: - bar_setup_rear @@ -65,13 +66,8 @@ set_fact: tripleo_backup_and_restore_exclude_paths: - "{{ tripleo_backup_and_restore_exclude_paths_common }}" - - "{{ (tripleo_backup_and_restore_exclude_paths_compute) if - (tripleo_backup_and_restore_enabled_services.stdout is search('nova_compute')) else [] }}" - - "{{ (tripleo_backup_and_restore_exclude_paths_controller) if - (tripleo_backup_and_restore_bootstrap_nodeid.stdout != tripleo_backup_and_restore_hostname.stdout and - tripleo_backup_and_restore_enabled_services.stdout is search('pacemaker') and - tripleo_backup_and_restore_enabled_services.stdout is search('mysql') and - tripleo_backup_and_restore_exclude_paths_controller_non_bootrapnode) else [] }}" + - "{{ tripleo_backup_and_restore_exclude_paths_compute }}" + - "{{ tripleo_backup_and_restore_exclude_paths_controller }}" tags: - bar_setup_rear diff --git a/tripleo_ansible/roles/backup_and_restore/tasks/main.yml b/tripleo_ansible/roles/backup_and_restore/tasks/main.yml index 5b9709a3e..ecd28f5cb 100644 --- a/tripleo_ansible/roles/backup_and_restore/tasks/main.yml +++ b/tripleo_ansible/roles/backup_and_restore/tasks/main.yml @@ -38,5 +38,20 @@ - name: Setup ReaR import_tasks: ../setup_rear/tasks/main.yml +- name: Service management + import_tasks: ../backup/tasks/service_manager_pause.yml + when: + - tripleo_backup_and_restore_service_manager + +- name: Backup the database + import_tasks: ../backup/tasks/db_backup.yml + when: + - tripleo_backup_and_restore_service_manager + - name: Create recovery images with ReaR import_tasks: ../backup/tasks/main.yml + +- name: Service management + import_tasks: ../backup/tasks/service_manager_unpause.yml + when: + - tripleo_backup_and_restore_service_manager