Add service management support.

The current role for backup and restore will need service
management to insure that all running services on the nodes
are stopped and restarted in order to capture a consistent state.
This change also includes database dump functionality which will
allow the operator to create a clean backup of the database.

Change-Id: Ic75f40a750880be4ea00ed79de4575cdbbe0f190

tripleo_ansible/roles/backup_and_restore/backup/tasks/db_backup.yml

@@ -0,0 +1,104 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Create a backup for each database into seperate files.
+- name: Get database root password
+  command: |
+    /bin/hiera -c /etc/puppet/hiera.yaml mysql::server::root_password
+  when: mysql_password is undefined
+  register: mysql_password
+  become: true
+  tags:
+    - bar_create_recover_image
+
+- name: Get galera bind_address
+  command: |
+    /bin/hiera -c /etc/puppet/hiera.yaml 'tripleo::profile::pacemaker::database::mysql::bind_address'
+  when: tripleo_backup_and_restore_pacemaker_galera_bind_address is undefined
+  register: tripleo_backup_and_restore_pacemaker_galera_bind_address
+  become: true
+  tags:
+    - bar_create_recover_image
+
+- name: Disable galera when there is no pacemaker mysql bind address
+  set_fact:
+    enabled_galera: false
+  when: tripleo_backup_and_restore_pacemaker_galera_bind_address.stdout == 'nil'
+  tags:
+    - bar_create_recover_image
+
+- name: Enable galera when there is pacemaker mysql bind address
+  set_fact:
+    enabled_galera: true
+  when: tripleo_backup_and_restore_pacemaker_galera_bind_address.stdout != 'nil'
+  tags:
+    - bar_create_recover_image
+
+- name: Get the mysql container id when galera is enabled
+  command: |
+    {{ tripleo_container_cli }} ps -a | grep galera | awk '{print $1}'
+  when: enabled_galera
+  register: galera_container_id
+  become: true
+  tags:
+    - bar_create_recover_image
+
+- name: Set the tripleo_backup_and_restore_mysql_container id
+  set_fact:
+    tripleo_backup_and_restore_mysql_container: "{{ galera_container_id.stdout }}"
+  when: enabled_galera
+  tags:
+    - bar_create_recover_image
+
+- name: MySQL Grants backup
+  shell: |
+    set -o pipefail
+    {{ tripleo_container_cli }} exec {{ tripleo_backup_and_restore_mysql_container }} bash -c "mysql -uroot \
+    -p{{ mysql_password.stdout }} -s -N \
+    -e \"SELECT CONCAT('\\\"SHOW GRANTS FOR ''',user,'''@''',host,''';\\\"') \
+    FROM mysql.user where (length(user) > 0 and user NOT LIKE 'root')\"  | xargs -n1 mysql \
+    -uroot -p{{ mysql_password.stdout }} -s -N -e | sed 's/$/;/' " > openstack-backup-mysql-grants.sql
+  when: mysql_password.stderr is defined
+  tags:
+    - bar_create_recover_image
+
+- name: MySQL BBDDs backup
+  shell: |
+    set -o pipefail
+    {{ tripleo_container_cli }} exec {{ tripleo_backup_and_restore_mysql_container }} bash -c "mysql -uroot \
+    -p{{ mysql_password.stdout }} -s -N \
+    -e \"select distinct table_schema from information_schema.tables \
+    where engine='innodb' and table_schema != 'mysql';\" | xargs mysqldump -uroot \
+    -p{{ mysql_password.stdout }}  --single-transaction --databases" > openstack-backup-mysql.sql
+  when: mysql_password.stderr is defined
+  tags:
+    - bar_create_recover_image
+
+- name: Pause mysql.
+  command: "{{ tripleo_container_cli }} pause {{ tripleo_backup_and_restore_mysql_container }}"
+  when:
+    - mysql_password.stderr is defined
+    - tripleo_backup_and_restore_mysql_container == "mysql"
+    - not enabled_galera
+  tags:
+    - bar_create_recover_image
+
+- name: Stop pacemaker
+  command: pcs cluster stop --all
+  when: enabled_galera
+  run_once: true
+  tags:
+    - bar_create_recover_image

tripleo_ansible/roles/backup_and_restore/backup/tasks/main.yml

@@ -32,16 +32,11 @@
   tags:
     - always
 
-- name: Create backup command
-  set_fact:
-    tripleo_backup_and_restore_rear_command: rear {{ '-s ' if tripleo_backup_and_restore_rear_simulate else '' }}-d -v mkbackup
-  tags:
-    - bar_create_recover_image
-
 - name: Create the node backup
   become: true
-  command: '{{ tripleo_backup_and_restore_rear_command }}'
+  command: rear {{ '-s ' if tripleo_backup_and_restore_rear_simulate else '' }}-d -v mkbackup
   register: tripleo_backup_and_restore_rear_output
+  when: tripleo_backup_and_restore_rear_output is undefined
   tags:
     - bar_create_recover_image
 

tripleo_ansible/roles/backup_and_restore/backup/tasks/service_manager_pause.yml

@@ -0,0 +1,50 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Call to podman to list running containers then commit all state to
+# disk. Once services state has been flushed dump the database then allow
+# the backup to start.
+
+- name: Get Container cli
+  command: /bin/hiera -c /etc/puppet/hiera.yaml container_cli
+  register: tripleo_backup_and_restore_container_cli
+  changed_when: tripleo_backup_and_restore_container_cli.stdout is undefined
+  tags:
+    - bar_create_recover_image
+
+- name: set tripleo_container_cli
+  set_fact:
+    tripleo_container_cli: "{{ tripleo_backup_and_restore_container_cli.stdout }}"
+  when:
+    - tripleo_backup_and_restore_container_cli.stdout != 'nil'
+  tags:
+    - bar_create_recover_image
+
+- name: Gather Container Service Name
+  shell: |
+    set -o pipefail
+    /usr/bin/{{ tripleo_container_cli }} ps --format '{{ '{{' }}.Names {{ '}}' }} ' | /usr/bin/egrep -v 'galera|mysql|bundle'
+  register: container_services
+  changed_when: container_services.stdout is undefined
+  tags:
+    - bar_create_recover_image
+
+- name: Pause containers for database backup.
+  command: "{{ tripleo_container_cli }} pause {{ item }}"
+  with_items: "{{ container_services.stdout_lines }}"
+  when: container_services is defined
+  tags:
+    - bar_create_recover_image

tripleo_ansible/roles/backup_and_restore/backup/tasks/service_manager_unpause.yml

@@ -0,0 +1,51 @@
+---
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Call to podman to list running containers then commit all state to
+# disk. Once services state has been flushed dump the database then allow
+# the backup to start.
+
+- name: Enable pacemaker
+  command: pcs cluster start --all
+  when: enabled_galera
+  run_once: true
+  tags:
+    - bar_create_recover_image
+
+- name: unPause database container
+  command: "{{ tripleo_container_cli }} unpause {{ tripleo_backup_and_restore_mysql_container }}"
+  when:
+    - tripleo_container_cli is defined
+    - not enabled_galera
+    - tripleo_backup_and_restore_mysql_container is defined
+  tags:
+    - bar_create_recover_image
+
+- name: Gather Container Service Name
+  shell: |
+    set -o pipefail
+    /usr/bin/{{ tripleo_container_cli }} ps -a --filter='status=paused' --format '{{ '{{' }}.Names {{ '}}' }} '
+  register: container_services
+  changed_when: container_services.stdout is defined
+  tags:
+    - bar_create_recover_image
+
+- name: Pause containers for database backup.
+  command: "{{ tripleo_container_cli }} unpause {{ item }}"
+  with_items: "{{ container_services.stdout_lines }}"
+  when: tripleo_container_cli is defined
+  tags:
+    - bar_create_recover_image

tripleo_ansible/roles/backup_and_restore/defaults/main.yml

@@ -17,6 +17,14 @@
 
 # All variables intended for modification should be placed in this file.
 
+# Set the container command line entry-point
+tripleo_container_cli: "{{ container_cli | default('docker') }}"
+# Stop and start all running services before backup is ran.
+tripleo_backup_and_restore_service_manager: true
+
+# Set the name of the mysql container
+tripleo_backup_and_restore_mysql_container: mysql
+
 # All variables within this role should have a prefix of "tripleo_backup_and_restore"
 tripleo_backup_and_restore_debug: false
 # By default this should be the Undercloud node

tripleo_ansible/roles/backup_and_restore/molecule/default/playbook.yml

@@ -22,6 +22,7 @@
     - role: backup_and_restore
       tripleo_backup_and_restore_nfs_server: undercloud
       tripleo_backup_and_restore_rear_simulate: true
+      tripleo_backup_and_restore_service_manager: false
       tripleo_backup_and_restore_hiera_config_file: "{{ ansible_user_dir }}/hiera.yaml"
   vars:
     ansible_python_interpreter: "{{ ansible_user_dir }}/test-python/bin/python"

tripleo_ansible/roles/backup_and_restore/setup_rear/tasks/main.yml

@@ -43,20 +43,21 @@
 - name: Get local hostname
   command: hostname
   register: tripleo_backup_and_restore_hostname
+  when: tripleo_backup_and_restore_hostname is undefined
   tags:
     - bar_setup_rear
 
-- name: Get bootstrap_nodeid
+- name: Set bootstrap nodeid
   become: true
-  command: "hiera -c {{ tripleo_backup_and_restore_hiera_config_file }} bootstrap_nodeid"
-  register: tripleo_backup_and_restore_bootstrap_nodeid
+  set_fact:
+    tripleo_backup_and_restore_bootstrap_nodeid: "{{ lookup('hiera', 'bootstrap_nodeid _hierarchy_file=/etc/puppet/hiera.yml') }}"
   tags:
     - bar_setup_rear
 
-- name: Get enabled services by node
+- name: List enabled services by node
   become: true
-  command: "hiera -c {{ tripleo_backup_and_restore_hiera_config_file }} service_names"
-  register: tripleo_backup_and_restore_enabled_services
+  set_fact:
+    tripleo_backup_and_restore_restore_enabled_services: "{{ lookup('hiera', 'service_names _hierarchy_file=/etc/puppet/hiera.yml') }}"
   tags:
     - bar_setup_rear
 
@@ -65,13 +66,8 @@
   set_fact:
     tripleo_backup_and_restore_exclude_paths:
       - "{{ tripleo_backup_and_restore_exclude_paths_common }}"
-      - "{{ (tripleo_backup_and_restore_exclude_paths_compute) if
-        (tripleo_backup_and_restore_enabled_services.stdout is search('nova_compute')) else [] }}"
-      - "{{ (tripleo_backup_and_restore_exclude_paths_controller) if
-        (tripleo_backup_and_restore_bootstrap_nodeid.stdout != tripleo_backup_and_restore_hostname.stdout and
-        tripleo_backup_and_restore_enabled_services.stdout is search('pacemaker') and
-        tripleo_backup_and_restore_enabled_services.stdout is search('mysql') and
-        tripleo_backup_and_restore_exclude_paths_controller_non_bootrapnode) else [] }}"
+      - "{{ tripleo_backup_and_restore_exclude_paths_compute }}"
+      - "{{ tripleo_backup_and_restore_exclude_paths_controller }}"
   tags:
     - bar_setup_rear
 

tripleo_ansible/roles/backup_and_restore/tasks/main.yml

@@ -38,5 +38,20 @@
 - name: Setup ReaR
   import_tasks: ../setup_rear/tasks/main.yml
 
+- name: Service management
+  import_tasks: ../backup/tasks/service_manager_pause.yml
+  when:
+    - tripleo_backup_and_restore_service_manager
+
+- name: Backup the database
+  import_tasks: ../backup/tasks/db_backup.yml
+  when:
+    - tripleo_backup_and_restore_service_manager
+
 - name: Create recovery images with ReaR
   import_tasks: ../backup/tasks/main.yml
+
+- name: Service management
+  import_tasks: ../backup/tasks/service_manager_unpause.yml
+  when:
+    - tripleo_backup_and_restore_service_manager