Add new option to perform an on-demand backup of MariaDB

blueprint database-backup-recovery

Introduce a new option, mariadb_backup, which takes a backup of all
databases hosted in MariaDB.

Backups are performed using XtraBackup, the output of which is saved to
a dedicated Docker volume on the target host (which defaults to the
first node in the MariaDB cluster).

It supports either full (the default) or incremental backups.

Change-Id: Ied224c0d19b8734aa72092aaddd530155999dbc3

ansible/group_vars/all.yml

@@ -545,6 +545,7 @@ enable_trove_singletenant: "no"
 enable_vitrage: "no"
 enable_vmtp: "no"
 enable_watcher: "no"
+enable_xtrabackup: "no"
 enable_zookeeper: "{{ enable_kafka | bool }}"
 enable_zun: "no"
 

ansible/mariadb_backup.yml

@@ -0,0 +1,23 @@
+---
+- name: Detect openstack_release variable
+  hosts: mariadb
+  gather_facts: false
+  tasks:
+    - name: Get current kolla-ansible version number
+      local_action: command python -c "import pbr.version; print(pbr.version.VersionInfo('kolla-ansible'))"
+      register: kolla_ansible_version
+      changed_when: false
+      when: openstack_release == "auto"
+
+    - name: Set openstack_release variable
+      set_fact:
+        openstack_release: "{{ kolla_ansible_version.stdout }}"
+      when: openstack_release == "auto"
+  tags: always
+
+- name: Backup MariaDB
+  hosts: mariadb
+  roles:
+    - { role: mariadb,
+        tags: mariadb,
+        when: enable_xtrabackup | bool }

ansible/roles/mariadb/defaults/main.yml

@@ -62,3 +62,16 @@ mariadb_dimensions: "{{ default_container_dimensions }}"
 # Vars used within recover_cluster.yml
 ########################################
 mariadb_service: "{{ mariadb_services['mariadb'] }}"
+
+####################
+# Backups
+####################
+xtrabackup_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-xtrabackup"
+xtrabackup_tag: "{{ openstack_release }}"
+xtrabackup_image_full: "{{ xtrabackup_image }}:{{ xtrabackup_tag }}"
+
+mariadb_backup_host: "{{ groups['mariadb'][0] }}"
+mariadb_backup_database_schema: "PERCONA_SCHEMA"
+mariadb_backup_database_user: "backup"
+mariadb_backup_database_address: "{{ database_address }}"
+mariadb_backup_type: "full"

ansible/roles/mariadb/tasks/backup.yml

@@ -0,0 +1,19 @@
+---
+- name: Taking {{ mariadb_backup_type }} database backup via XtraBackup
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ xtrabackup_image_full }}"
+    name: "xtrabackup"
+    restart_policy: "never"
+    remove_on_exit: True
+    environment:
+      BACKUP_TYPE: "{{ mariadb_backup_type }}"
+    volumes:
+      - "{{ node_config_directory }}xtrabackup:/etc/mysql:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "mariadb_backup:/backup"
+    volumes_from:
+      - "mariadb"
+  when:
+    - inventory_hostname == mariadb_backup_host

ansible/roles/mariadb/tasks/config.yml

@@ -12,6 +12,33 @@
     - item.value.enabled | bool
   with_dict: "{{ mariadb_services }}"
 
+- name: Ensuring database backup config directory exists
+  file:
+    path: "{{ node_config_directory }}xtrabackup"
+    state: "directory"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
+    mode: "0770"
+  become: true
+  when:
+    - enable_xtrabackup | bool
+    - inventory_hostname == mariadb_backup_host
+
+- name: Copying over my.cnf for xtrabackup
+  merge_configs:
+    sources:
+      - "{{ role_path }}/templates/backup.my.cnf.j2"
+      - "{{ node_custom_config }}/backup.my.cnf"
+      - "{{ node_custom_config }}/mariadb/{{ inventory_hostname }}/backup.my.cnf"
+    dest: "{{ node_config_directory }}xtrabackup/my.cnf"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
+    mode: "0660"
+  become: true
+  when:
+    - enable_xtrabackup | bool
+    - inventory_hostname == mariadb_backup_host
+
 - name: Copying over config.json files for services
   vars:
     service_name: "mariadb"

ansible/roles/mariadb/tasks/register.yml

@@ -13,6 +13,53 @@
       priv: "*.*:USAGE"
   run_once: True
 
+- name: Creating the Percona XtraBackup database
+  kolla_toolbox:
+    module_name: mysql_db
+    module_args:
+      login_host: "{{ database_address }}"
+      login_port: "{{ database_port }}"
+      login_user: "{{ database_user }}"
+      login_password: "{{ database_password }}"
+      name: "{{ mariadb_backup_database_schema }}"
+  run_once: True
+  when:
+    - enable_xtrabackup | bool
+
+- name: Creating database backup user and setting permissions
+  kolla_toolbox:
+    module_name: mysql_user
+    module_args:
+      login_host: "{{ database_address }}"
+      login_port: "{{ database_port }}"
+      login_user: "{{ database_user }}"
+      login_password: "{{ database_password }}"
+      name: "{{ mariadb_backup_database_user }}"
+      password: "{{ mariadb_backup_database_password }}"
+      host: "%"
+      priv: "*.*:CREATE TABLESPACE,RELOAD,PROCESS,SUPER,LOCK TABLES,REPLICATION CLIENT"
+      append_privs: True
+  run_once: True
+  when:
+    - enable_xtrabackup | bool
+
+- name: Granting permissions on XtraBackup database to backup user
+  kolla_toolbox:
+    module_name: mysql_user
+    module_args:
+      login_host: "{{ database_address }}"
+      login_port: "{{ database_port }}"
+      login_user: "{{ database_user }}"
+      login_password: "{{ database_password }}"
+      name: "{{ mariadb_backup_database_user }}"
+      password: "{{ mariadb_backup_database_password }}"
+      host: "%"
+      priv: "{{ mariadb_backup_database_schema }}.*:CREATE,INSERT,SELECT"
+      append_privs: True
+  run_once: True
+  when:
+    - enable_xtrabackup | bool
+
 - name: Cleaning up facts
   set_fact:
     delegate_host: "bootstraped"

ansible/roles/mariadb/templates/backup.my.cnf.j2

@@ -0,0 +1,6 @@
+[client]
+default-character-set=utf8
+user={{ mariadb_backup_database_user }}
+password={{ mariadb_backup_database_password }}
+host={{ database_address }}
+port={{ database_port }}

doc/source/admin/index.rst

@@ -6,5 +6,6 @@ Admin Guides
    :maxdepth: 2
 
    advanced-configuration
+   mariadb-backup-and-restore
    production-architecture-guide
    deployment-philosophy

doc/source/admin/mariadb-backup-and-restore.rst

@@ -0,0 +1,140 @@
+.. _mariadb-backup-and-restore:
+
+===================================
+MariaDB database backup and restore
+===================================
+
+Kolla-Ansible can facilitate either full or incremental backups of data
+hosted in MariaDB. It achieves this using Percona's Xtrabackup, a tool
+designed to allow for 'hot backups' - an approach which means that consistent
+backups can be taken without any downtime for your database or your cloud.
+
+.. note::
+
+   By default, backups will be performed on the first node in your Galera cluster
+   or on the MariaDB node itself if you just have the one. Backup files are saved
+   to a dedicated Docker volume - ``mariadb_backup`` - and it's the contents of
+   this that you should target for transferring backups elsewhere.
+
+Enabling Backup Functionality
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For backups to work, some reconfiguration of MariaDB is required - this is to
+enable appropriate permissions for the backup client, and also to create an
+additional database in order to store backup information.
+
+Firstly, enable backups via ``globals.yml``:
+
+.. code-block:: console
+
+   enable_xtrabackup: "yes"
+
+Then, kick off a reconfiguration of MariaDB:
+
+``kolla-ansible -i INVENTORY reconfigure -t mariadb``
+
+Once that has run successfully, you should then be able to take full and
+incremental backups as described below.
+
+Backup Procedure
+~~~~~~~~~~~~~~~~
+
+To perform a full backup, run the following command:
+
+``kolla-ansible -i INVENTORY mariadb_backup``
+
+Or to perform an incremental backup:
+
+``kolla-ansible -i INVENTORY mariadb_backup --incremental``
+
+Kolla doesn't currently manage the scheduling of these backups, so you'll
+need to configure an appropriate scheduler (i.e cron) to run these commands
+on your behalf should you require regular snapshots of your data. A suggested
+schedule would be:
+
+* Daily full, retained for two weeks
+* Hourly incremental, retained for one day
+
+Backups are performed on your behalf on the designated database node using
+permissions defined during the configuration step; no password is required to
+invoke these commands.
+
+Furthermore, backup actions can be triggered from a node with a minimal
+installation of Kolla-Ansible, specifically one which doesn't require a copy of
+``passwords.yml``.  This is of note if you're looking to implement automated
+backups scheduled via a cron job.
+
+Restoring backups
+~~~~~~~~~~~~~~~~~
+
+Owing to the way in which XtraBackup performs hot backups, there are some
+steps that must be performed in order to prepare your data before it can be
+copied into place for use by MariaDB. This process is currently manual, but
+the Kolla XtraBackup image includes the tooling necessary to successfully
+prepare backups. Two examples are given below.
+
+Full
+----
+
+For a full backup, start a new container using the XtraBackup image with the
+following options on the master database node:
+
+.. code-block:: console
+
+   docker run -it --volumes-from mariadb --name dbrestore \
+      -v mariadb_backup:/backup kolla/centos-binary-xtrabackup:rocky \
+      /bin/bash
+   cd /backup
+   mkdir -p /restore/full
+   cat mysqlbackup-04-10-2018.xbc.xbs | xbstream -x -C /restore/full/
+   innobackupex --decompress /restore/full
+   find /restore -name *.qp -exec rm {} \;
+   innobackupex --apply-log /restore/full
+
+Then stop the MariaDB instance, delete the old data files (or move
+them elsewhere), and copy the backup into place:
+
+.. code-block:: console
+
+   docker stop mariadb
+   rm -rf /var/lib/mysql/* /var/lib/mysql/.*
+   innobackupex --copy-back /restore/full
+
+Then you can restart MariaDB with the restored data in place:
+
+.. code-block:: console
+
+   docker start mariadb
+   docker logs mariadb
+   81004 15:48:27 mysqld_safe WSREP: Running position recovery with --log_error='/var/lib/mysql//wsrep_recovery.BDTAm8' --pid-file='/var/lib/mysql//scratch-recover.pid'
+   181004 15:48:30 mysqld_safe WSREP: Recovered position 9388319e-c7bd-11e8-b2ce-6e9ec70d9926:58
+
+Incremental
+-----------
+
+This starts off similar to the full backup restore procedure above, but we
+must apply the logs from the incremental backups first of all before doing
+the final preparation required prior to restore. In the example below, I have
+a full backup - ``mysqlbackup-06-11-2018-1541505206.qp.xbc.xbs``, and an
+incremental backup,
+``incremental-11-mysqlbackup-06-11-2018-1541505223.qp.xbc.xbs``.
+
+.. code-block:: console
+
+   docker run -it --volumes-from mariadb --name dbrestore \
+      -v mariadb_backup:/backup kolla/centos-binary-xtrabackup:rocky \
+      /bin/bash
+   cd /backup
+   mkdir -p /restore/full
+   mkdir -p /restore/inc/11
+   cat mysqlbackup-06-11-2018-1541505206.qp.xbc.xbs | xbstream -x -C /restore/full/
+   cat incremental-11-mysqlbackup-06-11-2018-1541505223.qp.xbc.xbs | xbstream -x -C /restore/inc/11
+   innobackupex --decompress /restore/full
+   innobackupex --decompress /restore/inc/11
+   find /restore -name *.qp -exec rm {} \;
+   innobackupex --apply-log --redo-only /restore/full
+   innobackupex --apply-log --redo-only --incremental-dir=/restore/inc/11 /restore/full
+   innobackupex --apply-log /restore/full
+
+At this point the backup is prepared and ready to be copied back into place,
+as per the previous example.

etc/kolla/globals.yml

@@ -284,6 +284,7 @@ kolla_internal_vip_address: "10.10.10.254"
 #enable_vitrage: "no"
 #enable_vmtp: "no"
 #enable_watcher: "no"
+#enable_xtrabackup: "no"
 #enable_zookeeper: "no"
 #enable_zun: "no"
 

etc/kolla/passwords.yml

@@ -15,6 +15,8 @@ cinder_rbd_secret_uuid:
 # Database options
 ####################
 database_password:
+# Password for the dedicated backup user account
+mariadb_backup_database_password:
 
 ####################
 # Docker options

releasenotes/notes/mariadb-xtrabackup-d4f48464dd6baaea.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    Supports taking a backup of all MariaDB-hosted databases using Percona XtraBackup.
+security:
+  - |
+    When the MariaDB backup option is enabled, it will create a new database
+    which is used to keep track of backup-related metadata, along with a new
+    backup user with a specific set of permissions limited to backup-related
+    actions only.

tools/kolla-ansible

@@ -54,6 +54,9 @@ Commands:
     prechecks           Do pre-deployment checks for hosts
     check               Do post-deployment smoke tests
     mariadb_recovery    Recover a completely stopped mariadb cluster
+    mariadb_backup      Take a backup of MariaDB databases
+                            --full (default)
+                            --incremental
     bootstrap-servers   Bootstrap servers with kolla deploy dependencies
     destroy             Destroy Kolla containers, volumes and host configuration
                             --include-images to also destroy Kolla images
@@ -91,6 +94,7 @@ cat <<EOF
 prechecks
 check
 mariadb_recovery
+mariadb_backup
 bootstrap-servers
 destroy
 deploy
@@ -107,7 +111,7 @@ EOF
 }
 
 SHORT_OPTS="hi:p:t:k:e:v"
-LONG_OPTS="help,inventory:,playbook:,skip-tags:,tags:,key:,extra:,verbose,configdir:,passwords:,limit:,forks:,vault-id:,ask-vault-pass,vault-password-file:,yes-i-really-really-mean-it,include-images,include-dev"
+LONG_OPTS="help,inventory:,playbook:,skip-tags:,tags:,key:,extra:,verbose,configdir:,passwords:,limit:,forks:,vault-id:,ask-vault-pass,vault-password-file:,yes-i-really-really-mean-it,include-images,include-dev:,full,incremental"
 
 RAW_ARGS="$*"
 ARGS=$(getopt -o "${SHORT_OPTS}" -l "${LONG_OPTS}" --name "$0" -- "$@") || { usage >&2; exit 2; }
@@ -125,6 +129,7 @@ PASSWORDS_FILE="${CONFIG_DIR}/passwords.yml"
 DANGER_CONFIRM=
 INCLUDE_IMAGES=
 INCLUDE_DEV=
+BACKUP_TYPE="full"
 # Serial is not recommended and disabled by default. Users can enable it by
 # configuring ANSIBLE_SERIAL variable.
 ANSIBLE_SERIAL=${ANSIBLE_SERIAL:-0}
@@ -221,6 +226,16 @@ while [ "$#" -gt 0 ]; do
             shift 2
             ;;
 
+    (--full)
+            BACKUP_TYPE="full"
+            shift 1
+            ;;
+
+    (--incremental)
+            BACKUP_TYPE="incremental"
+            shift 1
+            ;;
+
     (--help|-h)
             usage
             shift
@@ -254,6 +269,11 @@ case "$1" in
         EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=deploy -e common_run=true"
         PLAYBOOK="${BASEDIR}/ansible/mariadb_recovery.yml"
         ;;
+(mariadb_backup)
+        ACTION="Backup MariaDB databases"
+        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=backup -e mariadb_backup_type=${BACKUP_TYPE} -e common_run=true"
+        PLAYBOOK="${BASEDIR}/ansible/mariadb_backup.yml"
+        ;;
 (destroy)
         ACTION="Destroy Kolla containers, volumes and host configuration"
         PLAYBOOK="${BASEDIR}/ansible/destroy.yml"