Use mariabackup for database backups

Currently, Xtrabackup is used for database backups. However, Xtrabackup
is not compatible with MariaDB 10.3. This change switches to use
mariabackup [1], which is available in the mariadb image.

The documented full and incremental restore procedures have been
modified to use mariabackup, following [2] and [3].

[1] https://mariadb.com/kb/en/library/mariabackup-overview/
[2] https://mariadb.com/kb/en/library/full-backup-and-restore-with-mariabackup/
[3] https://mariadb.com/kb/en/library/incremental-backup-and-restore-with-mariabackup/

Change-Id: Id52b9b1f7b013277e401b1f6b8aed34473d2b2c4
Closes-Bug: #1843043
Depends-On: https://review.opendev.org/692996
(cherry picked from commit 7f47ddf7f4)

ansible/group_vars/all.yml

@@ -584,6 +584,9 @@ enable_manila_backend_generic: "no"
 enable_manila_backend_hnas: "no"
 enable_manila_backend_cephfs_native: "no"
 enable_manila_backend_cephfs_nfs: "no"
+# TODO(mgoddard): Change this to a plain "no" when support enable_xtrabackup
+# has been removed.
+enable_mariabackup: "{{ enable_xtrabackup | default('no') }}"
 enable_mistral: "no"
 enable_monasca: "no"
 enable_mongodb: "no"
@@ -631,7 +634,6 @@ enable_trove_singletenant: "no"
 enable_vitrage: "no"
 enable_vmtp: "no"
 enable_watcher: "no"
-enable_xtrabackup: "no"
 enable_zookeeper: "{{ enable_kafka | bool }}"
 enable_zun: "no"
 

ansible/mariadb_backup.yml

@@ -8,4 +8,4 @@
   roles:
     - { role: mariadb,
         tags: mariadb,
-        when: enable_xtrabackup | bool }
+        when: enable_mariabackup | bool }

ansible/roles/mariadb/defaults/main.yml

@@ -67,9 +67,9 @@ mariadb_service: "{{ mariadb_services['mariadb'] }}"
 ####################
 # Backups
 ####################
-xtrabackup_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ mariadb_install_type }}-xtrabackup"
-xtrabackup_tag: "{{ openstack_release }}"
-xtrabackup_image_full: "{{ xtrabackup_image }}:{{ xtrabackup_tag }}"
+mariabackup_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ mariadb_install_type }}-mariadb"
+mariabackup_tag: "{{ openstack_release }}"
+mariabackup_image_full: "{{ mariabackup_image }}:{{ mariabackup_tag }}"
 
 mariadb_backup_host: "{{ groups['mariadb'][0] }}"
 mariadb_backup_database_schema: "PERCONA_SCHEMA"

ansible/roles/mariadb/tasks/backup.yml

@@ -1,20 +1,22 @@
 ---
-- name: Taking {{ mariadb_backup_type }} database backup via XtraBackup
+- name: Taking {{ mariadb_backup_type }} database backup via Mariabackup
   become: true
   kolla_docker:
     action: "start_container"
+    command: "bash -c 'sudo -E kolla_set_configs && /usr/local/bin/kolla_mariadb_backup.sh'"
     common_options: "{{ docker_common_options }}"
-    image: "{{ xtrabackup_image_full }}"
-    name: "xtrabackup"
-    restart_policy: "never"
+    detach: False
+    image: "{{ mariabackup_image_full }}"
+    name: "mariabackup"
+    restart_policy: no
     remove_on_exit: True
     environment:
       BACKUP_TYPE: "{{ mariadb_backup_type }}"
     volumes:
-      - "{{ node_config_directory }}xtrabackup:/etc/mysql:ro"
+      - "{{ node_config_directory }}/mariabackup/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
+      - "mariadb:/var/lib/mysql"
       - "mariadb_backup:/backup"
-    volumes_from:
-      - "mariadb"
+      - "kolla_logs:/var/log/kolla/"
   when:
     - inventory_hostname == mariadb_backup_host

ansible/roles/mariadb/tasks/config.yml

@@ -14,29 +14,29 @@
 
 - name: Ensuring database backup config directory exists
   file:
-    path: "{{ node_config_directory }}xtrabackup"
+    path: "{{ node_config_directory }}/mariabackup"
     state: "directory"
     owner: "{{ config_owner_user }}"
     group: "{{ config_owner_group }}"
     mode: "0770"
   become: true
   when:
-    - enable_xtrabackup | bool
+    - enable_mariabackup | bool
     - inventory_hostname == mariadb_backup_host
 
-- name: Copying over my.cnf for xtrabackup
+- name: Copying over my.cnf for mariabackup
   merge_configs:
     sources:
       - "{{ role_path }}/templates/backup.my.cnf.j2"
       - "{{ node_custom_config }}/backup.my.cnf"
       - "{{ node_custom_config }}/mariadb/{{ inventory_hostname }}/backup.my.cnf"
-    dest: "{{ node_config_directory }}xtrabackup/my.cnf"
+    dest: "{{ node_config_directory }}/mariabackup/my.cnf"
     owner: "{{ config_owner_user }}"
     group: "{{ config_owner_group }}"
     mode: "0660"
   become: true
   when:
-    - enable_xtrabackup | bool
+    - enable_mariabackup | bool
     - inventory_hostname == mariadb_backup_host
 
 - name: Copying over config.json files for services
@@ -54,6 +54,18 @@
   notify:
     - restart mariadb
 
+- name: Copying over config.json files for mariabackup
+  vars:
+    service_name: "mariabackup"
+  template:
+    src: "{{ service_name }}.json.j2"
+    dest: "{{ node_config_directory }}/{{ service_name }}/config.json"
+    mode: "0660"
+  become: true
+  when:
+    - enable_mariabackup | bool
+    - inventory_hostname == mariadb_backup_host
+
 - name: Copying over galera.cnf
   vars:
     service_name: "mariadb"

ansible/roles/mariadb/tasks/register.yml

@@ -14,7 +14,7 @@
       priv: "*.*:USAGE"
   run_once: True
 
-- name: Creating the Percona XtraBackup database
+- name: Creating the Mariabackup database
   become: true
   kolla_toolbox:
     module_name: mysql_db
@@ -26,7 +26,7 @@
       name: "{{ mariadb_backup_database_schema }}"
   run_once: True
   when:
-    - enable_xtrabackup | bool
+    - enable_mariabackup | bool
 
 - name: Creating database backup user and setting permissions
   become: true
@@ -44,9 +44,9 @@
       append_privs: True
   run_once: True
   when:
-    - enable_xtrabackup | bool
+    - enable_mariabackup | bool
 
-- name: Granting permissions on XtraBackup database to backup user
+- name: Granting permissions on Mariabackup database to backup user
   become: true
   kolla_toolbox:
     module_name: mysql_user
@@ -62,7 +62,7 @@
       append_privs: True
   run_once: True
   when:
-    - enable_xtrabackup | bool
+    - enable_mariabackup | bool
 
 - name: Cleaning up facts
   set_fact:

ansible/roles/mariadb/templates/mariabackup.json.j2

@@ -0,0 +1,21 @@
+{
+    "command": "false",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/my.cnf",
+            "dest": "/etc/mysql/my.cnf",
+            "owner": "mysql",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/mariadb",
+            "owner": "mysql:mysql"
+        },
+        {
+            "path": "/backup",
+            "owner": "mysql:mysql"
+        }
+    ]
+}

doc/source/admin/mariadb-backup-and-restore.rst

@@ -5,7 +5,7 @@ MariaDB database backup and restore
 ===================================
 
 Kolla-Ansible can facilitate either full or incremental backups of data
-hosted in MariaDB. It achieves this using Percona's Xtrabackup, a tool
+hosted in MariaDB. It achieves this using Mariabackup, a tool
 designed to allow for 'hot backups' - an approach which means that consistent
 backups can be taken without any downtime for your database or your cloud.
 
@@ -27,7 +27,7 @@ Firstly, enable backups via ``globals.yml``:
 
 .. code-block:: console
 
-   enable_xtrabackup: "yes"
+   enable_mariabackup: "yes"
 
 Then, kick off a reconfiguration of MariaDB:
 
@@ -67,38 +67,49 @@ backups scheduled via a cron job.
 Restoring backups
 ~~~~~~~~~~~~~~~~~
 
-Owing to the way in which XtraBackup performs hot backups, there are some
+Owing to the way in which Mariabackup performs hot backups, there are some
 steps that must be performed in order to prepare your data before it can be
 copied into place for use by MariaDB. This process is currently manual, but
-the Kolla XtraBackup image includes the tooling necessary to successfully
+the Kolla Mariabackup image includes the tooling necessary to successfully
 prepare backups. Two examples are given below.
 
 Full
 ----
 
-For a full backup, start a new container using the XtraBackup image with the
+For a full backup, start a new container using the Mariabackup image with the
 following options on the master database node:
 
 .. code-block:: console
 
-   docker run -it --volumes-from mariadb --name dbrestore \
-      -v mariadb_backup:/backup kolla/centos-binary-xtrabackup:rocky \
+   docker run --rm -it --volumes-from mariadb --name dbrestore \
+      --volume mariadb_backup:/backup \
+      kolla/centos-binary-mariadb:train \
       /bin/bash
-   cd /backup
-   mkdir -p /restore/full
-   cat mysqlbackup-04-10-2018.xbc.xbs | xbstream -x -C /restore/full/
-   innobackupex --decompress /restore/full
-   find /restore -name *.qp -exec rm {} \;
-   innobackupex --apply-log /restore/full
+   (dbrestore) $ cd /backup
+   (dbrestore) $ rm -rf /backup/restore
+   (dbrestore) $ mkdir -p /backup/restore/full
+   (dbrestore) $ gunzip mysqlbackup-04-10-2018.xbc.xbs.gz
+   (dbrestore) $ mbstream -x -C /backup/restore/full/ < mysqlbackup-04-10-2018.xbc.xbs
+   (dbrestore) $ mariabackup --prepare --target-dir /backup/restore/full
 
-Then stop the MariaDB instance, delete the old data files (or move
-them elsewhere), and copy the backup into place:
+Stop the MariaDB instance.
 
 .. code-block:: console
 
    docker stop mariadb
-   rm -rf /var/lib/mysql/* /var/lib/mysql/.*
-   innobackupex --copy-back /restore/full
+
+Delete the old data files (or move them elsewhere), and copy the backup into
+place:
+
+.. code-block:: console
+
+   docker run --rm -it --volumes-from mariadb --name dbrestore \
+      --volume mariadb_backup:/backup \
+      kolla/centos-binary-mariadb:train \
+      /bin/bash
+   (dbrestore) $ rm -rf /var/lib/mysql/*
+   (dbrestore) $ rm -rf /var/lib/mysql/\.[^\.]*
+   (dbrestore) $ mariabackup --copy-back --target-dir /backup/restore/full
 
 Then you can restart MariaDB with the restored data in place:
 
@@ -121,20 +132,20 @@ incremental backup,
 
 .. code-block:: console
 
-   docker run -it --volumes-from mariadb --name dbrestore \
-      -v mariadb_backup:/backup kolla/centos-binary-xtrabackup:rocky \
+   docker run --rm -it --volumes-from mariadb --name dbrestore \
+      --volume mariadb_backup:/backup --tmpfs /backup/restore \
+      kolla/centos-binary-mariadb:train \
       /bin/bash
-   cd /backup
-   mkdir -p /restore/full
-   mkdir -p /restore/inc/11
-   cat mysqlbackup-06-11-2018-1541505206.qp.xbc.xbs | xbstream -x -C /restore/full/
-   cat incremental-11-mysqlbackup-06-11-2018-1541505223.qp.xbc.xbs | xbstream -x -C /restore/inc/11
-   innobackupex --decompress /restore/full
-   innobackupex --decompress /restore/inc/11
-   find /restore -name *.qp -exec rm {} \;
-   innobackupex --apply-log --redo-only /restore/full
-   innobackupex --apply-log --redo-only --incremental-dir=/restore/inc/11 /restore/full
-   innobackupex --apply-log /restore/full
+   (dbrestore) $ cd /backup
+   (dbrestore) $ rm -rf /backup/restore
+   (dbrestore) $ mkdir -p /backup/restore/full
+   (dbrestore) $ mkdir -p /backup/restore/inc
+   (dbrestore) $ gunzip mysqlbackup-06-11-2018-1541505206.qp.xbc.xbs.gz
+   (dbrestore) $ gunzip incremental-11-mysqlbackup-06-11-2018-1541505223.qp.xbc.xbs.gz
+   (dbrestore) $ mbstream -x -C /backup/restore/full/ < mysqlbackup-06-11-2018-1541505206.qp.xbc.xbs
+   (dbrestore) $ mbstream -x -C /backup/restore/inc < incremental-11-mysqlbackup-06-11-2018-1541505223.qp.xbc.xbs
+   (dbrestore) $ mariabackup --prepare --target-dir /backup/restore/full
+   (dbrestore) $ mariabackup --prepare --incremental-dir=/backup/restore/inc --target-dir /backup/restore/full
 
 At this point the backup is prepared and ready to be copied back into place,
 as per the previous example.

etc/kolla/globals.yml

@@ -270,6 +270,7 @@ kolla_internal_vip_address: "10.10.10.254"
 #enable_manila_backend_hnas: "no"
 #enable_manila_backend_cephfs_native: "no"
 #enable_manila_backend_cephfs_nfs: "no"
+#enable_mariabackup: "no"
 #enable_mistral: "no"
 #enable_monasca: "no"
 #enable_mongodb: "no"
@@ -316,7 +317,6 @@ kolla_internal_vip_address: "10.10.10.254"
 #enable_vitrage: "no"
 #enable_vmtp: "no"
 #enable_watcher: "no"
-#enable_xtrabackup: "no"
 #enable_zookeeper: "no"
 #enable_zun: "no"
 

releasenotes/notes/mariabackup-bd3b238823e589da.yaml

@@ -0,0 +1,13 @@
+---
+upgrade:
+  - |
+    Changes the database backup procedure to use ``mariabackup`` which is
+    compatible with MariaDB 10.3. The ``qpress`` based compression used
+    previously is now replaced with ``gzip``. The documented restore procedure
+    has been modified accordingly.  See the `Mariabackup documentation
+    <https://mariadb.com/kb/en/library/mariabackup-overview/>`__ for further
+    information.
+deprecations:
+  - |
+    The ``enable_xtrabackup`` variable is deprecated in favour of
+    ``enable_mariabackup``.