Browse Source

Add new option to perform an on-demand backup of MariaDB

blueprint database-backup-recovery

Introduce a new option, mariadb_backup, which takes a backup of all
databases hosted in MariaDB.

Backups are performed using XtraBackup, the output of which is saved to
a dedicated Docker volume on the target host (which defaults to the
first node in the MariaDB cluster).

It supports either full (the default) or incremental backups.

Change-Id: Ied224c0d19b8734aa72092aaddd530155999dbc3
tags/8.0.0.0b1
Nick Jones 1 year ago
parent
commit
f704a78029

+ 1
- 0
ansible/group_vars/all.yml View File

@@ -545,6 +545,7 @@ enable_trove_singletenant: "no"
545 545
 enable_vitrage: "no"
546 546
 enable_vmtp: "no"
547 547
 enable_watcher: "no"
548
+enable_xtrabackup: "no"
548 549
 enable_zookeeper: "{{ enable_kafka | bool }}"
549 550
 enable_zun: "no"
550 551
 

+ 23
- 0
ansible/mariadb_backup.yml View File

@@ -0,0 +1,23 @@
1
+---
2
+- name: Detect openstack_release variable
3
+  hosts: mariadb
4
+  gather_facts: false
5
+  tasks:
6
+    - name: Get current kolla-ansible version number
7
+      local_action: command python -c "import pbr.version; print(pbr.version.VersionInfo('kolla-ansible'))"
8
+      register: kolla_ansible_version
9
+      changed_when: false
10
+      when: openstack_release == "auto"
11
+
12
+    - name: Set openstack_release variable
13
+      set_fact:
14
+        openstack_release: "{{ kolla_ansible_version.stdout }}"
15
+      when: openstack_release == "auto"
16
+  tags: always
17
+
18
+- name: Backup MariaDB
19
+  hosts: mariadb
20
+  roles:
21
+    - { role: mariadb,
22
+        tags: mariadb,
23
+        when: enable_xtrabackup | bool }

+ 13
- 0
ansible/roles/mariadb/defaults/main.yml View File

@@ -62,3 +62,16 @@ mariadb_dimensions: "{{ default_container_dimensions }}"
62 62
 # Vars used within recover_cluster.yml
63 63
 ########################################
64 64
 mariadb_service: "{{ mariadb_services['mariadb'] }}"
65
+
66
+####################
67
+# Backups
68
+####################
69
+xtrabackup_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-xtrabackup"
70
+xtrabackup_tag: "{{ openstack_release }}"
71
+xtrabackup_image_full: "{{ xtrabackup_image }}:{{ xtrabackup_tag }}"
72
+
73
+mariadb_backup_host: "{{ groups['mariadb'][0] }}"
74
+mariadb_backup_database_schema: "PERCONA_SCHEMA"
75
+mariadb_backup_database_user: "backup"
76
+mariadb_backup_database_address: "{{ database_address }}"
77
+mariadb_backup_type: "full"

+ 19
- 0
ansible/roles/mariadb/tasks/backup.yml View File

@@ -0,0 +1,19 @@
1
+---
2
+- name: Taking {{ mariadb_backup_type }} database backup via XtraBackup
3
+  kolla_docker:
4
+    action: "start_container"
5
+    common_options: "{{ docker_common_options }}"
6
+    image: "{{ xtrabackup_image_full }}"
7
+    name: "xtrabackup"
8
+    restart_policy: "never"
9
+    remove_on_exit: True
10
+    environment:
11
+      BACKUP_TYPE: "{{ mariadb_backup_type }}"
12
+    volumes:
13
+      - "{{ node_config_directory }}xtrabackup:/etc/mysql:ro"
14
+      - "/etc/localtime:/etc/localtime:ro"
15
+      - "mariadb_backup:/backup"
16
+    volumes_from:
17
+      - "mariadb"
18
+  when:
19
+    - inventory_hostname == mariadb_backup_host

+ 27
- 0
ansible/roles/mariadb/tasks/config.yml View File

@@ -12,6 +12,33 @@
12 12
     - item.value.enabled | bool
13 13
   with_dict: "{{ mariadb_services }}"
14 14
 
15
+- name: Ensuring database backup config directory exists
16
+  file:
17
+    path: "{{ node_config_directory }}xtrabackup"
18
+    state: "directory"
19
+    owner: "{{ config_owner_user }}"
20
+    group: "{{ config_owner_group }}"
21
+    mode: "0770"
22
+  become: true
23
+  when:
24
+    - enable_xtrabackup | bool
25
+    - inventory_hostname == mariadb_backup_host
26
+
27
+- name: Copying over my.cnf for xtrabackup
28
+  merge_configs:
29
+    sources:
30
+      - "{{ role_path }}/templates/backup.my.cnf.j2"
31
+      - "{{ node_custom_config }}/backup.my.cnf"
32
+      - "{{ node_custom_config }}/mariadb/{{ inventory_hostname }}/backup.my.cnf"
33
+    dest: "{{ node_config_directory }}xtrabackup/my.cnf"
34
+    owner: "{{ config_owner_user }}"
35
+    group: "{{ config_owner_group }}"
36
+    mode: "0660"
37
+  become: true
38
+  when:
39
+    - enable_xtrabackup | bool
40
+    - inventory_hostname == mariadb_backup_host
41
+
15 42
 - name: Copying over config.json files for services
16 43
   vars:
17 44
     service_name: "mariadb"

+ 47
- 0
ansible/roles/mariadb/tasks/register.yml View File

@@ -13,6 +13,53 @@
13 13
       priv: "*.*:USAGE"
14 14
   run_once: True
15 15
 
16
+- name: Creating the Percona XtraBackup database
17
+  kolla_toolbox:
18
+    module_name: mysql_db
19
+    module_args:
20
+      login_host: "{{ database_address }}"
21
+      login_port: "{{ database_port }}"
22
+      login_user: "{{ database_user }}"
23
+      login_password: "{{ database_password }}"
24
+      name: "{{ mariadb_backup_database_schema }}"
25
+  run_once: True
26
+  when:
27
+    - enable_xtrabackup | bool
28
+
29
+- name: Creating database backup user and setting permissions
30
+  kolla_toolbox:
31
+    module_name: mysql_user
32
+    module_args:
33
+      login_host: "{{ database_address }}"
34
+      login_port: "{{ database_port }}"
35
+      login_user: "{{ database_user }}"
36
+      login_password: "{{ database_password }}"
37
+      name: "{{ mariadb_backup_database_user }}"
38
+      password: "{{ mariadb_backup_database_password }}"
39
+      host: "%"
40
+      priv: "*.*:CREATE TABLESPACE,RELOAD,PROCESS,SUPER,LOCK TABLES,REPLICATION CLIENT"
41
+      append_privs: True
42
+  run_once: True
43
+  when:
44
+    - enable_xtrabackup | bool
45
+
46
+- name: Granting permissions on XtraBackup database to backup user
47
+  kolla_toolbox:
48
+    module_name: mysql_user
49
+    module_args:
50
+      login_host: "{{ database_address }}"
51
+      login_port: "{{ database_port }}"
52
+      login_user: "{{ database_user }}"
53
+      login_password: "{{ database_password }}"
54
+      name: "{{ mariadb_backup_database_user }}"
55
+      password: "{{ mariadb_backup_database_password }}"
56
+      host: "%"
57
+      priv: "{{ mariadb_backup_database_schema }}.*:CREATE,INSERT,SELECT"
58
+      append_privs: True
59
+  run_once: True
60
+  when:
61
+    - enable_xtrabackup | bool
62
+
16 63
 - name: Cleaning up facts
17 64
   set_fact:
18 65
     delegate_host: "bootstraped"

+ 6
- 0
ansible/roles/mariadb/templates/backup.my.cnf.j2 View File

@@ -0,0 +1,6 @@
1
+[client]
2
+default-character-set=utf8
3
+user={{ mariadb_backup_database_user }}
4
+password={{ mariadb_backup_database_password }}
5
+host={{ database_address }}
6
+port={{ database_port }}

+ 1
- 0
doc/source/admin/index.rst View File

@@ -6,5 +6,6 @@ Admin Guides
6 6
    :maxdepth: 2
7 7
 
8 8
    advanced-configuration
9
+   mariadb-backup-and-restore
9 10
    production-architecture-guide
10 11
    deployment-philosophy

+ 140
- 0
doc/source/admin/mariadb-backup-and-restore.rst View File

@@ -0,0 +1,140 @@
1
+.. _mariadb-backup-and-restore:
2
+
3
+===================================
4
+MariaDB database backup and restore
5
+===================================
6
+
7
+Kolla-Ansible can facilitate either full or incremental backups of data
8
+hosted in MariaDB. It achieves this using Percona's Xtrabackup, a tool
9
+designed to allow for 'hot backups' - an approach which means that consistent
10
+backups can be taken without any downtime for your database or your cloud.
11
+
12
+.. note::
13
+
14
+   By default, backups will be performed on the first node in your Galera cluster
15
+   or on the MariaDB node itself if you just have the one. Backup files are saved
16
+   to a dedicated Docker volume - ``mariadb_backup`` - and it's the contents of
17
+   this that you should target for transferring backups elsewhere.
18
+
19
+Enabling Backup Functionality
20
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
21
+
22
+For backups to work, some reconfiguration of MariaDB is required - this is to
23
+enable appropriate permissions for the backup client, and also to create an
24
+additional database in order to store backup information.
25
+
26
+Firstly, enable backups via ``globals.yml``:
27
+
28
+.. code-block:: console
29
+
30
+   enable_xtrabackup: "yes"
31
+
32
+Then, kick off a reconfiguration of MariaDB:
33
+
34
+``kolla-ansible -i INVENTORY reconfigure -t mariadb``
35
+
36
+Once that has run successfully, you should then be able to take full and
37
+incremental backups as described below.
38
+
39
+Backup Procedure
40
+~~~~~~~~~~~~~~~~
41
+
42
+To perform a full backup, run the following command:
43
+
44
+``kolla-ansible -i INVENTORY mariadb_backup``
45
+
46
+Or to perform an incremental backup:
47
+
48
+``kolla-ansible -i INVENTORY mariadb_backup --incremental``
49
+
50
+Kolla doesn't currently manage the scheduling of these backups, so you'll
51
+need to configure an appropriate scheduler (i.e cron) to run these commands
52
+on your behalf should you require regular snapshots of your data. A suggested
53
+schedule would be:
54
+
55
+* Daily full, retained for two weeks
56
+* Hourly incremental, retained for one day
57
+
58
+Backups are performed on your behalf on the designated database node using
59
+permissions defined during the configuration step; no password is required to
60
+invoke these commands.
61
+
62
+Furthermore, backup actions can be triggered from a node with a minimal
63
+installation of Kolla-Ansible, specifically one which doesn't require a copy of
64
+``passwords.yml``.  This is of note if you're looking to implement automated
65
+backups scheduled via a cron job.
66
+
67
+Restoring backups
68
+~~~~~~~~~~~~~~~~~
69
+
70
+Owing to the way in which XtraBackup performs hot backups, there are some
71
+steps that must be performed in order to prepare your data before it can be
72
+copied into place for use by MariaDB. This process is currently manual, but
73
+the Kolla XtraBackup image includes the tooling necessary to successfully
74
+prepare backups. Two examples are given below.
75
+
76
+Full
77
+----
78
+
79
+For a full backup, start a new container using the XtraBackup image with the
80
+following options on the master database node:
81
+
82
+.. code-block:: console
83
+
84
+   docker run -it --volumes-from mariadb --name dbrestore \
85
+      -v mariadb_backup:/backup kolla/centos-binary-xtrabackup:rocky \
86
+      /bin/bash
87
+   cd /backup
88
+   mkdir -p /restore/full
89
+   cat mysqlbackup-04-10-2018.xbc.xbs | xbstream -x -C /restore/full/
90
+   innobackupex --decompress /restore/full
91
+   find /restore -name *.qp -exec rm {} \;
92
+   innobackupex --apply-log /restore/full
93
+
94
+Then stop the MariaDB instance, delete the old data files (or move
95
+them elsewhere), and copy the backup into place:
96
+
97
+.. code-block:: console
98
+
99
+   docker stop mariadb
100
+   rm -rf /var/lib/mysql/* /var/lib/mysql/.*
101
+   innobackupex --copy-back /restore/full
102
+
103
+Then you can restart MariaDB with the restored data in place:
104
+
105
+.. code-block:: console
106
+
107
+   docker start mariadb
108
+   docker logs mariadb
109
+   81004 15:48:27 mysqld_safe WSREP: Running position recovery with --log_error='/var/lib/mysql//wsrep_recovery.BDTAm8' --pid-file='/var/lib/mysql//scratch-recover.pid'
110
+   181004 15:48:30 mysqld_safe WSREP: Recovered position 9388319e-c7bd-11e8-b2ce-6e9ec70d9926:58
111
+
112
+Incremental
113
+-----------
114
+
115
+This starts off similar to the full backup restore procedure above, but we
116
+must apply the logs from the incremental backups first of all before doing
117
+the final preparation required prior to restore. In the example below, I have
118
+a full backup - ``mysqlbackup-06-11-2018-1541505206.qp.xbc.xbs``, and an
119
+incremental backup,
120
+``incremental-11-mysqlbackup-06-11-2018-1541505223.qp.xbc.xbs``.
121
+
122
+.. code-block:: console
123
+
124
+   docker run -it --volumes-from mariadb --name dbrestore \
125
+      -v mariadb_backup:/backup kolla/centos-binary-xtrabackup:rocky \
126
+      /bin/bash
127
+   cd /backup
128
+   mkdir -p /restore/full
129
+   mkdir -p /restore/inc/11
130
+   cat mysqlbackup-06-11-2018-1541505206.qp.xbc.xbs | xbstream -x -C /restore/full/
131
+   cat incremental-11-mysqlbackup-06-11-2018-1541505223.qp.xbc.xbs | xbstream -x -C /restore/inc/11
132
+   innobackupex --decompress /restore/full
133
+   innobackupex --decompress /restore/inc/11
134
+   find /restore -name *.qp -exec rm {} \;
135
+   innobackupex --apply-log --redo-only /restore/full
136
+   innobackupex --apply-log --redo-only --incremental-dir=/restore/inc/11 /restore/full
137
+   innobackupex --apply-log /restore/full
138
+
139
+At this point the backup is prepared and ready to be copied back into place,
140
+as per the previous example.

+ 1
- 0
etc/kolla/globals.yml View File

@@ -284,6 +284,7 @@ kolla_internal_vip_address: "10.10.10.254"
284 284
 #enable_vitrage: "no"
285 285
 #enable_vmtp: "no"
286 286
 #enable_watcher: "no"
287
+#enable_xtrabackup: "no"
287 288
 #enable_zookeeper: "no"
288 289
 #enable_zun: "no"
289 290
 

+ 2
- 0
etc/kolla/passwords.yml View File

@@ -15,6 +15,8 @@ cinder_rbd_secret_uuid:
15 15
 # Database options
16 16
 ####################
17 17
 database_password:
18
+# Password for the dedicated backup user account
19
+mariadb_backup_database_password:
18 20
 
19 21
 ####################
20 22
 # Docker options

+ 10
- 0
releasenotes/notes/mariadb-xtrabackup-d4f48464dd6baaea.yaml View File

@@ -0,0 +1,10 @@
1
+---
2
+features:
3
+  - |
4
+    Supports taking a backup of all MariaDB-hosted databases using Percona XtraBackup.
5
+security:
6
+  - |
7
+    When the MariaDB backup option is enabled, it will create a new database
8
+    which is used to keep track of backup-related metadata, along with a new
9
+    backup user with a specific set of permissions limited to backup-related
10
+    actions only.

+ 21
- 1
tools/kolla-ansible View File

@@ -54,6 +54,9 @@ Commands:
54 54
     prechecks           Do pre-deployment checks for hosts
55 55
     check               Do post-deployment smoke tests
56 56
     mariadb_recovery    Recover a completely stopped mariadb cluster
57
+    mariadb_backup      Take a backup of MariaDB databases
58
+                            --full (default)
59
+                            --incremental
57 60
     bootstrap-servers   Bootstrap servers with kolla deploy dependencies
58 61
     destroy             Destroy Kolla containers, volumes and host configuration
59 62
                             --include-images to also destroy Kolla images
@@ -91,6 +94,7 @@ cat <<EOF
91 94
 prechecks
92 95
 check
93 96
 mariadb_recovery
97
+mariadb_backup
94 98
 bootstrap-servers
95 99
 destroy
96 100
 deploy
@@ -107,7 +111,7 @@ EOF
107 111
 }
108 112
 
109 113
 SHORT_OPTS="hi:p:t:k:e:v"
110
-LONG_OPTS="help,inventory:,playbook:,skip-tags:,tags:,key:,extra:,verbose,configdir:,passwords:,limit:,forks:,vault-id:,ask-vault-pass,vault-password-file:,yes-i-really-really-mean-it,include-images,include-dev"
114
+LONG_OPTS="help,inventory:,playbook:,skip-tags:,tags:,key:,extra:,verbose,configdir:,passwords:,limit:,forks:,vault-id:,ask-vault-pass,vault-password-file:,yes-i-really-really-mean-it,include-images,include-dev:,full,incremental"
111 115
 
112 116
 RAW_ARGS="$*"
113 117
 ARGS=$(getopt -o "${SHORT_OPTS}" -l "${LONG_OPTS}" --name "$0" -- "$@") || { usage >&2; exit 2; }
@@ -125,6 +129,7 @@ PASSWORDS_FILE="${CONFIG_DIR}/passwords.yml"
125 129
 DANGER_CONFIRM=
126 130
 INCLUDE_IMAGES=
127 131
 INCLUDE_DEV=
132
+BACKUP_TYPE="full"
128 133
 # Serial is not recommended and disabled by default. Users can enable it by
129 134
 # configuring ANSIBLE_SERIAL variable.
130 135
 ANSIBLE_SERIAL=${ANSIBLE_SERIAL:-0}
@@ -221,6 +226,16 @@ while [ "$#" -gt 0 ]; do
221 226
             shift 2
222 227
             ;;
223 228
 
229
+    (--full)
230
+            BACKUP_TYPE="full"
231
+            shift 1
232
+            ;;
233
+
234
+    (--incremental)
235
+            BACKUP_TYPE="incremental"
236
+            shift 1
237
+            ;;
238
+
224 239
     (--help|-h)
225 240
             usage
226 241
             shift
@@ -254,6 +269,11 @@ case "$1" in
254 269
         EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=deploy -e common_run=true"
255 270
         PLAYBOOK="${BASEDIR}/ansible/mariadb_recovery.yml"
256 271
         ;;
272
+(mariadb_backup)
273
+        ACTION="Backup MariaDB databases"
274
+        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=backup -e mariadb_backup_type=${BACKUP_TYPE} -e common_run=true"
275
+        PLAYBOOK="${BASEDIR}/ansible/mariadb_backup.yml"
276
+        ;;
257 277
 (destroy)
258 278
         ACTION="Destroy Kolla containers, volumes and host configuration"
259 279
         PLAYBOOK="${BASEDIR}/ansible/destroy.yml"

Loading…
Cancel
Save