Add commands for database backup and recovery

Uses the support [1] for database backups added to Kolla Ansible in the
Stein release.

Adds the following new commands:

kayobe overcloud database backup [--incremental]
kayobe overcloud database recover [--force-recovery-host <host>]

The recovery command is for recovering a clustered database that has
lost quorum.

[1] https://docs.openstack.org/kolla-ansible/latest/admin/mariadb-backup-and-restore.html

Change-Id: Ie16354cd01ea7dd83cd3d4058dd8451b8387600b
Story: 2005015
Task: 29493
This commit is contained in:
Mark Goddard 2019-03-22 16:30:42 +00:00
parent 4495910428
commit e96179921f
14 changed files with 193 additions and 0 deletions

View File

@ -369,6 +369,7 @@ kolla_enable_skydive: "no"
kolla_enable_storm: "{{ 'yes' if kolla_enable_monasca | bool else 'no' }}"
kolla_enable_swift: "no"
kolla_enable_telegraf: "no"
kolla_enable_xtrabackup: "no"
kolla_enable_zookeeper: "{{ 'yes' if kolla_enable_kafka | bool or kolla_enable_storm | bool else 'no' }}"
###############################################################################

View File

@ -119,6 +119,7 @@
- { name: nova, file: nova.conf }
- { name: octavia, file: octavia.conf }
- { name: sahara, file: sahara.conf }
- { name: xtrabackup, file: backup.my.cnf }
- { name: zookeeper, file: zookeeper.cfg }
- name: Initialise a fact containing extra configuration
@ -223,5 +224,6 @@
kolla_extra_nova: "{{ kolla_extra_config.nova | default }}"
kolla_extra_octavia: "{{ kolla_extra_config.octavia | default }}"
kolla_extra_sahara: "{{ kolla_extra_config.sahara | default }}"
kolla_extra_xtrabackup: "{{ kolla_extra_config.xtrabackup | default }}"
kolla_extra_zookeeper: "{{ kolla_extra_config.zookeeper | default }}"
kolla_extra_config_path: "{{ kayobe_config_path }}/kolla/config"

View File

@ -152,5 +152,6 @@ kolla_feature_flags:
- vitrage
- vmtp
- watcher
- xtrabackup
- zookeeper
- zun

View File

@ -404,6 +404,15 @@ kolla_enable_storm:
# Whether to enable swift.
kolla_enable_swift:
###############################################################################
# Xtrabackup configuration.
# Whether to enable Xtrabackup.
kolla_enable_xtrabackup:
# Free form extra configuration to append to backup.my.cnf.
kolla_extra_xtrabackup:
###############################################################################
# Zookeeper configuration.

View File

@ -96,6 +96,10 @@ provisioner:
foo=bar
kolla_enable_swift: true
kolla_enable_storm: true
kolla_enable_xtrabackup: true
kolla_extra_xtrabackup: |
[extra-backup.my.cnf]
foo=bar
kolla_enable_zookeeper: true
kolla_extra_zookeeper: |
[extra-zookeeper.cfg]

View File

@ -78,6 +78,7 @@ def test_service_config_directory(host, path):
'nova.conf',
'octavia.conf',
'sahara.conf',
'backup.my.cnf',
'zookeeper.cfg'])
def test_service_ini_file(host, path):
# TODO(mgoddard): Check more of config file contents.

View File

@ -34,6 +34,7 @@
- { src: pxelinux.default.j2, dest: ironic/pxelinux.default, enabled: "{{ kolla_enable_ironic }}" }
- { src: inspector.ipxe.j2, dest: ironic/inspector.ipxe, enabled: "{{ kolla_enable_ironic_ipxe }}" }
- { src: sahara.conf.j2, dest: sahara.conf, enabled: "{{ kolla_enable_sahara }}" }
- { src: backup.my.cnf.j2, dest: backup.my.cnf, enabled: "{{ kolla_enable_xtrabackup }}" }
- { src: zookeeper.cfg.j2, dest: zookeeper.cfg, enabled: "{{ kolla_enable_zookeeper }}" }
when: item.enabled | bool

View File

@ -0,0 +1,9 @@
# {{ ansible_managed }}
{% if kolla_extra_xtrabackup %}
#######################
# Extra configuration
#######################
{{ kolla_extra_xtrabackup }}
{% endif %}

View File

@ -166,6 +166,11 @@ kolla_openstack_custom_config:
- container.ring.gz
- object.builder
- object.ring.gz
# Xtrabackup.
- src: "{{ kolla_extra_config_path }}/xtrabackup"
dest: "{{ kolla_node_custom_config_path }}/xtrabackup"
patterns: "*"
enabled: "{{ kolla_enable_xtrabackup }}"
# Zookeeper.
- src: "{{ kolla_extra_config_path }}/zookeeper"
dest: "{{ kolla_node_custom_config_path }}/zookeeper"

View File

@ -35,6 +35,8 @@ For example::
To execute the command with root privileges, add the ``--become`` argument.
Adding the ``--verbose`` argument allows the output of the command to be seen.
.. _overcloud-administration-reconfigure:
Reconfiguring Containerised Services
====================================
@ -129,3 +131,35 @@ The configuration will be generated remotely on the overcloud hosts in the
specified directory, with one subdirectory per container. This command may be
followed by ``kayobe ovecloud service configuration save`` to gather the
generated configuration to the Ansible control host.
Performing Database Backups
===========================
Database backups can be performed using the underlying support in Kolla
Ansible.
In order to enable backups, enable Xtrabackup in
``${KAYOBE_CONFIG_PATH}/kolla.yml``:
.. code-block:: console
kolla_enable_xtrabackup: true
To apply this change, use the :ref:`kayobe overcloud service reconfigure
<overcloud-administration-reconfigure>` command.
To perform a full backup, run the following command:
.. code-block:: console
kayobe overcloud database backup
Or to perform an incremental backup, run the following command:
.. code-block:: console
kayobe overcloud database backup --incremental
Further information on backing up and restoring the database is available in
the `Kolla Ansible documentation
<https://docs.openstack.org/kolla-ansible/latest/admin/mariadb-backup-and-restore.html>`__.

View File

@ -970,6 +970,49 @@ class OvercloudHostUpgrade(KayobeAnsibleMixin, VaultMixin, Command):
self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud")
class OvercloudDatabaseBackup(KollaAnsibleMixin, VaultMixin, Command):
"""Backup the overcloud database."""
def get_parser(self, prog_name):
parser = super(OvercloudDatabaseBackup, self).get_parser(prog_name)
group = parser.add_argument_group("Overcloud Database Backup")
group.add_argument("--incremental", action='store_true',
help="Whether to perform an incremental database "
"backup. Default is false.")
return parser
def take_action(self, parsed_args):
self.app.LOG.debug("Performing overcloud database backup")
extra_args = []
if parsed_args.incremental:
extra_args.append('--incremental')
self.run_kolla_ansible_overcloud(parsed_args, "mariadb_backup",
extra_args=extra_args)
class OvercloudDatabaseRecover(KollaAnsibleMixin, VaultMixin, Command):
"""Recover the overcloud database."""
def get_parser(self, prog_name):
parser = super(OvercloudDatabaseRecover, self).get_parser(prog_name)
group = parser.add_argument_group("Overcloud Database Recovery")
group.add_argument("--force-recovery-host",
help="Name of a host to use to perform the "
"recovery. By default kolla-ansible will "
"automatically determine which host to use, "
"and this option should not be used.")
return parser
def take_action(self, parsed_args):
self.app.LOG.debug("Performing overcloud database recovery")
extra_vars = {}
if parsed_args.force_recovery_host:
extra_vars['mariadb_recover_inventory_name'] = (
parsed_args.force_recovery_host)
self.run_kolla_ansible_overcloud(parsed_args, "mariadb_recovery",
extra_vars=extra_vars)
class OvercloudServiceConfigurationGenerate(KayobeAnsibleMixin,
KollaAnsibleMixin, VaultMixin,
Command):

View File

@ -1254,6 +1254,76 @@ class TestCase(unittest.TestCase):
]
self.assertEqual(expected_calls, mock_run.call_args_list)
@mock.patch.object(commands.KollaAnsibleMixin,
"run_kolla_ansible_overcloud")
def test_overcloud_database_backup(self, mock_run):
command = commands.OvercloudDatabaseBackup(TestApp(), [])
parser = command.get_parser("test")
parsed_args = parser.parse_args([])
result = command.run(parsed_args)
self.assertEqual(0, result)
expected_calls = [
mock.call(
mock.ANY,
"mariadb_backup",
extra_args=[]
),
]
self.assertEqual(expected_calls, mock_run.call_args_list)
@mock.patch.object(commands.KollaAnsibleMixin,
"run_kolla_ansible_overcloud")
def test_overcloud_database_backup_incremental(self, mock_run):
command = commands.OvercloudDatabaseBackup(TestApp(), [])
parser = command.get_parser("test")
parsed_args = parser.parse_args(["--incremental"])
result = command.run(parsed_args)
self.assertEqual(0, result)
expected_calls = [
mock.call(
mock.ANY,
"mariadb_backup",
extra_args=["--incremental"]
),
]
self.assertEqual(expected_calls, mock_run.call_args_list)
@mock.patch.object(commands.KollaAnsibleMixin,
"run_kolla_ansible_overcloud")
def test_overcloud_database_recover(self, mock_run):
command = commands.OvercloudDatabaseRecover(TestApp(), [])
parser = command.get_parser("test")
parsed_args = parser.parse_args([])
result = command.run(parsed_args)
self.assertEqual(0, result)
expected_calls = [
mock.call(
mock.ANY,
"mariadb_recovery",
extra_vars={}
),
]
self.assertEqual(expected_calls, mock_run.call_args_list)
@mock.patch.object(commands.KollaAnsibleMixin,
"run_kolla_ansible_overcloud")
def test_overcloud_database_recover_force_host(self, mock_run):
command = commands.OvercloudDatabaseRecover(TestApp(), [])
parser = command.get_parser("test")
parsed_args = parser.parse_args(["--force-recovery-host", "foo"])
result = command.run(parsed_args)
self.assertEqual(0, result)
expected_calls = [
mock.call(
mock.ANY,
"mariadb_recovery",
extra_vars={
"mariadb_recover_inventory_name": "foo"
}
),
]
self.assertEqual(expected_calls, mock_run.call_args_list)
@mock.patch.object(commands.KayobeAnsibleMixin,
"run_kayobe_playbooks")
def test_overcloud_service_configuration_save(self, mock_run):

View File

@ -0,0 +1,11 @@
---
features:
- |
Adds commands to make use of the database backup and recovery features in
Kolla Ansible.
``kayobe overcloud database backup [--incremental]`` can be used to take a
full or incremental backup of the database using Xtrabackup.
``kayobe overcloud database recover [--force-recovery-host <host>]`` can be
used to recover a database cluster that has lost Quorum.

View File

@ -54,6 +54,8 @@ kayobe.cli=
overcloud_bios_raid_configure = kayobe.cli.commands:OvercloudBIOSRAIDConfigure
overcloud_container_image_build = kayobe.cli.commands:OvercloudContainerImageBuild
overcloud_container_image_pull = kayobe.cli.commands:OvercloudContainerImagePull
overcloud_database_backup = kayobe.cli.commands:OvercloudDatabaseBackup
overcloud_database_recover = kayobe.cli.commands:OvercloudDatabaseRecover
overcloud_deployment_image_build = kayobe.cli.commands:OvercloudDeploymentImageBuild
overcloud_deprovision = kayobe.cli.commands:OvercloudDeprovision
overcloud_hardware_inspect = kayobe.cli.commands:OvercloudHardwareInspect