diff --git a/ansible/group_vars/all/kolla b/ansible/group_vars/all/kolla index 0970c0f35..58fce931e 100644 --- a/ansible/group_vars/all/kolla +++ b/ansible/group_vars/all/kolla @@ -369,6 +369,7 @@ kolla_enable_skydive: "no" kolla_enable_storm: "{{ 'yes' if kolla_enable_monasca | bool else 'no' }}" kolla_enable_swift: "no" kolla_enable_telegraf: "no" +kolla_enable_xtrabackup: "no" kolla_enable_zookeeper: "{{ 'yes' if kolla_enable_kafka | bool or kolla_enable_storm | bool else 'no' }}" ############################################################################### diff --git a/ansible/kolla-openstack.yml b/ansible/kolla-openstack.yml index e093406f5..f6c5bed1c 100644 --- a/ansible/kolla-openstack.yml +++ b/ansible/kolla-openstack.yml @@ -119,6 +119,7 @@ - { name: nova, file: nova.conf } - { name: octavia, file: octavia.conf } - { name: sahara, file: sahara.conf } + - { name: xtrabackup, file: backup.my.cnf } - { name: zookeeper, file: zookeeper.cfg } - name: Initialise a fact containing extra configuration @@ -223,5 +224,6 @@ kolla_extra_nova: "{{ kolla_extra_config.nova | default }}" kolla_extra_octavia: "{{ kolla_extra_config.octavia | default }}" kolla_extra_sahara: "{{ kolla_extra_config.sahara | default }}" + kolla_extra_xtrabackup: "{{ kolla_extra_config.xtrabackup | default }}" kolla_extra_zookeeper: "{{ kolla_extra_config.zookeeper | default }}" kolla_extra_config_path: "{{ kayobe_config_path }}/kolla/config" diff --git a/ansible/roles/kolla-ansible/vars/main.yml b/ansible/roles/kolla-ansible/vars/main.yml index 378e464a4..3aedbdf34 100644 --- a/ansible/roles/kolla-ansible/vars/main.yml +++ b/ansible/roles/kolla-ansible/vars/main.yml @@ -152,5 +152,6 @@ kolla_feature_flags: - vitrage - vmtp - watcher + - xtrabackup - zookeeper - zun diff --git a/ansible/roles/kolla-openstack/defaults/main.yml b/ansible/roles/kolla-openstack/defaults/main.yml index 368be6065..9e76d0828 100644 --- a/ansible/roles/kolla-openstack/defaults/main.yml +++ b/ansible/roles/kolla-openstack/defaults/main.yml @@ -404,6 +404,15 @@ kolla_enable_storm: # Whether to enable swift. kolla_enable_swift: +############################################################################### +# Xtrabackup configuration. + +# Whether to enable Xtrabackup. +kolla_enable_xtrabackup: + +# Free form extra configuration to append to backup.my.cnf. +kolla_extra_xtrabackup: + ############################################################################### # Zookeeper configuration. diff --git a/ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml b/ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml index 547745b98..58669cd54 100644 --- a/ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml +++ b/ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml @@ -96,6 +96,10 @@ provisioner: foo=bar kolla_enable_swift: true kolla_enable_storm: true + kolla_enable_xtrabackup: true + kolla_extra_xtrabackup: | + [extra-backup.my.cnf] + foo=bar kolla_enable_zookeeper: true kolla_extra_zookeeper: | [extra-zookeeper.cfg] diff --git a/ansible/roles/kolla-openstack/molecule/enable-everything/tests/test_default.py b/ansible/roles/kolla-openstack/molecule/enable-everything/tests/test_default.py index 2766f2525..2e308e72f 100644 --- a/ansible/roles/kolla-openstack/molecule/enable-everything/tests/test_default.py +++ b/ansible/roles/kolla-openstack/molecule/enable-everything/tests/test_default.py @@ -78,6 +78,7 @@ def test_service_config_directory(host, path): 'nova.conf', 'octavia.conf', 'sahara.conf', + 'backup.my.cnf', 'zookeeper.cfg']) def test_service_ini_file(host, path): # TODO(mgoddard): Check more of config file contents. diff --git a/ansible/roles/kolla-openstack/tasks/config.yml b/ansible/roles/kolla-openstack/tasks/config.yml index 324543d4a..815ea0ffc 100644 --- a/ansible/roles/kolla-openstack/tasks/config.yml +++ b/ansible/roles/kolla-openstack/tasks/config.yml @@ -34,6 +34,7 @@ - { src: pxelinux.default.j2, dest: ironic/pxelinux.default, enabled: "{{ kolla_enable_ironic }}" } - { src: inspector.ipxe.j2, dest: ironic/inspector.ipxe, enabled: "{{ kolla_enable_ironic_ipxe }}" } - { src: sahara.conf.j2, dest: sahara.conf, enabled: "{{ kolla_enable_sahara }}" } + - { src: backup.my.cnf.j2, dest: backup.my.cnf, enabled: "{{ kolla_enable_xtrabackup }}" } - { src: zookeeper.cfg.j2, dest: zookeeper.cfg, enabled: "{{ kolla_enable_zookeeper }}" } when: item.enabled | bool diff --git a/ansible/roles/kolla-openstack/templates/backup.my.cnf.j2 b/ansible/roles/kolla-openstack/templates/backup.my.cnf.j2 new file mode 100644 index 000000000..bb29de1f3 --- /dev/null +++ b/ansible/roles/kolla-openstack/templates/backup.my.cnf.j2 @@ -0,0 +1,9 @@ +# {{ ansible_managed }} + +{% if kolla_extra_xtrabackup %} +####################### +# Extra configuration +####################### + +{{ kolla_extra_xtrabackup }} +{% endif %} diff --git a/ansible/roles/kolla-openstack/vars/main.yml b/ansible/roles/kolla-openstack/vars/main.yml index 0e4fa7e8f..70e52d872 100644 --- a/ansible/roles/kolla-openstack/vars/main.yml +++ b/ansible/roles/kolla-openstack/vars/main.yml @@ -166,6 +166,11 @@ kolla_openstack_custom_config: - container.ring.gz - object.builder - object.ring.gz + # Xtrabackup. + - src: "{{ kolla_extra_config_path }}/xtrabackup" + dest: "{{ kolla_node_custom_config_path }}/xtrabackup" + patterns: "*" + enabled: "{{ kolla_enable_xtrabackup }}" # Zookeeper. - src: "{{ kolla_extra_config_path }}/zookeeper" dest: "{{ kolla_node_custom_config_path }}/zookeeper" diff --git a/doc/source/administration/overcloud.rst b/doc/source/administration/overcloud.rst index 2a3336ce2..8757b8d68 100644 --- a/doc/source/administration/overcloud.rst +++ b/doc/source/administration/overcloud.rst @@ -35,6 +35,8 @@ For example:: To execute the command with root privileges, add the ``--become`` argument. Adding the ``--verbose`` argument allows the output of the command to be seen. +.. _overcloud-administration-reconfigure: + Reconfiguring Containerised Services ==================================== @@ -129,3 +131,35 @@ The configuration will be generated remotely on the overcloud hosts in the specified directory, with one subdirectory per container. This command may be followed by ``kayobe ovecloud service configuration save`` to gather the generated configuration to the Ansible control host. + +Performing Database Backups +=========================== + +Database backups can be performed using the underlying support in Kolla +Ansible. + +In order to enable backups, enable Xtrabackup in +``${KAYOBE_CONFIG_PATH}/kolla.yml``: + +.. code-block:: console + + kolla_enable_xtrabackup: true + +To apply this change, use the :ref:`kayobe overcloud service reconfigure +` command. + +To perform a full backup, run the following command: + +.. code-block:: console + + kayobe overcloud database backup + +Or to perform an incremental backup, run the following command: + +.. code-block:: console + + kayobe overcloud database backup --incremental + +Further information on backing up and restoring the database is available in +the `Kolla Ansible documentation +`__. diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index c95c4544e..dbb8bcc3a 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -970,6 +970,49 @@ class OvercloudHostUpgrade(KayobeAnsibleMixin, VaultMixin, Command): self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud") +class OvercloudDatabaseBackup(KollaAnsibleMixin, VaultMixin, Command): + """Backup the overcloud database.""" + + def get_parser(self, prog_name): + parser = super(OvercloudDatabaseBackup, self).get_parser(prog_name) + group = parser.add_argument_group("Overcloud Database Backup") + group.add_argument("--incremental", action='store_true', + help="Whether to perform an incremental database " + "backup. Default is false.") + return parser + + def take_action(self, parsed_args): + self.app.LOG.debug("Performing overcloud database backup") + extra_args = [] + if parsed_args.incremental: + extra_args.append('--incremental') + self.run_kolla_ansible_overcloud(parsed_args, "mariadb_backup", + extra_args=extra_args) + + +class OvercloudDatabaseRecover(KollaAnsibleMixin, VaultMixin, Command): + """Recover the overcloud database.""" + + def get_parser(self, prog_name): + parser = super(OvercloudDatabaseRecover, self).get_parser(prog_name) + group = parser.add_argument_group("Overcloud Database Recovery") + group.add_argument("--force-recovery-host", + help="Name of a host to use to perform the " + "recovery. By default kolla-ansible will " + "automatically determine which host to use, " + "and this option should not be used.") + return parser + + def take_action(self, parsed_args): + self.app.LOG.debug("Performing overcloud database recovery") + extra_vars = {} + if parsed_args.force_recovery_host: + extra_vars['mariadb_recover_inventory_name'] = ( + parsed_args.force_recovery_host) + self.run_kolla_ansible_overcloud(parsed_args, "mariadb_recovery", + extra_vars=extra_vars) + + class OvercloudServiceConfigurationGenerate(KayobeAnsibleMixin, KollaAnsibleMixin, VaultMixin, Command): diff --git a/kayobe/tests/unit/cli/test_commands.py b/kayobe/tests/unit/cli/test_commands.py index 3261d0aad..b4788fdea 100644 --- a/kayobe/tests/unit/cli/test_commands.py +++ b/kayobe/tests/unit/cli/test_commands.py @@ -1254,6 +1254,76 @@ class TestCase(unittest.TestCase): ] self.assertEqual(expected_calls, mock_run.call_args_list) + @mock.patch.object(commands.KollaAnsibleMixin, + "run_kolla_ansible_overcloud") + def test_overcloud_database_backup(self, mock_run): + command = commands.OvercloudDatabaseBackup(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args([]) + result = command.run(parsed_args) + self.assertEqual(0, result) + expected_calls = [ + mock.call( + mock.ANY, + "mariadb_backup", + extra_args=[] + ), + ] + self.assertEqual(expected_calls, mock_run.call_args_list) + + @mock.patch.object(commands.KollaAnsibleMixin, + "run_kolla_ansible_overcloud") + def test_overcloud_database_backup_incremental(self, mock_run): + command = commands.OvercloudDatabaseBackup(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args(["--incremental"]) + result = command.run(parsed_args) + self.assertEqual(0, result) + expected_calls = [ + mock.call( + mock.ANY, + "mariadb_backup", + extra_args=["--incremental"] + ), + ] + self.assertEqual(expected_calls, mock_run.call_args_list) + + @mock.patch.object(commands.KollaAnsibleMixin, + "run_kolla_ansible_overcloud") + def test_overcloud_database_recover(self, mock_run): + command = commands.OvercloudDatabaseRecover(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args([]) + result = command.run(parsed_args) + self.assertEqual(0, result) + expected_calls = [ + mock.call( + mock.ANY, + "mariadb_recovery", + extra_vars={} + ), + ] + self.assertEqual(expected_calls, mock_run.call_args_list) + + @mock.patch.object(commands.KollaAnsibleMixin, + "run_kolla_ansible_overcloud") + def test_overcloud_database_recover_force_host(self, mock_run): + command = commands.OvercloudDatabaseRecover(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args(["--force-recovery-host", "foo"]) + result = command.run(parsed_args) + self.assertEqual(0, result) + expected_calls = [ + mock.call( + mock.ANY, + "mariadb_recovery", + extra_vars={ + "mariadb_recover_inventory_name": "foo" + } + ), + ] + self.assertEqual(expected_calls, mock_run.call_args_list) + @mock.patch.object(commands.KayobeAnsibleMixin, "run_kayobe_playbooks") def test_overcloud_service_configuration_save(self, mock_run): diff --git a/releasenotes/notes/db-backup-recovery-d4f1ced2ebd7dac4.yaml b/releasenotes/notes/db-backup-recovery-d4f1ced2ebd7dac4.yaml new file mode 100644 index 000000000..1d2ef04f9 --- /dev/null +++ b/releasenotes/notes/db-backup-recovery-d4f1ced2ebd7dac4.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + Adds commands to make use of the database backup and recovery features in + Kolla Ansible. + + ``kayobe overcloud database backup [--incremental]`` can be used to take a + full or incremental backup of the database using Xtrabackup. + + ``kayobe overcloud database recover [--force-recovery-host ]`` can be + used to recover a database cluster that has lost Quorum. diff --git a/setup.cfg b/setup.cfg index cbf3d9e25..f4aff103e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,6 +54,8 @@ kayobe.cli= overcloud_bios_raid_configure = kayobe.cli.commands:OvercloudBIOSRAIDConfigure overcloud_container_image_build = kayobe.cli.commands:OvercloudContainerImageBuild overcloud_container_image_pull = kayobe.cli.commands:OvercloudContainerImagePull + overcloud_database_backup = kayobe.cli.commands:OvercloudDatabaseBackup + overcloud_database_recover = kayobe.cli.commands:OvercloudDatabaseRecover overcloud_deployment_image_build = kayobe.cli.commands:OvercloudDeploymentImageBuild overcloud_deprovision = kayobe.cli.commands:OvercloudDeprovision overcloud_hardware_inspect = kayobe.cli.commands:OvercloudHardwareInspect