openstack/trove
Code Issues Proposed changes

Merge "Separate backup docker image for each database version"

Browse Source
This commit is contained in:
Zuul 2023-10-08 04:43:35 +00:00 committed by Gerrit Code Review
commit 3bbeeb87e0
7 changed files with 191 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
doc/source/admin/run_trove_in_production.rst
View File

@ -301,7 +301,50 @@ Some config options specifically for trove guest agent:
[mysql]
docker_image = your-registry/your-repo/mysql
backup_docker_image = your-registry/your-repo/db-backup-mysql:1.1.0
backup_docker_image = your-registry/your-repo/db-backup-mysql
Make Trove work with multiple versions for each datastore
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
When Trove do a backup/restore actions, The Trove guest agent pulls container
images with tags matching the datastore version of the database instance running.
To Ensure the trove guest agent can run backup/restore properly, you need to ensure
the images with the proper tags already exists in the registry.
Such as:
If your datastore manager is 'mariadb' and its name is 'MariaDB',
and it has 2 datastore versions:
.. code-block:: bash
openstack datastore version list MariaDB
+--------------------------------------+------+---------+
| ID | Name | Version |
+--------------------------------------+------+---------+
| 550aebf7-df97-49f1-bf24-7cd7b69fa365 | 10.3 | 10.3 |
| ee988cc3-bb30-4aaf-9837-e90a34f60d37 | 10.4 | 10.4 |
+--------------------------------------+------+---------+
Configure the ``backup_docker_image`` options like following:
.. path /etc/trove/trove-guestagent.conf
.. code-block:: ini
[mariadb]
# Database docker image. (string value)
docker_image = your-registry/your-repo/db-mariadb
# The docker image used for backup and restore. (string value)
backup_docker_image = your-registry/your-repo/db-backup-mariadb
.. note::
Do not configure the image tag for the image. because if the image doesn't
contain the tag, Trove will use the datastore version as the tag.
Administrators need to ensure that the Docker backup image has 2 tags (10.3 & 10.4)
in docker registry. For example:
your-registry/your-repo/db-backup-mariadb:10.3 & your-registry/your-repo/db-backup-mariadb:10.4
Finally, when trove-guestagent does backup/restore, it will pull this image with the tag equals datastore version.
Initialize Trove Database
~~~~~~~~~~~~~~~~~~~~~~~~~

6
releasenotes/notes/separate-backup-docker-image-884165.yaml Normal file
View File

@ -0,0 +1,6 @@
---
features:
- |
Add support for multiple datastore versions for each datastore.
more details:
`Story 2010770 <https://storyboard.openstack.org/#!/story/2010770>`__

24
trove/common/cfg.py
View File

@ -660,10 +660,12 @@ mysql_opts = [
help='Database docker image.'
),
cfg.StrOpt(
'backup_docker_image', default='openstacktrove/db-backup-mysql:1.1.0',
help='The docker image used for backup and restore. For mysql, '
'the minor version is added to the image name as a suffix before '
'creating container, e.g. openstacktrove/db-backup-mysql5.7:1.0.0'
'backup_docker_image',
sample_default='your-registry/your-repo/db-backup-mysql',
help='The docker image used for backup and restore. Trove will uses'
'datastore version as the image tag, for example: '
'your-registry/your-repo/db-backup-mysql:5.7 is used for mysql'
'datastore with version 5.7'
),
]
@ -1112,8 +1114,11 @@ postgresql_opts = [
),
cfg.StrOpt(
'backup_docker_image',
default='openstacktrove/db-backup-postgresql:1.1.2',
help='The docker image used for backup and restore.'
sample_default='your-registry/your-repo/db-backup-postgresql',
help='The docker image used for backup and restore. Trove will uses'
'datastore version as the image tag, for example: '
'your-registry/your-repo/db-backup-postgresql:12 is used for'
'postgresql datastore with version 12'
),
cfg.BoolOpt('icmp', default=False,
help='Whether to permit ICMP.',
@ -1437,8 +1442,11 @@ mariadb_opts = [
),
cfg.StrOpt(
'backup_docker_image',
default='openstacktrove/db-backup-mariadb:1.1.0',
help='The docker image used for backup and restore.'
sample_default='your-registry/your-repo/db-backup-mariadb',
help='The docker image used for backup and restore. Trove will uses'
'datastore version as the image tag, for example: '
'your-registry/your-repo/db-backup-mariadb:10.3 is used for '
'postgresql datastore with version 10.3'
),
]

17
trove/guestagent/datastore/mysql/service.py
View File

@ -65,15 +65,18 @@ class MySqlApp(service.BaseMySqlApp):
For example, this method converts openstacktrove/db-backup-mysql:1.0.0
to openstacktrove/db-backup-mysql5.7:1.0.0
**deprecated**: this function is for backward compatibility.
"""
image = cfg.get_configuration_property('backup_docker_image')
name, tag = image.rsplit(':', 1)
# Get minor version
cur_ver = semantic_version.Version.coerce(CONF.datastore_version)
minor_ver = f"{cur_ver.major}.{cur_ver.minor}"
return f"{name}{minor_ver}:{tag}"
if not self._image_has_tag(image):
return super().get_backup_image()
else:
name, tag = image.rsplit(':', 1)
# Get minor version
cur_ver = semantic_version.Version.coerce(CONF.datastore_version)
minor_ver = f"{cur_ver.major}.{cur_ver.minor}"
return f"{name}{minor_ver}:{tag}"
def get_backup_strategy(self):
"""Get backup strategy.

4
trove/guestagent/datastore/postgres/service.py
View File

@ -263,8 +263,8 @@ class PgSqlApp(service.BaseDbApp):
def restore_backup(self, context, backup_info, restore_location):
backup_id = backup_info['id']
storage_driver = CONF.storage_strategy
backup_driver = cfg.get_configuration_property('backup_strategy')
image = cfg.get_configuration_property('backup_docker_image')
backup_driver = self.get_backup_strategy()
image = self.get_backup_image()
name = 'db_restore'
volumes = {
'/var/lib/postgresql/data': {

26
trove/guestagent/datastore/service.py
View File

@ -415,8 +415,32 @@ class BaseDbApp(object):
self.reset_configuration(config_contents)
self.start_db(update_db=True, ds_version=ds_version)
@staticmethod
def _image_has_tag(image):
"""
Whether docker_image being config with tag
"example.domain:5000/repo/image_name:tag",
"example.domain:5000/repo/image-name:tag",
"example.domain:5000/repo/image-name:tag_tag",
"example.domain:5000/repo/image_name:tag-tag",
"example.domain:5000/repo/image-name",
"example.domain:5000/repo/image_name",
"example.domain:5000:5000/repo/image-name",
"example.domain/repo/image-name",
"example.domain/repo/image-name:tag"
Returns:
- True if match
"""
return image.split('/')[-1].find(':') > 0
def get_backup_image(self):
return cfg.get_configuration_property('backup_docker_image')
image = cfg.get_configuration_property('backup_docker_image')
if not self._image_has_tag(image):
ds_version = CONF.datastore_version
image = (f'{image}:latest' if not ds_version else
f'{image}:{ds_version}')
return image
def get_backup_strategy(self):
return cfg.get_configuration_property('backup_strategy')

88
trove/tests/unittests/guestagent/datastore/test_service.py Normal file
View File

@ -0,0 +1,88 @@
# Copyright 2023 BizflyCloud
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest import mock
from trove.common import cfg
from trove.guestagent.datastore.mariadb import service
from trove.guestagent.datastore.mysql import service as mysql_service
from trove.guestagent.datastore import service as base_service
from trove.tests.unittests import trove_testtools
CONF = cfg.CONF
class TestService(trove_testtools.TestCase):
def setUp(self):
super(TestService, self).setUp()
_docker_client = mock.MagicMock()
status = mock.MagicMock()
self.app = service.MariaDBApp(_docker_client, status)
self.mysql_app = mysql_service.MySqlApp(_docker_client, status)
def test_get_backup_image_with_tag(self):
self.patch_datastore_manager('mariadb')
CONF.set_override('backup_docker_image',
'example.domain/repo/mariadb:tag', 'mariadb')
image = self.app.get_backup_image()
self.assertEqual(CONF.mariadb.backup_docker_image, image)
def test_get_backup_image_without_tag(self):
self.patch_datastore_manager('mariadb')
CONF.set_override('backup_docker_image',
'example.domain/repo/mariadb', 'mariadb')
self.patch_conf_property('datastore_version', '10.4')
image = self.app.get_backup_image()
_img = f'{CONF.mariadb.backup_docker_image}:{CONF.datastore_version}'
self.assertEqual(_img, image)
def test_mysql_backup_image_with_tag(self):
self.patch_datastore_manager('mysql')
CONF.set_override('backup_docker_image',
'example.domain/repo/mysql:1.1.0', 'mysql')
self.patch_conf_property('datastore_version', '5.7')
image = self.mysql_app.get_backup_image()
self.assertEqual(image, "example.domain/repo/mysql5.7:1.1.0")
def test_mysql_backup_image_without_tag(self):
self.patch_datastore_manager('mysql')
CONF.set_override('backup_docker_image',
'example.domain/repo/mysql', 'mysql')
self.patch_conf_property('datastore_version', '5.7')
image = self.mysql_app.get_backup_image()
self.assertEqual(image, "example.domain/repo/mysql:5.7")
def test_image_has_tag(self):
fake_values = [
"example.domain:5000/repo/image_name:tag",
"example.domain:5000/repo/image-name:tag_tag",
"example.domain:5000/repo/image_name:tag-tag",
"example.domain:5000/repo/image-name",
"example.domain:5000/repo/image_name",
"example.domain/repo/image-name",
"example.domain/repo/image-name:tag"]
self.assertTrue(
base_service.BaseDbApp._image_has_tag(fake_values[0]))
self.assertTrue(
base_service.BaseDbApp._image_has_tag(fake_values[1]))
self.assertTrue(
base_service.BaseDbApp._image_has_tag(fake_values[2]))
self.assertFalse(
base_service.BaseDbApp._image_has_tag(fake_values[3]))
self.assertFalse(
base_service.BaseDbApp._image_has_tag(fake_values[4]))
self.assertFalse(
base_service.BaseDbApp._image_has_tag(fake_values[5]))
self.assertTrue(
base_service.BaseDbApp._image_has_tag(fake_values[6]))