diff --git a/sysinv/sysinv/sysinv/sysinv/common/constants.py b/sysinv/sysinv/sysinv/sysinv/common/constants.py index f9359a5661..f9285d951e 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/constants.py +++ b/sysinv/sysinv/sysinv/sysinv/common/constants.py @@ -17,8 +17,6 @@ SYSINV_CONFIG_FILE_LOCAL = '/etc/sysinv/sysinv.conf' SYSINV_CONF_DEFAULT_FILE = 'sysinv.conf.default' SYSINV_CONF_DEFAULT_PATH = os.path.join(SYSINV_CONFIG_PATH, SYSINV_CONF_DEFAULT_FILE) -SYSINV_RESTORE_FLAG = os.path.join(SYSINV_CONFIG_PATH, - ".restore_in_progress") HTTPS_CONFIG_REQUIRED = os.path.join(tsc.CONFIG_PATH, '.https_config_required') ADMIN_ENDPOINT_CONFIG_REQUIRED = os.path.join(tsc.CONFIG_PATH, '.admin_endpoint_config_required') @@ -1175,6 +1173,18 @@ UPGRADE_ABORTING = 'aborting' UPGRADE_ABORT_COMPLETING = 'abort-completing' UPGRADE_ABORTING_ROLLBACK = 'aborting-reinstall' +# Restore states +RESTORE_STATE_IN_PROGRESS = 'restore-in-progress' +RESTORE_STATE_COMPLETED = 'restore-completed' + +# Restore progress constants +RESTORE_PROGRESS_ALREADY_COMPLETED = "Restore procedure already completed" +RESTORE_PROGRESS_STARTED = "Restore procedure started" +RESTORE_PROGRESS_ALREADY_IN_PROGRESS = "Restore procedure already in progress" +RESTORE_PROGRESS_NOT_IN_PROGRESS = "Restore procedure is not in progress" +RESTORE_PROGRESS_IN_PROGRESS = "Restore procedure is in progress" +RESTORE_PROGRESS_COMPLETED = "Restore procedure completed" + # LLDP LLDP_OVS_PORT_PREFIX = 'lldp' LLDP_OVS_PORT_NAME_LEN = 15 diff --git a/sysinv/sysinv/sysinv/sysinv/common/exception.py b/sysinv/sysinv/sysinv/sysinv/common/exception.py index 302721b685..82cdc776e6 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/exception.py +++ b/sysinv/sysinv/sysinv/sysinv/common/exception.py @@ -1536,6 +1536,14 @@ class KubeNotConfigured(SysinvException): "will not be available.") +class RestoreAlreadyExists(Conflict): + message = _("A Restore with UUID %(uuid)s already exists.") + + +class RestoreNotFound(NotFound): + message = _("Restore with UUID %(uuid)s not found.") + + class LifecycleSemanticCheckException(SysinvException): message = _("Semantic check hook for app failed.") diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index 98c3134c7c..6302eb33b3 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -5689,9 +5689,16 @@ class ConductorManager(service.PeriodicService): return False - @staticmethod - def _verify_restore_in_progress(): - return os.path.isfile(constants.SYSINV_RESTORE_FLAG) + def _verify_restore_in_progress(self): + """Check if restore is in progress""" + + try: + self.dbapi.restore_get_one( + filters={'state': constants.RESTORE_STATE_IN_PROGRESS}) + except exception.NotFound: + return False + else: + return True @periodic_task.periodic_task(spacing=CONF.conductor.audit_interval, run_immediately=True) @@ -12934,11 +12941,17 @@ class ConductorManager(service.PeriodicService): :param context: request context. """ - LOG.info("Preparing for restore procedure. Creating flag file.") + LOG.info("Preparing for restore procedure.") + try: + self.dbapi.restore_get_one( + filters={'state': constants.RESTORE_STATE_IN_PROGRESS}) + except exception.NotFound: + self.dbapi.restore_create( + values={'state': constants.RESTORE_STATE_IN_PROGRESS}) + else: + return constants.RESTORE_PROGRESS_ALREADY_IN_PROGRESS - cutils.touch(constants.SYSINV_RESTORE_FLAG) - - return "Restore procedure started" + return constants.RESTORE_PROGRESS_STARTED def complete_restore(self, context): """Complete the restore @@ -12968,11 +12981,18 @@ class ConductorManager(service.PeriodicService): LOG.error(e) return message - LOG.info("Complete the restore procedure. Remove flag file.") + try: + restore = self.dbapi.restore_get_one( + filters={'state': constants.RESTORE_STATE_IN_PROGRESS}) + except exception.NotFound: + return constants.RESTORE_PROGRESS_ALREADY_COMPLETED + else: + self.dbapi.restore_update(restore.uuid, + values={'state': constants.RESTORE_STATE_COMPLETED}) - cutils.delete_if_exists(constants.SYSINV_RESTORE_FLAG) + LOG.info("Complete the restore procedure.") - return "Restore procedure completed" + return constants.RESTORE_PROGRESS_COMPLETED def get_restore_state(self, context): """Get the restore state @@ -12981,9 +13001,9 @@ class ConductorManager(service.PeriodicService): """ if self._verify_restore_in_progress(): - output = "Restore procedure is in progress" + output = constants.RESTORE_PROGRESS_IN_PROGRESS else: - output = "Restore procedure is not in progress" + output = constants.RESTORE_PROGRESS_NOT_IN_PROGRESS LOG.info(output) return output diff --git a/sysinv/sysinv/sysinv/sysinv/db/api.py b/sysinv/sysinv/sysinv/sysinv/db/api.py index 8f3d7ba7ae..1422863163 100644 --- a/sysinv/sysinv/sysinv/sysinv/db/api.py +++ b/sysinv/sysinv/sysinv/sysinv/db/api.py @@ -16,7 +16,7 @@ # License for the specific language governing permissions and limitations # under the License. # -# Copyright (c) 2013-2020 Wind River Systems, Inc. +# Copyright (c) 2013-2021 Wind River Systems, Inc. # @@ -4527,3 +4527,69 @@ class Connection(object): :param upgrade_id: The id or uuid of a kube_upgrade. """ + + @abc.abstractmethod + def restore_create(self, values): + """Create a new restore entry + + :param values: A dict containing several items used to identify + and track the entry. + + { + 'uuid': uuidutils.generate_uuid(), + } + :returns: A restore record. + """ + + @abc.abstractmethod + def restore_get(self, id): + """Return a restore entry for a given id + + :param _id: The id or uuid of a restore entry + :returns: a restore entry + """ + + @abc.abstractmethod + def restore_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + """Return a list of restore entries. + + :param limit: Maximum number of restore entries to return. + :param marker: the last item of the previous page; we return the next + result set. + :param sort_key: Attribute by which results should be sorted. + :param sort_dir: direction in which results should be sorted. + (asc, desc) + """ + + @abc.abstractmethod + def restore_get_one(self, filters): + """Return exactly one restore. + + :param filters: A dict of filters to apply on the query. + The key of the entry is the column to search in. + The value of the entry is the value to search for. + Capable of simple filtering equivalent to `value in [values]`. + Eg: filters={'state': 'some-state-value'} is equivalent to + `model.MyModel.state in ['some-state-value']` + + :returns: A restore. + """ + + @abc.abstractmethod + def restore_update(self, uuid, values): + """Update properties of a restore. + + :param node: The uuid of a restore entry. + :param values: Dict of values to update. + {'state': constants.RESTORE_STATE_COMPLETED + } + :returns: A restore entry. + """ + + @abc.abstractmethod + def restore_destroy(self, id): + """Destroy a restore entry. + + :param id: The id or uuid of a restore entry. + """ diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py index 3c9692a966..e5319104f1 100644 --- a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py @@ -15,7 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. # -# Copyright (c) 2013-2020 Wind River Systems, Inc. +# Copyright (c) 2013-2021 Wind River Systems, Inc. # """SQLAlchemy storage backend.""" @@ -8706,3 +8706,79 @@ class Connection(api.Connection): else: query = query.filter_by(status=status) return query.all() + + def _restore_get(self, id): + query = model_query(models.Restore) + if utils.is_uuid_like(id): + query = query.filter_by(uuid=id) + else: + query = query.filter_by(id=id) + + try: + result = query.one() + except NoResultFound: + raise exception.RestoreNotFound(uuid=id) + + return result + + @objects.objectify(objects.restore) + def restore_create(self, values): + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + restore = models.Restore() + restore.update(values) + with _session_for_write() as session: + try: + session.add(restore) + session.flush() + except db_exc.DBDuplicateEntry: + raise exception.RestoreAlreadyExists(uuid=values['uuid']) + + return restore + + @objects.objectify(objects.restore) + def restore_get(self, id): + return self._restore_get(id) + + @objects.objectify(objects.restore) + def restore_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.Restore) + + return _paginate_query(models.Restore, limit, marker, + sort_key, sort_dir, query) + + @objects.objectify(objects.restore) + def restore_get_one(self, filters): + query = model_query(models.Restore) + + for key in filters if filters else {}: + query = query.filter(getattr(models.Restore, key).in_([filters[key]])) + + try: + return query.one() + except NoResultFound: + raise exception.NotFound() + + @objects.objectify(objects.restore) + def restore_update(self, uuid, values): + with _session_for_write() as session: + query = model_query(models.Restore, session=session) + query = query.filter_by(uuid=uuid) + + count = query.update(values, synchronize_session='fetch') + if count != 1: + raise exception.RestoreNotFound(uuid=uuid) + return query.one() + + def restore_destroy(self, id): + with _session_for_write() as session: + query = model_query(models.Restore, session=session) + query = query.filter_by(uuid=id) + + try: + query.one() + except NoResultFound: + raise exception.RestoreNotFound(uuid=id) + + query.delete() diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/112_add_backup_restore_table.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/112_add_backup_restore_table.py new file mode 100644 index 0000000000..bd3d6a6e70 --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/112_add_backup_restore_table.py @@ -0,0 +1,39 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright (c) 2021 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from sqlalchemy import DateTime, String, Integer, Text +from sqlalchemy import Column, MetaData, Table + +ENGINE = 'InnoDB' +CHARSET = 'utf8' + + +def upgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + backup_restore = Table( + 'backup_restore', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('id', Integer, primary_key=True, nullable=False), + Column('uuid', String(36), unique=True), + Column('state', String(128), nullable=False), + Column('capabilities', Text), + + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + backup_restore.create() + + +def downgrade(migrate_engine): + # Downgrade is unsupported in this release. + raise NotImplementedError('SysInv database downgrade is unsupported.') diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py index 996d5445d9..a885137199 100644 --- a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py @@ -15,7 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. # -# Copyright (c) 2013-2020 Wind River Systems, Inc. +# Copyright (c) 2013-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -1616,6 +1616,15 @@ class SoftwareUpgrade(Base): foreign_keys=[to_load]) +class Restore(Base): + __tablename__ = 'backup_restore' + + id = Column('id', Integer, primary_key=True, nullable=False) + uuid = Column('uuid', String(36), unique=True) + state = Column('state', String(128), nullable=False) + capabilities = Column(JSONEncodedDict) + + class HostUpgrade(Base): __tablename__ = 'host_upgrade' diff --git a/sysinv/sysinv/sysinv/sysinv/objects/__init__.py b/sysinv/sysinv/sysinv/sysinv/objects/__init__.py index bd7bf8523b..59da4896ee 100644 --- a/sysinv/sysinv/sysinv/sysinv/objects/__init__.py +++ b/sysinv/sysinv/sysinv/sysinv/objects/__init__.py @@ -12,7 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. # -# Copyright (c) 2013-2020 Wind River Systems, Inc. +# Copyright (c) 2013-2021 Wind River Systems, Inc. # @@ -97,6 +97,7 @@ from sysinv.objects import storage_tier from sysinv.objects import storage_ceph_external from sysinv.objects import storage_ceph_rook from sysinv.objects import host_fs +from sysinv.objects import restore def objectify(klass): @@ -203,6 +204,7 @@ device_image_label = device_image_label.DeviceImageLabel device_image_state = device_image_state.DeviceImageState device_label = device_label.DeviceLabel fpga_device = fpga_device.FPGADevice +restore = restore.Restore __all__ = ("system", "cluster", @@ -279,6 +281,7 @@ __all__ = ("system", "device_image_label", "device_label", "fpga_device", + "restore", # alias objects for RPC compatibility "ihost", "ilvg", diff --git a/sysinv/sysinv/sysinv/sysinv/objects/restore.py b/sysinv/sysinv/sysinv/sysinv/objects/restore.py new file mode 100644 index 0000000000..7838510634 --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/objects/restore.py @@ -0,0 +1,33 @@ +# Copyright (c) 2021 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# + +from sysinv.db import api as db_api +from sysinv.objects import base +from sysinv.objects import utils + + +class Restore(base.SysinvObject): + # VERSION 1.0: Initial version + VERSION = '1.0' + + dbapi = db_api.get_instance() + + fields = {'id': int, + 'uuid': utils.uuid_or_none, + 'state': utils.str_or_none, + 'capabilities': utils.dict_or_none, + } + + @base.remotable_classmethod + def get_by_uuid(cls, context, uuid): + return cls.dbapi.restore_get(uuid) + + def save_changes(self, context, updates): + self.dbapi.restore_update(self.uuid, # pylint: disable=no-member + updates) diff --git a/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_restore.py b/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_restore.py new file mode 100644 index 0000000000..cef69673df --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_restore.py @@ -0,0 +1,110 @@ +# +# Copyright (c) 2021 Wind River Systems, Inc. +# +# SPDX-License-Identilfier: Apache-2.0 +# + +""" +Tests for the restore logic +""" + +from sysinv.common import constants +from sysinv.conductor import manager +from sysinv.db import api as dbapi +from sysinv.openstack.common import context +from sysinv.tests.db import base + + +class RestoreTestCase(base.BaseHostTestCase): + + def setUp(self): + super(RestoreTestCase, self).setUp() + + # Set up objects for testing + self.service = manager.ConductorManager('test-host', 'test-topic') + self.service.dbapi = dbapi.get_instance() + self.context = context.get_admin_context() + self.valid_restore_states = [ + constants.RESTORE_PROGRESS_ALREADY_COMPLETED, + constants.RESTORE_PROGRESS_STARTED, + constants.RESTORE_PROGRESS_ALREADY_IN_PROGRESS, + constants.RESTORE_PROGRESS_NOT_IN_PROGRESS, + constants.RESTORE_PROGRESS_IN_PROGRESS, + constants.RESTORE_PROGRESS_COMPLETED] + + def tearDown(self): + super(RestoreTestCase, self).tearDown() + + def _create_controller(self, which, **kw): + return self._create_test_host( + personality=constants.CONTROLLER, + subfunction=None, + numa_nodes=1, + unit=which, + **kw) + + def test_restore_transitions(self): + # Create controller-0 + _ = self._create_controller( + which=0, + invprovision=constants.PROVISIONED, + administrative=constants.ADMIN_UNLOCKED, + operational=constants.OPERATIONAL_ENABLED, + availability=constants.AVAILABILITY_AVAILABLE) + + self.assertEqual(self.service.get_restore_state(self.context), + constants.RESTORE_PROGRESS_NOT_IN_PROGRESS) + self.assertEqual(self.service.complete_restore(self.context), + constants.RESTORE_PROGRESS_ALREADY_COMPLETED) + + self.assertEqual(self.service.start_restore(self.context), + constants.RESTORE_PROGRESS_STARTED) + self.assertEqual(self.service.get_restore_state(self.context), + constants.RESTORE_PROGRESS_IN_PROGRESS) + + self.assertEqual(self.service.start_restore(self.context), + constants.RESTORE_PROGRESS_ALREADY_IN_PROGRESS) + self.assertEqual(self.service.get_restore_state(self.context), + constants.RESTORE_PROGRESS_IN_PROGRESS) + + self.assertEqual(self.service.complete_restore(self.context), + constants.RESTORE_PROGRESS_COMPLETED) + self.assertEqual(self.service.get_restore_state(self.context), + constants.RESTORE_PROGRESS_NOT_IN_PROGRESS) + + def test_restore_complete_rejection(self): + # Create controller-0 + _ = self._create_controller( + which=0, + invprovision=constants.PROVISIONED, + administrative=constants.ADMIN_UNLOCKED, + operational=constants.OPERATIONAL_ENABLED, + availability=constants.AVAILABILITY_AVAILABLE) + + # Create controller-1 + _ = self._create_controller( + which=1, + invprovision=constants.PROVISIONED, + administrative=constants.ADMIN_UNLOCKED, + operational=constants.OPERATIONAL_DISABLED, + availability=constants.AVAILABILITY_OFFLINE) + + self.assertEqual(self.service.get_restore_state(self.context), + constants.RESTORE_PROGRESS_NOT_IN_PROGRESS) + self.assertTrue(self.service.complete_restore(self.context) + not in self.valid_restore_states) + + self.assertEqual(self.service.start_restore(self.context), + constants.RESTORE_PROGRESS_STARTED) + self.assertEqual(self.service.get_restore_state(self.context), + constants.RESTORE_PROGRESS_IN_PROGRESS) + + self.assertEqual(self.service.start_restore(self.context), + constants.RESTORE_PROGRESS_ALREADY_IN_PROGRESS) + self.assertEqual(self.service.get_restore_state(self.context), + constants.RESTORE_PROGRESS_IN_PROGRESS) + + self.assertTrue(self.service.complete_restore(self.context) + not in self.valid_restore_states) + self.assertEqual(self.service.get_restore_state(self.context), + constants.RESTORE_PROGRESS_IN_PROGRESS)