Simplex Upgrade: Threaded backup file generation

Move the call to generate the backup file to the sysinv agent.

This prevents blocking the sysinv conductor while the backup is being
generated.

Change-Id: I2b455396a8ebebe61f973a29935046066dfa2af1
Story: 2002886
Task: 22847
Signed-off-by: Jack Ding <jack.ding@windriver.com>

controllerconfig/controllerconfig/controllerconfig/upgrades/management.py

@@ -7,7 +7,6 @@
 #
 # This file contains functions used by sysinv to manage upgrades.
 #
-
 import json
 import glob
 import os
@@ -196,12 +195,9 @@ def prepare_upgrade(from_load, to_load, i_system):
     LOG.info("Finished upgrade preparations")
 
 
-def create_simplex_backup(controller_fs, software_upgrade):
+def create_simplex_backup(software_upgrade):
     """Creates the upgrade metadata and creates the system backup"""
     backup_data = {}
-    fs_data = {}
-    fs_data['database_gib'] = controller_fs.database_gib * 2
-    backup_data['filesystem'] = fs_data
     upgrade_data = software_upgrade.as_dict()
     if upgrade_data['created_at']:
         upgrade_data['created_at'] = \

sysinv/sysinv/sysinv/sysinv/agent/manager.py

@@ -1466,6 +1466,42 @@ class AgentManager(service.PeriodicService):
 
         return
 
+    def create_simplex_backup(self, context, software_upgrade):
+        """Creates the upgrade metadata and creates the system backup
+
+        :param context: request context.
+        :param software_upgrade: software_upgrade object
+        :returns: none
+        """
+        try:
+            from controllerconfig import backup_restore
+            from controllerconfig.upgrades import \
+                management as upgrades_management
+        except ImportError:
+            LOG.error("Attempt to import during create_simplex_backup failed")
+            return
+
+        if tsc.system_mode != constants.SYSTEM_MODE_SIMPLEX:
+            LOG.error("create_simplex_backup called for non-simplex system")
+            return
+
+        LOG.info("Starting simplex upgrade data collection")
+        success = True
+        try:
+            upgrades_management.create_simplex_backup(software_upgrade)
+        except Exception as ex:
+            LOG.info("Exception during simplex upgrade data collection")
+            LOG.exception(ex)
+            success = False
+        else:
+            LOG.info("Simplex upgrade data collection complete")
+
+        rpcapi = conductor_rpcapi.ConductorAPI(
+            topic=conductor_rpcapi.MANAGER_TOPIC)
+        rpcapi.complete_simplex_backup(context, success=success)
+
+        return
+
     def _audit_tpm_device(self, context, host_id):
         """ Audit the tpmdevice status on this host and update. """
         rpcapi = conductor_rpcapi.ConductorAPI(

sysinv/sysinv/sysinv/sysinv/agent/rpcapi.py

@@ -162,6 +162,20 @@ class AgentAPI(sysinv.openstack.common.rpc.proxy.RpcProxy):
 
         return retval
 
+    def create_simplex_backup(self, context, software_upgrade):
+        """Asynchronously, have the agent create the simplex backup data
+
+        :param context: request context.
+        :param software_upgrade: software_upgrade object
+        :returns: none
+        """
+        retval = self.fanout_cast(context,
+                                  self.make_msg(
+                                      'create_simplex_backup',
+                                      software_upgrade=software_upgrade))
+
+        return retval
+
     def apply_tpm_config(self, context, tpm_context):
         """Asynchronously, have the agent apply the tpm config
 

sysinv/sysinv/sysinv/sysinv/conductor/manager.py

@@ -8310,12 +8310,12 @@ class ConductorManager(service.PeriodicService):
                                    '-r', to_version])
 
             if tsc.system_mode == constants.SYSTEM_MODE_SIMPLEX:
-                LOG.info("Creating upgrade backup")
-                backup_data = {}
-                controller_fs = self.dbapi.controller_fs_get_one()
+                LOG.info("Generating agent request to create simplex upgrade "
+                         "data")
                 software_upgrade = self.dbapi.software_upgrade_get_one()
-                upgrades_management.create_simplex_backup(controller_fs,
-                                                          software_upgrade)
+                rpcapi = agent_rpcapi.AgentAPI()
+                rpcapi.create_simplex_backup(context, software_upgrade)
+                return
             else:
                 i_system = self.dbapi.isystem_get_one()
                 upgrades_management.prepare_upgrade(
@@ -8353,12 +8353,6 @@ class ConductorManager(service.PeriodicService):
         self.dbapi.software_upgrade_update(
             upgrade.uuid, {'state': constants.UPGRADE_STARTED})
 
-        if tsc.system_mode == constants.SYSTEM_MODE_SIMPLEX:
-            controller_fs = self.dbapi.controller_fs_get()
-            software_upgrade = self.dbapi.software_upgrade_get_one()
-            upgrades_management.create_simplex_backup(controller_fs,
-                                                      software_upgrade)
-
     def activate_upgrade(self, context, upgrade):
         """Activate the upgrade. Generate and apply new manifests.
 
@@ -8535,6 +8529,53 @@ class ConductorManager(service.PeriodicService):
 
         return rpc_upgrade
 
+    def complete_simplex_backup(self, context, success):
+        """Complete the simplex upgrade start process
+
+        :param context: request context.
+        :param success: If the create_simplex_backup call completed
+        """
+        try:
+            upgrade = self.dbapi.software_upgrade_get_one()
+        except exception.NotFound:
+            LOG.error("Software upgrade record not found")
+            return
+
+        from_version = upgrade.from_release
+        to_version = upgrade.to_release
+
+        if not success:
+            # The upgrade start data collection failed, stop the upgrade
+            upgrades_management.abort_upgrade(from_version, to_version,
+                                              upgrade)
+            # Delete upgrade record
+            self.dbapi.software_upgrade_destroy(upgrade.uuid)
+            LOG.info("Simplex upgrade start failed")
+        else:
+            LOG.info("Simplex upgrade start completed")
+            # Raise alarm to show an upgrade is in progress
+            entity_instance_id = "%s=%s" % (fm_constants.FM_ENTITY_TYPE_HOST,
+                                            constants.CONTROLLER_HOSTNAME)
+            fault = fm_api.Fault(
+                alarm_id=fm_constants.FM_ALARM_ID_UPGRADE_IN_PROGRESS,
+                alarm_state=fm_constants.FM_ALARM_STATE_SET,
+                entity_type_id=fm_constants.FM_ENTITY_TYPE_HOST,
+                entity_instance_id=entity_instance_id,
+                severity=fm_constants.FM_ALARM_SEVERITY_MINOR,
+                reason_text="System Upgrade in progress.",
+                # operational
+                alarm_type=fm_constants.FM_ALARM_TYPE_7,
+                # congestion
+                probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_8,
+                proposed_repair_action="No action required.",
+                service_affecting=False)
+            fm_api.FaultAPIs().set_fault(fault)
+
+            self.dbapi.software_upgrade_update(
+                upgrade.uuid, {'state': constants.UPGRADE_STARTED})
+
+        return
+
     def get_system_health(self, context, force=False, upgrade=False):
         """
         Performs a system health check.

sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py

@@ -1205,6 +1205,15 @@ class ConductorAPI(sysinv.openstack.common.rpc.proxy.RpcProxy):
         return self.call(context, self.make_msg('abort_upgrade',
                                                 upgrade=upgrade))
 
+    def complete_simplex_backup(self, context, success):
+        """Asynchronously, complete the simplex upgrade start process
+
+        :param context: request context.
+        :param success: If the create_simplex_backup call completed
+                """
+        return self.cast(context, self.make_msg('complete_simplex_backup',
+                                                success=success))
+
     def get_system_health(self, context, force=False, upgrade=False):
         """
         Performs a system health check.