Merge "Support async for sw-deploy start and activate"

nfv/nfv-common/nfv_common/tasks/_task_future.py

@@ -9,9 +9,7 @@ from nfv_common.tasks._task_work import TaskWork
 
 DLOG = debug.debug_get_logger('nfv_common.tasks.task_future')
 
-# TODO(vselvara)The timeout to be changed back to 20sec when the start
+TASK_TIMEOUT = 20
-# software-api is async for patch-release
-TASK_TIMEOUT = 600
 
 
 class TaskFuture(object):

nfv/nfv-common/nfv_common/thread/_thread.py

@@ -94,7 +94,7 @@ class Thread(object):
                         self._stall_timestamp_ms = \
                             timers.get_monotonic_timestamp_in_ms()
 
-                    DLOG.warn("Thread %s stalled, progress_marker=%s, "
+                    DLOG.error("Thread %s stalled, progress_marker=%s, "
                                "elapsed_secs=%s." % (self._name,
                                                      self._progress_marker.value,
                                                      self.stall_elapsed_secs))

nfv/nfv-plugins/nfv_plugins/nfvi_plugins/config.ini

@@ -123,14 +123,12 @@ port=30004
 
 # WARNING: Any changes to these timeouts must be reflected in the timeouts
 # used for the associated REST API calls.
-# TODO(vselvara) sysinv and usm timeout to be changed back to 60 once 
-# the sw-deploy start API for patch release is made asynchronous. 
 [nfvi-timeouts]
 openstack.get_token=10
 neutron.disable_host_services=40
 neutron.delete_host_services=40
 glance.upload_image_data_by_file=180
 glance.upload_image_data_by_url=180
-sysinv=600
+sysinv=60
 patching.apply_patch=180
-usm=600
+usm=60

nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py

@@ -2362,6 +2362,19 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
                 DLOG.error("Software release not found.")
                 return
 
+            future.work(usm.sw_deploy_show, self._platform_token)
+            future.result = (yield)
+
+            if not future.result.is_complete():
+                DLOG.error("USM software deploy get deploy did not complete.")
+                return
+
+            deploy_info = future.result.data
+            if deploy_info:
+                deploy_info = deploy_info[0]
+            else:
+                deploy_info = None
+
             future.work(usm.sw_deploy_host_list, self._platform_token)
             future.result = (yield)
 
@@ -2376,6 +2389,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
             upgrade_obj = nfvi.objects.v1.Upgrade(
                 release,
                 release_info,
+                deploy_info,
                 hosts_info_data,
             )
 
@@ -2400,7 +2414,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
             callback.send(response)
             callback.close()
 
-    def sw_deploy_precheck(self, future, release, callback):
+    def sw_deploy_precheck(self, future, release, force, callback):
         """
         Precheck a USM software deploy
         """
@@ -2423,7 +2437,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
                 self._platform_token = future.result.data
 
-            future.work(usm.sw_deploy_precheck, self._platform_token, release)
+            future.work(usm.sw_deploy_precheck, self._platform_token, release, force)
             future.result = (yield)
             if not future.result.is_complete():
                 DLOG.error("USM software deploy precheck did not complete.")
@@ -2452,7 +2466,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
             callback.send(response)
             callback.close()
 
-    def sw_deploy_start(self, future, release, callback):
+    def sw_deploy_start(self, future, release, force, callback):
         """
         Start a USM software deploy
         """
@@ -2474,41 +2488,13 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
                 self._platform_token = future.result.data
 
-            future.work(usm.sw_deploy_start, self._platform_token, release)
+            future.work(usm.sw_deploy_start, self._platform_token, release, force)
             future.result = (yield)
             if not future.result.is_complete():
                 DLOG.error("USM software deploy start did not complete.")
                 return
 
-            # TODO(vselvara): remove the state check here once the api is changed
+            response['result-data'] = future.result.data
-            # to async. state check to be done in _strategy_steps.py
-            future.work(usm.sw_deploy_host_list, self._platform_token)
-            future.result = (yield)
-
-            if not future.result.is_complete():
-                DLOG.error("USM software deploy host list did not complete.")
-                return
-
-            hosts_info_data = future.result.data
-
-            future.work(usm.sw_deploy_show, self._platform_token)
-            future.result = (yield)
-            if not future.result.is_complete():
-                DLOG.error("USM software deploy show did not complete.")
-                return
-            state_info = future.result.data
-            if state_info:
-                state_info = state_info[0]
-            else:
-                state_info = None
-
-            upgrade_obj = nfvi.objects.v1.Upgrade(
-                release,
-                state_info,
-                hosts_info_data,
-            )
-
-            response['result-data'] = upgrade_obj
             response['completed'] = True
 
         except exceptions.OpenStackRestAPIException as e:
@@ -2575,6 +2561,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
             upgrade_obj = nfvi.objects.v1.Upgrade(
                 release,
                 state_info,
+                None,
                 None)
 
             response['result-data'] = upgrade_obj
@@ -2649,6 +2636,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
             upgrade_obj = nfvi.objects.v1.Upgrade(
                 release,
                 release_info,
+                None,
                 None)
 
             response['result-data'] = upgrade_obj

nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/usm.py

@@ -13,7 +13,6 @@ from nfv_plugins.nfvi_plugins.openstack.rest_api import rest_api_request
 
 
 REST_API_REQUEST_TIMEOUT = 60
-REST_API_START_REQUEST_TIMEOUT = 600
 
 DLOG = debug.debug_get_logger('nfv_plugins.nfvi_plugins.openstack.usm')
 
@@ -38,6 +37,7 @@ def _api_get(token, url):
     """
     Perform a generic GET for a particular API endpoint
     """
+
     response = rest_api_request(token,
                                 "GET",
                                 url,
@@ -99,10 +99,10 @@ def sw_deploy_precheck(token, release, force=False):
     Ask USM to precheck before a deployment
     """
 
-    uri = (f"deploy/{release}/precheck/force" if
+    uri = f"deploy/{release}/precheck"
-            force else f"deploy/{release}/precheck")
+    data = {"force": force} if force else {}
     url = _usm_api_cmd(token, uri)
-    response = _api_post(token, url, {})
+    response = _api_post(token, url, data)
     return response
 
 
@@ -111,9 +111,10 @@ def sw_deploy_start(token, release, force=False):
     Ask USM to start a deployment
     """
 
-    uri = f"deploy/{release}/start/force" if force else f"deploy/{release}/start"
+    uri = f"deploy/{release}/start"
     url = _usm_api_cmd(token, uri)
-    response = _api_post(token, url, {}, None, REST_API_START_REQUEST_TIMEOUT)
+    data = {"force": force} if force else {}
+    response = _api_post(token, url, data)
     return response
 
 

nfv/nfv-tests/nfv_unit_tests/tests/test_sw_deploy_strategy.py

@@ -3,6 +3,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 #
+
 from unittest import mock
 import uuid
 
@@ -234,6 +235,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
                     'reboot_required': False,
                 },
                 None,
+                None,
             )
         )
 
@@ -280,6 +282,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
                     'reboot_required': False,
                 },
                 None,
+                None,
             )
         )
 
@@ -327,6 +330,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
                     'reboot_required': False,
                 },
                 None,
+                None,
             )
         )
 
@@ -379,6 +383,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
                     'reboot_required': False,
                 },
                 None,
+                None,
             )
         )
 
@@ -427,6 +432,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
                     'reboot_required': False,
                 },
                 None,
+                None,
             )
         )
 
@@ -476,6 +482,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
                     'reboot_required': False,
                 },
                 None,
+                None,
             )
         )
 
@@ -925,6 +932,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
                 '13.01',
                 {'state': 'deployed'},
                 None,
+                None,
             )
         )
 
@@ -952,6 +960,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
                 '13.01',
                 {'state': 'committed'},
                 None,
+                None,
             )
         )
 
@@ -979,6 +988,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
                 '13.01',
                 None,
                 None,
+                None,
             )
         )
 
@@ -1014,6 +1024,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
                     'reboot_required': False,
                 },
                 None,
+                None,
             )
         )
 
@@ -1079,6 +1090,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
                     'reboot_required': True,
                 },
                 None,
+                None,
             )
         )
 
@@ -1148,6 +1160,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
                     'reboot_required': False,
                 },
                 None,
+                None,
             )
         )
 
@@ -1222,6 +1235,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
                     'reboot_required': True,
                 },
                 None,
+                None,
             )
         )
 
@@ -1308,6 +1322,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
                     'reboot_required': False,
                 },
                 None,
+                None,
             )
         )
 
@@ -1404,6 +1419,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
                     'reboot_required': True,
                 },
                 None,
+                None,
             )
         )
 

nfv/nfv-tests/nfv_unit_tests/tests/test_sw_upgrade_strategy.py

@@ -83,6 +83,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
                     'reboot_required': True,
                 },
                 None,
+                None,
             )
 
         strategy.nfvi_upgrade = nfvi_upgrade

nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py

@@ -397,22 +397,24 @@ def nfvi_get_upgrade(release, callback):
     return cmd_id
 
 
-def nfvi_sw_deploy_precheck(release, callback):
+def nfvi_sw_deploy_precheck(release, force, callback):
     """
     Software deploy precheck
     """
     cmd_id = _infrastructure_plugin.invoke_plugin('sw_deploy_precheck',
                                                   release,
+                                                  force,
                                                   callback=callback)
     return cmd_id
 
 
-def nfvi_upgrade_start(release, callback):
+def nfvi_upgrade_start(release, force, callback):
     """
     Software deploy start
     """
     cmd_id = _infrastructure_plugin.invoke_plugin('sw_deploy_start',
                                                   release,
+                                                  force,
                                                   callback=callback)
     return cmd_id
 

nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_upgrade.py

@@ -4,39 +4,70 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+from enum import Enum
+
 from nfv_vim.nfvi.objects.v1._object import ObjectData
 
-# USM states
+
-USM_ABORTING = 'aborting'
+# Enums from https://opendev.org/starlingx/update/src/branch/master/software/software/states.py
-USM_AVAILABLE = 'available'
+class RELEASE_STATES(Enum):
-USM_COMMITTED = 'committed'
+    AVAILABLE = 'available'
-USM_DEPLOYED = 'deployed'
+    UNAVAILABLE = 'unavailable'
-USM_DEPLOYING_ACTIVATE = 'deploying-activate'
+    DEPLOYING = 'deploying'
-USM_DEPLOYING_COMPLETE = 'deploying-complete'
+    DEPLOYED = 'deployed'
-USM_DEPLOYING_HOST = 'deploying-host'
+    REMOVING = 'removing'
-USM_DEPLOYING_START = 'deploying-start'
+    COMMITTED = 'committed'
-USM_REMOVING = 'removing'
+
-USM_UNAVAILABLE = 'unavailable'
+
-USM_UNKNOWN = 'n/a'
+class DEPLOY_STATES(Enum):
+    START = 'start'
+    START_DONE = 'start-done'
+    START_FAILED = 'start-failed'
+
+    HOST = 'host'
+    HOST_DONE = 'host-done'
+    HOST_FAILED = 'host-failed'
+
+    ACTIVATE = 'activate'
+    ACTIVATE_DONE = 'activate-done'
+    ACTIVATE_FAILED = 'activate-failed'
+
+    ABORT = 'abort'
+    ABORT_DONE = 'abort-done'
+
+
+class DEPLOY_HOST_STATES(Enum):
+    DEPLOYED = 'deployed'
+    DEPLOYING = 'deploying'
+    FAILED = 'failed'
+    PENDING = 'pending'
 
 
 class Upgrade(ObjectData):
     """
     NFVI Upgrade Object
     """
-    def __init__(self, release, release_info, hosts_info):
+    def __init__(self, release, release_info, deploy_info, hosts_info):
         super(Upgrade, self).__init__('1.0.0')
         self.update(dict(release=release,
                          release_info=release_info,
+                         deploy_info=deploy_info,
                          hosts_info=hosts_info))
 
     @property
-    def state(self):
+    def release_state(self):
         if not self.release_info:
             return None
 
         return self.release_info["state"]
 
+    @property
+    def deploy_state(self):
+        if not self.deploy_info:
+            return None
+
+        return self.deploy_info["state"]
+
     @property
     def reboot_required(self):
         if not self.release_info:
@@ -46,34 +77,48 @@ class Upgrade(ObjectData):
 
     @property
     def is_available(self):
-        return self.state == USM_AVAILABLE
+        return self.release_state == RELEASE_STATES.AVAILABLE.value
 
     @property
     def is_deployed(self):
-        return self.state == USM_DEPLOYED
+        return self.release_state == RELEASE_STATES.DEPLOYED.value
 
     @property
-    def is_commited(self):
+    def is_committed(self):
-        return self.state == USM_COMMITTED
+        return self.release_state == RELEASE_STATES.COMMITTED.value
 
     @property
-    def is_started(self):
+    def is_starting(self):
-        return self.state == USM_DEPLOYING_START
+        return self.deploy_state == DEPLOY_STATES.START.value
 
     @property
-    def is_activated(self):
+    def is_start_done(self):
-        return self.state == USM_DEPLOYING_ACTIVATE
+        return self.deploy_state == DEPLOY_STATES.START_DONE.value
 
-    def is_host_deployed(self, hostname):
+    @property
-        """Return if hostname is deployed
+    def is_start_failed(self):
+        return self.deploy_state == DEPLOY_STATES.START_FAILED.value
 
-        If is missing it is assumed not deployed.
+    @property
-        """
+    def is_deploying_hosts(self):
+        return self.deploy_state == DEPLOY_STATES.HOST.value
 
-        if self.hosts_info is None:
+    @property
-            return False
+    def is_deploy_hosts_done(self):
+        return self.deploy_state == DEPLOY_STATES.HOST_DONE.value
 
-        if hostname not in self.hosts_info:
+    @property
-            raise EnvironmentError(f"{hostname} does not exist on system")
+    def is_deploy_hosts_failed(self):
+        return self.deploy_state == DEPLOY_STATES.HOST_FAILED.value
 
-        return self.hosts_info[hostname]["state"] == USM_DEPLOYED
+    @property
+    def is_activating(self):
+        return self.deploy_state == DEPLOY_STATES.ACTIVATE.value
+
+    @property
+    def is_activate_done(self):
+        return self.deploy_state == DEPLOY_STATES.ACTIVATE_DONE.value
+
+    @property
+    def is_activate_failed(self):
+        return self.deploy_state == DEPLOY_STATES.ACTIVATE_FAILED.value

nfv/nfv-vim/nfv_vim/strategy/_strategy.py

@@ -1810,7 +1810,7 @@ class SwUpgradeStrategy(
 
         stage = strategy.StrategyStage(strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_QUERY)
         stage.add_step(strategy.QueryAlarmsStep(ignore_alarms=self._ignore_alarms))
-        stage.add_step(strategy.SwDeployPrecheckStep(release=self._release))
+        stage.add_step(strategy.SwDeployPrecheckStep(release=self._release, force=self._ignore_alarms))
         stage.add_step(strategy.QueryUpgradeStep(release=self._release))
         self.build_phase.add_stage(stage)
         super(SwUpgradeStrategy, self).build()
@@ -1889,13 +1889,13 @@ class SwUpgradeStrategy(
 
         stage = strategy.StrategyStage(strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_START)
 
-        # Do not ignore any alarms when starting an upgrade
         stage.add_step(strategy.QueryAlarmsStep(True, ignore_alarms=self._ignore_alarms))
-        if self.nfvi_upgrade.is_available:
+        # If the release is not available the deployment is already started
-            # sw-deploy start must be done on controller-0
+        # sw-deploy start must be done on controller-0
-            self._swact_fix(stage, HOST_NAME.CONTROLLER_1)
+        self._swact_fix(stage, HOST_NAME.CONTROLLER_1)
-            stage.add_step(strategy.UpgradeStartStep(release=self._release))
+        stage.add_step(strategy.UpgradeStartStep(release=self._release, force=self._ignore_alarms))
-            stage.add_step(strategy.SystemStabilizeStep())
+        stage.add_step(strategy.SystemStabilizeStep())
+        # TODO(jkraitbe): This SWACT should be part of deploy hosts logic
         # sw-deploy host must first be on controller-1
         self._swact_fix(stage, HOST_NAME.CONTROLLER_0)
         self.apply_phase.add_stage(stage)
@@ -1908,10 +1908,10 @@ class SwUpgradeStrategy(
 
         stage = strategy.StrategyStage(strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_COMPLETE)
         stage.add_step(strategy.QueryAlarmsStep(ignore_alarms=self._ignore_alarms))
+        # TODO(jkraitbe): This SWACT should be part of deploy hosts logic
         # sw-deploy complete should be on controller-0
         self._swact_fix(stage, HOST_NAME.CONTROLLER_1)
-        if not self.nfvi_upgrade.is_activated:
+        stage.add_step(strategy.UpgradeActivateStep(release=self._release))
-            stage.add_step(strategy.UpgradeActivateStep(release=self._release))
         stage.add_step(strategy.UpgradeCompleteStep(release=self._release))
         stage.add_step(strategy.SystemStabilizeStep())
         self.apply_phase.add_stage(stage)
@@ -1939,7 +1939,7 @@ class SwUpgradeStrategy(
                 self.save()
                 return
 
-            if self.nfvi_upgrade.is_deployed or self.nfvi_upgrade.is_commited:
+            if self.nfvi_upgrade.is_deployed or self.nfvi_upgrade.is_committed:
                 reason = "Software release is already deployed or committed."
                 DLOG.warn(reason)
                 self._state = strategy.STRATEGY_STATE.BUILD_FAILED
@@ -1988,61 +1988,60 @@ class SwUpgradeStrategy(
 
             self._add_upgrade_start_stage()
 
-            if not self.nfvi_upgrade.is_activated:
+            # TODO(jkraitbe): Exclude hosts that are already deployed.
-                # TODO(jkraitbe): Exclude hosts that are already deployed.
+            # The hosts states are found in self.nfvi_upgrade.hosts_info.
-                # The hosts states are found in self.nfvi_upgrade.hosts_info.
+            # None means deployment hasn't started.
-                # None means deployment hasn't started.
 
-                # TODO(jkraitbe): SWACT to controller-1 should be
+            # TODO(jkraitbe): SWACT to controller-1 should be
-                # here instead of in _add_upgrade_start_stage.
+            # here instead of in _add_upgrade_start_stage.
 
-                for host in host_table.values():
+            for host in host_table.values():
-                    if HOST_PERSONALITY.CONTROLLER in host.personality:
+                if HOST_PERSONALITY.CONTROLLER in host.personality:
-                        controllers_hosts.append(host)
+                    controllers_hosts.append(host)
-                        if HOST_PERSONALITY.WORKER in host.personality:
+                    if HOST_PERSONALITY.WORKER in host.personality:
-                            # We need to use this strategy on AIO type
+                        # We need to use this strategy on AIO type
-                            controller_strategy = self._add_worker_strategy_stages
+                        controller_strategy = self._add_worker_strategy_stages
 
-                    elif HOST_PERSONALITY.STORAGE in host.personality:
+                elif HOST_PERSONALITY.STORAGE in host.personality:
-                        storage_hosts.append(host)
+                    storage_hosts.append(host)
 
-                    elif HOST_PERSONALITY.WORKER in host.personality:
+                elif HOST_PERSONALITY.WORKER in host.personality:
-                        worker_hosts.append(host)
+                    worker_hosts.append(host)
 
-                    else:
+                else:
-                        DLOG.error(f"Unsupported personality for host {host.name}.")
+                    DLOG.error(f"Unsupported personality for host {host.name}.")
+                    self._state = strategy.STRATEGY_STATE.BUILD_FAILED
+                    self.build_phase.result = \
+                        strategy.STRATEGY_PHASE_RESULT.FAILED
+                    self.build_phase.result_reason = \
+                        'Unsupported personality for host'
+                    self.sw_update_obj.strategy_build_complete(
+                        False, self.build_phase.result_reason)
+                    self.save()
+                    return
+
+            # Reverse controller hosts so controller-1 is first
+            controllers_hosts = controllers_hosts[::-1]
+
+            strategy_pairs = [
+                (controller_strategy, controllers_hosts),
+                (self._add_storage_strategy_stages, storage_hosts),
+                (self._add_worker_strategy_stages, worker_hosts)
+            ]
+
+            for stage_func, host_list in strategy_pairs:
+                if host_list:
+                    success, reason = stage_func(host_list, reboot_required)
+                    if not success:
                         self._state = strategy.STRATEGY_STATE.BUILD_FAILED
                         self.build_phase.result = \
                             strategy.STRATEGY_PHASE_RESULT.FAILED
-                        self.build_phase.result_reason = \
+                        self.build_phase.result_reason = reason
-                            'Unsupported personality for host'
                         self.sw_update_obj.strategy_build_complete(
                             False, self.build_phase.result_reason)
                         self.save()
                         return
 
-                # Reverse controller hosts so controller-1 is first
-                controllers_hosts = controllers_hosts[::-1]
-
-                strategy_pairs = [
-                    (controller_strategy, controllers_hosts),
-                    (self._add_storage_strategy_stages, storage_hosts),
-                    (self._add_worker_strategy_stages, worker_hosts)
-                ]
-
-                for stage_func, host_list in strategy_pairs:
-                    if host_list:
-                        success, reason = stage_func(host_list, reboot_required)
-                        if not success:
-                            self._state = strategy.STRATEGY_STATE.BUILD_FAILED
-                            self.build_phase.result = \
-                                strategy.STRATEGY_PHASE_RESULT.FAILED
-                            self.build_phase.result_reason = reason
-                            self.sw_update_obj.strategy_build_complete(
-                                False, self.build_phase.result_reason)
-                            self.save()
-                            return
-
             self._add_upgrade_complete_stage()
 
             if 0 == len(self.apply_phase.stages):
@@ -2097,6 +2096,7 @@ class SwUpgradeStrategy(
             self._nfvi_upgrade = nfvi.objects.v1.Upgrade(
                 nfvi_upgrade_data['release'],
                 nfvi_upgrade_data['release_info'],
+                nfvi_upgrade_data['deploy_info'],
                 nfvi_upgrade_data['hosts_info'])
         else:
             self._nfvi_upgrade = None

nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py

@@ -927,10 +927,11 @@ class SwDeployPrecheckStep(strategy.StrategyStep):
     """
     Software Deploy Precheck - Strategy Step
     """
-    def __init__(self, release):
+    def __init__(self, release, force):
         super(SwDeployPrecheckStep, self).__init__(
             STRATEGY_STEP_NAME.SW_DEPLOY_PRECHECK, timeout_in_secs=60)
         self._release = release
+        self._force = force
 
     @coroutine
     def _sw_deploy_precheck_callback(self):
@@ -942,7 +943,9 @@ class SwDeployPrecheckStep(strategy.StrategyStep):
 
         if response['completed']:
             if not response['result-data']:
-                DLOG.debug("sw-deploy precheck completed %s" % response['result-data'])
+                DLOG.debug("sw-deploy precheck completed")
+                result = strategy.STRATEGY_STEP_RESULT.SUCCESS
+                self.stage.step_complete(result, '')
             else:
                 reason = "sw-deploy precheck failed"
                 # TODO(vselvara) to display the entire error response
@@ -957,7 +960,7 @@ class SwDeployPrecheckStep(strategy.StrategyStep):
         from nfv_vim import nfvi
 
         DLOG.info("Step (%s) apply." % self._name)
-        nfvi.nfvi_sw_deploy_precheck(self._release, self._sw_deploy_precheck_callback())
+        nfvi.nfvi_sw_deploy_precheck(self._release, self._force, self._sw_deploy_precheck_callback())
         return strategy.STRATEGY_STEP_RESULT.WAIT, ""
 
     def from_dict(self, data):
@@ -967,6 +970,7 @@ class SwDeployPrecheckStep(strategy.StrategyStep):
         """
         super(SwDeployPrecheckStep, self).from_dict(data)
         self._release = data["release"]
+        self._force = data["force"]
         return self
 
     def as_dict(self):
@@ -975,6 +979,7 @@ class SwDeployPrecheckStep(strategy.StrategyStep):
         """
         data = super(SwDeployPrecheckStep, self).as_dict()
         data['release'] = self._release
+        data['force'] = self._force
         data['entity_type'] = ''
         data['entity_names'] = list()
         data['entity_uuids'] = list()
@@ -1099,11 +1104,13 @@ class UpgradeStartStep(strategy.StrategyStep):
     """
     Upgrade Start - Strategy Step
     """
-    def __init__(self, release):
+    def __init__(self, release, force):
         super(UpgradeStartStep, self).__init__(
-            STRATEGY_STEP_NAME.START_UPGRADE, timeout_in_secs=600)
+            STRATEGY_STEP_NAME.START_UPGRADE, timeout_in_secs=1200)
 
         self._release = release
+        self._force = force
+        self._query_inprogress = False
 
     @coroutine
     def _start_upgrade_callback(self):
@@ -1112,20 +1119,63 @@ class UpgradeStartStep(strategy.StrategyStep):
         """
         response = (yield)
         DLOG.debug("Start-Upgrade callback response=%s." % response)
-        if response['completed']:
+
-            state_info = response['result-data']['release_info']['state']
+        if not response['completed']:
-            if state_info == SW_DEPLOY_START:
+            reason = (
-                result = strategy.STRATEGY_STEP_RESULT.SUCCESS
+                "Unknown error while trying to start the software deployment, "
-                self.stage.step_complete(result, "")
+                "check /var/log/nfv-vim.log for more information."
-            else:
+            )
-                reason = "Software deploy not started. Current state:%s" % state_info
-                result = strategy.STRATEGY_STEP_RESULT.FAILED
-                self.stage.step_complete(result, reason)
-        else:
-            reason = "Software deploy start not completed"
             result = strategy.STRATEGY_STEP_RESULT.FAILED
             self.stage.step_complete(result, reason)
 
+        elif not (info := response["result-data"]["info"].strip()).endswith("started"):
+            # TODO(jkraitbe): This will likely change in future
+            # Failed/Rejected starts return HTTP 200 so we need to parse result
+            reason = f"Failed deploy start: {info}"
+            result = strategy.STRATEGY_STEP_RESULT.FAILED
+            self.stage.step_complete(result, reason)
+
+    @coroutine
+    def _handle_start_upgrade_callback(self):
+        """
+        Handle Start Upgrade Callback
+        """
+        response = (yield)
+        DLOG.debug("Handle Start-Upgrade callback response=%s." % response)
+
+        self._query_inprogress = False
+
+        if not response['completed']:
+            # Something went wrong while collecting state info
+            return
+
+        self.strategy.nfvi_upgrade = response['result-data']
+
+        if self.strategy.nfvi_upgrade.is_start_done:
+            DLOG.debug("Handle Start-Upgrade callback, deploy start is done")
+            reason = ""
+            result = strategy.STRATEGY_STEP_RESULT.SUCCESS
+            self.stage.step_complete(result, reason)
+
+        elif self.strategy.nfvi_upgrade.is_start_failed:
+            reason = (
+                "Software deployment start failed, "
+                "check /var/log/software.log for more information."
+            )
+            result = strategy.STRATEGY_STEP_RESULT.FAILED
+            self.stage.step_complete(result, reason)
+
+        elif not self.strategy.nfvi_upgrade.is_starting:
+            reason = (
+                "Unknown error while starting the software deployment, "
+                "check /var/log/software.log for more information."
+            )
+            result = strategy.STRATEGY_STEP_RESULT.FAILED
+            self.stage.step_complete(result, reason)
+
+        else:
+            DLOG.debug("Handle Start-Upgrade callback, deploy start in progress...")
+
     def apply(self):
         """
         Upgrade Start
@@ -1133,9 +1183,25 @@ class UpgradeStartStep(strategy.StrategyStep):
         from nfv_vim import nfvi
 
         DLOG.info("Step (%s) apply." % self._name)
-        nfvi.nfvi_upgrade_start(self._release, self._start_upgrade_callback())
+        nfvi.nfvi_upgrade_start(self._release, self._force, self._start_upgrade_callback())
         return strategy.STRATEGY_STEP_RESULT.WAIT, ""
 
+    def handle_event(self, event, event_data=None):
+        """
+        Handle Host events
+        """
+        from nfv_vim import nfvi
+
+        DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
+
+        if event == STRATEGY_EVENT.HOST_AUDIT:
+            if not self._query_inprogress:
+                self._query_inprogress = True
+                nfvi.nfvi_get_upgrade(self._release, self._handle_start_upgrade_callback())
+            return True
+
+        return False
+
     def from_dict(self, data):
         """
         Returns the upgrade start step object initialized using the given
@@ -1143,6 +1209,8 @@ class UpgradeStartStep(strategy.StrategyStep):
         """
         super(UpgradeStartStep, self).from_dict(data)
         self._release = data["release"]
+        self._force = data["force"]
+        self._query_inprogress = False
         return self
 
     def as_dict(self):
@@ -1154,6 +1222,7 @@ class UpgradeStartStep(strategy.StrategyStep):
         data['entity_names'] = list()
         data['entity_uuids'] = list()
         data['release'] = self._release
+        data['force'] = self._force
         return data
 
 
@@ -1161,34 +1230,74 @@ class UpgradeActivateStep(strategy.StrategyStep):
     """
     Upgrade Activate - Strategy Step
     """
-
     def __init__(self, release):
         super(UpgradeActivateStep, self).__init__(
-            STRATEGY_STEP_NAME.ACTIVATE_UPGRADE, timeout_in_secs=900)
+            STRATEGY_STEP_NAME.ACTIVATE_UPGRADE, timeout_in_secs=1500)
 
         self._release = release
+        self._query_inprogress = False
 
     @coroutine
     def _activate_upgrade_callback(self):
         """
         Activate Upgrade Callback
         """
+
         response = (yield)
-        self.strategy.nfvi_upgrade = response['result-data']
+        DLOG.debug("Activate-Upgrade callback response=%s." % response)
-        if response['completed']:
+
-            state_info = self.strategy.nfvi_upgrade['release_info']['state']
+        if not response['completed']:
-            if state_info == SW_DEPLOY_ACTIVATE_DONE:
+            reason = (
-                result = strategy.STRATEGY_STEP_RESULT.SUCCESS
+                "Unknown error while trying to activate the software deployment, "
-                self.stage.step_complete(result, "")
+                "check /var/log/nfv-vim.log for more information."
-            else:
+            )
-                reason = "SW Deploy Activate not successful. Current state:%s" % state_info
-                result = strategy.STRATEGY_STEP_RESULT.FAILED
-                self.stage.step_complete(result, reason)
-        else:
-            reason = "SW Deploy Activate not complete. Current response:%s" % response['result-data']
             result = strategy.STRATEGY_STEP_RESULT.FAILED
             self.stage.step_complete(result, reason)
 
+        # TODO(jkraitbe): This will change in future
+
+    @coroutine
+    def _handle_activate_upgrade_callback(self):
+        """
+        Handle Activate Upgrade Callback
+        """
+
+        response = (yield)
+        DLOG.debug("Handle Activate-Upgrade callback response=%s." % response)
+
+        self._query_inprogress = False
+
+        if not response['completed']:
+            # Something went wrong while collecting state info
+            return
+
+        self.strategy.nfvi_upgrade = response['result-data']
+
+        if self.strategy.nfvi_upgrade.is_activate_done:
+            DLOG.debug("Handle Activate-Upgrade callback, deploy activate is done")
+            reason = ""
+            result = strategy.STRATEGY_STEP_RESULT.SUCCESS
+            self.stage.step_complete(result, reason)
+
+        elif self.strategy.nfvi_upgrade.is_activate_failed:
+            reason = (
+                "Software deployment activate failed, "
+                "check /var/log/software.log for more information."
+            )
+            result = strategy.STRATEGY_STEP_RESULT.FAILED
+            self.stage.step_complete(result, reason)
+
+        elif not self.strategy.nfvi_upgrade.is_activating:
+            reason = (
+                "Unknown error while activating the software deployment, "
+                "check /var/log/software.log for more information."
+            )
+            result = strategy.STRATEGY_STEP_RESULT.FAILED
+            self.stage.step_complete(result, reason)
+
+        else:
+            DLOG.debug("Handle Activate-Upgrade callback, deploy activate in progress...")
+
     def apply(self):
         """
         Upgrade Activate
@@ -1199,6 +1308,22 @@ class UpgradeActivateStep(strategy.StrategyStep):
         nfvi.nfvi_upgrade_activate(self._release, self._activate_upgrade_callback())
         return strategy.STRATEGY_STEP_RESULT.WAIT, ""
 
+    def handle_event(self, event, event_data=None):
+        """
+        Handle Host events
+        """
+        from nfv_vim import nfvi
+
+        DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
+
+        if event == STRATEGY_EVENT.HOST_AUDIT:
+            if not self._query_inprogress:
+                self._query_inprogress = True
+                nfvi.nfvi_get_upgrade(self._release, self._handle_activate_upgrade_callback())
+            return True
+
+        return False
+
     def from_dict(self, data):
         """
         Returns the upgrade activate step object initialized using the given
@@ -1206,6 +1331,7 @@ class UpgradeActivateStep(strategy.StrategyStep):
         """
         super(UpgradeActivateStep, self).from_dict(data)
         self._release = data["release"]
+        self._query_inprogress = False
         return self
 
     def as_dict(self):
@@ -1238,6 +1364,7 @@ class UpgradeCompleteStep(strategy.StrategyStep):
         """
         response = (yield)
         DLOG.debug("Complete-Upgrade callback response=%s." % response)
+
         self.strategy.nfvi_upgrade = response['result-data']
         if response['completed']:
             state_info = self.strategy.nfvi_upgrade['release_info']['state']