DC upgrade state machine states will select next state

Rather than having the state machine states as an ordered list, this modification allows each state to specify the next state. This enhancement allows future features like compound states or a pre-check state to enable skipping over known states or alternative transitions. This adds a pre-check state. Only the success path has been tested so far. Change-Id: Ideae9885b6026cb441bcb6ac08204784b876f3a9 Story: 2007403 Task: 40347 Signed-off-by: albailey <Al.Bailey@windriver.com>
2020-06-17 11:05:01 -05:00
parent a7065cf6b7
commit 4e4355458d
16 changed files with 290 additions and 87 deletions
--- a/distributedcloud/dcmanager/common/consts.py
+++ b/distributedcloud/dcmanager/common/consts.py
@@ -105,6 +105,7 @@ STRATEGY_STATE_COMPLETE = "complete"
 STRATEGY_STATE_ABORTED = "aborted"
 STRATEGY_STATE_FAILED = "failed"

+STRATEGY_STATE_PRE_CHECK = "pre check"
 STRATEGY_STATE_INSTALLING_LICENSE = "installing license"
 STRATEGY_STATE_IMPORTING_LOAD = "importing load"
 STRATEGY_STATE_STARTING_UPGRADE = "starting upgrade"
--- a/distributedcloud/dcmanager/common/exceptions.py
+++ b/distributedcloud/dcmanager/common/exceptions.py
@@ -178,6 +178,11 @@ class LicenseMissingError(DCManagerException):
    message = _("License does not exist on subcloud: %(subcloud_id)s")


+class ManualRecoveryRequiredException(DCManagerException):
+    message = _("Offline Subcloud: %(subcloud)s needs manual recovery from "
+                "deploy state:%(deploy_state)s")
+
+
 class VaultLoadMissingError(DCManagerException):
    message = _("No matching: %(file_type) found in vault: %(vault_dir)")

--- a/distributedcloud/dcmanager/manager/states/base.py
+++ b/distributedcloud/dcmanager/manager/states/base.py
@@ -20,11 +20,15 @@ LOG = logging.getLogger(__name__)
@six.add_metaclass(abc.ABCMeta)
 class BaseState(object):

-    def __init__(self):
+    def __init__(self, next_state):
        super(BaseState, self).__init__()
+        self.next_state = next_state
        self.context = context.get_admin_context()
        self._stop = None

+    def override_next_state(self, next_state):
+        self.next_state = next_state
+
    def registerStopEvent(self, stop_event):
        """Store an orch_thread threading.Event to detect stop."""
        self._stop = stop_event
@@ -95,5 +99,9 @@ class BaseState(object):

    @abc.abstractmethod
    def perform_state_action(self, strategy_step):
-        """Perform the action for this state on the strategy_step"""
+        """Perform the action for this state on the strategy_step
+
+        Returns the next state in the state machine on success.
+        Any exceptions raised by this method set the strategy to FAILED.
+        """
        pass
--- a/distributedcloud/dcmanager/manager/states/lock_host.py
+++ b/distributedcloud/dcmanager/manager/states/lock_host.py
@@ -5,7 +5,7 @@
 #
 import time

-from dcmanager.common.consts import ADMIN_LOCKED
+from dcmanager.common import consts
 from dcmanager.common.exceptions import StrategyStoppedException
 from dcmanager.manager.states.base import BaseState

@@ -17,9 +17,9 @@ DEFAULT_SLEEP_DURATION = 10
 class LockHostState(BaseState):
    """Orchestration state for locking a host"""

-    def __init__(self,
-                 hostname='controller-0'):
-        super(LockHostState, self).__init__()
+    def __init__(self, hostname='controller-0'):
+        super(LockHostState, self).__init__(
+            next_state=consts.STRATEGY_STATE_UPGRADING_SIMPLEX)
        self.target_hostname = hostname
        # max time to wait (in seconds) is: sleep_duration * max_queries
        self.sleep_duration = DEFAULT_SLEEP_DURATION
@@ -28,8 +28,8 @@ class LockHostState(BaseState):
    def perform_state_action(self, strategy_step):
        """Locks a host on the subcloud

-        Any exceptions raised by this method set the strategy to FAILED
-        Returning normally from this method set the strategy to the next step
+        Returns the next state in the state machine on success.
+        Any exceptions raised by this method set the strategy to FAILED.
        """

        # Create a sysinv client on the subcloud
@@ -40,11 +40,11 @@ class LockHostState(BaseState):
        host = sysinv_client.get_host(self.target_hostname)

        # if the host is already in the desired state, no need for action
-        if host.administrative == ADMIN_LOCKED:
+        if host.administrative == consts.ADMIN_LOCKED:
            msg = "Host: %s already: %s." % (self.target_hostname,
                                             host.administrative)
            self.info_log(strategy_step, msg)
-            return True
+            return self.next_state

        # Invoke the action
        # ihost_action is 'lock' and task is set to 'Locking'
@@ -60,7 +60,7 @@ class LockHostState(BaseState):
                raise StrategyStoppedException()
            # query the administrative state to see if it is the new state.
            host = sysinv_client.get_host(self.target_hostname)
-            if host.administrative == ADMIN_LOCKED:
+            if host.administrative == consts.ADMIN_LOCKED:
                msg = "Host: %s is now: %s" % (self.target_hostname,
                                               host.administrative)
                self.info_log(strategy_step, msg)
@@ -73,4 +73,4 @@ class LockHostState(BaseState):
        # If we are here, the loop broke out cleanly and the action succeeded
        # When we return from this method without throwing an exception, the
        # state machine can proceed to the next state
-        return True
+        return self.next_state
--- a/distributedcloud/dcmanager/manager/states/unlock_host.py
+++ b/distributedcloud/dcmanager/manager/states/unlock_host.py
@@ -5,8 +5,7 @@
 #
 import time

-from dcmanager.common.consts import ADMIN_UNLOCKED
-from dcmanager.common.consts import OPERATIONAL_ENABLED
+from dcmanager.common import consts
 from dcmanager.common.exceptions import StrategyStoppedException
 from dcmanager.manager.states.base import BaseState

@@ -27,7 +26,8 @@ class UnlockHostState(BaseState):
    """Orchestration state for unlocking a host."""

    def __init__(self, hostname='controller-0'):
-        super(UnlockHostState, self).__init__()
+        super(UnlockHostState, self).__init__(
+            next_state=consts.STRATEGY_STATE_ACTIVATING_UPGRADE)
        self.target_hostname = hostname
        self.max_api_queries = DEFAULT_MAX_API_QUERIES
        self.api_sleep_duration = DEFAULT_API_SLEEP
@@ -37,8 +37,8 @@ class UnlockHostState(BaseState):
    def perform_state_action(self, strategy_step):
        """Unlocks a host on the subcloud

-        Any exceptions raised by this method set the strategy to FAILED
-        Returning normally from this method set the strategy to the next step
+        Returns the next state in the state machine on success.
+        Any exceptions raised by this method set the strategy to FAILED.
        """

        # Create a sysinv client on the subcloud
@@ -49,11 +49,11 @@ class UnlockHostState(BaseState):
        host = sysinv_client.get_host(self.target_hostname)

        # if the host is already in the desired state, no need for action
-        if host.administrative == ADMIN_UNLOCKED:
+        if host.administrative == consts.ADMIN_UNLOCKED:
            msg = "Host: %s already: %s." % (self.target_hostname,
                                             host.administrative)
            self.info_log(strategy_step, msg)
-            return True
+            return self.next_state

        # Invoke the action
        # ihost_action is 'unlock' and task is set to 'Unlocking'
@@ -78,8 +78,8 @@ class UnlockHostState(BaseState):
            try:
                # query the administrative state to see if it is the new state.
                host = sysinv_client.get_host(self.target_hostname)
-                if (host.administrative == ADMIN_UNLOCKED and
-                        host.operational == OPERATIONAL_ENABLED):
+                if (host.administrative == consts.ADMIN_UNLOCKED and
+                        host.operational == consts.OPERATIONAL_ENABLED):
                    # Success. Break out of the loop.
                    msg = "Host: %s is now: %s %s" % (self.target_hostname,
                                                      host.administrative,
@@ -126,4 +126,4 @@ class UnlockHostState(BaseState):
        # If we are here, the loop broke out cleanly and the action succeeded
        # When we return from this method without throwing an exception, the
        # state machine can proceed to the next state
-        return True
+        return self.next_state
--- a/distributedcloud/dcmanager/manager/states/upgrade/activating.py
+++ b/distributedcloud/dcmanager/manager/states/upgrade/activating.py
@@ -5,6 +5,7 @@
 #
 import time

+from dcmanager.common import consts
 from dcmanager.common.exceptions import StrategyStoppedException
 from dcmanager.manager.states.base import BaseState

@@ -22,7 +23,8 @@ class ActivatingUpgradeState(BaseState):
    """Upgrade state actions for activating an upgrade"""

    def __init__(self):
-        super(ActivatingUpgradeState, self).__init__()
+        super(ActivatingUpgradeState, self).__init__(
+            next_state=consts.STRATEGY_STATE_COMPLETING_UPGRADE)
        # max time to wait (in seconds) is: sleep_duration * max_queries
        self.sleep_duration = DEFAULT_SLEEP_DURATION
        self.max_queries = DEFAULT_MAX_QUERIES
@@ -38,8 +40,8 @@ class ActivatingUpgradeState(BaseState):
    def perform_state_action(self, strategy_step):
        """Activate an upgrade on a subcloud

-        Any exceptions raised by this method set the strategy to FAILED
-        Returning normally from this method set the strategy to the next step
+        Returns the next state in the state machine on success.
+        Any exceptions raised by this method set the strategy to FAILED.
        """
        # get the keystone and sysinv clients for the subcloud
        ks_client = self.get_keystone_client(strategy_step.subcloud.name)
@@ -52,7 +54,7 @@ class ActivatingUpgradeState(BaseState):
        if upgrade_state in ACTIVATING_COMPLETED_STATES:
            self.info_log(strategy_step,
                          "Already in an activating state:%s" % upgrade_state)
-            return True
+            return self.next_state

        # invoke the API 'upgrade-activate'.
        # Throws an exception on failure (no upgrade found, bad host state)
@@ -78,4 +80,4 @@ class ActivatingUpgradeState(BaseState):

        # When we return from this method without throwing an exception, the
        # state machine can proceed to the next state
-        return True
+        return self.next_state
--- a/distributedcloud/dcmanager/manager/states/upgrade/completing.py
+++ b/distributedcloud/dcmanager/manager/states/upgrade/completing.py
@@ -5,6 +5,7 @@
 #
 import time

+from dcmanager.common import consts
 from dcmanager.common.exceptions import StrategyStoppedException
 from dcmanager.manager.states.base import BaseState

@@ -17,7 +18,8 @@ class CompletingUpgradeState(BaseState):
    """Upgrade state actions for completing an upgrade"""

    def __init__(self):
-        super(CompletingUpgradeState, self).__init__()
+        super(CompletingUpgradeState, self).__init__(
+            next_state=consts.STRATEGY_STATE_COMPLETE)
        # max time to wait (in seconds) is: sleep_duration * max_queries
        self.sleep_duration = DEFAULT_SLEEP_DURATION
        self.max_queries = DEFAULT_MAX_QUERIES
@@ -25,8 +27,8 @@ class CompletingUpgradeState(BaseState):
    def perform_state_action(self, strategy_step):
        """Complete an upgrade on a subcloud

-        Any exceptions raised by this method set the strategy to FAILED
-        Returning normally from this method set the strategy to the next step
+        Returns the next state in the state machine on success.
+        Any exceptions raised by this method set the strategy to FAILED.
        """
        # get the keystone and sysinv clients for the subcloud
        ks_client = self.get_keystone_client(strategy_step.subcloud.name)
@@ -40,7 +42,7 @@ class CompletingUpgradeState(BaseState):
        if len(upgrades) == 0:
            self.info_log(strategy_step,
                          "No upgrades exist. Nothing needs completing")
-            return True
+            return self.next_state

        # invoke the API 'upgrade-complete'
        # This is a partially blocking call that raises exception on failure.
@@ -64,4 +66,4 @@ class CompletingUpgradeState(BaseState):

        # When we return from this method without throwing an exception, the
        # state machine can proceed to the next state
-        return True
+        return self.next_state
--- a/distributedcloud/dcmanager/manager/states/upgrade/importing_load.py
+++ b/distributedcloud/dcmanager/manager/states/upgrade/importing_load.py
@@ -19,7 +19,8 @@ class ImportingLoadState(BaseState):
    """Upgrade state for importing a load"""

    def __init__(self):
-        super(ImportingLoadState, self).__init__()
+        super(ImportingLoadState, self).__init__(
+            next_state=consts.STRATEGY_STATE_STARTING_UPGRADE)
        # max time to wait (in seconds) is: sleep_duration * max_queries
        self.sleep_duration = DEFAULT_SLEEP_DURATION
        self.max_queries = DEFAULT_MAX_QUERIES
@@ -27,8 +28,8 @@ class ImportingLoadState(BaseState):
    def perform_state_action(self, strategy_step):
        """Import a load on a subcloud

-        Any exceptions raised by this method set the strategy to FAILED
-        Returning normally from this method set the strategy to the next step
+        Returns the next state in the state machine on success.
+        Any exceptions raised by this method set the strategy to FAILED.
        """
        # determine the version of the system controller in region one
        local_ks_client = self.get_keystone_client()
@@ -48,7 +49,7 @@ class ImportingLoadState(BaseState):
            if load.software_version == target_version:
                self.info_log(strategy_step,
                              "Load:%s already found" % target_version)
-                return True
+                return self.next_state

        # If we are here, the load needs to be imported
        # ISO and SIG files are found in the vault under a version directory
@@ -82,4 +83,4 @@ class ImportingLoadState(BaseState):

        # When we return from this method without throwing an exception, the
        # state machine can proceed to the next state
-        return True
+        return self.next_state
--- a/distributedcloud/dcmanager/manager/states/upgrade/installing_license.py
+++ b/distributedcloud/dcmanager/manager/states/upgrade/installing_license.py
@@ -15,7 +15,8 @@ class InstallingLicenseState(BaseState):
    """Upgrade state action for installing a license"""

    def __init__(self):
-        super(InstallingLicenseState, self).__init__()
+        super(InstallingLicenseState, self).__init__(
+            next_state=consts.STRATEGY_STATE_IMPORTING_LOAD)

    @staticmethod
    def license_up_to_date(target_license, existing_license):
@@ -24,8 +25,8 @@ class InstallingLicenseState(BaseState):
    def perform_state_action(self, strategy_step):
        """Install the License for a software upgrade in this subcloud

-        Any exceptions raised by this method set the strategy to FAILED
-        Returning normally from this method set the strategy to the next step
+        Returns the next state in the state machine on success.
+        Any exceptions raised by this method set the strategy to FAILED.
        """

        # check if the the system controller has a license
@@ -47,7 +48,7 @@ class InstallingLicenseState(BaseState):
                self.info_log(strategy_step,
                              "System Controller License missing: %s."
                              % target_error)
-                return True
+                return self.next_state
            else:
                # An unexpected error occurred querying the license
                raise exceptions.LicenseInstallError(
@@ -68,7 +69,7 @@ class InstallingLicenseState(BaseState):
        if len(subcloud_error) == 0:
            if self.license_up_to_date(target_license, subcloud_license):
                self.info_log(strategy_step, "License up to date.")
-                return True
+                return self.next_state
            else:
                self.debug_log(strategy_step, "License mismatch. Updating.")
        else:
@@ -83,4 +84,4 @@ class InstallingLicenseState(BaseState):

        # The license has been successfully installed. Move to the next stage
        self.info_log(strategy_step, "License installed.")
-        return True
+        return self.next_state
--- a/distributedcloud/dcmanager/manager/states/upgrade/migrating_data.py
+++ b/distributedcloud/dcmanager/manager/states/upgrade/migrating_data.py
@@ -53,7 +53,8 @@ class MigratingDataState(BaseState):
    """Upgrade step for migrating data"""

    def __init__(self):
-        super(MigratingDataState, self).__init__()
+        super(MigratingDataState, self).__init__(
+            next_state=consts.STRATEGY_STATE_UNLOCKING_CONTROLLER)

        self.ansible_sleep = DEFAULT_ANSIBLE_SLEEP
        self.max_api_queries = DEFAULT_MAX_API_QUERIES
@@ -164,13 +165,13 @@ class MigratingDataState(BaseState):
    def perform_state_action(self, strategy_step):
        """Migrate data for an upgrade on a subcloud

-        Any exceptions raised by this method set the strategy to FAILED
-        Returning normally from this method set the strategy to the next step
+        Returns the next state in the state machine on success.
+        Any exceptions raised by this method set the strategy to FAILED.
        """

        if not self.is_subcloud_data_migration_required(strategy_step):
            self.info_log(strategy_step, "Data migration is already done.")
-            return True
+            return self.next_state

        self.info_log(strategy_step, "Start migrating data...")
        db_api.subcloud_update(
@@ -209,4 +210,4 @@ class MigratingDataState(BaseState):
        self.wait_for_unlock(strategy_step)

        self.info_log(strategy_step, "Data migration completed.")
-        return True
+        return self.next_state
--- a/distributedcloud/dcmanager/manager/states/upgrade/pre_check.py
+++ b/distributedcloud/dcmanager/manager/states/upgrade/pre_check.py
@@ -0,0 +1,51 @@
+#
+# Copyright (c) 2020 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+from dcmanager.common import consts
+from dcmanager.common.exceptions import ManualRecoveryRequiredException
+from dcmanager.db import api as db_api
+from dcmanager.manager.states.base import BaseState
+
+# These deploy states should transition to the 'upgrading' state
+VALID_UPGRADE_STATES = [consts.DEPLOY_STATE_INSTALL_FAILED,
+                        consts.DEPLOY_STATE_DATA_MIGRATION_FAILED, ]
+
+# These deploy states should transition to the 'migrating_data' state
+VALID_MIGRATE_DATA_STATES = [consts.DEPLOY_STATE_INSTALLED, ]
+
+
+class PreCheckState(BaseState):
+    """This State skips to the appropriate later state based on subcloud"""
+
+    def __init__(self):
+        super(PreCheckState, self).__init__(
+            next_state=consts.STRATEGY_STATE_INSTALLING_LICENSE)
+
+    def perform_state_action(self, strategy_step):
+        """This state will check if the subcloud is offline:
+
+        if online, proceed to INSTALLING_LICENSE state
+        if offline, check the deploy_status and transfer to the correct state.
+        if an unsupported deploy_status is encountered, fail the upgrade
+        """
+        subcloud = db_api.subcloud_get(self.context, strategy_step.subcloud.id)
+        if subcloud.availability_status == consts.AVAILABILITY_ONLINE:
+            return self.next_state
+
+        # it is offline.
+        if subcloud.deploy_status in VALID_UPGRADE_STATES:
+            self.override_next_state(consts.STRATEGY_STATE_UPGRADING_SIMPLEX)
+            return self.next_state
+
+        if subcloud.deploy_status in VALID_MIGRATE_DATA_STATES:
+            self.override_next_state(consts.STRATEGY_STATE_MIGRATING_DATA)
+            return self.next_state
+
+        # FAIL: We are offline and encountered an un-recoverable deploy status
+        self.info_log(strategy_step,
+                      "Un-handled deploy_status: %s" % subcloud.deploy_status)
+        raise ManualRecoveryRequiredException(
+            subcloud=strategy_step.subcloud.name,
+            deploy_status=subcloud.deploy_status)
--- a/distributedcloud/dcmanager/manager/states/upgrade/starting_upgrade.py
+++ b/distributedcloud/dcmanager/manager/states/upgrade/starting_upgrade.py
@@ -6,6 +6,7 @@
 import time

 from dccommon.drivers.openstack.vim import ALARM_RESTRICTIONS_RELAXED
+from dcmanager.common import consts
 from dcmanager.common.exceptions import StrategyStoppedException
 from dcmanager.common import utils
 from dcmanager.manager.states.base import BaseState
@@ -21,7 +22,8 @@ class StartingUpgradeState(BaseState):
    """Upgrade state for starting an upgrade on a subcloud"""

    def __init__(self):
-        super(StartingUpgradeState, self).__init__()
+        super(StartingUpgradeState, self).__init__(
+            next_state=consts.STRATEGY_STATE_LOCKING_CONTROLLER)
        self.sleep_duration = DEFAULT_SLEEP_DURATION
        self.max_queries = DEFAULT_MAX_QUERIES

@@ -35,8 +37,8 @@ class StartingUpgradeState(BaseState):
    def perform_state_action(self, strategy_step):
        """Start an upgrade on a subcloud

-        Any exceptions raised by this method set the strategy to FAILED
-        Returning normally from this method set the strategy to the next step
+        Returns the next state in the state machine on success.
+        Any exceptions raised by this method set the strategy to FAILED.
        """
        # get the keystone and sysinv clients for the subcloud
        ks_client = self.get_keystone_client(strategy_step.subcloud.name)
@@ -51,7 +53,7 @@ class StartingUpgradeState(BaseState):
                # If a previous upgrade exists (even one that failed) skip
                self.info_log(strategy_step,
                              "An upgrade already exists: %s" % upgrade)
-                return True
+                return self.next_state
        else:
            # invoke the API 'upgrade-start'.
            # query the alarm_restriction_type from DB SwUpdateOpts
@@ -85,4 +87,4 @@ class StartingUpgradeState(BaseState):

        # When we return from this method without throwing an exception, the
        # state machine can proceed to the next state
-        return True
+        return self.next_state
--- a/distributedcloud/dcmanager/manager/states/upgrade/upgrading_simplex.py
+++ b/distributedcloud/dcmanager/manager/states/upgrade/upgrading_simplex.py
@@ -11,7 +11,6 @@ from dccommon.install_consts import ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK
 from dccommon.subcloud_install import SubcloudInstall

 from dcmanager.common import consts
-from dcmanager.common.consts import INVENTORY_FILE_POSTFIX
 from dcmanager.common import utils
 from dcmanager.db import api as db_api
 from dcmanager.manager.states.base import BaseState
@@ -27,13 +26,14 @@ class UpgradingSimplexState(BaseState):
    """Upgrade state for upgrading a simplex subcloud host"""

    def __init__(self):
-        super(UpgradingSimplexState, self).__init__()
+        super(UpgradingSimplexState, self).__init__(
+            next_state=consts.STRATEGY_STATE_MIGRATING_DATA)

    def perform_state_action(self, strategy_step):
        """Upgrade a simplex host on a subcloud

-        Any exceptions raised by this method set the strategy to FAILED
-        Returning normally from this method set the strategy to the next step
+        Returns the next state in the state machine on success.
+        Any exceptions raised by this method set the strategy to FAILED.
        """

        LOG.info("Performing simplex upgrade for subcloud %s" %
@@ -65,7 +65,7 @@ class UpgradingSimplexState(BaseState):
                target_version, subcloud_sysinv_client):
            self.info_log(strategy_step,
                          "Load:%s already active" % target_version)
-            return True
+            return self.next_state

        # Check whether subcloud supports redfish, and if not, fail.
        # This needs to be inferred from absence of install_values as
@@ -78,6 +78,7 @@ class UpgradingSimplexState(BaseState):
        # Upgrade the subcloud to the install_values image
        self.perform_subcloud_install(
            strategy_step, local_ks_client.session, install_values)
+        return self.next_state

    def _check_load_already_active(self, target_version, subcloud_sysinv_client):
        """Check if the target_version is already active in subcloud"""
@@ -347,7 +348,7 @@ class UpgradingSimplexState(BaseState):

        ansible_subcloud_inventory_file = os.path.join(
            consts.ANSIBLE_OVERRIDES_PATH,
-            strategy_step.subcloud.name + INVENTORY_FILE_POSTFIX)
+            strategy_step.subcloud.name + consts.INVENTORY_FILE_POSTFIX)

        # Create the ansible inventory for the upgrade subcloud
        utils.create_subcloud_inventory(install_values,
--- a/distributedcloud/dcmanager/manager/sw_upgrade_orch_thread.py
+++ b/distributedcloud/dcmanager/manager/sw_upgrade_orch_thread.py
@@ -39,6 +39,7 @@ from dcmanager.manager.states.upgrade.installing_license \
    import InstallingLicenseState
 from dcmanager.manager.states.upgrade.migrating_data \
    import MigratingDataState
+from dcmanager.manager.states.upgrade.pre_check import PreCheckState
 from dcmanager.manager.states.upgrade.starting_upgrade \
    import StartingUpgradeState
 from dcmanager.manager.states.upgrade.upgrading_simplex \
@@ -46,21 +47,9 @@ from dcmanager.manager.states.upgrade.upgrading_simplex \

 LOG = logging.getLogger(__name__)

-# The state machine transition order for an APPLY
-ORDERED_STATES = [
-    consts.STRATEGY_STATE_INSTALLING_LICENSE,
-    consts.STRATEGY_STATE_IMPORTING_LOAD,
-    consts.STRATEGY_STATE_STARTING_UPGRADE,
-    consts.STRATEGY_STATE_LOCKING_CONTROLLER,
-    consts.STRATEGY_STATE_UPGRADING_SIMPLEX,
-    consts.STRATEGY_STATE_MIGRATING_DATA,
-    consts.STRATEGY_STATE_UNLOCKING_CONTROLLER,
-    consts.STRATEGY_STATE_ACTIVATING_UPGRADE,
-    consts.STRATEGY_STATE_COMPLETING_UPGRADE,
-]
-
-# every state in ORDERED_STATES should have an operator
+# every state should have an operator
 STATE_OPERATORS = {
+    consts.STRATEGY_STATE_PRE_CHECK: PreCheckState,
    consts.STRATEGY_STATE_INSTALLING_LICENSE: InstallingLicenseState,
    consts.STRATEGY_STATE_IMPORTING_LOAD: ImportingLoadState,
    consts.STRATEGY_STATE_STARTING_UPGRADE: StartingUpgradeState,
@@ -105,7 +94,7 @@ class SwUpgradeOrchThread(threading.Thread):
        self.subcloud_workers = dict()

        # When an upgrade is initiated, this is the first state
-        self.starting_state = ORDERED_STATES[0]
+        self.starting_state = consts.STRATEGY_STATE_PRE_CHECK

    def stopped(self):
        return self._stop.isSet()
@@ -144,17 +133,6 @@ class SwUpgradeOrchThread(threading.Thread):
        # instantiate and return the state_operator class
        return state_operator()

-    @staticmethod
-    def determine_next_state(strategy_step):
-        """Return next state for the strategy step based on current state."""
-        # todo(abailey): next_state may differ for AIO, STD, etc.. subclouds
-        next_index = ORDERED_STATES.index(strategy_step.state) + 1
-        if next_index < len(ORDERED_STATES):
-            next_state = ORDERED_STATES[next_index]
-        else:
-            next_state = consts.STRATEGY_STATE_COMPLETE
-        return next_state
-
    def strategy_step_update(self, subcloud_id, state=None, details=None):
        """Update the strategy step in the DB

@@ -405,9 +383,8 @@ class SwUpgradeOrchThread(threading.Thread):
            # Instantiate the state operator and perform the state actions
            state_operator = self.determine_state_operator(strategy_step)
            state_operator.registerStopEvent(self._stop)
-            state_operator.perform_state_action(strategy_step)
+            next_state = state_operator.perform_state_action(strategy_step)
            # If we get here without an exception raised, proceed to next state
-            next_state = self.determine_next_state(strategy_step)
            self.strategy_step_update(strategy_step.subcloud_id,
                                      state=next_state)
        except Exception as e:
--- a/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_base.py
+++ b/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_base.py
@@ -8,6 +8,7 @@ import uuid

 from dcmanager.common import consts
 from dcmanager.manager.states.base import BaseState
+from oslo_utils import timeutils

 from dcmanager.tests.unit.manager.test_sw_upgrade import TestSwUpgrade

@@ -87,6 +88,37 @@ class FakeController(object):
        self.task = task


+class FakeSubcloud(object):
+    def __init__(self,
+                 subcloud_id=1,
+                 name='subcloud1',
+                 description='subcloud',
+                 location='A location',
+                 software_version='12.34',
+                 management_state=consts.MANAGEMENT_MANAGED,
+                 availability_status=consts.AVAILABILITY_ONLINE,
+                 deploy_status=consts.DEPLOY_STATE_DONE):
+        self.id = subcloud_id
+        self.name = name
+        self.description = description
+        self.location = location
+        self.software_version = software_version
+        self.management_state = management_state
+        self.availability_status = availability_status
+        self.deploy_status = deploy_status
+        # todo(abailey): add these and re-factor other unit tests to use
+        # self.management_subnet = management_subnet
+        # self.management_gateway_ip = management_gateway_ip
+        # self.management_start_ip = management_start_ip
+        # self.management_end_ip = management_end_ip
+        # self.external_oam_subnet = external_oam_subnet
+        # self.external_oam_gateway_address = external_oam_gateway_address
+        # self.external_oam_floating_address = external_oam_floating_address
+        # self.systemcontroller_gateway_ip = systemcontroller_gateway_ip
+        self.created_at = timeutils.utcnow()
+        self.updated_at = timeutils.utcnow()
+
+
 class TestSwUpgradeState(TestSwUpgrade):
    def setUp(self):
        super(TestSwUpgradeState, self).setUp()
--- a/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_pre_check.py
+++ b/distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_pre_check.py
@@ -0,0 +1,119 @@
+#
+# Copyright (c) 2020 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+import mock
+
+from dcmanager.common import consts
+
+from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeSubcloud
+from dcmanager.tests.unit.manager.states.upgrade.test_base \
+    import TestSwUpgradeState
+
+
+class TestSwUpgradePreCheckStage(TestSwUpgradeState):
+
+    def setUp(self):
+        super(TestSwUpgradePreCheckStage, self).setUp()
+
+        # Add the strategy_step state being processed by this unit test
+        self.strategy_step = \
+            self.setup_strategy_step(consts.STRATEGY_STATE_PRE_CHECK)
+
+        # Mock the db API call. Each test will override the return value
+        p = mock.patch('dcmanager.db.api.subcloud_get')
+        self.mock_db_query = p.start()
+        self.addCleanup(p.stop)
+
+    def test_upgrade_pre_check_subcloud_online(self):
+        """Test pre check step where the subcloud is online.
+
+        The pre-check should transition in this scenario to the first state
+        of a normal upgrade orchestation which is 'installing license'.
+        """
+
+        # subcloud is online
+        self.mock_db_query.return_value = \
+            FakeSubcloud(availability_status=consts.AVAILABILITY_ONLINE)
+
+        # invoke the strategy state operation on the orch thread
+        self.worker.perform_state_action(self.strategy_step)
+
+        # verify the DB query was invoked
+        self.mock_db_query.assert_called()
+
+        # Verify the expected next state happened (installing license)
+        self.assert_step_updated(self.strategy_step.subcloud_id,
+                                 consts.STRATEGY_STATE_INSTALLING_LICENSE)
+
+    def test_upgrade_pre_check_subcloud_jumps_to_migrating(self):
+        """Test pre check step which jumps to the migrating data state
+
+        The pre-check should transition in this scenario to the migrating data
+        state if the subcloud is now offline, and the deploy status can be
+        handled by that state.
+        """
+
+        # subcloud is offline but deploy_state of 'installed' should allow
+        # the upgrade to resume at the 'migrating data' state
+        self.mock_db_query.return_value = FakeSubcloud(
+            availability_status=consts.AVAILABILITY_OFFLINE,
+            deploy_status=consts.DEPLOY_STATE_INSTALLED)
+
+        # invoke the strategy state operation on the orch thread
+        self.worker.perform_state_action(self.strategy_step)
+
+        # verify the DB query was invoked
+        self.mock_db_query.assert_called()
+
+        # Verify the expected next state happened (migrating data)
+        self.assert_step_updated(self.strategy_step.subcloud_id,
+                                 consts.STRATEGY_STATE_MIGRATING_DATA)
+
+    def test_upgrade_pre_check_subcloud_jumps_to_upgrading(self):
+        """Test pre check step which jumps to the upgrading state
+
+        The pre-check should transition in this scenario to the upgrading
+        state if the subcloud is now offline, and the deploy status can be
+        handled by that state.
+        """
+
+        # subcloud is offline but deploy_status of 'migration failed'
+        # should be recoverable by an upgrade
+        self.mock_db_query.return_value = FakeSubcloud(
+            availability_status=consts.AVAILABILITY_OFFLINE,
+            deploy_status=consts.DEPLOY_STATE_DATA_MIGRATION_FAILED)
+
+        # invoke the strategy state operation on the orch thread
+        self.worker.perform_state_action(self.strategy_step)
+
+        # verify the DB query was invoked
+        self.mock_db_query.assert_called()
+
+        # Verify the expected next state happened (upgrading)
+        self.assert_step_updated(self.strategy_step.subcloud_id,
+                                 consts.STRATEGY_STATE_UPGRADING_SIMPLEX)
+
+    def test_upgrade_pre_check_subcloud_cannot_proceed(self):
+        """Test pre check step which requires manual intervention to proceed
+
+        The pre-check should raise an exception and transition to the failed
+        state when an offline subcloud is not in a deploy_status that has a
+        known recovery route.
+        """
+
+        # subcloud is offline and there does not appear to be a way to revover
+        self.mock_db_query.return_value = FakeSubcloud(
+            availability_status=consts.AVAILABILITY_OFFLINE,
+            deploy_status=consts.DEPLOY_STATE_BOOTSTRAP_FAILED)
+
+        # invoke the strategy state operation on the orch thread
+        self.worker.perform_state_action(self.strategy_step)
+
+        # verify the DB query was invoked
+        self.mock_db_query.assert_called()
+
+        # Verify the exception caused the state to go to failed
+        self.assert_step_updated(self.strategy_step.subcloud_id,
+                                 consts.STRATEGY_STATE_FAILED)