Merge "Implement import-load and starting-upgrade strategy states"
This commit is contained in:
commit
94f6dc9fb4
@ -235,6 +235,11 @@ class SysinvClient(base.DriverBase):
|
|||||||
LOG.error("delete_load exception={}".format(e))
|
LOG.error("delete_load exception={}".format(e))
|
||||||
raise e
|
raise e
|
||||||
|
|
||||||
|
def import_load(self, path_to_iso, path_to_sig):
|
||||||
|
"""Import the particular software load."""
|
||||||
|
return self.sysinv_client.load.import_load(path_to_iso=path_to_iso,
|
||||||
|
path_to_sig=path_to_sig)
|
||||||
|
|
||||||
def get_hosts(self):
|
def get_hosts(self):
|
||||||
"""Get a list of hosts."""
|
"""Get a list of hosts."""
|
||||||
return self.sysinv_client.ihost.list()
|
return self.sysinv_client.ihost.list()
|
||||||
@ -243,6 +248,21 @@ class SysinvClient(base.DriverBase):
|
|||||||
"""Get a list of upgrades."""
|
"""Get a list of upgrades."""
|
||||||
return self.sysinv_client.upgrade.list()
|
return self.sysinv_client.upgrade.list()
|
||||||
|
|
||||||
|
def upgrade_activate(self):
|
||||||
|
"""Invoke the API for 'system upgrade-activate', which is an update """
|
||||||
|
patch = [{'op': 'replace',
|
||||||
|
'path': '/state',
|
||||||
|
'value': 'activation-requested'}, ]
|
||||||
|
return self.sysinv_client.upgrade.update(patch)
|
||||||
|
|
||||||
|
def upgrade_complete(self):
|
||||||
|
"""Invoke the API for 'system upgrade-complete', which is a delete"""
|
||||||
|
return self.sysinv_client.upgrade.delete()
|
||||||
|
|
||||||
|
def upgrade_start(self, force=False):
|
||||||
|
"""Invoke the API for 'system upgrade-start', which is a create"""
|
||||||
|
return self.sysinv_client.upgrade.create(force)
|
||||||
|
|
||||||
def get_applications(self):
|
def get_applications(self):
|
||||||
"""Get a list of containerized applications"""
|
"""Get a list of containerized applications"""
|
||||||
|
|
||||||
|
@ -108,7 +108,8 @@ STRATEGY_STATE_LOCKING_CONTROLLER = "locking controller"
|
|||||||
STRATEGY_STATE_UPGRADING_SIMPLEX = "upgrading simplex"
|
STRATEGY_STATE_UPGRADING_SIMPLEX = "upgrading simplex"
|
||||||
STRATEGY_STATE_MIGRATING_DATA = "migrating data"
|
STRATEGY_STATE_MIGRATING_DATA = "migrating data"
|
||||||
STRATEGY_STATE_UNLOCKING_CONTROLLER = "unlocking controller"
|
STRATEGY_STATE_UNLOCKING_CONTROLLER = "unlocking controller"
|
||||||
STRATEGY_STATE_ACTIVATING = "activating"
|
STRATEGY_STATE_ACTIVATING_UPGRADE = "activating upgrade"
|
||||||
|
STRATEGY_STATE_COMPLETING_UPGRADE = "completing upgrade"
|
||||||
|
|
||||||
# Subcloud deploy status states
|
# Subcloud deploy status states
|
||||||
DEPLOY_STATE_NONE = 'not-deployed'
|
DEPLOY_STATE_NONE = 'not-deployed'
|
||||||
|
@ -178,6 +178,10 @@ class LicenseMissingError(DCManagerException):
|
|||||||
message = _("License does not exist on subcloud: %(subcloud_id)s")
|
message = _("License does not exist on subcloud: %(subcloud_id)s")
|
||||||
|
|
||||||
|
|
||||||
|
class VaultLoadMissingError(DCManagerException):
|
||||||
|
message = _("No matching: %(file_type) found in vault: %(vault_dir)")
|
||||||
|
|
||||||
|
|
||||||
class StrategyStepNotFound(NotFound):
|
class StrategyStepNotFound(NotFound):
|
||||||
message = _("StrategyStep with subcloud_id %(subcloud_id)s "
|
message = _("StrategyStep with subcloud_id %(subcloud_id)s "
|
||||||
"doesn't exist.")
|
"doesn't exist.")
|
||||||
|
@ -28,6 +28,13 @@ class BaseState(object):
|
|||||||
self.get_region_name(strategy_step),
|
self.get_region_name(strategy_step),
|
||||||
details))
|
details))
|
||||||
|
|
||||||
|
def info_log(self, strategy_step, details):
|
||||||
|
LOG.info("Stage: %s, State: %s, Subcloud: %s, Details: %s"
|
||||||
|
% (strategy_step.stage,
|
||||||
|
strategy_step.state,
|
||||||
|
self.get_region_name(strategy_step),
|
||||||
|
details))
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_region_name(strategy_step):
|
def get_region_name(strategy_step):
|
||||||
"""Get the region name for a strategy step"""
|
"""Get the region name for a strategy step"""
|
||||||
|
@ -5,12 +5,9 @@
|
|||||||
#
|
#
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from oslo_log import log as logging
|
|
||||||
|
|
||||||
from dcmanager.common.consts import ADMIN_LOCKED
|
from dcmanager.common.consts import ADMIN_LOCKED
|
||||||
from dcmanager.manager.states.base import BaseState
|
from dcmanager.manager.states.base import BaseState
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
|
||||||
DEFAULT_MAX_QUERIES = 6
|
DEFAULT_MAX_QUERIES = 6
|
||||||
DEFAULT_SLEEP_DURATION = 10
|
DEFAULT_SLEEP_DURATION = 10
|
||||||
|
|
||||||
@ -46,7 +43,7 @@ class LockHostState(BaseState):
|
|||||||
if host.administrative == ADMIN_LOCKED:
|
if host.administrative == ADMIN_LOCKED:
|
||||||
msg = "Host: %s already: %s." % (self.target_hostname,
|
msg = "Host: %s already: %s." % (self.target_hostname,
|
||||||
host.administrative)
|
host.administrative)
|
||||||
self.debug_log(strategy_step, msg)
|
self.info_log(strategy_step, msg)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
# Invoke the action
|
# Invoke the action
|
||||||
@ -63,7 +60,7 @@ class LockHostState(BaseState):
|
|||||||
if host.administrative == ADMIN_LOCKED:
|
if host.administrative == ADMIN_LOCKED:
|
||||||
msg = "Host: %s is now: %s" % (self.target_hostname,
|
msg = "Host: %s is now: %s" % (self.target_hostname,
|
||||||
host.administrative)
|
host.administrative)
|
||||||
self.debug_log(strategy_step, msg)
|
self.info_log(strategy_step, msg)
|
||||||
break
|
break
|
||||||
counter += 1
|
counter += 1
|
||||||
if counter >= self.max_queries:
|
if counter >= self.max_queries:
|
||||||
|
@ -5,12 +5,9 @@
|
|||||||
#
|
#
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from oslo_log import log as logging
|
|
||||||
|
|
||||||
from dcmanager.common.consts import ADMIN_UNLOCKED
|
from dcmanager.common.consts import ADMIN_UNLOCKED
|
||||||
from dcmanager.manager.states.base import BaseState
|
from dcmanager.manager.states.base import BaseState
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
|
||||||
DEFAULT_MAX_QUERIES = 6
|
DEFAULT_MAX_QUERIES = 6
|
||||||
DEFAULT_SLEEP_DURATION = 10
|
DEFAULT_SLEEP_DURATION = 10
|
||||||
|
|
||||||
@ -51,7 +48,7 @@ class UnlockHostState(BaseState):
|
|||||||
if host.administrative == ADMIN_UNLOCKED:
|
if host.administrative == ADMIN_UNLOCKED:
|
||||||
msg = "Host: %s already: %s." % (self.target_hostname,
|
msg = "Host: %s already: %s." % (self.target_hostname,
|
||||||
host.administrative)
|
host.administrative)
|
||||||
self.debug_log(strategy_step, msg)
|
self.info_log(strategy_step, msg)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
# Invoke the action
|
# Invoke the action
|
||||||
@ -68,7 +65,7 @@ class UnlockHostState(BaseState):
|
|||||||
if host.administrative == ADMIN_UNLOCKED:
|
if host.administrative == ADMIN_UNLOCKED:
|
||||||
msg = "Host: %s is now: %s" % (self.target_hostname,
|
msg = "Host: %s is now: %s" % (self.target_hostname,
|
||||||
host.administrative)
|
host.administrative)
|
||||||
self.debug_log(strategy_step, msg)
|
self.info_log(strategy_step, msg)
|
||||||
break
|
break
|
||||||
async_counter += 1
|
async_counter += 1
|
||||||
# check_async_counter throws exception if loops exceeded or aborted
|
# check_async_counter throws exception if loops exceeded or aborted
|
||||||
|
@ -3,18 +3,19 @@
|
|||||||
#
|
#
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
#
|
#
|
||||||
from oslo_log import log as logging
|
|
||||||
|
|
||||||
from dcmanager.manager.states.base import BaseState
|
from dcmanager.manager.states.base import BaseState
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
ALREADY_ACTIVATING_STATES = ['activation-requested',
|
||||||
|
'activation-failed',
|
||||||
|
'activation-complete',
|
||||||
|
'activating']
|
||||||
|
|
||||||
|
|
||||||
class ActivatingState(BaseState):
|
class ActivatingUpgradeState(BaseState):
|
||||||
"""Upgrade state actions for activating an upgrade"""
|
"""Upgrade state actions for activating an upgrade"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super(ActivatingState, self).__init__()
|
super(ActivatingUpgradeState, self).__init__()
|
||||||
|
|
||||||
def perform_state_action(self, strategy_step):
|
def perform_state_action(self, strategy_step):
|
||||||
"""Activate an upgrade on a subcloud
|
"""Activate an upgrade on a subcloud
|
||||||
@ -22,9 +23,28 @@ class ActivatingState(BaseState):
|
|||||||
Any exceptions raised by this method set the strategy to FAILED
|
Any exceptions raised by this method set the strategy to FAILED
|
||||||
Returning normally from this method set the strategy to the next step
|
Returning normally from this method set the strategy to the next step
|
||||||
"""
|
"""
|
||||||
LOG.warning("ActivatingState has not been implemented yet.")
|
# get the keystone and sysinv clients for the subcloud
|
||||||
|
ks_client = self.get_keystone_client(strategy_step.subcloud.name)
|
||||||
|
sysinv_client = self.get_sysinv_client(strategy_step.subcloud.name,
|
||||||
|
ks_client.session)
|
||||||
|
upgrades = sysinv_client.get_upgrades()
|
||||||
|
|
||||||
|
# If there are no existing upgrades, there is nothing to activate
|
||||||
|
if len(upgrades) == 0:
|
||||||
|
raise Exception("No upgrades were found to activate")
|
||||||
|
|
||||||
|
# The list of upgrades will never contain more than one entry.
|
||||||
|
for upgrade in upgrades:
|
||||||
|
# Check if an existing upgrade is already activated
|
||||||
|
if upgrade.state in ALREADY_ACTIVATING_STATES:
|
||||||
|
self.info_log(strategy_step,
|
||||||
|
"Already in activating state:%s" % upgrade.state)
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
# invoke the API 'upgrade-activate'.
|
||||||
|
# Throws an exception on failure (no upgrade found, bad host state)
|
||||||
|
sysinv_client.upgrade_activate()
|
||||||
|
|
||||||
# When we return from this method without throwing an exception, the
|
# When we return from this method without throwing an exception, the
|
||||||
# state machine can proceed to the next state
|
# state machine can proceed to the next state
|
||||||
LOG.warning("Faking transition to next state")
|
|
||||||
return True
|
return True
|
||||||
|
@ -3,18 +3,14 @@
|
|||||||
#
|
#
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
#
|
#
|
||||||
from oslo_log import log as logging
|
|
||||||
|
|
||||||
from dcmanager.manager.states.base import BaseState
|
from dcmanager.manager.states.base import BaseState
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
class CompletingUpgradeState(BaseState):
|
||||||
class CompletingState(BaseState):
|
|
||||||
"""Upgrade state actions for completing an upgrade"""
|
"""Upgrade state actions for completing an upgrade"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super(CompletingState, self).__init__()
|
super(CompletingUpgradeState, self).__init__()
|
||||||
|
|
||||||
def perform_state_action(self, strategy_step):
|
def perform_state_action(self, strategy_step):
|
||||||
"""Complete an upgrade on a subcloud
|
"""Complete an upgrade on a subcloud
|
||||||
@ -22,9 +18,24 @@ class CompletingState(BaseState):
|
|||||||
Any exceptions raised by this method set the strategy to FAILED
|
Any exceptions raised by this method set the strategy to FAILED
|
||||||
Returning normally from this method set the strategy to the next step
|
Returning normally from this method set the strategy to the next step
|
||||||
"""
|
"""
|
||||||
LOG.warning("CompletingState has not been implemented yet.")
|
# get the keystone and sysinv clients for the subcloud
|
||||||
|
ks_client = self.get_keystone_client(strategy_step.subcloud.name)
|
||||||
|
sysinv_client = self.get_sysinv_client(strategy_step.subcloud.name,
|
||||||
|
ks_client.session)
|
||||||
|
|
||||||
|
# upgrade-complete causes the upgrade to be deleted.
|
||||||
|
# if no upgrade exists, there is no need to call it.
|
||||||
|
# The API should always return a list, but check for None anyways
|
||||||
|
upgrades = sysinv_client.get_upgrades()
|
||||||
|
if len(upgrades) == 0:
|
||||||
|
self.info_log(strategy_step,
|
||||||
|
"No upgrades exist. Nothing needs completing")
|
||||||
|
return True
|
||||||
|
|
||||||
|
# invoke the API 'upgrade-complete'
|
||||||
|
# This is a blocking call that raises an exception on failure.
|
||||||
|
sysinv_client.upgrade_complete()
|
||||||
|
|
||||||
# When we return from this method without throwing an exception, the
|
# When we return from this method without throwing an exception, the
|
||||||
# state machine can proceed to the next state
|
# state machine can proceed to the next state
|
||||||
LOG.warning("Faking transition to next state")
|
|
||||||
return True
|
return True
|
||||||
|
@ -1,30 +0,0 @@
|
|||||||
#
|
|
||||||
# Copyright (c) 2020 Wind River Systems, Inc.
|
|
||||||
#
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
#
|
|
||||||
from oslo_log import log as logging
|
|
||||||
|
|
||||||
from dcmanager.manager.states.base import BaseState
|
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
class ImportLoadState(BaseState):
|
|
||||||
"""Upgrade state for importing a load"""
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
super(ImportLoadState, self).__init__()
|
|
||||||
|
|
||||||
def perform_state_action(self, strategy_step):
|
|
||||||
"""Import a load on a subcloud
|
|
||||||
|
|
||||||
Any exceptions raised by this method set the strategy to FAILED
|
|
||||||
Returning normally from this method set the strategy to the next step
|
|
||||||
"""
|
|
||||||
LOG.warning("ImportLoadState has not been implemented yet.")
|
|
||||||
|
|
||||||
# When we return from this method without throwing an exception, the
|
|
||||||
# state machine can proceed to the next state
|
|
||||||
LOG.warning("Faking transition to next state")
|
|
||||||
return True
|
|
@ -0,0 +1,55 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
#
|
||||||
|
from dcmanager.common import consts
|
||||||
|
from dcmanager.manager.states.base import BaseState
|
||||||
|
from dcmanager.manager.states.upgrade import utils
|
||||||
|
|
||||||
|
|
||||||
|
class ImportingLoadState(BaseState):
|
||||||
|
"""Upgrade state for importing a load"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super(ImportingLoadState, self).__init__()
|
||||||
|
|
||||||
|
def perform_state_action(self, strategy_step):
|
||||||
|
"""Import a load on a subcloud
|
||||||
|
|
||||||
|
Any exceptions raised by this method set the strategy to FAILED
|
||||||
|
Returning normally from this method set the strategy to the next step
|
||||||
|
"""
|
||||||
|
# determine the version of the system controller in region one
|
||||||
|
local_ks_client = self.get_keystone_client()
|
||||||
|
local_sysinv_client = \
|
||||||
|
self.get_sysinv_client(consts.DEFAULT_REGION_NAME,
|
||||||
|
local_ks_client.session)
|
||||||
|
target_version = local_sysinv_client.get_system().software_version
|
||||||
|
|
||||||
|
# get the keystone and sysinv clients for the subcloud
|
||||||
|
ks_client = self.get_keystone_client(strategy_step.subcloud.name)
|
||||||
|
sysinv_client = self.get_sysinv_client(strategy_step.subcloud.name,
|
||||||
|
ks_client.session)
|
||||||
|
|
||||||
|
# Check if the load is already imported by checking what the version is
|
||||||
|
current_loads = sysinv_client.get_loads()
|
||||||
|
for load in current_loads:
|
||||||
|
if load.software_version == target_version:
|
||||||
|
self.info_log(strategy_step,
|
||||||
|
"Load:%s already found" % target_version)
|
||||||
|
return True
|
||||||
|
|
||||||
|
# If we are here, the load needs to be imported
|
||||||
|
# ISO and SIG files are found in the vault under a version directory
|
||||||
|
iso_path, sig_path = utils.get_vault_load_files(target_version)
|
||||||
|
|
||||||
|
# Call the API.
|
||||||
|
imported_load = sysinv_client.import_load(iso_path, sig_path)
|
||||||
|
new_load = imported_load.get('new_load', {})
|
||||||
|
if new_load.get('software_version') != target_version:
|
||||||
|
raise Exception("The imported load was not the expected version")
|
||||||
|
|
||||||
|
# When we return from this method without throwing an exception, the
|
||||||
|
# state machine can proceed to the next state
|
||||||
|
return True
|
@ -0,0 +1,86 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
#
|
||||||
|
from dcmanager.common import consts
|
||||||
|
from dcmanager.common import exceptions
|
||||||
|
from dcmanager.manager.states.base import BaseState
|
||||||
|
|
||||||
|
# When a license is not installed, this will be part of the API error string
|
||||||
|
LICENSE_FILE_NOT_FOUND_SUBSTRING = "License file not found"
|
||||||
|
|
||||||
|
|
||||||
|
class InstallingLicenseState(BaseState):
|
||||||
|
"""Upgrade state action for installing a license"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super(InstallingLicenseState, self).__init__()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def license_up_to_date(target_license, existing_license):
|
||||||
|
return target_license == existing_license
|
||||||
|
|
||||||
|
def perform_state_action(self, strategy_step):
|
||||||
|
"""Install the License for a software upgrade in this subcloud
|
||||||
|
|
||||||
|
Any exceptions raised by this method set the strategy to FAILED
|
||||||
|
Returning normally from this method set the strategy to the next step
|
||||||
|
"""
|
||||||
|
|
||||||
|
# check if the the system controller has a license
|
||||||
|
local_ks_client = self.get_keystone_client()
|
||||||
|
local_sysinv_client = \
|
||||||
|
self.get_sysinv_client(consts.DEFAULT_REGION_NAME,
|
||||||
|
local_ks_client.session)
|
||||||
|
system_controller_license = local_sysinv_client.get_license()
|
||||||
|
# get_license returns a dictionary with keys: content and error
|
||||||
|
# 'content' can be an empty string in success or failure case.
|
||||||
|
# 'error' is an empty string only in success case.
|
||||||
|
target_license = system_controller_license.get('content')
|
||||||
|
target_error = system_controller_license.get('error')
|
||||||
|
|
||||||
|
# If the system controller does not have a license, do not attempt
|
||||||
|
# to install licenses on subclouds, simply proceed to the next stage
|
||||||
|
if len(target_error) != 0:
|
||||||
|
if LICENSE_FILE_NOT_FOUND_SUBSTRING in target_error:
|
||||||
|
self.info_log(strategy_step,
|
||||||
|
"System Controller License missing: %s."
|
||||||
|
% target_error)
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
# An unexpected error occurred querying the license
|
||||||
|
raise exceptions.LicenseInstallError(
|
||||||
|
subcloud_id=consts.SYSTEM_CONTROLLER_NAME)
|
||||||
|
|
||||||
|
# retrieve the keystone session for the subcloud and query its license
|
||||||
|
subcloud_ks_client = \
|
||||||
|
self.get_keystone_client(strategy_step.subcloud.name)
|
||||||
|
subcloud_sysinv_client = \
|
||||||
|
self.get_sysinv_client(strategy_step.subcloud.name,
|
||||||
|
subcloud_ks_client.session)
|
||||||
|
subcloud_license_response = subcloud_sysinv_client.get_license()
|
||||||
|
subcloud_license = subcloud_license_response.get('content')
|
||||||
|
subcloud_error = subcloud_license_response.get('error')
|
||||||
|
|
||||||
|
# Skip license install if the license is already up to date
|
||||||
|
# If there was not an error, there might be a license
|
||||||
|
if len(subcloud_error) == 0:
|
||||||
|
if self.license_up_to_date(target_license, subcloud_license):
|
||||||
|
self.info_log(strategy_step, "License up to date.")
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
self.debug_log(strategy_step, "License mismatch. Updating.")
|
||||||
|
else:
|
||||||
|
self.debug_log(strategy_step, "License missing. Installing.")
|
||||||
|
|
||||||
|
# Install the license
|
||||||
|
install_rc = subcloud_sysinv_client.install_license(target_license)
|
||||||
|
install_error = install_rc.get('error')
|
||||||
|
if len(install_error) != 0:
|
||||||
|
raise exceptions.LicenseInstallError(
|
||||||
|
subcloud_id=strategy_step.subcloud_id)
|
||||||
|
|
||||||
|
# The license has been successfully installed. Move to the next stage
|
||||||
|
self.info_log(strategy_step, "License installed.")
|
||||||
|
return True
|
@ -3,18 +3,15 @@
|
|||||||
#
|
#
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
#
|
#
|
||||||
from oslo_log import log as logging
|
|
||||||
|
|
||||||
from dcmanager.manager.states.base import BaseState
|
from dcmanager.manager.states.base import BaseState
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
class StartingUpgradeState(BaseState):
|
class StartingUpgradeState(BaseState):
|
||||||
"""Upgrade state for starting an upgrade on a subcloud"""
|
"""Upgrade state for starting an upgrade on a subcloud"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self, force=False):
|
||||||
super(StartingUpgradeState, self).__init__()
|
super(StartingUpgradeState, self).__init__()
|
||||||
|
self.force = force
|
||||||
|
|
||||||
def perform_state_action(self, strategy_step):
|
def perform_state_action(self, strategy_step):
|
||||||
"""Start an upgrade on a subcloud
|
"""Start an upgrade on a subcloud
|
||||||
@ -22,9 +19,24 @@ class StartingUpgradeState(BaseState):
|
|||||||
Any exceptions raised by this method set the strategy to FAILED
|
Any exceptions raised by this method set the strategy to FAILED
|
||||||
Returning normally from this method set the strategy to the next step
|
Returning normally from this method set the strategy to the next step
|
||||||
"""
|
"""
|
||||||
LOG.warning("StartingUpgradeState has not been implemented yet.")
|
# get the keystone and sysinv clients for the subcloud
|
||||||
|
ks_client = self.get_keystone_client(strategy_step.subcloud.name)
|
||||||
|
sysinv_client = self.get_sysinv_client(strategy_step.subcloud.name,
|
||||||
|
ks_client.session)
|
||||||
|
|
||||||
|
# Check if an existing upgrade is already in progress.
|
||||||
|
# The list of upgrades will never contain more than one entry.
|
||||||
|
upgrades = sysinv_client.get_upgrades()
|
||||||
|
if upgrades is not None and len(upgrades) > 0:
|
||||||
|
for upgrade in upgrades:
|
||||||
|
# If a previous upgrade exists (even one that failed) skip
|
||||||
|
self.info_log(strategy_step,
|
||||||
|
"An upgrade already exists: %s" % upgrade)
|
||||||
|
else:
|
||||||
|
# invoke the API 'upgrade-start'.
|
||||||
|
# This call is synchronous and throws an exception on failure.
|
||||||
|
sysinv_client.upgrade_start(self.force)
|
||||||
|
|
||||||
# When we return from this method without throwing an exception, the
|
# When we return from this method without throwing an exception, the
|
||||||
# state machine can proceed to the next state
|
# state machine can proceed to the next state
|
||||||
LOG.warning("Faking transition to next state")
|
|
||||||
return True
|
return True
|
||||||
|
38
distributedcloud/dcmanager/manager/states/upgrade/utils.py
Normal file
38
distributedcloud/dcmanager/manager/states/upgrade/utils.py
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
#
|
||||||
|
import os
|
||||||
|
|
||||||
|
from dcmanager.common.exceptions import VaultLoadMissingError
|
||||||
|
|
||||||
|
VAULT_LOADS_PATH = '/opt/dc-vault/loads'
|
||||||
|
|
||||||
|
|
||||||
|
def get_vault_load_files(target_version):
|
||||||
|
"""Return a tuple for the ISO and SIG for this load version from the vault.
|
||||||
|
|
||||||
|
The files can be imported to the vault using any name, but must end
|
||||||
|
in 'iso' or 'sig'.
|
||||||
|
: param target_version: The software version to search under the vault
|
||||||
|
"""
|
||||||
|
vault_dir = "{}/{}/".format(VAULT_LOADS_PATH, target_version)
|
||||||
|
|
||||||
|
matching_iso = None
|
||||||
|
matching_sig = None
|
||||||
|
for a_file in os.listdir(vault_dir):
|
||||||
|
if a_file.lower().endswith(".iso"):
|
||||||
|
matching_iso = os.path.join(vault_dir, a_file)
|
||||||
|
continue
|
||||||
|
elif a_file.lower().endswith(".sig"):
|
||||||
|
matching_sig = os.path.join(vault_dir, a_file)
|
||||||
|
continue
|
||||||
|
# If no .iso or .sig is found, raise an exception
|
||||||
|
if matching_iso is None:
|
||||||
|
raise VaultLoadMissingError(file_type='.iso', vault_dir=vault_dir)
|
||||||
|
if matching_sig is None:
|
||||||
|
raise VaultLoadMissingError(file_type='.sig', vault_dir=vault_dir)
|
||||||
|
|
||||||
|
# return the iso and sig for this load
|
||||||
|
return (matching_iso, matching_sig)
|
@ -25,19 +25,52 @@ import time
|
|||||||
|
|
||||||
from oslo_log import log as logging
|
from oslo_log import log as logging
|
||||||
|
|
||||||
from dccommon.drivers.openstack.sdk_platform import OpenStackDriver
|
|
||||||
from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
|
|
||||||
|
|
||||||
from dcmanager.common import consts
|
from dcmanager.common import consts
|
||||||
from dcmanager.common import context
|
from dcmanager.common import context
|
||||||
from dcmanager.common import exceptions
|
from dcmanager.common import exceptions
|
||||||
from dcmanager.common import scheduler
|
from dcmanager.common import scheduler
|
||||||
from dcmanager.db import api as db_api
|
from dcmanager.db import api as db_api
|
||||||
|
from dcmanager.manager.states.lock_host import LockHostState
|
||||||
|
from dcmanager.manager.states.unlock_host import UnlockHostState
|
||||||
|
from dcmanager.manager.states.upgrade.activating import ActivatingUpgradeState
|
||||||
|
from dcmanager.manager.states.upgrade.completing import CompletingUpgradeState
|
||||||
|
from dcmanager.manager.states.upgrade.importing_load import ImportingLoadState
|
||||||
|
from dcmanager.manager.states.upgrade.installing_license \
|
||||||
|
import InstallingLicenseState
|
||||||
|
from dcmanager.manager.states.upgrade.migrating_data \
|
||||||
|
import MigratingDataState
|
||||||
|
from dcmanager.manager.states.upgrade.starting_upgrade \
|
||||||
|
import StartingUpgradeState
|
||||||
|
from dcmanager.manager.states.upgrade.upgrading_simplex \
|
||||||
|
import UpgradingSimplexState
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
# When a license is not installed, this will be part of the API error string
|
# The state machine transition order for an APPLY
|
||||||
LICENSE_FILE_NOT_FOUND_SUBSTRING = "License file not found"
|
ORDERED_STATES = [
|
||||||
|
consts.STRATEGY_STATE_INSTALLING_LICENSE,
|
||||||
|
consts.STRATEGY_STATE_IMPORTING_LOAD,
|
||||||
|
consts.STRATEGY_STATE_STARTING_UPGRADE,
|
||||||
|
consts.STRATEGY_STATE_LOCKING_CONTROLLER,
|
||||||
|
consts.STRATEGY_STATE_UPGRADING_SIMPLEX,
|
||||||
|
consts.STRATEGY_STATE_MIGRATING_DATA,
|
||||||
|
consts.STRATEGY_STATE_UNLOCKING_CONTROLLER,
|
||||||
|
consts.STRATEGY_STATE_ACTIVATING_UPGRADE,
|
||||||
|
consts.STRATEGY_STATE_COMPLETING_UPGRADE,
|
||||||
|
]
|
||||||
|
|
||||||
|
# every state in ORDERED_STATES should have an operator
|
||||||
|
STATE_OPERATORS = {
|
||||||
|
consts.STRATEGY_STATE_INSTALLING_LICENSE: InstallingLicenseState,
|
||||||
|
consts.STRATEGY_STATE_IMPORTING_LOAD: ImportingLoadState,
|
||||||
|
consts.STRATEGY_STATE_STARTING_UPGRADE: StartingUpgradeState,
|
||||||
|
consts.STRATEGY_STATE_LOCKING_CONTROLLER: LockHostState,
|
||||||
|
consts.STRATEGY_STATE_UPGRADING_SIMPLEX: UpgradingSimplexState,
|
||||||
|
consts.STRATEGY_STATE_MIGRATING_DATA: MigratingDataState,
|
||||||
|
consts.STRATEGY_STATE_UNLOCKING_CONTROLLER: UnlockHostState,
|
||||||
|
consts.STRATEGY_STATE_ACTIVATING_UPGRADE: ActivatingUpgradeState,
|
||||||
|
consts.STRATEGY_STATE_COMPLETING_UPGRADE: CompletingUpgradeState,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class SwUpgradeOrchThread(threading.Thread):
|
class SwUpgradeOrchThread(threading.Thread):
|
||||||
@ -72,7 +105,7 @@ class SwUpgradeOrchThread(threading.Thread):
|
|||||||
self.subcloud_workers = dict()
|
self.subcloud_workers = dict()
|
||||||
|
|
||||||
# When an upgrade is initiated, this is the first state
|
# When an upgrade is initiated, this is the first state
|
||||||
self.starting_state = consts.STRATEGY_STATE_INSTALLING_LICENSE
|
self.starting_state = ORDERED_STATES[0]
|
||||||
|
|
||||||
def stopped(self):
|
def stopped(self):
|
||||||
return self._stop.isSet()
|
return self._stop.isSet()
|
||||||
@ -87,22 +120,6 @@ class SwUpgradeOrchThread(threading.Thread):
|
|||||||
self.thread_group_manager.stop()
|
self.thread_group_manager.stop()
|
||||||
LOG.info("SwUpgradeOrchThread Stopped")
|
LOG.info("SwUpgradeOrchThread Stopped")
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def get_ks_client(region_name=consts.DEFAULT_REGION_NAME):
|
|
||||||
"""This will get a cached keystone client (and token)"""
|
|
||||||
try:
|
|
||||||
os_client = OpenStackDriver(
|
|
||||||
region_name=region_name,
|
|
||||||
region_clients=None)
|
|
||||||
return os_client.keystone_client
|
|
||||||
except Exception:
|
|
||||||
LOG.warn('Failure initializing KeystoneClient')
|
|
||||||
raise
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def get_sysinv_client(region_name, session):
|
|
||||||
return SysinvClient(region_name, session)
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_region_name(strategy_step):
|
def get_region_name(strategy_step):
|
||||||
"""Get the region name for a strategy step"""
|
"""Get the region name for a strategy step"""
|
||||||
@ -121,8 +138,22 @@ class SwUpgradeOrchThread(threading.Thread):
|
|||||||
return details
|
return details
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def license_up_to_date(target_license, existing_license):
|
def determine_state_operator(strategy_step):
|
||||||
return target_license == existing_license
|
"""Return the state operator for the current state"""
|
||||||
|
state_operator = STATE_OPERATORS.get(strategy_step.state)
|
||||||
|
# instantiate and return the state_operator class
|
||||||
|
return state_operator()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def determine_next_state(strategy_step):
|
||||||
|
"""Return next state for the strategy step based on current state."""
|
||||||
|
# todo(abailey): next_state may differ for AIO, STD, etc.. subclouds
|
||||||
|
next_index = ORDERED_STATES.index(strategy_step.state) + 1
|
||||||
|
if next_index < len(ORDERED_STATES):
|
||||||
|
next_state = ORDERED_STATES[next_index]
|
||||||
|
else:
|
||||||
|
next_state = consts.STRATEGY_STATE_COMPLETE
|
||||||
|
return next_state
|
||||||
|
|
||||||
def strategy_step_update(self, subcloud_id, state=None, details=None):
|
def strategy_step_update(self, subcloud_id, state=None, details=None):
|
||||||
"""Update the strategy step in the DB
|
"""Update the strategy step in the DB
|
||||||
@ -137,13 +168,13 @@ class SwUpgradeOrchThread(threading.Thread):
|
|||||||
consts.STRATEGY_STATE_ABORTED,
|
consts.STRATEGY_STATE_ABORTED,
|
||||||
consts.STRATEGY_STATE_FAILED]:
|
consts.STRATEGY_STATE_FAILED]:
|
||||||
finished_at = datetime.datetime.now()
|
finished_at = datetime.datetime.now()
|
||||||
db_api.strategy_step_update(
|
# Return the updated object, in case we need to use its updated values
|
||||||
self.context,
|
return db_api.strategy_step_update(self.context,
|
||||||
subcloud_id,
|
subcloud_id,
|
||||||
state=state,
|
state=state,
|
||||||
details=details,
|
details=details,
|
||||||
started_at=started_at,
|
started_at=started_at,
|
||||||
finished_at=finished_at)
|
finished_at=finished_at)
|
||||||
|
|
||||||
def upgrade_orch(self):
|
def upgrade_orch(self):
|
||||||
while not self.stopped():
|
while not self.stopped():
|
||||||
@ -288,216 +319,23 @@ class SwUpgradeOrchThread(threading.Thread):
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
# We are just getting started, enter the first state
|
# We are just getting started, enter the first state
|
||||||
self.strategy_step_update(
|
# Use the updated value for calling process_upgrade_step
|
||||||
|
strategy_step = self.strategy_step_update(
|
||||||
strategy_step.subcloud_id,
|
strategy_step.subcloud_id,
|
||||||
state=consts.STRATEGY_STATE_INSTALLING_LICENSE)
|
state=self.starting_state)
|
||||||
|
# Starting state should log an error if greenthread exists
|
||||||
# Initial step should log an error if a greenthread exists
|
|
||||||
# All other steps should not.
|
|
||||||
self.process_upgrade_step(region,
|
self.process_upgrade_step(region,
|
||||||
strategy_step,
|
strategy_step,
|
||||||
self.install_subcloud_license,
|
|
||||||
log_error=True)
|
log_error=True)
|
||||||
# todo(abailey): state and their method invoked can be managed
|
|
||||||
# using a dictionary to make this more maintainable.
|
|
||||||
elif strategy_step.state == \
|
|
||||||
consts.STRATEGY_STATE_INSTALLING_LICENSE:
|
|
||||||
self.process_upgrade_step(region,
|
|
||||||
strategy_step,
|
|
||||||
self.install_subcloud_license,
|
|
||||||
log_error=False)
|
|
||||||
elif strategy_step.state == \
|
|
||||||
consts.STRATEGY_STATE_IMPORTING_LOAD:
|
|
||||||
self.process_upgrade_step(region,
|
|
||||||
strategy_step,
|
|
||||||
self.import_subcloud_load,
|
|
||||||
log_error=False)
|
|
||||||
elif strategy_step.state == \
|
|
||||||
consts.STRATEGY_STATE_STARTING_UPGRADE:
|
|
||||||
self.process_upgrade_step(region,
|
|
||||||
strategy_step,
|
|
||||||
self.start_subcloud_upgrade,
|
|
||||||
log_error=False)
|
|
||||||
elif strategy_step.state == \
|
|
||||||
consts.STRATEGY_STATE_LOCKING_CONTROLLER:
|
|
||||||
self.process_upgrade_step(region,
|
|
||||||
strategy_step,
|
|
||||||
self.lock_subcloud_controller,
|
|
||||||
log_error=False)
|
|
||||||
elif strategy_step.state == \
|
|
||||||
consts.STRATEGY_STATE_UPGRADING_SIMPLEX:
|
|
||||||
self.process_upgrade_step(region,
|
|
||||||
strategy_step,
|
|
||||||
self.upgrade_subcloud_simplex,
|
|
||||||
log_error=False)
|
|
||||||
elif strategy_step.state == \
|
|
||||||
consts.STRATEGY_STATE_MIGRATING_DATA:
|
|
||||||
self.process_upgrade_step(region,
|
|
||||||
strategy_step,
|
|
||||||
self.migrate_subcloud_data,
|
|
||||||
log_error=False)
|
|
||||||
elif strategy_step.state == \
|
|
||||||
consts.STRATEGY_STATE_UNLOCKING_CONTROLLER:
|
|
||||||
self.process_upgrade_step(region,
|
|
||||||
strategy_step,
|
|
||||||
self.unlock_subcloud_controller,
|
|
||||||
log_error=False)
|
|
||||||
elif strategy_step.state == \
|
|
||||||
consts.STRATEGY_STATE_ACTIVATING:
|
|
||||||
self.process_upgrade_step(region,
|
|
||||||
strategy_step,
|
|
||||||
self.activate_subcloud,
|
|
||||||
log_error=False)
|
|
||||||
# todo(abailey): Add calls to self.process_upgrade_step
|
|
||||||
# for each additional state, with the appropriate thread
|
|
||||||
# method called.
|
|
||||||
else:
|
else:
|
||||||
LOG.error("Unimplemented state %s" % strategy_step.state)
|
self.process_upgrade_step(region,
|
||||||
self.strategy_step_update(
|
strategy_step,
|
||||||
strategy_step.subcloud_id,
|
log_error=False)
|
||||||
state=consts.STRATEGY_STATE_FAILED,
|
|
||||||
details=("Upgrade state not implemented: %s"
|
|
||||||
% strategy_step.state))
|
|
||||||
|
|
||||||
if self.stopped():
|
if self.stopped():
|
||||||
LOG.info("Exiting because task is stopped")
|
LOG.info("Exiting because task is stopped")
|
||||||
return
|
return
|
||||||
|
|
||||||
def process_upgrade_step(self,
|
|
||||||
region,
|
|
||||||
strategy_step,
|
|
||||||
upgrade_thread_method,
|
|
||||||
log_error=False):
|
|
||||||
if region in self.subcloud_workers:
|
|
||||||
# A worker already exists. Let it finish whatever it was doing.
|
|
||||||
if log_error:
|
|
||||||
LOG.error("Worker should not exist for %s." % region)
|
|
||||||
else:
|
|
||||||
LOG.debug("Update worker exists for %s." % region)
|
|
||||||
else:
|
|
||||||
# Create a greenthread to start processing the upgrade for the
|
|
||||||
# subcloud and invoke the specified upgrade_thread_method
|
|
||||||
self.subcloud_workers[region] = \
|
|
||||||
self.thread_group_manager.start(upgrade_thread_method,
|
|
||||||
strategy_step)
|
|
||||||
|
|
||||||
def install_subcloud_license(self, strategy_step):
|
|
||||||
"""Install the license for the upgrade in this subcloud
|
|
||||||
|
|
||||||
Removes the worker reference after the operation is complete.
|
|
||||||
"""
|
|
||||||
|
|
||||||
try:
|
|
||||||
LOG.info("Stage: %s for subcloud %s"
|
|
||||||
% (strategy_step.stage,
|
|
||||||
self.get_region_name(strategy_step)))
|
|
||||||
self.do_install_subcloud_license(strategy_step)
|
|
||||||
except Exception:
|
|
||||||
# Catch ALL exceptions and set the strategy to failed
|
|
||||||
LOG.exception("Install license failed for %s"
|
|
||||||
% self.get_region_name(strategy_step))
|
|
||||||
self.strategy_step_update(strategy_step.subcloud_id,
|
|
||||||
state=consts.STRATEGY_STATE_FAILED,
|
|
||||||
details=("Install license failed"))
|
|
||||||
finally:
|
|
||||||
# The worker is done.
|
|
||||||
region = self.get_region_name(strategy_step)
|
|
||||||
if region in self.subcloud_workers:
|
|
||||||
del self.subcloud_workers[region]
|
|
||||||
|
|
||||||
def do_install_subcloud_license(self, strategy_step):
|
|
||||||
"""Install the License for a software upgrade in this subcloud"""
|
|
||||||
|
|
||||||
# Note: no need to catch exceptions in this method.
|
|
||||||
|
|
||||||
# next_state is the next state that the strategy will use on
|
|
||||||
# successful completion of this state
|
|
||||||
next_state = consts.STRATEGY_STATE_IMPORTING_LOAD
|
|
||||||
|
|
||||||
# We check the system controller license for system controller and
|
|
||||||
# subclouds
|
|
||||||
local_ks_client = self.get_ks_client()
|
|
||||||
local_sysinv_client = \
|
|
||||||
self.get_sysinv_client(consts.DEFAULT_REGION_NAME,
|
|
||||||
local_ks_client.session)
|
|
||||||
system_controller_license = local_sysinv_client.get_license()
|
|
||||||
# get_license returns a dictionary with keys: content and error
|
|
||||||
# 'content' can be an empty string in success or failure case.
|
|
||||||
# 'error' is an empty string only in success case.
|
|
||||||
target_license = system_controller_license.get('content')
|
|
||||||
target_error = system_controller_license.get('error')
|
|
||||||
|
|
||||||
# If the system controller does not have a license, do not attempt
|
|
||||||
# to install licenses on subclouds, and simply proceed to the next stage
|
|
||||||
if len(target_error) != 0:
|
|
||||||
if LICENSE_FILE_NOT_FOUND_SUBSTRING in target_error:
|
|
||||||
LOG.debug("Stage:<%s>, Subcloud:<%s>. "
|
|
||||||
"System Controller License missing: %s."
|
|
||||||
% (strategy_step.stage,
|
|
||||||
self.get_region_name(strategy_step),
|
|
||||||
target_error))
|
|
||||||
self.strategy_step_update(strategy_step.subcloud_id,
|
|
||||||
state=next_state)
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
# An unexpected API error was returned. Fail this stage.
|
|
||||||
LOG.warning("Stage:<%s>, Subcloud:<%s>. "
|
|
||||||
"System Controller License query failed: %s."
|
|
||||||
% (strategy_step.stage,
|
|
||||||
self.get_region_name(strategy_step),
|
|
||||||
target_error))
|
|
||||||
raise exceptions.LicenseMissingError(
|
|
||||||
subcloud_id=consts.SYSTEM_CONTROLLER_NAME)
|
|
||||||
|
|
||||||
# retrieve the keystone session for the subcloud and query its license
|
|
||||||
subcloud_ks_client = self.get_ks_client(strategy_step.subcloud.name)
|
|
||||||
subcloud_sysinv_client = \
|
|
||||||
self.get_sysinv_client(strategy_step.subcloud.name,
|
|
||||||
subcloud_ks_client.session)
|
|
||||||
subcloud_license_response = subcloud_sysinv_client.get_license()
|
|
||||||
subcloud_license = subcloud_license_response.get('content')
|
|
||||||
subcloud_error = subcloud_license_response.get('error')
|
|
||||||
|
|
||||||
# Skip license install if the license is already up to date
|
|
||||||
# If there was not an error, there might be a license
|
|
||||||
if len(subcloud_error) == 0:
|
|
||||||
if self.license_up_to_date(target_license, subcloud_license):
|
|
||||||
LOG.debug("Stage:<%s>, Subcloud:<%s>. License up to date."
|
|
||||||
% (strategy_step.stage,
|
|
||||||
self.get_region_name(strategy_step)))
|
|
||||||
self.strategy_step_update(strategy_step.subcloud_id,
|
|
||||||
state=next_state)
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
LOG.debug("Stage:<%s>, Subcloud:<%s>. "
|
|
||||||
"License mismatch. Updating."
|
|
||||||
% (strategy_step.stage,
|
|
||||||
self.get_region_name(strategy_step)))
|
|
||||||
else:
|
|
||||||
LOG.debug("Stage:<%s>, Subcloud:<%s>. "
|
|
||||||
"License missing. Installing."
|
|
||||||
% (strategy_step.stage,
|
|
||||||
self.get_region_name(strategy_step)))
|
|
||||||
|
|
||||||
# Install the license
|
|
||||||
install_rc = subcloud_sysinv_client.install_license(target_license)
|
|
||||||
install_error = install_rc.get('error')
|
|
||||||
if len(install_error) != 0:
|
|
||||||
LOG.warning("Stage:<%s>, Subcloud:<%s>. "
|
|
||||||
"License install failed:<%s>."
|
|
||||||
% (strategy_step.stage,
|
|
||||||
self.get_region_name(strategy_step),
|
|
||||||
install_error))
|
|
||||||
raise exceptions.LicenseInstallError(
|
|
||||||
subcloud_id=strategy_step.subcloud_id)
|
|
||||||
|
|
||||||
# The license has been successfully installed. Move to the next stage
|
|
||||||
LOG.debug("Stage:<%s>, Subcloud:<%s>. "
|
|
||||||
"License installed."
|
|
||||||
% (strategy_step.stage,
|
|
||||||
self.get_region_name(strategy_step)))
|
|
||||||
self.strategy_step_update(strategy_step.subcloud_id, state=next_state)
|
|
||||||
|
|
||||||
def abort(self, sw_update_strategy):
|
def abort(self, sw_update_strategy):
|
||||||
"""Abort an upgrade strategy"""
|
"""Abort an upgrade strategy"""
|
||||||
|
|
||||||
@ -536,15 +374,33 @@ class SwUpgradeOrchThread(threading.Thread):
|
|||||||
LOG.exception(e)
|
LOG.exception(e)
|
||||||
raise e
|
raise e
|
||||||
|
|
||||||
def perform_state_action(self, strategy_step, state_operator, next_state):
|
def process_upgrade_step(self, region, strategy_step, log_error=False):
|
||||||
|
"""manage the green thread for calling perform_state_action"""
|
||||||
|
if region in self.subcloud_workers:
|
||||||
|
# A worker already exists. Let it finish whatever it was doing.
|
||||||
|
if log_error:
|
||||||
|
LOG.error("Worker should not exist for %s." % region)
|
||||||
|
else:
|
||||||
|
LOG.debug("Update worker exists for %s." % region)
|
||||||
|
else:
|
||||||
|
# Create a greenthread to start processing the upgrade for the
|
||||||
|
# subcloud and invoke the specified upgrade_thread_method
|
||||||
|
self.subcloud_workers[region] = \
|
||||||
|
self.thread_group_manager.start(self.perform_state_action,
|
||||||
|
strategy_step)
|
||||||
|
|
||||||
|
def perform_state_action(self, strategy_step):
|
||||||
"""Extensible state handler for processing and transitioning states """
|
"""Extensible state handler for processing and transitioning states """
|
||||||
try:
|
try:
|
||||||
LOG.info("Stage: %s, State: %s, Subcloud: %s"
|
LOG.info("Stage: %s, State: %s, Subcloud: %s"
|
||||||
% (strategy_step.stage,
|
% (strategy_step.stage,
|
||||||
strategy_step.state,
|
strategy_step.state,
|
||||||
self.get_region_name(strategy_step)))
|
self.get_region_name(strategy_step)))
|
||||||
|
# Instantiate the state operator and perform the state actions
|
||||||
|
state_operator = self.determine_state_operator(strategy_step)
|
||||||
state_operator.perform_state_action(strategy_step)
|
state_operator.perform_state_action(strategy_step)
|
||||||
# If we get here without an exception raised, proceed to next state
|
# If we get here without an exception raised, proceed to next state
|
||||||
|
next_state = self.determine_next_state(strategy_step)
|
||||||
self.strategy_step_update(strategy_step.subcloud_id,
|
self.strategy_step_update(strategy_step.subcloud_id,
|
||||||
state=next_state)
|
state=next_state)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@ -562,47 +418,3 @@ class SwUpgradeOrchThread(threading.Thread):
|
|||||||
region = self.get_region_name(strategy_step)
|
region = self.get_region_name(strategy_step)
|
||||||
if region in self.subcloud_workers:
|
if region in self.subcloud_workers:
|
||||||
del self.subcloud_workers[region]
|
del self.subcloud_workers[region]
|
||||||
|
|
||||||
# todo(abailey): convert license install to the same pattern as the other states
|
|
||||||
|
|
||||||
def import_subcloud_load(self, strategy_step):
|
|
||||||
from dcmanager.manager.states.upgrade.import_load import ImportLoadState
|
|
||||||
self.perform_state_action(strategy_step,
|
|
||||||
ImportLoadState(),
|
|
||||||
consts.STRATEGY_STATE_STARTING_UPGRADE)
|
|
||||||
|
|
||||||
def start_subcloud_upgrade(self, strategy_step):
|
|
||||||
from dcmanager.manager.states.upgrade.starting_upgrade import StartingUpgradeState
|
|
||||||
self.perform_state_action(strategy_step,
|
|
||||||
StartingUpgradeState(),
|
|
||||||
consts.STRATEGY_STATE_LOCKING_CONTROLLER)
|
|
||||||
|
|
||||||
def lock_subcloud_controller(self, strategy_step):
|
|
||||||
from dcmanager.manager.states.lock_host import LockHostState
|
|
||||||
self.perform_state_action(strategy_step,
|
|
||||||
LockHostState(),
|
|
||||||
consts.STRATEGY_STATE_UPGRADING_SIMPLEX)
|
|
||||||
|
|
||||||
def upgrade_subcloud_simplex(self, strategy_step):
|
|
||||||
from dcmanager.manager.states.upgrade.upgrading_simplex import UpgradingSimplexState
|
|
||||||
self.perform_state_action(strategy_step,
|
|
||||||
UpgradingSimplexState(),
|
|
||||||
consts.STRATEGY_STATE_MIGRATING_DATA)
|
|
||||||
|
|
||||||
def migrate_subcloud_data(self, strategy_step):
|
|
||||||
from dcmanager.manager.states.upgrade.migrating_data import MigratingDataState
|
|
||||||
self.perform_state_action(strategy_step,
|
|
||||||
MigratingDataState(),
|
|
||||||
consts.STRATEGY_STATE_UNLOCKING_CONTROLLER)
|
|
||||||
|
|
||||||
def unlock_subcloud_controller(self, strategy_step):
|
|
||||||
from dcmanager.manager.states.unlock_host import UnlockHostState
|
|
||||||
self.perform_state_action(strategy_step,
|
|
||||||
UnlockHostState(),
|
|
||||||
consts.STRATEGY_STATE_ACTIVATING)
|
|
||||||
|
|
||||||
def activate_subcloud(self, strategy_step):
|
|
||||||
from dcmanager.manager.states.upgrade.activating import ActivatingState
|
|
||||||
self.perform_state_action(strategy_step,
|
|
||||||
ActivatingState(),
|
|
||||||
consts.STRATEGY_STATE_COMPLETE)
|
|
||||||
|
@ -44,7 +44,7 @@ SUBCLOUD_SAMPLE_DATA_0 = [
|
|||||||
"subcloud-4", # name
|
"subcloud-4", # name
|
||||||
"demo subcloud", # description
|
"demo subcloud", # description
|
||||||
"Ottawa-Lab-Aisle_3-Rack_C", # location
|
"Ottawa-Lab-Aisle_3-Rack_C", # location
|
||||||
"20.01", # software-version
|
"12.34", # software-version
|
||||||
"managed", # management-state
|
"managed", # management-state
|
||||||
"online", # availability-status
|
"online", # availability-status
|
||||||
"fd01:3::0/64", # management_subnet
|
"fd01:3::0/64", # management_subnet
|
||||||
|
@ -56,7 +56,7 @@ FAKE_SUBCLOUD_DATA = {"name": "subcloud1",
|
|||||||
|
|
||||||
FAKE_SUBCLOUD_INSTALL_VALUES = {
|
FAKE_SUBCLOUD_INSTALL_VALUES = {
|
||||||
"image": "http://192.168.101.2:8080/iso/bootimage.iso",
|
"image": "http://192.168.101.2:8080/iso/bootimage.iso",
|
||||||
"software_version": "20.01",
|
"software_version": "12.34",
|
||||||
"bootstrap_interface": "eno1",
|
"bootstrap_interface": "eno1",
|
||||||
"bootstrap_address": "128.224.151.183",
|
"bootstrap_address": "128.224.151.183",
|
||||||
"bootstrap_address_prefix": 23,
|
"bootstrap_address_prefix": 23,
|
||||||
|
@ -0,0 +1,91 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
#
|
||||||
|
import mock
|
||||||
|
|
||||||
|
from dcmanager.common import consts
|
||||||
|
|
||||||
|
from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeUpgrade
|
||||||
|
from dcmanager.tests.unit.manager.states.upgrade.test_base \
|
||||||
|
import TestSwUpgradeState
|
||||||
|
|
||||||
|
VALID_UPGRADE = FakeUpgrade(state='imported')
|
||||||
|
ACTIVATING_UPGRADE = FakeUpgrade(state='activation-requested')
|
||||||
|
ALREADY_ACTIVATED_UPGRADE = FakeUpgrade(state='activation-complete')
|
||||||
|
|
||||||
|
|
||||||
|
class TestSwUpgradeActivatingStage(TestSwUpgradeState):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(TestSwUpgradeActivatingStage, self).setUp()
|
||||||
|
|
||||||
|
# next state after activating an upgrade is 'completing'
|
||||||
|
self.on_success_state = consts.STRATEGY_STATE_COMPLETING_UPGRADE
|
||||||
|
|
||||||
|
# Add the strategy_step state being processed by this unit test
|
||||||
|
self.strategy_step = \
|
||||||
|
self.setup_strategy_step(consts.STRATEGY_STATE_ACTIVATING_UPGRADE)
|
||||||
|
|
||||||
|
# Add mock API endpoints for sysinv client calls invcked by this state
|
||||||
|
self.sysinv_client.upgrade_activate = mock.MagicMock()
|
||||||
|
self.sysinv_client.get_upgrades = mock.MagicMock()
|
||||||
|
|
||||||
|
def test_upgrade_subcloud_activating_upgrade_failure(self):
|
||||||
|
"""Test the activating upgrade API call fails."""
|
||||||
|
|
||||||
|
# upgrade_activate will only be called if an appropriate upgrade exists
|
||||||
|
self.sysinv_client.get_upgrades.return_value = [VALID_UPGRADE, ]
|
||||||
|
|
||||||
|
# API call raises an exception when it is rejected
|
||||||
|
self.sysinv_client.upgrade_activate.side_effect = \
|
||||||
|
Exception("upgrade activate failed for some reason")
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# verify the expected API call was invoked
|
||||||
|
self.sysinv_client.upgrade_activate.assert_called()
|
||||||
|
|
||||||
|
# Verify the state moves to 'failed'
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
consts.STRATEGY_STATE_FAILED)
|
||||||
|
|
||||||
|
def test_upgrade_subcloud_activating_upgrade_success(self):
|
||||||
|
"""Test the activating upgrade step succeeds."""
|
||||||
|
|
||||||
|
# upgrade_activate will only be called if an appropriate upgrade exists
|
||||||
|
self.sysinv_client.get_upgrades.return_value = [VALID_UPGRADE, ]
|
||||||
|
|
||||||
|
# API call will not raise an exception, and will return an upgrade
|
||||||
|
self.sysinv_client.upgrade_activate.return_value = ACTIVATING_UPGRADE
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# verify the API cvall was invoked
|
||||||
|
self.sysinv_client.upgrade_activate.assert_called()
|
||||||
|
|
||||||
|
# On success, the state should be updated to the next state
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
self.on_success_state)
|
||||||
|
|
||||||
|
def test_upgrade_subcloud_activating_upgrade_skip_already_activated(self):
|
||||||
|
"""Test the activating upgrade step skipped if already activated."""
|
||||||
|
|
||||||
|
# upgrade_activate will only be called if an appropriate upgrade exists
|
||||||
|
self.sysinv_client.get_upgrades.return_value = \
|
||||||
|
[ALREADY_ACTIVATED_UPGRADE, ]
|
||||||
|
|
||||||
|
# API call will not be invoked, so no need to mock it
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# upgrade is already in one of the activating states so skip activating
|
||||||
|
self.sysinv_client.upgrade_activate.assert_not_called()
|
||||||
|
|
||||||
|
# On success, the state is set to the next state
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
self.on_success_state)
|
@ -4,14 +4,15 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
#
|
#
|
||||||
import mock
|
import mock
|
||||||
|
import uuid
|
||||||
|
|
||||||
from dcmanager.manager.states.base import BaseState
|
from dcmanager.manager.states.base import BaseState
|
||||||
from sysinv.common import constants as sysinv_constants
|
from sysinv.common import constants as sysinv_constants
|
||||||
|
|
||||||
from dcmanager.tests.unit.manager.test_sw_upgrade import TestSwUpgrade
|
from dcmanager.tests.unit.manager.test_sw_upgrade import TestSwUpgrade
|
||||||
|
|
||||||
CURRENT_LOAD = '20.01'
|
PREVIOUS_VERSION = '12.34'
|
||||||
UPDATED_LOAD = '20.06'
|
UPGRADED_VERSION = '56.78'
|
||||||
|
|
||||||
|
|
||||||
class FakeKeystoneClient(object):
|
class FakeKeystoneClient(object):
|
||||||
@ -19,6 +20,47 @@ class FakeKeystoneClient(object):
|
|||||||
self.session = mock.MagicMock()
|
self.session = mock.MagicMock()
|
||||||
|
|
||||||
|
|
||||||
|
class FakeLoad(object):
|
||||||
|
def __init__(self,
|
||||||
|
obj_id,
|
||||||
|
compatible_version='N/A',
|
||||||
|
required_patches='N/A',
|
||||||
|
software_version=PREVIOUS_VERSION,
|
||||||
|
state='active',
|
||||||
|
created_at=None,
|
||||||
|
updated_at=None):
|
||||||
|
self.id = obj_id
|
||||||
|
self.uuid = uuid.uuid4()
|
||||||
|
self.required_patches = required_patches
|
||||||
|
self.software_version = software_version
|
||||||
|
self.state = state
|
||||||
|
self.created_at = created_at
|
||||||
|
self.updated_at = updated_at
|
||||||
|
|
||||||
|
|
||||||
|
class FakeSystem(object):
|
||||||
|
def __init__(self,
|
||||||
|
obj_id=1,
|
||||||
|
software_version=UPGRADED_VERSION):
|
||||||
|
self.id = obj_id
|
||||||
|
self.uuid = uuid.uuid4()
|
||||||
|
self.software_version = software_version
|
||||||
|
|
||||||
|
|
||||||
|
class FakeUpgrade(object):
|
||||||
|
def __init__(self,
|
||||||
|
obj_id=1,
|
||||||
|
state='completed',
|
||||||
|
from_release=PREVIOUS_VERSION,
|
||||||
|
to_release=UPGRADED_VERSION):
|
||||||
|
self.id = obj_id
|
||||||
|
self.uuid = uuid.uuid4()
|
||||||
|
self.state = state
|
||||||
|
self.from_release = from_release
|
||||||
|
self.to_release = to_release
|
||||||
|
self.links = []
|
||||||
|
|
||||||
|
|
||||||
class FakeSysinvClient(object):
|
class FakeSysinvClient(object):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
pass
|
pass
|
||||||
@ -31,7 +73,7 @@ class FakeController(object):
|
|||||||
administrative=sysinv_constants.ADMIN_UNLOCKED,
|
administrative=sysinv_constants.ADMIN_UNLOCKED,
|
||||||
availability=sysinv_constants.AVAILABILITY_AVAILABLE,
|
availability=sysinv_constants.AVAILABILITY_AVAILABLE,
|
||||||
ihost_action=None,
|
ihost_action=None,
|
||||||
target_load=CURRENT_LOAD,
|
target_load=UPGRADED_VERSION,
|
||||||
task=None):
|
task=None):
|
||||||
self.id = host_id
|
self.id = host_id
|
||||||
self.hostname = hostname
|
self.hostname = hostname
|
||||||
|
@ -0,0 +1,90 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
#
|
||||||
|
import mock
|
||||||
|
|
||||||
|
from dcmanager.common import consts
|
||||||
|
|
||||||
|
from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeUpgrade
|
||||||
|
from dcmanager.tests.unit.manager.states.upgrade.test_base \
|
||||||
|
import TestSwUpgradeState
|
||||||
|
|
||||||
|
VALID_UPGRADE = FakeUpgrade(state='activation-complete')
|
||||||
|
INVALID_UPGRADE = FakeUpgrade(state='aborting')
|
||||||
|
|
||||||
|
|
||||||
|
class TestSwUpgradeCompletingStage(TestSwUpgradeState):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(TestSwUpgradeCompletingStage, self).setUp()
|
||||||
|
|
||||||
|
# next state after completing an upgrade is 'complete'
|
||||||
|
self.on_success_state = consts.STRATEGY_STATE_COMPLETE
|
||||||
|
|
||||||
|
# Add the strategy_step state being processed by this unit test
|
||||||
|
self.strategy_step = \
|
||||||
|
self.setup_strategy_step(consts.STRATEGY_STATE_COMPLETING_UPGRADE)
|
||||||
|
|
||||||
|
# Add mock API endpoints for sysinv client calls invcked by this state
|
||||||
|
self.sysinv_client.upgrade_complete = mock.MagicMock()
|
||||||
|
self.sysinv_client.get_upgrades = mock.MagicMock()
|
||||||
|
|
||||||
|
def test_upgrade_subcloud_completing_upgrade_failure(self):
|
||||||
|
"""Test the completing upgrade API call fails."""
|
||||||
|
|
||||||
|
# upgrade_complete will only be called if an appropriate upgrade exists
|
||||||
|
self.sysinv_client.get_upgrades.return_value = [VALID_UPGRADE, ]
|
||||||
|
|
||||||
|
# API call raises an exception when it is rejected
|
||||||
|
self.sysinv_client.upgrade_complete.side_effect = \
|
||||||
|
Exception("upgrade complete failed for some reason")
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# verify the expected API call was invoked
|
||||||
|
self.sysinv_client.upgrade_complete.assert_called()
|
||||||
|
|
||||||
|
# Verify the state moves to 'failed'
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
consts.STRATEGY_STATE_FAILED)
|
||||||
|
|
||||||
|
def test_upgrade_subcloud_completing_upgrade_success(self):
|
||||||
|
"""Test the completing upgrade step succeeds."""
|
||||||
|
|
||||||
|
# upgrade_complete will only be called if an appropriate upgrade exists
|
||||||
|
self.sysinv_client.get_upgrades.return_value = [VALID_UPGRADE, ]
|
||||||
|
|
||||||
|
# API call will not raise an exception. It will delete the upgrade
|
||||||
|
self.sysinv_client.upgrade_complete.return_value = None
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# verify the API cvall was invoked
|
||||||
|
self.sysinv_client.upgrade_complete.assert_called()
|
||||||
|
|
||||||
|
# On success, the state should be updated to the next state
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
self.on_success_state)
|
||||||
|
|
||||||
|
def test_upgrade_subcloud_completing_upgrade_skip_already_completed(self):
|
||||||
|
"""Test the completing upgrade step skipped if already completed."""
|
||||||
|
|
||||||
|
# upgrade_complete will only be called if an appropriate upgrade exists
|
||||||
|
# If the upgrade has been deleted, there is nothing to complete
|
||||||
|
self.sysinv_client.get_upgrades.return_value = []
|
||||||
|
|
||||||
|
# API call will not be invoked, so no need to mock it
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# upgrade is already in one of the completing states so skip completing
|
||||||
|
self.sysinv_client.upgrade_complete.assert_not_called()
|
||||||
|
|
||||||
|
# On success, the state is set to the next state
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
self.on_success_state)
|
@ -0,0 +1,155 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
#
|
||||||
|
import mock
|
||||||
|
|
||||||
|
from dcmanager.common import consts
|
||||||
|
from dcmanager.common.exceptions import VaultLoadMissingError
|
||||||
|
|
||||||
|
from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeLoad
|
||||||
|
from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeSystem
|
||||||
|
from dcmanager.tests.unit.manager.states.upgrade.test_base \
|
||||||
|
import PREVIOUS_VERSION
|
||||||
|
from dcmanager.tests.unit.manager.states.upgrade.test_base \
|
||||||
|
import TestSwUpgradeState
|
||||||
|
from dcmanager.tests.unit.manager.states.upgrade.test_base \
|
||||||
|
import UPGRADED_VERSION
|
||||||
|
|
||||||
|
PREVIOUS_LOAD = FakeLoad(1, software_version=PREVIOUS_VERSION)
|
||||||
|
UPGRADED_LOAD = FakeLoad(2,
|
||||||
|
compatible_version=PREVIOUS_VERSION,
|
||||||
|
software_version=UPGRADED_VERSION)
|
||||||
|
|
||||||
|
DEST_LOAD_EXISTS = [PREVIOUS_LOAD, UPGRADED_LOAD, ]
|
||||||
|
DEST_LOAD_MISSING = [PREVIOUS_LOAD, ]
|
||||||
|
|
||||||
|
FAKE_ISO = '/opt/dc-vault/loads/' + UPGRADED_VERSION + '/bootimage.iso'
|
||||||
|
FAKE_SIG = '/opt/dc-vault/loads/' + UPGRADED_VERSION + '/bootimage.sig'
|
||||||
|
|
||||||
|
FAILED_IMPORT_RESPONSE = 'kaboom'
|
||||||
|
SUCCESS_IMPORT_RESPONSE = {
|
||||||
|
'new_load': {
|
||||||
|
'id': 2,
|
||||||
|
'uuid': 'aaa4b4c6-8536-41f6-87ea-211d208a723b',
|
||||||
|
'compatible_version': PREVIOUS_VERSION,
|
||||||
|
'required_patches': '',
|
||||||
|
'software_version': UPGRADED_VERSION,
|
||||||
|
'state': 'importing',
|
||||||
|
'created_at': '2020-06-01 12:12:12+00:00',
|
||||||
|
'updated_at': None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class TestSwUpgradeImportingLoadStage(TestSwUpgradeState):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(TestSwUpgradeImportingLoadStage, self).setUp()
|
||||||
|
|
||||||
|
# next state after 'importing load' is 'starting upgrade'
|
||||||
|
self.on_success_state = consts.STRATEGY_STATE_STARTING_UPGRADE
|
||||||
|
|
||||||
|
# Add the strategy_step state being processed by this unit test
|
||||||
|
self.strategy_step = \
|
||||||
|
self.setup_strategy_step(consts.STRATEGY_STATE_IMPORTING_LOAD)
|
||||||
|
|
||||||
|
# Add mock API endpoints for sysinv client calls invcked by this state
|
||||||
|
self.sysinv_client.get_system = mock.MagicMock()
|
||||||
|
self.sysinv_client.get_system.return_value = FakeSystem()
|
||||||
|
self.sysinv_client.get_loads = mock.MagicMock()
|
||||||
|
self.sysinv_client.import_load = mock.MagicMock()
|
||||||
|
|
||||||
|
@mock.patch('dcmanager.manager.states.upgrade.utils.get_vault_load_files')
|
||||||
|
def test_upgrade_subcloud_importing_load_failure(self, mock_vault_files):
|
||||||
|
"""Test importing load step where the import_load API call fails."""
|
||||||
|
|
||||||
|
# simulate determine_matching_load finding the iso and sig in the vault
|
||||||
|
mock_vault_files.return_value = (FAKE_ISO, FAKE_SIG)
|
||||||
|
|
||||||
|
# Simulate the target load has not been imported yet on the subcloud
|
||||||
|
self.sysinv_client.get_loads.return_value = DEST_LOAD_MISSING
|
||||||
|
|
||||||
|
# Simulate an API failure on the subcloud.
|
||||||
|
self.sysinv_client.import_load.return_value = FAILED_IMPORT_RESPONSE
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# verify the import load API call was invoked
|
||||||
|
self.sysinv_client.import_load.assert_called()
|
||||||
|
|
||||||
|
# Verify a failure leads to a state failure
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
consts.STRATEGY_STATE_FAILED)
|
||||||
|
|
||||||
|
@mock.patch('dcmanager.manager.states.upgrade.utils.get_vault_load_files')
|
||||||
|
def test_upgrade_subcloud_importing_load_success(self, mock_vault_files):
|
||||||
|
"""Test the importing load step succeeds.
|
||||||
|
|
||||||
|
The load will be imported on the subcloud when the subcloud does not
|
||||||
|
have the load already imported, and the API call succeeds to import it.
|
||||||
|
"""
|
||||||
|
# simulate determine_matching_load finding the iso and sig in the vault
|
||||||
|
mock_vault_files.return_value = (FAKE_ISO, FAKE_SIG)
|
||||||
|
|
||||||
|
# Simulate the target load has not been imported yet on the subcloud
|
||||||
|
self.sysinv_client.get_loads.return_value = DEST_LOAD_MISSING
|
||||||
|
|
||||||
|
# Simulate an API success on the subcloud.
|
||||||
|
self.sysinv_client.import_load.return_value = SUCCESS_IMPORT_RESPONSE
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# verify the import load API call was invoked
|
||||||
|
self.sysinv_client.import_load.assert_called()
|
||||||
|
|
||||||
|
# On success, should have moved to the next state
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
self.on_success_state)
|
||||||
|
|
||||||
|
@mock.patch('dcmanager.manager.states.upgrade.utils.get_vault_load_files')
|
||||||
|
def test_upgrade_subcloud_importing_load_fails_missing_vault_files(
|
||||||
|
self,
|
||||||
|
mock_determine_matching_load):
|
||||||
|
"""Test importing load fails when files are not in the vault."""
|
||||||
|
|
||||||
|
mock_determine_matching_load.side_effect = \
|
||||||
|
VaultLoadMissingError(file_type='.iso', vault_dir='/mock/vault/')
|
||||||
|
|
||||||
|
# Simulate the target load has not been imported yet on the subcloud
|
||||||
|
self.sysinv_client.get_loads.return_value = DEST_LOAD_MISSING
|
||||||
|
|
||||||
|
# Simulate an API success on the subcloud. It should not get here.
|
||||||
|
self.sysinv_client.import_load.return_value = SUCCESS_IMPORT_RESPONSE
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# verify the import load API call was never invoked
|
||||||
|
self.sysinv_client.import_load.assert_not_called()
|
||||||
|
|
||||||
|
# Verify a failure leads to a state failure
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
consts.STRATEGY_STATE_FAILED)
|
||||||
|
|
||||||
|
def test_upgrade_subcloud_importing_load_skip_existing(self):
|
||||||
|
"""Test the importing load step skipped due to load already there"""
|
||||||
|
|
||||||
|
# Simulate the target load has been previously imported on the subcloud
|
||||||
|
self.sysinv_client.get_loads.return_value = DEST_LOAD_EXISTS
|
||||||
|
|
||||||
|
# Simulate an API failure for import_load. It should not be called.
|
||||||
|
self.sysinv_client.import_load.return_value = FAILED_IMPORT_RESPONSE
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# The import_load should not have been attempted
|
||||||
|
self.sysinv_client.import_load.assert_not_called()
|
||||||
|
|
||||||
|
# On success, should have moved to the next state
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
self.on_success_state)
|
@ -0,0 +1,158 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
#
|
||||||
|
import mock
|
||||||
|
|
||||||
|
from dcmanager.common import consts
|
||||||
|
|
||||||
|
from dcmanager.tests.unit.manager.states.upgrade.test_base \
|
||||||
|
import TestSwUpgradeState
|
||||||
|
|
||||||
|
MISSING_LICENSE_RESPONSE = {
|
||||||
|
u'content': u'',
|
||||||
|
u'error': u'License file not found. A license may not have been installed.'
|
||||||
|
}
|
||||||
|
|
||||||
|
LICENSE_VALID_RESPONSE = {
|
||||||
|
u'content': u'A valid license',
|
||||||
|
u'error': u''
|
||||||
|
}
|
||||||
|
|
||||||
|
ALTERNATE_LICENSE_RESPONSE = {
|
||||||
|
u'content': u'A different valid license',
|
||||||
|
u'error': u''
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class TestSwUpgradeInstallingLicenseStage(TestSwUpgradeState):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(TestSwUpgradeInstallingLicenseStage, self).setUp()
|
||||||
|
|
||||||
|
# next state after installing a license is 'importing load'
|
||||||
|
self.on_success_state = consts.STRATEGY_STATE_IMPORTING_LOAD
|
||||||
|
|
||||||
|
# Add the strategy_step state being processed by this unit test
|
||||||
|
self.strategy_step = \
|
||||||
|
self.setup_strategy_step(consts.STRATEGY_STATE_INSTALLING_LICENSE)
|
||||||
|
|
||||||
|
# Add mock API endpoints for sysinv client calls invcked by this state
|
||||||
|
self.sysinv_client.get_license = mock.MagicMock()
|
||||||
|
self.sysinv_client.install_license = mock.MagicMock()
|
||||||
|
|
||||||
|
def test_upgrade_subcloud_license_install_failure(self):
|
||||||
|
"""Test the installing license step where the install fails.
|
||||||
|
|
||||||
|
The system controller has a license, but the API call to install on the
|
||||||
|
subcloud fails.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Order of get_license calls:
|
||||||
|
# first license query is to system controller
|
||||||
|
# second license query is to subcloud (should be missing)
|
||||||
|
self.sysinv_client.get_license.side_effect = [LICENSE_VALID_RESPONSE,
|
||||||
|
MISSING_LICENSE_RESPONSE]
|
||||||
|
|
||||||
|
# Simulate a license install failure on the subcloud
|
||||||
|
self.sysinv_client.install_license.return_value = \
|
||||||
|
MISSING_LICENSE_RESPONSE
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# verify the license install was invoked
|
||||||
|
self.sysinv_client.install_license.assert_called()
|
||||||
|
|
||||||
|
# Verify a install_license failure leads to a state failure
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
consts.STRATEGY_STATE_FAILED)
|
||||||
|
|
||||||
|
def test_upgrade_subcloud_license_install_success(self):
|
||||||
|
"""Test the install license step succeeds.
|
||||||
|
|
||||||
|
The license will be installed on the subcloud when system controller
|
||||||
|
has a license, the subcloud does not have a license, and the API call
|
||||||
|
succeeds.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Order of get_license calls:
|
||||||
|
# first license query is to system controller
|
||||||
|
# second license query is to subcloud (should be missing)
|
||||||
|
self.sysinv_client.get_license.side_effect = [LICENSE_VALID_RESPONSE,
|
||||||
|
MISSING_LICENSE_RESPONSE]
|
||||||
|
|
||||||
|
# A license install should return a success
|
||||||
|
self.sysinv_client.install_license.return_value = \
|
||||||
|
LICENSE_VALID_RESPONSE
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# verify the license install was invoked
|
||||||
|
self.sysinv_client.install_license.assert_called()
|
||||||
|
|
||||||
|
# On success, the next state after installing license is importing load
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
self.on_success_state)
|
||||||
|
|
||||||
|
def test_upgrade_subcloud_license_skip_existing(self):
|
||||||
|
"""Test the install license step skipped due to license up to date"""
|
||||||
|
|
||||||
|
# Order of get_license calls:
|
||||||
|
# first license query is to system controller
|
||||||
|
# second license query is to subcloud
|
||||||
|
self.sysinv_client.get_license.side_effect = [LICENSE_VALID_RESPONSE,
|
||||||
|
LICENSE_VALID_RESPONSE]
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# A license install should not have been attempted due to the license
|
||||||
|
# already being up to date
|
||||||
|
self.sysinv_client.install_license.assert_not_called()
|
||||||
|
|
||||||
|
# On success, the next state after installing license is importing load
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
self.on_success_state)
|
||||||
|
|
||||||
|
def test_upgrade_subcloud_license_overrides_mismatched_license(self):
|
||||||
|
"""Test the install license overrides a mismatched license"""
|
||||||
|
|
||||||
|
# Order of get_license calls:
|
||||||
|
# first license query is to system controller
|
||||||
|
# second license query is to subcloud (should be valid but different)
|
||||||
|
self.sysinv_client.get_license.side_effect = \
|
||||||
|
[LICENSE_VALID_RESPONSE,
|
||||||
|
ALTERNATE_LICENSE_RESPONSE]
|
||||||
|
|
||||||
|
# A license install should return a success
|
||||||
|
self.sysinv_client.install_license.return_value = \
|
||||||
|
LICENSE_VALID_RESPONSE
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# verify the license install was invoked
|
||||||
|
self.sysinv_client.install_license.assert_called()
|
||||||
|
|
||||||
|
# Verify it successfully moves to the next step
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
self.on_success_state)
|
||||||
|
|
||||||
|
def test_upgrade_subcloud_license_skip_when_no_sys_controller_lic(self):
|
||||||
|
"""Test license install skipped when no license on system controller."""
|
||||||
|
|
||||||
|
# Only makes one query: to system controller
|
||||||
|
self.sysinv_client.get_license.return_value = MISSING_LICENSE_RESPONSE
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# Should skip install_license API call
|
||||||
|
self.sysinv_client.install_license.assert_not_called()
|
||||||
|
|
||||||
|
# Verify it successfully moves to the next step
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
self.on_success_state)
|
@ -56,8 +56,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState):
|
|||||||
# mock the API call as failed on the subcloud
|
# mock the API call as failed on the subcloud
|
||||||
self.sysinv_client.lock_host.return_value = CONTROLLER_0_LOCKING
|
self.sysinv_client.lock_host.return_value = CONTROLLER_0_LOCKING
|
||||||
|
|
||||||
# invoke the strategy state operation
|
# invoke the strategy state operation on the orch thread
|
||||||
self.worker.lock_subcloud_controller(self.strategy_step)
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
# verify the lock command was actually attempted
|
# verify the lock command was actually attempted
|
||||||
self.sysinv_client.lock_host.assert_called()
|
self.sysinv_client.lock_host.assert_called()
|
||||||
@ -72,8 +72,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState):
|
|||||||
# mock the controller host query as being already locked
|
# mock the controller host query as being already locked
|
||||||
self.sysinv_client.get_host.return_value = CONTROLLER_0_LOCKED
|
self.sysinv_client.get_host.return_value = CONTROLLER_0_LOCKED
|
||||||
|
|
||||||
# invoke the strategy state operation
|
# invoke the strategy state operation on the orch thread
|
||||||
self.worker.lock_subcloud_controller(self.strategy_step)
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
# verify the lock command was never attempted
|
# verify the lock command was never attempted
|
||||||
self.sysinv_client.lock_host.assert_not_called()
|
self.sysinv_client.lock_host.assert_not_called()
|
||||||
@ -95,8 +95,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState):
|
|||||||
# mock the API call as successful on the subcloud
|
# mock the API call as successful on the subcloud
|
||||||
self.sysinv_client.lock_host.return_value = CONTROLLER_0_LOCKING
|
self.sysinv_client.lock_host.return_value = CONTROLLER_0_LOCKING
|
||||||
|
|
||||||
# invoke the strategy state operation
|
# invoke the strategy state operation on the orch thread
|
||||||
self.worker.lock_subcloud_controller(self.strategy_step)
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
# verify the lock command was actually attempted
|
# verify the lock command was actually attempted
|
||||||
self.sysinv_client.lock_host.assert_called()
|
self.sysinv_client.lock_host.assert_called()
|
||||||
@ -118,8 +118,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState):
|
|||||||
# mock the API call as failed on the subcloud
|
# mock the API call as failed on the subcloud
|
||||||
self.sysinv_client.lock_host.return_value = CONTROLLER_0_LOCKING_FAILED
|
self.sysinv_client.lock_host.return_value = CONTROLLER_0_LOCKING_FAILED
|
||||||
|
|
||||||
# invoke the strategy state operation
|
# invoke the strategy state operation on the orch thread
|
||||||
self.worker.lock_subcloud_controller(self.strategy_step)
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
# verify the lock command was actually attempted
|
# verify the lock command was actually attempted
|
||||||
self.sysinv_client.lock_host.assert_called()
|
self.sysinv_client.lock_host.assert_called()
|
||||||
@ -135,8 +135,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState):
|
|||||||
self.sysinv_client.get_host.side_effect = \
|
self.sysinv_client.get_host.side_effect = \
|
||||||
Exception("Unable to find host controller-0")
|
Exception("Unable to find host controller-0")
|
||||||
|
|
||||||
# invoke the strategy state operation
|
# invoke the strategy state operation on the orch thread
|
||||||
self.worker.lock_subcloud_controller(self.strategy_step)
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
# verify the lock command was never attempted
|
# verify the lock command was never attempted
|
||||||
self.sysinv_client.lock_host.assert_not_called()
|
self.sysinv_client.lock_host.assert_not_called()
|
||||||
|
@ -0,0 +1,124 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
#
|
||||||
|
import mock
|
||||||
|
|
||||||
|
from dcmanager.common import consts
|
||||||
|
|
||||||
|
from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeUpgrade
|
||||||
|
from dcmanager.tests.unit.manager.states.upgrade.test_base \
|
||||||
|
import TestSwUpgradeState
|
||||||
|
|
||||||
|
UPGRADE_ABORTING = [FakeUpgrade(state='aborting'), ]
|
||||||
|
UPGRADE_STARTED = [FakeUpgrade(state='started'), ]
|
||||||
|
SUCCESS_UPGRADE_START = 'I do not know what this looks like yet'
|
||||||
|
|
||||||
|
|
||||||
|
class TestSwUpgradeStartingUpgradeStage(TestSwUpgradeState):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(TestSwUpgradeStartingUpgradeStage, self).setUp()
|
||||||
|
|
||||||
|
# next state after 'starting upgrade' is 'migrating data'
|
||||||
|
self.on_success_state = consts.STRATEGY_STATE_LOCKING_CONTROLLER
|
||||||
|
|
||||||
|
# Add the strategy_step state being processed by this unit test
|
||||||
|
self.strategy_step = \
|
||||||
|
self.setup_strategy_step(consts.STRATEGY_STATE_STARTING_UPGRADE)
|
||||||
|
|
||||||
|
# Add mock API endpoints for sysinv client calls invcked by this state
|
||||||
|
self.sysinv_client.upgrade_start = mock.MagicMock()
|
||||||
|
self.sysinv_client.get_upgrades = mock.MagicMock()
|
||||||
|
|
||||||
|
def test_upgrade_subcloud_upgrade_start_failure(self):
|
||||||
|
"""Test the upgrade_start where the API call fails.
|
||||||
|
|
||||||
|
The upgrade_start call fails due to a validation check such as from
|
||||||
|
the health-query check.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# No upgrades should yet exist in the DB / API
|
||||||
|
self.sysinv_client.get_upgrades.return_value = []
|
||||||
|
|
||||||
|
# Simulate an upgrade_start failure on the subcloud.
|
||||||
|
# The API throws an exception rather than returning an error response
|
||||||
|
self.sysinv_client.upgrade_start.side_effect = \
|
||||||
|
Exception("HTTPBadRequest: upgrade-start rejected: "
|
||||||
|
"System is not in a valid state for upgrades. "
|
||||||
|
"Run system health-query-upgrade for more details.")
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# verify the API call that failed was invoked
|
||||||
|
self.sysinv_client.upgrade_start.assert_called()
|
||||||
|
|
||||||
|
# Verify the API failure leads to a state failure
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
consts.STRATEGY_STATE_FAILED)
|
||||||
|
|
||||||
|
def test_upgrade_subcloud_upgrade_start_success(self):
|
||||||
|
"""Test upgrade_start where the API call succeeds.
|
||||||
|
|
||||||
|
This will result in an upgrade being created with the appropriate
|
||||||
|
state.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# No upgrades should yet exist in the DB / API
|
||||||
|
self.sysinv_client.get_upgrades.return_value = []
|
||||||
|
|
||||||
|
# Simulate an upgrade_start succeeds on the subcloud
|
||||||
|
self.sysinv_client.upgrade_start.return_value = SUCCESS_UPGRADE_START
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# verify the API call that succeeded was actually invoked
|
||||||
|
self.sysinv_client.upgrade_start.assert_called()
|
||||||
|
|
||||||
|
# On success, the state should transition to the next state
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
self.on_success_state)
|
||||||
|
|
||||||
|
def test_upgrade_subcloud_upgrade_start_skip_already_started(self):
|
||||||
|
"""Test upgrade_start where the upgrade is already started."""
|
||||||
|
|
||||||
|
# An already started upgrade exists in the DB"""
|
||||||
|
self.sysinv_client.get_upgrades.return_value = [UPGRADE_STARTED, ]
|
||||||
|
|
||||||
|
# upgrade_start should not be invoked, so can be mocked as 'failed'
|
||||||
|
# by raising an exception
|
||||||
|
self.sysinv_client.upgrade_start.side_effect = \
|
||||||
|
Exception("HTTPBadRequest: this is a fake exception")
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# upgrade_start API call should not have been attempted due to the
|
||||||
|
# existing upgrade already in started state.
|
||||||
|
self.sysinv_client.upgrade_start.assert_not_called()
|
||||||
|
|
||||||
|
# On success, the state should transition to the next state
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
self.on_success_state)
|
||||||
|
|
||||||
|
def test_upgrade_subcloud_upgrade_start_fails_bad_existing_upgrade(self):
|
||||||
|
"""Test the upgrade_start fails due to a bad existing upgrade."""
|
||||||
|
|
||||||
|
# An already started upgrade exists in the DB but is in bad shape."""
|
||||||
|
self.sysinv_client.get_upgrades.return_value = [UPGRADE_ABORTING, ]
|
||||||
|
|
||||||
|
# upgrade_start will NOT be invoked. No need to mock it.
|
||||||
|
|
||||||
|
# invoke the strategy state operation on the orch thread
|
||||||
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
|
# upgrade_start API call should not have been attempted due to the
|
||||||
|
# invalid existing upgrade that needs to be cleaned up.
|
||||||
|
self.sysinv_client.upgrade_start.assert_not_called()
|
||||||
|
|
||||||
|
# Verify it failed and moves to the next step
|
||||||
|
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||||
|
self.on_success_state)
|
@ -31,7 +31,7 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState):
|
|||||||
super(TestSwUpgradeUnlockControllerStage, self).setUp()
|
super(TestSwUpgradeUnlockControllerStage, self).setUp()
|
||||||
|
|
||||||
# next state after a successful unlock is 'activating'
|
# next state after a successful unlock is 'activating'
|
||||||
self.on_success_state = consts.STRATEGY_STATE_ACTIVATING
|
self.on_success_state = consts.STRATEGY_STATE_ACTIVATING_UPGRADE
|
||||||
|
|
||||||
# Add the strategy_step state being processed by this unit test
|
# Add the strategy_step state being processed by this unit test
|
||||||
self.strategy_step = self.setup_strategy_step(
|
self.strategy_step = self.setup_strategy_step(
|
||||||
@ -56,8 +56,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState):
|
|||||||
# mock the API call as failed on the subcloud
|
# mock the API call as failed on the subcloud
|
||||||
self.sysinv_client.unlock_host.return_value = CONTROLLER_0_UNLOCKING
|
self.sysinv_client.unlock_host.return_value = CONTROLLER_0_UNLOCKING
|
||||||
|
|
||||||
# invoke the strategy state operation
|
# invoke the strategy state operation on the orch thread
|
||||||
self.worker.unlock_subcloud_controller(self.strategy_step)
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
# verify the unlock command was actually attempted
|
# verify the unlock command was actually attempted
|
||||||
self.sysinv_client.unlock_host.assert_called()
|
self.sysinv_client.unlock_host.assert_called()
|
||||||
@ -72,8 +72,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState):
|
|||||||
# mock the controller host query as being already unlocked
|
# mock the controller host query as being already unlocked
|
||||||
self.sysinv_client.get_host.return_value = CONTROLLER_0_UNLOCKED
|
self.sysinv_client.get_host.return_value = CONTROLLER_0_UNLOCKED
|
||||||
|
|
||||||
# invoke the strategy state operation
|
# invoke the strategy state operation on the orch thread
|
||||||
self.worker.unlock_subcloud_controller(self.strategy_step)
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
# verify the unlock command was never attempted
|
# verify the unlock command was never attempted
|
||||||
self.sysinv_client.unlock_host.assert_not_called()
|
self.sysinv_client.unlock_host.assert_not_called()
|
||||||
@ -95,8 +95,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState):
|
|||||||
# mock the API call as successful on the subcloud
|
# mock the API call as successful on the subcloud
|
||||||
self.sysinv_client.unlock_host.return_value = CONTROLLER_0_UNLOCKING
|
self.sysinv_client.unlock_host.return_value = CONTROLLER_0_UNLOCKING
|
||||||
|
|
||||||
# invoke the strategy state operation
|
# invoke the strategy state operation on the orch thread
|
||||||
self.worker.unlock_subcloud_controller(self.strategy_step)
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
# verify the lock command was actually attempted
|
# verify the lock command was actually attempted
|
||||||
self.sysinv_client.unlock_host.assert_called()
|
self.sysinv_client.unlock_host.assert_called()
|
||||||
@ -119,8 +119,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState):
|
|||||||
self.sysinv_client.unlock_host.return_value = \
|
self.sysinv_client.unlock_host.return_value = \
|
||||||
CONTROLLER_0_UNLOCKING_FAILED
|
CONTROLLER_0_UNLOCKING_FAILED
|
||||||
|
|
||||||
# invoke the strategy state operation
|
# invoke the strategy state operation on the orch thread
|
||||||
self.worker.unlock_subcloud_controller(self.strategy_step)
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
# verify the unlock command was actually attempted
|
# verify the unlock command was actually attempted
|
||||||
self.sysinv_client.unlock_host.assert_called()
|
self.sysinv_client.unlock_host.assert_called()
|
||||||
@ -136,8 +136,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState):
|
|||||||
self.sysinv_client.get_host.side_effect = \
|
self.sysinv_client.get_host.side_effect = \
|
||||||
Exception("Unable to find host controller-0")
|
Exception("Unable to find host controller-0")
|
||||||
|
|
||||||
# invoke the strategy state operation
|
# invoke the strategy state operation on the orch thread
|
||||||
self.worker.unlock_subcloud_controller(self.strategy_step)
|
self.worker.perform_state_action(self.strategy_step)
|
||||||
|
|
||||||
# verify the unlock command was never attempted
|
# verify the unlock command was never attempted
|
||||||
self.sysinv_client.unlock_host.assert_not_called()
|
self.sysinv_client.unlock_host.assert_not_called()
|
||||||
|
@ -33,7 +33,6 @@ from dcmanager.tests.unit.manager.test_sw_update_manager \
|
|||||||
import Subcloud
|
import Subcloud
|
||||||
from dcmanager.tests import utils
|
from dcmanager.tests import utils
|
||||||
|
|
||||||
|
|
||||||
CONF = cfg.CONF
|
CONF = cfg.CONF
|
||||||
FAKE_ID = '1'
|
FAKE_ID = '1'
|
||||||
FAKE_SW_UPDATE_DATA = {
|
FAKE_SW_UPDATE_DATA = {
|
||||||
@ -53,29 +52,6 @@ FAKE_STRATEGY_STEP_DATA = {
|
|||||||
"subcloud": None
|
"subcloud": None
|
||||||
}
|
}
|
||||||
|
|
||||||
MISSING_LICENSE_RESPONSE = {
|
|
||||||
u'content': u'',
|
|
||||||
u'error': u'License file not found. A license may not have been installed.'
|
|
||||||
}
|
|
||||||
|
|
||||||
LICENSE_VALID_RESPONSE = {
|
|
||||||
u'content': u'A valid license',
|
|
||||||
u'error': u''
|
|
||||||
}
|
|
||||||
|
|
||||||
ALTERNATE_LICENSE_RESPONSE = {
|
|
||||||
u'content': u'A different valid license',
|
|
||||||
u'error': u''
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class FakeSysinvClient(object):
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
super(FakeSysinvClient, self).__init__()
|
|
||||||
self.get_license = mock.MagicMock()
|
|
||||||
self.install_license = mock.MagicMock()
|
|
||||||
|
|
||||||
|
|
||||||
class TestSwUpgrade(base.DCManagerTestCase):
|
class TestSwUpgrade(base.DCManagerTestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
@ -99,14 +75,6 @@ class TestSwUpgrade(base.DCManagerTestCase):
|
|||||||
self.fake_patch_orch_thread
|
self.fake_patch_orch_thread
|
||||||
self.addCleanup(p.stop)
|
self.addCleanup(p.stop)
|
||||||
|
|
||||||
# Mock the sysinv client
|
|
||||||
self.fake_sysinv_client = FakeSysinvClient()
|
|
||||||
p = mock.patch.object(sw_upgrade_orch_thread.SwUpgradeOrchThread,
|
|
||||||
'get_sysinv_client')
|
|
||||||
self.mock_sysinv_client = p.start()
|
|
||||||
self.mock_sysinv_client.return_value = self.fake_sysinv_client
|
|
||||||
self.addCleanup(p.stop)
|
|
||||||
|
|
||||||
# Mock db_api
|
# Mock db_api
|
||||||
p = mock.patch.object(sw_upgrade_orch_thread, 'db_api')
|
p = mock.patch.object(sw_upgrade_orch_thread, 'db_api')
|
||||||
self.mock_db_api = p.start()
|
self.mock_db_api = p.start()
|
||||||
@ -128,7 +96,6 @@ class TestSwUpgrade(base.DCManagerTestCase):
|
|||||||
mock_dcmanager_audit_api = mock.Mock()
|
mock_dcmanager_audit_api = mock.Mock()
|
||||||
worker = sw_update_manager.SwUpgradeOrchThread(mock_strategy_lock,
|
worker = sw_update_manager.SwUpgradeOrchThread(mock_strategy_lock,
|
||||||
mock_dcmanager_audit_api)
|
mock_dcmanager_audit_api)
|
||||||
worker.get_ks_client = mock.Mock()
|
|
||||||
return worker
|
return worker
|
||||||
|
|
||||||
def assert_step_updated(self, subcloud_id, update_state):
|
def assert_step_updated(self, subcloud_id, update_state):
|
||||||
@ -140,168 +107,3 @@ class TestSwUpgrade(base.DCManagerTestCase):
|
|||||||
started_at=mock.ANY,
|
started_at=mock.ANY,
|
||||||
finished_at=mock.ANY,
|
finished_at=mock.ANY,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestSwUpgradeLicenseStage(TestSwUpgrade):
|
|
||||||
|
|
||||||
def setUp(self):
|
|
||||||
super(TestSwUpgradeLicenseStage, self).setUp()
|
|
||||||
self.strategy_step = \
|
|
||||||
self.setup_strategy_step(consts.STRATEGY_STATE_INSTALLING_LICENSE)
|
|
||||||
|
|
||||||
def test_upgrade_subcloud_license_install_failure(self):
|
|
||||||
# Test the install subcloud license step where the system controller
|
|
||||||
# license is valid, and the subcloud license install fails
|
|
||||||
|
|
||||||
# Order of get_license calls:
|
|
||||||
# first license query is to system controller
|
|
||||||
# second license query is to subcloud (should be missing)
|
|
||||||
self.fake_sysinv_client.get_license.side_effect = \
|
|
||||||
[LICENSE_VALID_RESPONSE,
|
|
||||||
MISSING_LICENSE_RESPONSE]
|
|
||||||
|
|
||||||
# Simulate a license install failure on the subcloud
|
|
||||||
self.fake_sysinv_client.install_license.return_value = \
|
|
||||||
MISSING_LICENSE_RESPONSE
|
|
||||||
|
|
||||||
self.worker.install_subcloud_license(self.strategy_step)
|
|
||||||
|
|
||||||
# verify the license install was invoked
|
|
||||||
self.fake_sysinv_client.install_license.assert_called()
|
|
||||||
|
|
||||||
# Verify a install_license failure leads to a state failure
|
|
||||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
|
||||||
consts.STRATEGY_STATE_FAILED)
|
|
||||||
|
|
||||||
def test_upgrade_subcloud_license_install_success(self):
|
|
||||||
# Test the install subcloud license step where the system controller
|
|
||||||
# license is valid, and the subcloud installation succeeds
|
|
||||||
|
|
||||||
# Order of get_license calls:
|
|
||||||
# first license query is to system controller
|
|
||||||
# second license query is to subcloud (should be missing)
|
|
||||||
self.fake_sysinv_client.get_license.side_effect = \
|
|
||||||
[LICENSE_VALID_RESPONSE,
|
|
||||||
MISSING_LICENSE_RESPONSE]
|
|
||||||
|
|
||||||
# A license install should return a success
|
|
||||||
self.fake_sysinv_client.install_license.return_value = \
|
|
||||||
LICENSE_VALID_RESPONSE
|
|
||||||
|
|
||||||
self.worker.install_subcloud_license(self.strategy_step)
|
|
||||||
|
|
||||||
# verify the license install was invoked
|
|
||||||
self.fake_sysinv_client.install_license.assert_called()
|
|
||||||
|
|
||||||
# On success, the next state after installing license is importing load
|
|
||||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
|
||||||
consts.STRATEGY_STATE_IMPORTING_LOAD)
|
|
||||||
|
|
||||||
def test_upgrade_subcloud_license_skip_existing(self):
|
|
||||||
# Test the install subcloud license step where the system controller
|
|
||||||
# license is valid, and the subcloud already has the same license
|
|
||||||
|
|
||||||
# Order of get_license calls:
|
|
||||||
# first license query is to system controller
|
|
||||||
# second license query is to subcloud
|
|
||||||
self.fake_sysinv_client.get_license.side_effect = \
|
|
||||||
[LICENSE_VALID_RESPONSE,
|
|
||||||
LICENSE_VALID_RESPONSE]
|
|
||||||
self.worker.install_subcloud_license(self.strategy_step)
|
|
||||||
|
|
||||||
# A license install should not have been attempted due to the license
|
|
||||||
# already being up to date
|
|
||||||
self.fake_sysinv_client.install_license.assert_not_called()
|
|
||||||
# On success, the next state after installing license is importing load
|
|
||||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
|
||||||
consts.STRATEGY_STATE_IMPORTING_LOAD)
|
|
||||||
|
|
||||||
def test_upgrade_subcloud_license_overrides_mismatched_license(self):
|
|
||||||
# Test the install subcloud license step where the system controller
|
|
||||||
# license is valid, and the subcloud has a differnt license which
|
|
||||||
# should be overridden
|
|
||||||
|
|
||||||
# Order of get_license calls:
|
|
||||||
# first license query is to system controller
|
|
||||||
# second license query is to subcloud (should be valid but different)
|
|
||||||
self.fake_sysinv_client.get_license.side_effect = \
|
|
||||||
[LICENSE_VALID_RESPONSE,
|
|
||||||
ALTERNATE_LICENSE_RESPONSE]
|
|
||||||
|
|
||||||
# A license install should return a success
|
|
||||||
self.fake_sysinv_client.install_license.return_value = \
|
|
||||||
LICENSE_VALID_RESPONSE
|
|
||||||
|
|
||||||
self.worker.install_subcloud_license(self.strategy_step)
|
|
||||||
|
|
||||||
# verify the license install was invoked
|
|
||||||
self.fake_sysinv_client.install_license.assert_called()
|
|
||||||
|
|
||||||
# On success, the next state after installing license is importing load
|
|
||||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
|
||||||
consts.STRATEGY_STATE_IMPORTING_LOAD)
|
|
||||||
|
|
||||||
def test_upgrade_subcloud_license_skip_when_no_sys_controller_lic(self):
|
|
||||||
# Test the install subcloud license step is skipped and proceeds
|
|
||||||
# to the next state when there is no license on system controller
|
|
||||||
|
|
||||||
# Only makes one query to system controller
|
|
||||||
self.fake_sysinv_client.get_license.side_effect = \
|
|
||||||
[MISSING_LICENSE_RESPONSE, ]
|
|
||||||
# Test the install subcloud license stage
|
|
||||||
self.worker.install_subcloud_license(self.strategy_step)
|
|
||||||
|
|
||||||
# A license install should proceed to the next state without
|
|
||||||
# calling a license install
|
|
||||||
self.fake_sysinv_client.install_license.assert_not_called()
|
|
||||||
# Skip license install and move to next state
|
|
||||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
|
||||||
consts.STRATEGY_STATE_IMPORTING_LOAD)
|
|
||||||
|
|
||||||
def test_upgrade_subcloud_license_handle_failure(self):
|
|
||||||
# Test the install subcloud license step where the system controller
|
|
||||||
# license is valid, and the subcloud license install fails
|
|
||||||
|
|
||||||
# Order of get_license calls:
|
|
||||||
# first license query is to system controller
|
|
||||||
# second license query is to subcloud (should be missing)
|
|
||||||
self.fake_sysinv_client.get_license.side_effect = \
|
|
||||||
[LICENSE_VALID_RESPONSE,
|
|
||||||
MISSING_LICENSE_RESPONSE]
|
|
||||||
|
|
||||||
# Simulate a license install failure on the subcloud
|
|
||||||
self.fake_sysinv_client.install_license.return_value = \
|
|
||||||
MISSING_LICENSE_RESPONSE
|
|
||||||
|
|
||||||
self.worker.install_subcloud_license(self.strategy_step)
|
|
||||||
|
|
||||||
# verify the license install was invoked
|
|
||||||
self.fake_sysinv_client.install_license.assert_called()
|
|
||||||
|
|
||||||
# Verify a install_license failure leads to a state failure
|
|
||||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
|
||||||
consts.STRATEGY_STATE_FAILED)
|
|
||||||
|
|
||||||
def test_upgrade_subcloud_license_installs(self):
|
|
||||||
# Test the install subcloud license step where the system controller
|
|
||||||
# license is valid, and the subcloud installation succeeds
|
|
||||||
|
|
||||||
# Order of get_license calls:
|
|
||||||
# first license query is to system controller
|
|
||||||
# second license query is to subcloud (should be missing)
|
|
||||||
self.fake_sysinv_client.get_license.side_effect = \
|
|
||||||
[LICENSE_VALID_RESPONSE,
|
|
||||||
MISSING_LICENSE_RESPONSE]
|
|
||||||
|
|
||||||
# A license install should return a success
|
|
||||||
self.fake_sysinv_client.install_license.return_value = \
|
|
||||||
LICENSE_VALID_RESPONSE
|
|
||||||
|
|
||||||
self.worker.install_subcloud_license(self.strategy_step)
|
|
||||||
|
|
||||||
# verify the license install was invoked
|
|
||||||
self.fake_sysinv_client.install_license.assert_called()
|
|
||||||
|
|
||||||
# On success, the next state after installing license is importing load
|
|
||||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
|
||||||
consts.STRATEGY_STATE_IMPORTING_LOAD)
|
|
||||||
|
Loading…
Reference in New Issue
Block a user