starlingx/distcloud
Code Issues Proposed changes

Merge "Implement import-load and starting-upgrade strategy states"

Browse Source
This commit is contained in:
Zuul 2020-06-10 15:03:36 +00:00 committed by Gerrit Code Review
commit 94f6dc9fb4
25 changed files with 1055 additions and 563 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
distributedcloud/dccommon/drivers/openstack/sysinv_v1.py
View File

@ -235,6 +235,11 @@ class SysinvClient(base.DriverBase):
LOG.error("delete_load exception={}".format(e))
raise e
def import_load(self, path_to_iso, path_to_sig):
"""Import the particular software load."""
return self.sysinv_client.load.import_load(path_to_iso=path_to_iso,
path_to_sig=path_to_sig)
def get_hosts(self):
"""Get a list of hosts."""
return self.sysinv_client.ihost.list()
@ -243,6 +248,21 @@ class SysinvClient(base.DriverBase):
"""Get a list of upgrades."""
return self.sysinv_client.upgrade.list()
def upgrade_activate(self):
"""Invoke the API for 'system upgrade-activate', which is an update """
patch = [{'op': 'replace',
'path': '/state',
'value': 'activation-requested'}, ]
return self.sysinv_client.upgrade.update(patch)
def upgrade_complete(self):
"""Invoke the API for 'system upgrade-complete', which is a delete"""
return self.sysinv_client.upgrade.delete()
def upgrade_start(self, force=False):
"""Invoke the API for 'system upgrade-start', which is a create"""
return self.sysinv_client.upgrade.create(force)
def get_applications(self):
"""Get a list of containerized applications"""

3
distributedcloud/dcmanager/common/consts.py
View File

@ -108,7 +108,8 @@ STRATEGY_STATE_LOCKING_CONTROLLER = "locking controller"
STRATEGY_STATE_UPGRADING_SIMPLEX = "upgrading simplex"
STRATEGY_STATE_MIGRATING_DATA = "migrating data"
STRATEGY_STATE_UNLOCKING_CONTROLLER = "unlocking controller"
STRATEGY_STATE_ACTIVATING = "activating"
STRATEGY_STATE_ACTIVATING_UPGRADE = "activating upgrade"
STRATEGY_STATE_COMPLETING_UPGRADE = "completing upgrade"
# Subcloud deploy status states
DEPLOY_STATE_NONE = 'not-deployed'

4
distributedcloud/dcmanager/common/exceptions.py
View File

@ -178,6 +178,10 @@ class LicenseMissingError(DCManagerException):
message = _("License does not exist on subcloud: %(subcloud_id)s")
class VaultLoadMissingError(DCManagerException):
message = _("No matching: %(file_type) found in vault: %(vault_dir)")
class StrategyStepNotFound(NotFound):
message = _("StrategyStep with subcloud_id %(subcloud_id)s "
"doesn't exist.")

7
distributedcloud/dcmanager/manager/states/base.py
View File

@ -28,6 +28,13 @@ class BaseState(object):
self.get_region_name(strategy_step),
details))
def info_log(self, strategy_step, details):
LOG.info("Stage: %s, State: %s, Subcloud: %s, Details: %s"
% (strategy_step.stage,
strategy_step.state,
self.get_region_name(strategy_step),
details))
@staticmethod
def get_region_name(strategy_step):
"""Get the region name for a strategy step"""

7
distributedcloud/dcmanager/manager/states/lock_host.py
View File

@ -5,12 +5,9 @@
#
import time
from oslo_log import log as logging
from dcmanager.common.consts import ADMIN_LOCKED
from dcmanager.manager.states.base import BaseState
LOG = logging.getLogger(__name__)
DEFAULT_MAX_QUERIES = 6
DEFAULT_SLEEP_DURATION = 10
@ -46,7 +43,7 @@ class LockHostState(BaseState):
if host.administrative == ADMIN_LOCKED:
msg = "Host: %s already: %s." % (self.target_hostname,
host.administrative)
self.debug_log(strategy_step, msg)
self.info_log(strategy_step, msg)
return True
# Invoke the action
@ -63,7 +60,7 @@ class LockHostState(BaseState):
if host.administrative == ADMIN_LOCKED:
msg = "Host: %s is now: %s" % (self.target_hostname,
host.administrative)
self.debug_log(strategy_step, msg)
self.info_log(strategy_step, msg)
break
counter += 1
if counter >= self.max_queries:

7
distributedcloud/dcmanager/manager/states/unlock_host.py
View File

@ -5,12 +5,9 @@
#
import time
from oslo_log import log as logging
from dcmanager.common.consts import ADMIN_UNLOCKED
from dcmanager.manager.states.base import BaseState
LOG = logging.getLogger(__name__)
DEFAULT_MAX_QUERIES = 6
DEFAULT_SLEEP_DURATION = 10
@ -51,7 +48,7 @@ class UnlockHostState(BaseState):
if host.administrative == ADMIN_UNLOCKED:
msg = "Host: %s already: %s." % (self.target_hostname,
host.administrative)
self.debug_log(strategy_step, msg)
self.info_log(strategy_step, msg)
return True
# Invoke the action
@ -68,7 +65,7 @@ class UnlockHostState(BaseState):
if host.administrative == ADMIN_UNLOCKED:
msg = "Host: %s is now: %s" % (self.target_hostname,
host.administrative)
self.debug_log(strategy_step, msg)
self.info_log(strategy_step, msg)
break
async_counter += 1
# check_async_counter throws exception if loops exceeded or aborted

34
distributedcloud/dcmanager/manager/states/upgrade/activating.py
View File

@ -3,18 +3,19 @@
#
# SPDX-License-Identifier: Apache-2.0
#
from oslo_log import log as logging
from dcmanager.manager.states.base import BaseState
LOG = logging.getLogger(__name__)
ALREADY_ACTIVATING_STATES = ['activation-requested',
'activation-failed',
'activation-complete',
'activating']
class ActivatingState(BaseState):
class ActivatingUpgradeState(BaseState):
"""Upgrade state actions for activating an upgrade"""
def __init__(self):
super(ActivatingState, self).__init__()
super(ActivatingUpgradeState, self).__init__()
def perform_state_action(self, strategy_step):
"""Activate an upgrade on a subcloud
@ -22,9 +23,28 @@ class ActivatingState(BaseState):
Any exceptions raised by this method set the strategy to FAILED
Returning normally from this method set the strategy to the next step
"""
LOG.warning("ActivatingState has not been implemented yet.")
# get the keystone and sysinv clients for the subcloud
ks_client = self.get_keystone_client(strategy_step.subcloud.name)
sysinv_client = self.get_sysinv_client(strategy_step.subcloud.name,
ks_client.session)
upgrades = sysinv_client.get_upgrades()
# If there are no existing upgrades, there is nothing to activate
if len(upgrades) == 0:
raise Exception("No upgrades were found to activate")
# The list of upgrades will never contain more than one entry.
for upgrade in upgrades:
# Check if an existing upgrade is already activated
if upgrade.state in ALREADY_ACTIVATING_STATES:
self.info_log(strategy_step,
"Already in activating state:%s" % upgrade.state)
break
else:
# invoke the API 'upgrade-activate'.
# Throws an exception on failure (no upgrade found, bad host state)
sysinv_client.upgrade_activate()
# When we return from this method without throwing an exception, the
# state machine can proceed to the next state
LOG.warning("Faking transition to next state")
return True

27
distributedcloud/dcmanager/manager/states/upgrade/completing.py
View File

@ -3,18 +3,14 @@
#
# SPDX-License-Identifier: Apache-2.0
#
from oslo_log import log as logging
from dcmanager.manager.states.base import BaseState
LOG = logging.getLogger(__name__)
class CompletingState(BaseState):
class CompletingUpgradeState(BaseState):
"""Upgrade state actions for completing an upgrade"""
def __init__(self):
super(CompletingState, self).__init__()
super(CompletingUpgradeState, self).__init__()
def perform_state_action(self, strategy_step):
"""Complete an upgrade on a subcloud
@ -22,9 +18,24 @@ class CompletingState(BaseState):
Any exceptions raised by this method set the strategy to FAILED
Returning normally from this method set the strategy to the next step
"""
LOG.warning("CompletingState has not been implemented yet.")
# get the keystone and sysinv clients for the subcloud
ks_client = self.get_keystone_client(strategy_step.subcloud.name)
sysinv_client = self.get_sysinv_client(strategy_step.subcloud.name,
ks_client.session)
# upgrade-complete causes the upgrade to be deleted.
# if no upgrade exists, there is no need to call it.
# The API should always return a list, but check for None anyways
upgrades = sysinv_client.get_upgrades()
if len(upgrades) == 0:
self.info_log(strategy_step,
"No upgrades exist. Nothing needs completing")
return True
# invoke the API 'upgrade-complete'
# This is a blocking call that raises an exception on failure.
sysinv_client.upgrade_complete()
# When we return from this method without throwing an exception, the
# state machine can proceed to the next state
LOG.warning("Faking transition to next state")
return True

30
distributedcloud/dcmanager/manager/states/upgrade/import_load.py
View File

@ -1,30 +0,0 @@
#
# Copyright (c) 2020 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
from oslo_log import log as logging
from dcmanager.manager.states.base import BaseState
LOG = logging.getLogger(__name__)
class ImportLoadState(BaseState):
"""Upgrade state for importing a load"""
def __init__(self):
super(ImportLoadState, self).__init__()
def perform_state_action(self, strategy_step):
"""Import a load on a subcloud
Any exceptions raised by this method set the strategy to FAILED
Returning normally from this method set the strategy to the next step
"""
LOG.warning("ImportLoadState has not been implemented yet.")
# When we return from this method without throwing an exception, the
# state machine can proceed to the next state
LOG.warning("Faking transition to next state")
return True

55
distributedcloud/dcmanager/manager/states/upgrade/importing_load.py Normal file
View File

@ -0,0 +1,55 @@
#
# Copyright (c) 2020 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
from dcmanager.common import consts
from dcmanager.manager.states.base import BaseState
from dcmanager.manager.states.upgrade import utils
class ImportingLoadState(BaseState):
"""Upgrade state for importing a load"""
def __init__(self):
super(ImportingLoadState, self).__init__()
def perform_state_action(self, strategy_step):
"""Import a load on a subcloud
Any exceptions raised by this method set the strategy to FAILED
Returning normally from this method set the strategy to the next step
"""
# determine the version of the system controller in region one
local_ks_client = self.get_keystone_client()
local_sysinv_client = \
self.get_sysinv_client(consts.DEFAULT_REGION_NAME,
local_ks_client.session)
target_version = local_sysinv_client.get_system().software_version
# get the keystone and sysinv clients for the subcloud
ks_client = self.get_keystone_client(strategy_step.subcloud.name)
sysinv_client = self.get_sysinv_client(strategy_step.subcloud.name,
ks_client.session)
# Check if the load is already imported by checking what the version is
current_loads = sysinv_client.get_loads()
for load in current_loads:
if load.software_version == target_version:
self.info_log(strategy_step,
"Load:%s already found" % target_version)
return True
# If we are here, the load needs to be imported
# ISO and SIG files are found in the vault under a version directory
iso_path, sig_path = utils.get_vault_load_files(target_version)
# Call the API.
imported_load = sysinv_client.import_load(iso_path, sig_path)
new_load = imported_load.get('new_load', {})
if new_load.get('software_version') != target_version:
raise Exception("The imported load was not the expected version")
# When we return from this method without throwing an exception, the
# state machine can proceed to the next state
return True

86
distributedcloud/dcmanager/manager/states/upgrade/installing_license.py Normal file
View File

@ -0,0 +1,86 @@
#
# Copyright (c) 2020 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
from dcmanager.common import consts
from dcmanager.common import exceptions
from dcmanager.manager.states.base import BaseState
# When a license is not installed, this will be part of the API error string
LICENSE_FILE_NOT_FOUND_SUBSTRING = "License file not found"
class InstallingLicenseState(BaseState):
"""Upgrade state action for installing a license"""
def __init__(self):
super(InstallingLicenseState, self).__init__()
@staticmethod
def license_up_to_date(target_license, existing_license):
return target_license == existing_license
def perform_state_action(self, strategy_step):
"""Install the License for a software upgrade in this subcloud
Any exceptions raised by this method set the strategy to FAILED
Returning normally from this method set the strategy to the next step
"""
# check if the the system controller has a license
local_ks_client = self.get_keystone_client()
local_sysinv_client = \
self.get_sysinv_client(consts.DEFAULT_REGION_NAME,
local_ks_client.session)
system_controller_license = local_sysinv_client.get_license()
# get_license returns a dictionary with keys: content and error
# 'content' can be an empty string in success or failure case.
# 'error' is an empty string only in success case.
target_license = system_controller_license.get('content')
target_error = system_controller_license.get('error')
# If the system controller does not have a license, do not attempt
# to install licenses on subclouds, simply proceed to the next stage
if len(target_error) != 0:
if LICENSE_FILE_NOT_FOUND_SUBSTRING in target_error:
self.info_log(strategy_step,
"System Controller License missing: %s."
% target_error)
return True
else:
# An unexpected error occurred querying the license
raise exceptions.LicenseInstallError(
subcloud_id=consts.SYSTEM_CONTROLLER_NAME)
# retrieve the keystone session for the subcloud and query its license
subcloud_ks_client = \
self.get_keystone_client(strategy_step.subcloud.name)
subcloud_sysinv_client = \
self.get_sysinv_client(strategy_step.subcloud.name,
subcloud_ks_client.session)
subcloud_license_response = subcloud_sysinv_client.get_license()
subcloud_license = subcloud_license_response.get('content')
subcloud_error = subcloud_license_response.get('error')
# Skip license install if the license is already up to date
# If there was not an error, there might be a license
if len(subcloud_error) == 0:
if self.license_up_to_date(target_license, subcloud_license):
self.info_log(strategy_step, "License up to date.")
return True
else:
self.debug_log(strategy_step, "License mismatch. Updating.")
else:
self.debug_log(strategy_step, "License missing. Installing.")
# Install the license
install_rc = subcloud_sysinv_client.install_license(target_license)
install_error = install_rc.get('error')
if len(install_error) != 0:
raise exceptions.LicenseInstallError(
subcloud_id=strategy_step.subcloud_id)
# The license has been successfully installed. Move to the next stage
self.info_log(strategy_step, "License installed.")
return True

26
distributedcloud/dcmanager/manager/states/upgrade/starting_upgrade.py
View File

@ -3,18 +3,15 @@
#
# SPDX-License-Identifier: Apache-2.0
#
from oslo_log import log as logging
from dcmanager.manager.states.base import BaseState
LOG = logging.getLogger(__name__)
class StartingUpgradeState(BaseState):
"""Upgrade state for starting an upgrade on a subcloud"""
def __init__(self):
def __init__(self, force=False):
super(StartingUpgradeState, self).__init__()
self.force = force
def perform_state_action(self, strategy_step):
"""Start an upgrade on a subcloud
@ -22,9 +19,24 @@ class StartingUpgradeState(BaseState):
Any exceptions raised by this method set the strategy to FAILED
Returning normally from this method set the strategy to the next step
"""
LOG.warning("StartingUpgradeState has not been implemented yet.")
# get the keystone and sysinv clients for the subcloud
ks_client = self.get_keystone_client(strategy_step.subcloud.name)
sysinv_client = self.get_sysinv_client(strategy_step.subcloud.name,
ks_client.session)
# Check if an existing upgrade is already in progress.
# The list of upgrades will never contain more than one entry.
upgrades = sysinv_client.get_upgrades()
if upgrades is not None and len(upgrades) > 0:
for upgrade in upgrades:
# If a previous upgrade exists (even one that failed) skip
self.info_log(strategy_step,
"An upgrade already exists: %s" % upgrade)
else:
# invoke the API 'upgrade-start'.
# This call is synchronous and throws an exception on failure.
sysinv_client.upgrade_start(self.force)
# When we return from this method without throwing an exception, the
# state machine can proceed to the next state
LOG.warning("Faking transition to next state")
return True

38
distributedcloud/dcmanager/manager/states/upgrade/utils.py Normal file
View File

@ -0,0 +1,38 @@
#
# Copyright (c) 2020 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import os
from dcmanager.common.exceptions import VaultLoadMissingError
VAULT_LOADS_PATH = '/opt/dc-vault/loads'
def get_vault_load_files(target_version):
"""Return a tuple for the ISO and SIG for this load version from the vault.
The files can be imported to the vault using any name, but must end
in 'iso' or 'sig'.
: param target_version: The software version to search under the vault
"""
vault_dir = "{}/{}/".format(VAULT_LOADS_PATH, target_version)
matching_iso = None
matching_sig = None
for a_file in os.listdir(vault_dir):
if a_file.lower().endswith(".iso"):
matching_iso = os.path.join(vault_dir, a_file)
continue
elif a_file.lower().endswith(".sig"):
matching_sig = os.path.join(vault_dir, a_file)
continue
# If no .iso or .sig is found, raise an exception
if matching_iso is None:
raise VaultLoadMissingError(file_type='.iso', vault_dir=vault_dir)
if matching_sig is None:
raise VaultLoadMissingError(file_type='.sig', vault_dir=vault_dir)
# return the iso and sig for this load
return (matching_iso, matching_sig)

364
distributedcloud/dcmanager/manager/sw_upgrade_orch_thread.py
View File

@ -25,19 +25,52 @@ import time
from oslo_log import log as logging
from dccommon.drivers.openstack.sdk_platform import OpenStackDriver
from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
from dcmanager.common import consts
from dcmanager.common import context
from dcmanager.common import exceptions
from dcmanager.common import scheduler
from dcmanager.db import api as db_api
from dcmanager.manager.states.lock_host import LockHostState
from dcmanager.manager.states.unlock_host import UnlockHostState
from dcmanager.manager.states.upgrade.activating import ActivatingUpgradeState
from dcmanager.manager.states.upgrade.completing import CompletingUpgradeState
from dcmanager.manager.states.upgrade.importing_load import ImportingLoadState
from dcmanager.manager.states.upgrade.installing_license \
import InstallingLicenseState
from dcmanager.manager.states.upgrade.migrating_data \
import MigratingDataState
from dcmanager.manager.states.upgrade.starting_upgrade \
import StartingUpgradeState
from dcmanager.manager.states.upgrade.upgrading_simplex \
import UpgradingSimplexState
LOG = logging.getLogger(__name__)
# When a license is not installed, this will be part of the API error string
LICENSE_FILE_NOT_FOUND_SUBSTRING = "License file not found"
# The state machine transition order for an APPLY
ORDERED_STATES = [
consts.STRATEGY_STATE_INSTALLING_LICENSE,
consts.STRATEGY_STATE_IMPORTING_LOAD,
consts.STRATEGY_STATE_STARTING_UPGRADE,
consts.STRATEGY_STATE_LOCKING_CONTROLLER,
consts.STRATEGY_STATE_UPGRADING_SIMPLEX,
consts.STRATEGY_STATE_MIGRATING_DATA,
consts.STRATEGY_STATE_UNLOCKING_CONTROLLER,
consts.STRATEGY_STATE_ACTIVATING_UPGRADE,
consts.STRATEGY_STATE_COMPLETING_UPGRADE,
]
# every state in ORDERED_STATES should have an operator
STATE_OPERATORS = {
consts.STRATEGY_STATE_INSTALLING_LICENSE: InstallingLicenseState,
consts.STRATEGY_STATE_IMPORTING_LOAD: ImportingLoadState,
consts.STRATEGY_STATE_STARTING_UPGRADE: StartingUpgradeState,
consts.STRATEGY_STATE_LOCKING_CONTROLLER: LockHostState,
consts.STRATEGY_STATE_UPGRADING_SIMPLEX: UpgradingSimplexState,
consts.STRATEGY_STATE_MIGRATING_DATA: MigratingDataState,
consts.STRATEGY_STATE_UNLOCKING_CONTROLLER: UnlockHostState,
consts.STRATEGY_STATE_ACTIVATING_UPGRADE: ActivatingUpgradeState,
consts.STRATEGY_STATE_COMPLETING_UPGRADE: CompletingUpgradeState,
}
class SwUpgradeOrchThread(threading.Thread):
@ -72,7 +105,7 @@ class SwUpgradeOrchThread(threading.Thread):
self.subcloud_workers = dict()
# When an upgrade is initiated, this is the first state
self.starting_state = consts.STRATEGY_STATE_INSTALLING_LICENSE
self.starting_state = ORDERED_STATES[0]
def stopped(self):
return self._stop.isSet()
@ -87,22 +120,6 @@ class SwUpgradeOrchThread(threading.Thread):
self.thread_group_manager.stop()
LOG.info("SwUpgradeOrchThread Stopped")
@staticmethod
def get_ks_client(region_name=consts.DEFAULT_REGION_NAME):
"""This will get a cached keystone client (and token)"""
try:
os_client = OpenStackDriver(
region_name=region_name,
region_clients=None)
return os_client.keystone_client
except Exception:
LOG.warn('Failure initializing KeystoneClient')
raise
@staticmethod
def get_sysinv_client(region_name, session):
return SysinvClient(region_name, session)
@staticmethod
def get_region_name(strategy_step):
"""Get the region name for a strategy step"""
@ -121,8 +138,22 @@ class SwUpgradeOrchThread(threading.Thread):
return details
@staticmethod
def license_up_to_date(target_license, existing_license):
return target_license == existing_license
def determine_state_operator(strategy_step):
"""Return the state operator for the current state"""
state_operator = STATE_OPERATORS.get(strategy_step.state)
# instantiate and return the state_operator class
return state_operator()
@staticmethod
def determine_next_state(strategy_step):
"""Return next state for the strategy step based on current state."""
# todo(abailey): next_state may differ for AIO, STD, etc.. subclouds
next_index = ORDERED_STATES.index(strategy_step.state) + 1
if next_index < len(ORDERED_STATES):
next_state = ORDERED_STATES[next_index]
else:
next_state = consts.STRATEGY_STATE_COMPLETE
return next_state
def strategy_step_update(self, subcloud_id, state=None, details=None):
"""Update the strategy step in the DB
@ -137,13 +168,13 @@ class SwUpgradeOrchThread(threading.Thread):
consts.STRATEGY_STATE_ABORTED,
consts.STRATEGY_STATE_FAILED]:
finished_at = datetime.datetime.now()
db_api.strategy_step_update(
self.context,
subcloud_id,
state=state,
details=details,
started_at=started_at,
finished_at=finished_at)
# Return the updated object, in case we need to use its updated values
return db_api.strategy_step_update(self.context,
subcloud_id,
state=state,
details=details,
started_at=started_at,
finished_at=finished_at)
def upgrade_orch(self):
while not self.stopped():
@ -288,216 +319,23 @@ class SwUpgradeOrchThread(threading.Thread):
continue
# We are just getting started, enter the first state
self.strategy_step_update(
# Use the updated value for calling process_upgrade_step
strategy_step = self.strategy_step_update(
strategy_step.subcloud_id,
state=consts.STRATEGY_STATE_INSTALLING_LICENSE)
# Initial step should log an error if a greenthread exists
# All other steps should not.
state=self.starting_state)
# Starting state should log an error if greenthread exists
self.process_upgrade_step(region,
strategy_step,
self.install_subcloud_license,
log_error=True)
# todo(abailey): state and their method invoked can be managed
# using a dictionary to make this more maintainable.
elif strategy_step.state == \
consts.STRATEGY_STATE_INSTALLING_LICENSE:
self.process_upgrade_step(region,
strategy_step,
self.install_subcloud_license,
log_error=False)
elif strategy_step.state == \
consts.STRATEGY_STATE_IMPORTING_LOAD:
self.process_upgrade_step(region,
strategy_step,
self.import_subcloud_load,
log_error=False)
elif strategy_step.state == \
consts.STRATEGY_STATE_STARTING_UPGRADE:
self.process_upgrade_step(region,
strategy_step,
self.start_subcloud_upgrade,
log_error=False)
elif strategy_step.state == \
consts.STRATEGY_STATE_LOCKING_CONTROLLER:
self.process_upgrade_step(region,
strategy_step,
self.lock_subcloud_controller,
log_error=False)
elif strategy_step.state == \
consts.STRATEGY_STATE_UPGRADING_SIMPLEX:
self.process_upgrade_step(region,
strategy_step,
self.upgrade_subcloud_simplex,
log_error=False)
elif strategy_step.state == \
consts.STRATEGY_STATE_MIGRATING_DATA:
self.process_upgrade_step(region,
strategy_step,
self.migrate_subcloud_data,
log_error=False)
elif strategy_step.state == \
consts.STRATEGY_STATE_UNLOCKING_CONTROLLER:
self.process_upgrade_step(region,
strategy_step,
self.unlock_subcloud_controller,
log_error=False)
elif strategy_step.state == \
consts.STRATEGY_STATE_ACTIVATING:
self.process_upgrade_step(region,
strategy_step,
self.activate_subcloud,
log_error=False)
# todo(abailey): Add calls to self.process_upgrade_step
# for each additional state, with the appropriate thread
# method called.
else:
LOG.error("Unimplemented state %s" % strategy_step.state)
self.strategy_step_update(
strategy_step.subcloud_id,
state=consts.STRATEGY_STATE_FAILED,
details=("Upgrade state not implemented: %s"
% strategy_step.state))
self.process_upgrade_step(region,
strategy_step,
log_error=False)
if self.stopped():
LOG.info("Exiting because task is stopped")
return
def process_upgrade_step(self,
region,
strategy_step,
upgrade_thread_method,
log_error=False):
if region in self.subcloud_workers:
# A worker already exists. Let it finish whatever it was doing.
if log_error:
LOG.error("Worker should not exist for %s." % region)
else:
LOG.debug("Update worker exists for %s." % region)
else:
# Create a greenthread to start processing the upgrade for the
# subcloud and invoke the specified upgrade_thread_method
self.subcloud_workers[region] = \
self.thread_group_manager.start(upgrade_thread_method,
strategy_step)
def install_subcloud_license(self, strategy_step):
"""Install the license for the upgrade in this subcloud
Removes the worker reference after the operation is complete.
"""
try:
LOG.info("Stage: %s for subcloud %s"
% (strategy_step.stage,
self.get_region_name(strategy_step)))
self.do_install_subcloud_license(strategy_step)
except Exception:
# Catch ALL exceptions and set the strategy to failed
LOG.exception("Install license failed for %s"
% self.get_region_name(strategy_step))
self.strategy_step_update(strategy_step.subcloud_id,
state=consts.STRATEGY_STATE_FAILED,
details=("Install license failed"))
finally:
# The worker is done.
region = self.get_region_name(strategy_step)
if region in self.subcloud_workers:
del self.subcloud_workers[region]
def do_install_subcloud_license(self, strategy_step):
"""Install the License for a software upgrade in this subcloud"""
# Note: no need to catch exceptions in this method.
# next_state is the next state that the strategy will use on
# successful completion of this state
next_state = consts.STRATEGY_STATE_IMPORTING_LOAD
# We check the system controller license for system controller and
# subclouds
local_ks_client = self.get_ks_client()
local_sysinv_client = \
self.get_sysinv_client(consts.DEFAULT_REGION_NAME,
local_ks_client.session)
system_controller_license = local_sysinv_client.get_license()
# get_license returns a dictionary with keys: content and error
# 'content' can be an empty string in success or failure case.
# 'error' is an empty string only in success case.
target_license = system_controller_license.get('content')
target_error = system_controller_license.get('error')
# If the system controller does not have a license, do not attempt
# to install licenses on subclouds, and simply proceed to the next stage
if len(target_error) != 0:
if LICENSE_FILE_NOT_FOUND_SUBSTRING in target_error:
LOG.debug("Stage:<%s>, Subcloud:<%s>. "
"System Controller License missing: %s."
% (strategy_step.stage,
self.get_region_name(strategy_step),
target_error))
self.strategy_step_update(strategy_step.subcloud_id,
state=next_state)
return
else:
# An unexpected API error was returned. Fail this stage.
LOG.warning("Stage:<%s>, Subcloud:<%s>. "
"System Controller License query failed: %s."
% (strategy_step.stage,
self.get_region_name(strategy_step),
target_error))
raise exceptions.LicenseMissingError(
subcloud_id=consts.SYSTEM_CONTROLLER_NAME)
# retrieve the keystone session for the subcloud and query its license
subcloud_ks_client = self.get_ks_client(strategy_step.subcloud.name)
subcloud_sysinv_client = \
self.get_sysinv_client(strategy_step.subcloud.name,
subcloud_ks_client.session)
subcloud_license_response = subcloud_sysinv_client.get_license()
subcloud_license = subcloud_license_response.get('content')
subcloud_error = subcloud_license_response.get('error')
# Skip license install if the license is already up to date
# If there was not an error, there might be a license
if len(subcloud_error) == 0:
if self.license_up_to_date(target_license, subcloud_license):
LOG.debug("Stage:<%s>, Subcloud:<%s>. License up to date."
% (strategy_step.stage,
self.get_region_name(strategy_step)))
self.strategy_step_update(strategy_step.subcloud_id,
state=next_state)
return
else:
LOG.debug("Stage:<%s>, Subcloud:<%s>. "
"License mismatch. Updating."
% (strategy_step.stage,
self.get_region_name(strategy_step)))
else:
LOG.debug("Stage:<%s>, Subcloud:<%s>. "
"License missing. Installing."
% (strategy_step.stage,
self.get_region_name(strategy_step)))
# Install the license
install_rc = subcloud_sysinv_client.install_license(target_license)
install_error = install_rc.get('error')
if len(install_error) != 0:
LOG.warning("Stage:<%s>, Subcloud:<%s>. "
"License install failed:<%s>."
% (strategy_step.stage,
self.get_region_name(strategy_step),
install_error))
raise exceptions.LicenseInstallError(
subcloud_id=strategy_step.subcloud_id)
# The license has been successfully installed. Move to the next stage
LOG.debug("Stage:<%s>, Subcloud:<%s>. "
"License installed."
% (strategy_step.stage,
self.get_region_name(strategy_step)))
self.strategy_step_update(strategy_step.subcloud_id, state=next_state)
def abort(self, sw_update_strategy):
"""Abort an upgrade strategy"""
@ -536,15 +374,33 @@ class SwUpgradeOrchThread(threading.Thread):
LOG.exception(e)
raise e
def perform_state_action(self, strategy_step, state_operator, next_state):
def process_upgrade_step(self, region, strategy_step, log_error=False):
"""manage the green thread for calling perform_state_action"""
if region in self.subcloud_workers:
# A worker already exists. Let it finish whatever it was doing.
if log_error:
LOG.error("Worker should not exist for %s." % region)
else:
LOG.debug("Update worker exists for %s." % region)
else:
# Create a greenthread to start processing the upgrade for the
# subcloud and invoke the specified upgrade_thread_method
self.subcloud_workers[region] = \
self.thread_group_manager.start(self.perform_state_action,
strategy_step)
def perform_state_action(self, strategy_step):
"""Extensible state handler for processing and transitioning states """
try:
LOG.info("Stage: %s, State: %s, Subcloud: %s"
% (strategy_step.stage,
strategy_step.state,
self.get_region_name(strategy_step)))
# Instantiate the state operator and perform the state actions
state_operator = self.determine_state_operator(strategy_step)
state_operator.perform_state_action(strategy_step)
# If we get here without an exception raised, proceed to next state
next_state = self.determine_next_state(strategy_step)
self.strategy_step_update(strategy_step.subcloud_id,
state=next_state)
except Exception as e:
@ -562,47 +418,3 @@ class SwUpgradeOrchThread(threading.Thread):
region = self.get_region_name(strategy_step)
if region in self.subcloud_workers:
del self.subcloud_workers[region]
# todo(abailey): convert license install to the same pattern as the other states
def import_subcloud_load(self, strategy_step):
from dcmanager.manager.states.upgrade.import_load import ImportLoadState
self.perform_state_action(strategy_step,
ImportLoadState(),
consts.STRATEGY_STATE_STARTING_UPGRADE)
def start_subcloud_upgrade(self, strategy_step):
from dcmanager.manager.states.upgrade.starting_upgrade import StartingUpgradeState
self.perform_state_action(strategy_step,
StartingUpgradeState(),
consts.STRATEGY_STATE_LOCKING_CONTROLLER)
def lock_subcloud_controller(self, strategy_step):
from dcmanager.manager.states.lock_host import LockHostState
self.perform_state_action(strategy_step,
LockHostState(),
consts.STRATEGY_STATE_UPGRADING_SIMPLEX)
def upgrade_subcloud_simplex(self, strategy_step):
from dcmanager.manager.states.upgrade.upgrading_simplex import UpgradingSimplexState
self.perform_state_action(strategy_step,
UpgradingSimplexState(),
consts.STRATEGY_STATE_MIGRATING_DATA)
def migrate_subcloud_data(self, strategy_step):
from dcmanager.manager.states.upgrade.migrating_data import MigratingDataState
self.perform_state_action(strategy_step,
MigratingDataState(),
consts.STRATEGY_STATE_UNLOCKING_CONTROLLER)
def unlock_subcloud_controller(self, strategy_step):
from dcmanager.manager.states.unlock_host import UnlockHostState
self.perform_state_action(strategy_step,
UnlockHostState(),
consts.STRATEGY_STATE_ACTIVATING)
def activate_subcloud(self, strategy_step):
from dcmanager.manager.states.upgrade.activating import ActivatingState
self.perform_state_action(strategy_step,
ActivatingState(),
consts.STRATEGY_STATE_COMPLETE)

2
distributedcloud/dcmanager/tests/base.py
View File

@ -44,7 +44,7 @@ SUBCLOUD_SAMPLE_DATA_0 = [
"subcloud-4", # name
"demo subcloud", # description
"Ottawa-Lab-Aisle_3-Rack_C", # location
"20.01", # software-version
"12.34", # software-version
"managed", # management-state
"online", # availability-status
"fd01:3::0/64", # management_subnet

2
distributedcloud/dcmanager/tests/unit/api/v1/controllers/test_subclouds.py
View File

@ -56,7 +56,7 @@ FAKE_SUBCLOUD_DATA = {"name": "subcloud1",
FAKE_SUBCLOUD_INSTALL_VALUES = {
"image": "http://192.168.101.2:8080/iso/bootimage.iso",
"software_version": "20.01",
"software_version": "12.34",
"bootstrap_interface": "eno1",
"bootstrap_address": "128.224.151.183",
"bootstrap_address_prefix": 23,

91
distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_activating_upgrade.py Normal file
View File

@ -0,0 +1,91 @@
#
# Copyright (c) 2020 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import mock
from dcmanager.common import consts
from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeUpgrade
from dcmanager.tests.unit.manager.states.upgrade.test_base \
import TestSwUpgradeState
VALID_UPGRADE = FakeUpgrade(state='imported')
ACTIVATING_UPGRADE = FakeUpgrade(state='activation-requested')
ALREADY_ACTIVATED_UPGRADE = FakeUpgrade(state='activation-complete')
class TestSwUpgradeActivatingStage(TestSwUpgradeState):
def setUp(self):
super(TestSwUpgradeActivatingStage, self).setUp()
# next state after activating an upgrade is 'completing'
self.on_success_state = consts.STRATEGY_STATE_COMPLETING_UPGRADE
# Add the strategy_step state being processed by this unit test
self.strategy_step = \
self.setup_strategy_step(consts.STRATEGY_STATE_ACTIVATING_UPGRADE)
# Add mock API endpoints for sysinv client calls invcked by this state
self.sysinv_client.upgrade_activate = mock.MagicMock()
self.sysinv_client.get_upgrades = mock.MagicMock()
def test_upgrade_subcloud_activating_upgrade_failure(self):
"""Test the activating upgrade API call fails."""
# upgrade_activate will only be called if an appropriate upgrade exists
self.sysinv_client.get_upgrades.return_value = [VALID_UPGRADE, ]
# API call raises an exception when it is rejected
self.sysinv_client.upgrade_activate.side_effect = \
Exception("upgrade activate failed for some reason")
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the expected API call was invoked
self.sysinv_client.upgrade_activate.assert_called()
# Verify the state moves to 'failed'
self.assert_step_updated(self.strategy_step.subcloud_id,
consts.STRATEGY_STATE_FAILED)
def test_upgrade_subcloud_activating_upgrade_success(self):
"""Test the activating upgrade step succeeds."""
# upgrade_activate will only be called if an appropriate upgrade exists
self.sysinv_client.get_upgrades.return_value = [VALID_UPGRADE, ]
# API call will not raise an exception, and will return an upgrade
self.sysinv_client.upgrade_activate.return_value = ACTIVATING_UPGRADE
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the API cvall was invoked
self.sysinv_client.upgrade_activate.assert_called()
# On success, the state should be updated to the next state
self.assert_step_updated(self.strategy_step.subcloud_id,
self.on_success_state)
def test_upgrade_subcloud_activating_upgrade_skip_already_activated(self):
"""Test the activating upgrade step skipped if already activated."""
# upgrade_activate will only be called if an appropriate upgrade exists
self.sysinv_client.get_upgrades.return_value = \
[ALREADY_ACTIVATED_UPGRADE, ]
# API call will not be invoked, so no need to mock it
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# upgrade is already in one of the activating states so skip activating
self.sysinv_client.upgrade_activate.assert_not_called()
# On success, the state is set to the next state
self.assert_step_updated(self.strategy_step.subcloud_id,
self.on_success_state)

48
distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_base.py
View File

@ -4,14 +4,15 @@
# SPDX-License-Identifier: Apache-2.0
#
import mock
import uuid
from dcmanager.manager.states.base import BaseState
from sysinv.common import constants as sysinv_constants
from dcmanager.tests.unit.manager.test_sw_upgrade import TestSwUpgrade
CURRENT_LOAD = '20.01'
UPDATED_LOAD = '20.06'
PREVIOUS_VERSION = '12.34'
UPGRADED_VERSION = '56.78'
class FakeKeystoneClient(object):
@ -19,6 +20,47 @@ class FakeKeystoneClient(object):
self.session = mock.MagicMock()
class FakeLoad(object):
def __init__(self,
obj_id,
compatible_version='N/A',
required_patches='N/A',
software_version=PREVIOUS_VERSION,
state='active',
created_at=None,
updated_at=None):
self.id = obj_id
self.uuid = uuid.uuid4()
self.required_patches = required_patches
self.software_version = software_version
self.state = state
self.created_at = created_at
self.updated_at = updated_at
class FakeSystem(object):
def __init__(self,
obj_id=1,
software_version=UPGRADED_VERSION):
self.id = obj_id
self.uuid = uuid.uuid4()
self.software_version = software_version
class FakeUpgrade(object):
def __init__(self,
obj_id=1,
state='completed',
from_release=PREVIOUS_VERSION,
to_release=UPGRADED_VERSION):
self.id = obj_id
self.uuid = uuid.uuid4()
self.state = state
self.from_release = from_release
self.to_release = to_release
self.links = []
class FakeSysinvClient(object):
def __init__(self):
pass
@ -31,7 +73,7 @@ class FakeController(object):
administrative=sysinv_constants.ADMIN_UNLOCKED,
availability=sysinv_constants.AVAILABILITY_AVAILABLE,
ihost_action=None,
target_load=CURRENT_LOAD,
target_load=UPGRADED_VERSION,
task=None):
self.id = host_id
self.hostname = hostname

90
distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_completing_upgrade.py Normal file
View File

@ -0,0 +1,90 @@
#
# Copyright (c) 2020 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import mock
from dcmanager.common import consts
from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeUpgrade
from dcmanager.tests.unit.manager.states.upgrade.test_base \
import TestSwUpgradeState
VALID_UPGRADE = FakeUpgrade(state='activation-complete')
INVALID_UPGRADE = FakeUpgrade(state='aborting')
class TestSwUpgradeCompletingStage(TestSwUpgradeState):
def setUp(self):
super(TestSwUpgradeCompletingStage, self).setUp()
# next state after completing an upgrade is 'complete'
self.on_success_state = consts.STRATEGY_STATE_COMPLETE
# Add the strategy_step state being processed by this unit test
self.strategy_step = \
self.setup_strategy_step(consts.STRATEGY_STATE_COMPLETING_UPGRADE)
# Add mock API endpoints for sysinv client calls invcked by this state
self.sysinv_client.upgrade_complete = mock.MagicMock()
self.sysinv_client.get_upgrades = mock.MagicMock()
def test_upgrade_subcloud_completing_upgrade_failure(self):
"""Test the completing upgrade API call fails."""
# upgrade_complete will only be called if an appropriate upgrade exists
self.sysinv_client.get_upgrades.return_value = [VALID_UPGRADE, ]
# API call raises an exception when it is rejected
self.sysinv_client.upgrade_complete.side_effect = \
Exception("upgrade complete failed for some reason")
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the expected API call was invoked
self.sysinv_client.upgrade_complete.assert_called()
# Verify the state moves to 'failed'
self.assert_step_updated(self.strategy_step.subcloud_id,
consts.STRATEGY_STATE_FAILED)
def test_upgrade_subcloud_completing_upgrade_success(self):
"""Test the completing upgrade step succeeds."""
# upgrade_complete will only be called if an appropriate upgrade exists
self.sysinv_client.get_upgrades.return_value = [VALID_UPGRADE, ]
# API call will not raise an exception. It will delete the upgrade
self.sysinv_client.upgrade_complete.return_value = None
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the API cvall was invoked
self.sysinv_client.upgrade_complete.assert_called()
# On success, the state should be updated to the next state
self.assert_step_updated(self.strategy_step.subcloud_id,
self.on_success_state)
def test_upgrade_subcloud_completing_upgrade_skip_already_completed(self):
"""Test the completing upgrade step skipped if already completed."""
# upgrade_complete will only be called if an appropriate upgrade exists
# If the upgrade has been deleted, there is nothing to complete
self.sysinv_client.get_upgrades.return_value = []
# API call will not be invoked, so no need to mock it
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# upgrade is already in one of the completing states so skip completing
self.sysinv_client.upgrade_complete.assert_not_called()
# On success, the state is set to the next state
self.assert_step_updated(self.strategy_step.subcloud_id,
self.on_success_state)

155
distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_importing_load.py Normal file
View File

@ -0,0 +1,155 @@
#
# Copyright (c) 2020 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import mock
from dcmanager.common import consts
from dcmanager.common.exceptions import VaultLoadMissingError
from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeLoad
from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeSystem
from dcmanager.tests.unit.manager.states.upgrade.test_base \
import PREVIOUS_VERSION
from dcmanager.tests.unit.manager.states.upgrade.test_base \
import TestSwUpgradeState
from dcmanager.tests.unit.manager.states.upgrade.test_base \
import UPGRADED_VERSION
PREVIOUS_LOAD = FakeLoad(1, software_version=PREVIOUS_VERSION)
UPGRADED_LOAD = FakeLoad(2,
compatible_version=PREVIOUS_VERSION,
software_version=UPGRADED_VERSION)
DEST_LOAD_EXISTS = [PREVIOUS_LOAD, UPGRADED_LOAD, ]
DEST_LOAD_MISSING = [PREVIOUS_LOAD, ]
FAKE_ISO = '/opt/dc-vault/loads/' + UPGRADED_VERSION + '/bootimage.iso'
FAKE_SIG = '/opt/dc-vault/loads/' + UPGRADED_VERSION + '/bootimage.sig'
FAILED_IMPORT_RESPONSE = 'kaboom'
SUCCESS_IMPORT_RESPONSE = {
'new_load': {
'id': 2,
'uuid': 'aaa4b4c6-8536-41f6-87ea-211d208a723b',
'compatible_version': PREVIOUS_VERSION,
'required_patches': '',
'software_version': UPGRADED_VERSION,
'state': 'importing',
'created_at': '2020-06-01 12:12:12+00:00',
'updated_at': None
}
}
class TestSwUpgradeImportingLoadStage(TestSwUpgradeState):
def setUp(self):
super(TestSwUpgradeImportingLoadStage, self).setUp()
# next state after 'importing load' is 'starting upgrade'
self.on_success_state = consts.STRATEGY_STATE_STARTING_UPGRADE
# Add the strategy_step state being processed by this unit test
self.strategy_step = \
self.setup_strategy_step(consts.STRATEGY_STATE_IMPORTING_LOAD)
# Add mock API endpoints for sysinv client calls invcked by this state
self.sysinv_client.get_system = mock.MagicMock()
self.sysinv_client.get_system.return_value = FakeSystem()
self.sysinv_client.get_loads = mock.MagicMock()
self.sysinv_client.import_load = mock.MagicMock()
@mock.patch('dcmanager.manager.states.upgrade.utils.get_vault_load_files')
def test_upgrade_subcloud_importing_load_failure(self, mock_vault_files):
"""Test importing load step where the import_load API call fails."""
# simulate determine_matching_load finding the iso and sig in the vault
mock_vault_files.return_value = (FAKE_ISO, FAKE_SIG)
# Simulate the target load has not been imported yet on the subcloud
self.sysinv_client.get_loads.return_value = DEST_LOAD_MISSING
# Simulate an API failure on the subcloud.
self.sysinv_client.import_load.return_value = FAILED_IMPORT_RESPONSE
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the import load API call was invoked
self.sysinv_client.import_load.assert_called()
# Verify a failure leads to a state failure
self.assert_step_updated(self.strategy_step.subcloud_id,
consts.STRATEGY_STATE_FAILED)
@mock.patch('dcmanager.manager.states.upgrade.utils.get_vault_load_files')
def test_upgrade_subcloud_importing_load_success(self, mock_vault_files):
"""Test the importing load step succeeds.
The load will be imported on the subcloud when the subcloud does not
have the load already imported, and the API call succeeds to import it.
"""
# simulate determine_matching_load finding the iso and sig in the vault
mock_vault_files.return_value = (FAKE_ISO, FAKE_SIG)
# Simulate the target load has not been imported yet on the subcloud
self.sysinv_client.get_loads.return_value = DEST_LOAD_MISSING
# Simulate an API success on the subcloud.
self.sysinv_client.import_load.return_value = SUCCESS_IMPORT_RESPONSE
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the import load API call was invoked
self.sysinv_client.import_load.assert_called()
# On success, should have moved to the next state
self.assert_step_updated(self.strategy_step.subcloud_id,
self.on_success_state)
@mock.patch('dcmanager.manager.states.upgrade.utils.get_vault_load_files')
def test_upgrade_subcloud_importing_load_fails_missing_vault_files(
self,
mock_determine_matching_load):
"""Test importing load fails when files are not in the vault."""
mock_determine_matching_load.side_effect = \
VaultLoadMissingError(file_type='.iso', vault_dir='/mock/vault/')
# Simulate the target load has not been imported yet on the subcloud
self.sysinv_client.get_loads.return_value = DEST_LOAD_MISSING
# Simulate an API success on the subcloud. It should not get here.
self.sysinv_client.import_load.return_value = SUCCESS_IMPORT_RESPONSE
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the import load API call was never invoked
self.sysinv_client.import_load.assert_not_called()
# Verify a failure leads to a state failure
self.assert_step_updated(self.strategy_step.subcloud_id,
consts.STRATEGY_STATE_FAILED)
def test_upgrade_subcloud_importing_load_skip_existing(self):
"""Test the importing load step skipped due to load already there"""
# Simulate the target load has been previously imported on the subcloud
self.sysinv_client.get_loads.return_value = DEST_LOAD_EXISTS
# Simulate an API failure for import_load. It should not be called.
self.sysinv_client.import_load.return_value = FAILED_IMPORT_RESPONSE
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# The import_load should not have been attempted
self.sysinv_client.import_load.assert_not_called()
# On success, should have moved to the next state
self.assert_step_updated(self.strategy_step.subcloud_id,
self.on_success_state)

158
distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_installing_license.py Normal file
View File

@ -0,0 +1,158 @@
#
# Copyright (c) 2020 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import mock
from dcmanager.common import consts
from dcmanager.tests.unit.manager.states.upgrade.test_base \
import TestSwUpgradeState
MISSING_LICENSE_RESPONSE = {
u'content': u'',
u'error': u'License file not found. A license may not have been installed.'
}
LICENSE_VALID_RESPONSE = {
u'content': u'A valid license',
u'error': u''
}
ALTERNATE_LICENSE_RESPONSE = {
u'content': u'A different valid license',
u'error': u''
}
class TestSwUpgradeInstallingLicenseStage(TestSwUpgradeState):
def setUp(self):
super(TestSwUpgradeInstallingLicenseStage, self).setUp()
# next state after installing a license is 'importing load'
self.on_success_state = consts.STRATEGY_STATE_IMPORTING_LOAD
# Add the strategy_step state being processed by this unit test
self.strategy_step = \
self.setup_strategy_step(consts.STRATEGY_STATE_INSTALLING_LICENSE)
# Add mock API endpoints for sysinv client calls invcked by this state
self.sysinv_client.get_license = mock.MagicMock()
self.sysinv_client.install_license = mock.MagicMock()
def test_upgrade_subcloud_license_install_failure(self):
"""Test the installing license step where the install fails.
The system controller has a license, but the API call to install on the
subcloud fails.
"""
# Order of get_license calls:
# first license query is to system controller
# second license query is to subcloud (should be missing)
self.sysinv_client.get_license.side_effect = [LICENSE_VALID_RESPONSE,
MISSING_LICENSE_RESPONSE]
# Simulate a license install failure on the subcloud
self.sysinv_client.install_license.return_value = \
MISSING_LICENSE_RESPONSE
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the license install was invoked
self.sysinv_client.install_license.assert_called()
# Verify a install_license failure leads to a state failure
self.assert_step_updated(self.strategy_step.subcloud_id,
consts.STRATEGY_STATE_FAILED)
def test_upgrade_subcloud_license_install_success(self):
"""Test the install license step succeeds.
The license will be installed on the subcloud when system controller
has a license, the subcloud does not have a license, and the API call
succeeds.
"""
# Order of get_license calls:
# first license query is to system controller
# second license query is to subcloud (should be missing)
self.sysinv_client.get_license.side_effect = [LICENSE_VALID_RESPONSE,
MISSING_LICENSE_RESPONSE]
# A license install should return a success
self.sysinv_client.install_license.return_value = \
LICENSE_VALID_RESPONSE
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the license install was invoked
self.sysinv_client.install_license.assert_called()
# On success, the next state after installing license is importing load
self.assert_step_updated(self.strategy_step.subcloud_id,
self.on_success_state)
def test_upgrade_subcloud_license_skip_existing(self):
"""Test the install license step skipped due to license up to date"""
# Order of get_license calls:
# first license query is to system controller
# second license query is to subcloud
self.sysinv_client.get_license.side_effect = [LICENSE_VALID_RESPONSE,
LICENSE_VALID_RESPONSE]
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# A license install should not have been attempted due to the license
# already being up to date
self.sysinv_client.install_license.assert_not_called()
# On success, the next state after installing license is importing load
self.assert_step_updated(self.strategy_step.subcloud_id,
self.on_success_state)
def test_upgrade_subcloud_license_overrides_mismatched_license(self):
"""Test the install license overrides a mismatched license"""
# Order of get_license calls:
# first license query is to system controller
# second license query is to subcloud (should be valid but different)
self.sysinv_client.get_license.side_effect = \
[LICENSE_VALID_RESPONSE,
ALTERNATE_LICENSE_RESPONSE]
# A license install should return a success
self.sysinv_client.install_license.return_value = \
LICENSE_VALID_RESPONSE
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the license install was invoked
self.sysinv_client.install_license.assert_called()
# Verify it successfully moves to the next step
self.assert_step_updated(self.strategy_step.subcloud_id,
self.on_success_state)
def test_upgrade_subcloud_license_skip_when_no_sys_controller_lic(self):
"""Test license install skipped when no license on system controller."""
# Only makes one query: to system controller
self.sysinv_client.get_license.return_value = MISSING_LICENSE_RESPONSE
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# Should skip install_license API call
self.sysinv_client.install_license.assert_not_called()
# Verify it successfully moves to the next step
self.assert_step_updated(self.strategy_step.subcloud_id,
self.on_success_state)

20
distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_lock_controller.py
View File

@ -56,8 +56,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState):
# mock the API call as failed on the subcloud
self.sysinv_client.lock_host.return_value = CONTROLLER_0_LOCKING
# invoke the strategy state operation
self.worker.lock_subcloud_controller(self.strategy_step)
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the lock command was actually attempted
self.sysinv_client.lock_host.assert_called()
@ -72,8 +72,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState):
# mock the controller host query as being already locked
self.sysinv_client.get_host.return_value = CONTROLLER_0_LOCKED
# invoke the strategy state operation
self.worker.lock_subcloud_controller(self.strategy_step)
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the lock command was never attempted
self.sysinv_client.lock_host.assert_not_called()
@ -95,8 +95,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState):
# mock the API call as successful on the subcloud
self.sysinv_client.lock_host.return_value = CONTROLLER_0_LOCKING
# invoke the strategy state operation
self.worker.lock_subcloud_controller(self.strategy_step)
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the lock command was actually attempted
self.sysinv_client.lock_host.assert_called()
@ -118,8 +118,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState):
# mock the API call as failed on the subcloud
self.sysinv_client.lock_host.return_value = CONTROLLER_0_LOCKING_FAILED
# invoke the strategy state operation
self.worker.lock_subcloud_controller(self.strategy_step)
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the lock command was actually attempted
self.sysinv_client.lock_host.assert_called()
@ -135,8 +135,8 @@ class TestSwUpgradeLockControllerStage(TestSwUpgradeState):
self.sysinv_client.get_host.side_effect = \
Exception("Unable to find host controller-0")
# invoke the strategy state operation
self.worker.lock_subcloud_controller(self.strategy_step)
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the lock command was never attempted
self.sysinv_client.lock_host.assert_not_called()

124
distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_starting_upgrade.py Normal file
View File

@ -0,0 +1,124 @@
#
# Copyright (c) 2020 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import mock
from dcmanager.common import consts
from dcmanager.tests.unit.manager.states.upgrade.test_base import FakeUpgrade
from dcmanager.tests.unit.manager.states.upgrade.test_base \
import TestSwUpgradeState
UPGRADE_ABORTING = [FakeUpgrade(state='aborting'), ]
UPGRADE_STARTED = [FakeUpgrade(state='started'), ]
SUCCESS_UPGRADE_START = 'I do not know what this looks like yet'
class TestSwUpgradeStartingUpgradeStage(TestSwUpgradeState):
def setUp(self):
super(TestSwUpgradeStartingUpgradeStage, self).setUp()
# next state after 'starting upgrade' is 'migrating data'
self.on_success_state = consts.STRATEGY_STATE_LOCKING_CONTROLLER
# Add the strategy_step state being processed by this unit test
self.strategy_step = \
self.setup_strategy_step(consts.STRATEGY_STATE_STARTING_UPGRADE)
# Add mock API endpoints for sysinv client calls invcked by this state
self.sysinv_client.upgrade_start = mock.MagicMock()
self.sysinv_client.get_upgrades = mock.MagicMock()
def test_upgrade_subcloud_upgrade_start_failure(self):
"""Test the upgrade_start where the API call fails.
The upgrade_start call fails due to a validation check such as from
the health-query check.
"""
# No upgrades should yet exist in the DB / API
self.sysinv_client.get_upgrades.return_value = []
# Simulate an upgrade_start failure on the subcloud.
# The API throws an exception rather than returning an error response
self.sysinv_client.upgrade_start.side_effect = \
Exception("HTTPBadRequest: upgrade-start rejected: "
"System is not in a valid state for upgrades. "
"Run system health-query-upgrade for more details.")
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the API call that failed was invoked
self.sysinv_client.upgrade_start.assert_called()
# Verify the API failure leads to a state failure
self.assert_step_updated(self.strategy_step.subcloud_id,
consts.STRATEGY_STATE_FAILED)
def test_upgrade_subcloud_upgrade_start_success(self):
"""Test upgrade_start where the API call succeeds.
This will result in an upgrade being created with the appropriate
state.
"""
# No upgrades should yet exist in the DB / API
self.sysinv_client.get_upgrades.return_value = []
# Simulate an upgrade_start succeeds on the subcloud
self.sysinv_client.upgrade_start.return_value = SUCCESS_UPGRADE_START
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the API call that succeeded was actually invoked
self.sysinv_client.upgrade_start.assert_called()
# On success, the state should transition to the next state
self.assert_step_updated(self.strategy_step.subcloud_id,
self.on_success_state)
def test_upgrade_subcloud_upgrade_start_skip_already_started(self):
"""Test upgrade_start where the upgrade is already started."""
# An already started upgrade exists in the DB"""
self.sysinv_client.get_upgrades.return_value = [UPGRADE_STARTED, ]
# upgrade_start should not be invoked, so can be mocked as 'failed'
# by raising an exception
self.sysinv_client.upgrade_start.side_effect = \
Exception("HTTPBadRequest: this is a fake exception")
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# upgrade_start API call should not have been attempted due to the
# existing upgrade already in started state.
self.sysinv_client.upgrade_start.assert_not_called()
# On success, the state should transition to the next state
self.assert_step_updated(self.strategy_step.subcloud_id,
self.on_success_state)
def test_upgrade_subcloud_upgrade_start_fails_bad_existing_upgrade(self):
"""Test the upgrade_start fails due to a bad existing upgrade."""
# An already started upgrade exists in the DB but is in bad shape."""
self.sysinv_client.get_upgrades.return_value = [UPGRADE_ABORTING, ]
# upgrade_start will NOT be invoked. No need to mock it.
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# upgrade_start API call should not have been attempted due to the
# invalid existing upgrade that needs to be cleaned up.
self.sysinv_client.upgrade_start.assert_not_called()
# Verify it failed and moves to the next step
self.assert_step_updated(self.strategy_step.subcloud_id,
self.on_success_state)

22
distributedcloud/dcmanager/tests/unit/manager/states/upgrade/test_unlock_controller.py
View File

@ -31,7 +31,7 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState):
super(TestSwUpgradeUnlockControllerStage, self).setUp()
# next state after a successful unlock is 'activating'
self.on_success_state = consts.STRATEGY_STATE_ACTIVATING
self.on_success_state = consts.STRATEGY_STATE_ACTIVATING_UPGRADE
# Add the strategy_step state being processed by this unit test
self.strategy_step = self.setup_strategy_step(
@ -56,8 +56,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState):
# mock the API call as failed on the subcloud
self.sysinv_client.unlock_host.return_value = CONTROLLER_0_UNLOCKING
# invoke the strategy state operation
self.worker.unlock_subcloud_controller(self.strategy_step)
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the unlock command was actually attempted
self.sysinv_client.unlock_host.assert_called()
@ -72,8 +72,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState):
# mock the controller host query as being already unlocked
self.sysinv_client.get_host.return_value = CONTROLLER_0_UNLOCKED
# invoke the strategy state operation
self.worker.unlock_subcloud_controller(self.strategy_step)
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the unlock command was never attempted
self.sysinv_client.unlock_host.assert_not_called()
@ -95,8 +95,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState):
# mock the API call as successful on the subcloud
self.sysinv_client.unlock_host.return_value = CONTROLLER_0_UNLOCKING
# invoke the strategy state operation
self.worker.unlock_subcloud_controller(self.strategy_step)
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the lock command was actually attempted
self.sysinv_client.unlock_host.assert_called()
@ -119,8 +119,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState):
self.sysinv_client.unlock_host.return_value = \
CONTROLLER_0_UNLOCKING_FAILED
# invoke the strategy state operation
self.worker.unlock_subcloud_controller(self.strategy_step)
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the unlock command was actually attempted
self.sysinv_client.unlock_host.assert_called()
@ -136,8 +136,8 @@ class TestSwUpgradeUnlockControllerStage(TestSwUpgradeState):
self.sysinv_client.get_host.side_effect = \
Exception("Unable to find host controller-0")
# invoke the strategy state operation
self.worker.unlock_subcloud_controller(self.strategy_step)
# invoke the strategy state operation on the orch thread
self.worker.perform_state_action(self.strategy_step)
# verify the unlock command was never attempted
self.sysinv_client.unlock_host.assert_not_called()

198
distributedcloud/dcmanager/tests/unit/manager/test_sw_upgrade.py
View File

@ -33,7 +33,6 @@ from dcmanager.tests.unit.manager.test_sw_update_manager \
import Subcloud
from dcmanager.tests import utils
CONF = cfg.CONF
FAKE_ID = '1'
FAKE_SW_UPDATE_DATA = {
@ -53,29 +52,6 @@ FAKE_STRATEGY_STEP_DATA = {
"subcloud": None
}
MISSING_LICENSE_RESPONSE = {
u'content': u'',
u'error': u'License file not found. A license may not have been installed.'
}
LICENSE_VALID_RESPONSE = {
u'content': u'A valid license',
u'error': u''
}
ALTERNATE_LICENSE_RESPONSE = {
u'content': u'A different valid license',
u'error': u''
}
class FakeSysinvClient(object):
def __init__(self):
super(FakeSysinvClient, self).__init__()
self.get_license = mock.MagicMock()
self.install_license = mock.MagicMock()
class TestSwUpgrade(base.DCManagerTestCase):
def setUp(self):
@ -99,14 +75,6 @@ class TestSwUpgrade(base.DCManagerTestCase):
self.fake_patch_orch_thread
self.addCleanup(p.stop)
# Mock the sysinv client
self.fake_sysinv_client = FakeSysinvClient()
p = mock.patch.object(sw_upgrade_orch_thread.SwUpgradeOrchThread,
'get_sysinv_client')
self.mock_sysinv_client = p.start()
self.mock_sysinv_client.return_value = self.fake_sysinv_client
self.addCleanup(p.stop)
# Mock db_api
p = mock.patch.object(sw_upgrade_orch_thread, 'db_api')
self.mock_db_api = p.start()
@ -128,7 +96,6 @@ class TestSwUpgrade(base.DCManagerTestCase):
mock_dcmanager_audit_api = mock.Mock()
worker = sw_update_manager.SwUpgradeOrchThread(mock_strategy_lock,
mock_dcmanager_audit_api)
worker.get_ks_client = mock.Mock()
return worker
def assert_step_updated(self, subcloud_id, update_state):
@ -140,168 +107,3 @@ class TestSwUpgrade(base.DCManagerTestCase):
started_at=mock.ANY,
finished_at=mock.ANY,
)
class TestSwUpgradeLicenseStage(TestSwUpgrade):
def setUp(self):
super(TestSwUpgradeLicenseStage, self).setUp()
self.strategy_step = \
self.setup_strategy_step(consts.STRATEGY_STATE_INSTALLING_LICENSE)
def test_upgrade_subcloud_license_install_failure(self):
# Test the install subcloud license step where the system controller
# license is valid, and the subcloud license install fails
# Order of get_license calls:
# first license query is to system controller
# second license query is to subcloud (should be missing)
self.fake_sysinv_client.get_license.side_effect = \
[LICENSE_VALID_RESPONSE,
MISSING_LICENSE_RESPONSE]
# Simulate a license install failure on the subcloud
self.fake_sysinv_client.install_license.return_value = \
MISSING_LICENSE_RESPONSE
self.worker.install_subcloud_license(self.strategy_step)
# verify the license install was invoked
self.fake_sysinv_client.install_license.assert_called()
# Verify a install_license failure leads to a state failure
self.assert_step_updated(self.strategy_step.subcloud_id,
consts.STRATEGY_STATE_FAILED)
def test_upgrade_subcloud_license_install_success(self):
# Test the install subcloud license step where the system controller
# license is valid, and the subcloud installation succeeds
# Order of get_license calls:
# first license query is to system controller
# second license query is to subcloud (should be missing)
self.fake_sysinv_client.get_license.side_effect = \
[LICENSE_VALID_RESPONSE,
MISSING_LICENSE_RESPONSE]
# A license install should return a success
self.fake_sysinv_client.install_license.return_value = \
LICENSE_VALID_RESPONSE
self.worker.install_subcloud_license(self.strategy_step)
# verify the license install was invoked
self.fake_sysinv_client.install_license.assert_called()
# On success, the next state after installing license is importing load
self.assert_step_updated(self.strategy_step.subcloud_id,
consts.STRATEGY_STATE_IMPORTING_LOAD)
def test_upgrade_subcloud_license_skip_existing(self):
# Test the install subcloud license step where the system controller
# license is valid, and the subcloud already has the same license
# Order of get_license calls:
# first license query is to system controller
# second license query is to subcloud
self.fake_sysinv_client.get_license.side_effect = \
[LICENSE_VALID_RESPONSE,
LICENSE_VALID_RESPONSE]
self.worker.install_subcloud_license(self.strategy_step)
# A license install should not have been attempted due to the license
# already being up to date
self.fake_sysinv_client.install_license.assert_not_called()
# On success, the next state after installing license is importing load
self.assert_step_updated(self.strategy_step.subcloud_id,
consts.STRATEGY_STATE_IMPORTING_LOAD)
def test_upgrade_subcloud_license_overrides_mismatched_license(self):
# Test the install subcloud license step where the system controller
# license is valid, and the subcloud has a differnt license which
# should be overridden
# Order of get_license calls:
# first license query is to system controller
# second license query is to subcloud (should be valid but different)
self.fake_sysinv_client.get_license.side_effect = \
[LICENSE_VALID_RESPONSE,
ALTERNATE_LICENSE_RESPONSE]
# A license install should return a success
self.fake_sysinv_client.install_license.return_value = \
LICENSE_VALID_RESPONSE
self.worker.install_subcloud_license(self.strategy_step)
# verify the license install was invoked
self.fake_sysinv_client.install_license.assert_called()
# On success, the next state after installing license is importing load
self.assert_step_updated(self.strategy_step.subcloud_id,
consts.STRATEGY_STATE_IMPORTING_LOAD)
def test_upgrade_subcloud_license_skip_when_no_sys_controller_lic(self):
# Test the install subcloud license step is skipped and proceeds
# to the next state when there is no license on system controller
# Only makes one query to system controller
self.fake_sysinv_client.get_license.side_effect = \
[MISSING_LICENSE_RESPONSE, ]
# Test the install subcloud license stage
self.worker.install_subcloud_license(self.strategy_step)
# A license install should proceed to the next state without
# calling a license install
self.fake_sysinv_client.install_license.assert_not_called()
# Skip license install and move to next state
self.assert_step_updated(self.strategy_step.subcloud_id,
consts.STRATEGY_STATE_IMPORTING_LOAD)
def test_upgrade_subcloud_license_handle_failure(self):
# Test the install subcloud license step where the system controller
# license is valid, and the subcloud license install fails
# Order of get_license calls:
# first license query is to system controller
# second license query is to subcloud (should be missing)
self.fake_sysinv_client.get_license.side_effect = \
[LICENSE_VALID_RESPONSE,
MISSING_LICENSE_RESPONSE]
# Simulate a license install failure on the subcloud
self.fake_sysinv_client.install_license.return_value = \
MISSING_LICENSE_RESPONSE
self.worker.install_subcloud_license(self.strategy_step)
# verify the license install was invoked
self.fake_sysinv_client.install_license.assert_called()
# Verify a install_license failure leads to a state failure
self.assert_step_updated(self.strategy_step.subcloud_id,
consts.STRATEGY_STATE_FAILED)
def test_upgrade_subcloud_license_installs(self):
# Test the install subcloud license step where the system controller
# license is valid, and the subcloud installation succeeds
# Order of get_license calls:
# first license query is to system controller
# second license query is to subcloud (should be missing)
self.fake_sysinv_client.get_license.side_effect = \
[LICENSE_VALID_RESPONSE,
MISSING_LICENSE_RESPONSE]
# A license install should return a success
self.fake_sysinv_client.install_license.return_value = \
LICENSE_VALID_RESPONSE
self.worker.install_subcloud_license(self.strategy_step)
# verify the license install was invoked
self.fake_sysinv_client.install_license.assert_called()
# On success, the next state after installing license is importing load
self.assert_step_updated(self.strategy_step.subcloud_id,
consts.STRATEGY_STATE_IMPORTING_LOAD)