Refactor distributed cloud patch orchestration
Makes PatchOrchThread a subclass of the OrchThread, following the same design used in the other types of orchestration. It also removes the strategy step to orchestrate the system controller patching, as it should be done separately, before initializing the patch orchestration to patch the subclouds. The alarm checks that were executed inside the 'updating' state are now part of a new 'pre-check' state that runs at the beggining of the orchestration process. It also refactors the unit tests to be in line with the unit tests of the other orchestrator types by using the TestSwUpdate base class. Test Plan: 1. PASS - Verify successfull patch orchestration by applying and removing a NRR patch; 2. PASS - Verify successfull patch orchestration by applying and removing a RR patch; 3. PASS - Induce a management affecting alarm in a subcloud and verify that orchestration fails for that subcloud; 4. PASS - Induce a 900.001 alarm by partially apply a patch in a subcloud beforehand and verify that orchestration completes successfully for that subcloud; 5. PASS - Create a DC orch patch strategy, then manually patch a subcloud using the sw-manager patch-strategy command and then apply the DC orch patch strategy, verifying if the state machine skips from the 'creating VIM patch strategy' state directly to the 'finishing patch strategy' state; 6. PASS - Verify that no strategy step is created for the system controller; 7. PASS - Execute another orchestration type (e.g. prestage) and verify that it still works as expected. Story: 2010584 Task: 47371 Co-Authored-By: Al Bailey <al.bailey@windriver.com> Signed-off-by: Gustavo Herzmann <gustavo.herzmann@windriver.com> Change-Id: I2c37bd59696e6f9e4fd706f3b3c97f8f9e4499b0
This commit is contained in:
parent
c9d5cb2080
commit
bd7d2a6922
|
@ -1,5 +1,5 @@
|
|||
# Copyright (c) 2016 Ericsson AB.
|
||||
# Copyright (c) 2017-2022 Wind River Systems, Inc.
|
||||
# Copyright (c) 2017-2023 Wind River Systems, Inc.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
|
@ -92,16 +92,18 @@ DEFAULT_SUBCLOUD_GROUP_DESCRIPTION = 'Default Subcloud Group'
|
|||
DEFAULT_SUBCLOUD_GROUP_UPDATE_APPLY_TYPE = SUBCLOUD_APPLY_TYPE_PARALLEL
|
||||
DEFAULT_SUBCLOUD_GROUP_MAX_PARALLEL_SUBCLOUDS = 2
|
||||
|
||||
# Strategy step states
|
||||
# Common strategy step states
|
||||
STRATEGY_STATE_INITIAL = "initial"
|
||||
STRATEGY_STATE_UPDATING_PATCHES = "updating patches"
|
||||
STRATEGY_STATE_CREATING_STRATEGY = "creating strategy"
|
||||
STRATEGY_STATE_APPLYING_STRATEGY = "applying strategy"
|
||||
STRATEGY_STATE_FINISHING = "finishing"
|
||||
STRATEGY_STATE_COMPLETE = "complete"
|
||||
STRATEGY_STATE_ABORTED = "aborted"
|
||||
STRATEGY_STATE_FAILED = "failed"
|
||||
|
||||
# Patch orchestrations states
|
||||
STRATEGY_STATE_CREATING_VIM_PATCH_STRATEGY = "creating VIM patch strategy"
|
||||
STRATEGY_STATE_DELETING_VIM_PATCH_STRATEGY = "deleting VIM patch strategy"
|
||||
STRATEGY_STATE_APPLYING_VIM_PATCH_STRATEGY = "applying VIM patch strategy"
|
||||
|
||||
# Upgrade orchestration states
|
||||
STRATEGY_STATE_PRE_CHECK = "pre check"
|
||||
STRATEGY_STATE_INSTALLING_LICENSE = "installing license"
|
||||
STRATEGY_STATE_IMPORTING_LOAD = "importing load"
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
# Copyright 2017 Ericsson AB.
|
||||
# Copyright (c) 2017-2022 Wind River Systems, Inc.
|
||||
# Copyright (c) 2017-2023 Wind River Systems, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
@ -23,7 +23,9 @@ from keystoneauth1 import exceptions as keystone_exceptions
|
|||
from oslo_log import log as logging
|
||||
|
||||
from dccommon import consts as dccommon_consts
|
||||
from dccommon.drivers.openstack.patching_v1 import PatchingClient
|
||||
from dccommon.drivers.openstack.sdk_platform import OpenStackDriver
|
||||
from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
|
||||
from dccommon.drivers.openstack import vim
|
||||
from dcmanager.common import consts
|
||||
from dcmanager.common import context
|
||||
|
@ -78,6 +80,8 @@ class OrchThread(threading.Thread):
|
|||
thread_pool_size=500)
|
||||
# Track worker created for each subcloud.
|
||||
self.subcloud_workers = dict()
|
||||
# Track if the strategy setup function was executed
|
||||
self._setup = False
|
||||
|
||||
@abc.abstractmethod
|
||||
def trigger_audit(self):
|
||||
|
@ -85,6 +89,28 @@ class OrchThread(threading.Thread):
|
|||
LOG.warn("(%s) OrchThread subclass must override trigger_audit"
|
||||
% self.update_type)
|
||||
|
||||
def _pre_apply_setup(self):
|
||||
"""Setup performed once before a strategy starts to apply"""
|
||||
if not self._setup:
|
||||
LOG.info("(%s) OrchThread Pre-Apply Setup" % self.update_type)
|
||||
self._setup = True
|
||||
self.pre_apply_setup()
|
||||
|
||||
def pre_apply_setup(self):
|
||||
"""Subclass can override this method"""
|
||||
pass
|
||||
|
||||
def _post_delete_teardown(self):
|
||||
"""Cleanup code executed once after deleting a strategy"""
|
||||
if self._setup:
|
||||
LOG.info("(%s) OrchThread Post-Delete Teardown" % self.update_type)
|
||||
self._setup = False
|
||||
self.post_delete_teardown()
|
||||
|
||||
def post_delete_teardown(self):
|
||||
"""Subclass can override this method"""
|
||||
pass
|
||||
|
||||
def stopped(self):
|
||||
return self._stop.isSet()
|
||||
|
||||
|
@ -114,6 +140,19 @@ class OrchThread(threading.Thread):
|
|||
ks_client = OrchThread.get_ks_client(region_name)
|
||||
return vim.VimClient(region_name, ks_client.session)
|
||||
|
||||
@staticmethod
|
||||
def get_sysinv_client(region_name=dccommon_consts.DEFAULT_REGION_NAME):
|
||||
ks_client = OrchThread.get_ks_client(region_name)
|
||||
endpoint = ks_client.endpoint_cache.get_endpoint('sysinv')
|
||||
return SysinvClient(region_name,
|
||||
ks_client.session,
|
||||
endpoint=endpoint)
|
||||
|
||||
@staticmethod
|
||||
def get_patching_client(region_name=dccommon_consts.DEFAULT_REGION_NAME):
|
||||
ks_client = OrchThread.get_ks_client(region_name)
|
||||
return PatchingClient(region_name, ks_client.session)
|
||||
|
||||
@staticmethod
|
||||
def get_region_name(strategy_step):
|
||||
"""Get the region name for a strategy step"""
|
||||
|
@ -184,6 +223,7 @@ class OrchThread(threading.Thread):
|
|||
if sw_update_strategy.state in [
|
||||
consts.SW_UPDATE_STATE_APPLYING,
|
||||
consts.SW_UPDATE_STATE_ABORTING]:
|
||||
self._pre_apply_setup()
|
||||
self.apply(sw_update_strategy)
|
||||
elif sw_update_strategy.state == \
|
||||
consts.SW_UPDATE_STATE_ABORT_REQUESTED:
|
||||
|
@ -191,6 +231,7 @@ class OrchThread(threading.Thread):
|
|||
elif sw_update_strategy.state == \
|
||||
consts.SW_UPDATE_STATE_DELETING:
|
||||
self.delete(sw_update_strategy)
|
||||
self._post_delete_teardown()
|
||||
|
||||
except exceptions.NotFound:
|
||||
# Nothing to do if a strategy doesn't exist
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -1,5 +1,5 @@
|
|||
#
|
||||
# Copyright (c) 2020-2022 Wind River Systems, Inc.
|
||||
# Copyright (c) 2020-2023 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
@ -31,10 +31,15 @@ class BaseState(object):
|
|||
self._stop = None
|
||||
self.region_name = region_name
|
||||
self._shared_caches = None
|
||||
self._job_data = None
|
||||
|
||||
def override_next_state(self, next_state):
|
||||
self.next_state = next_state
|
||||
|
||||
def set_job_data(self, job_data):
|
||||
"""Store an orch_thread job data object"""
|
||||
self._job_data = job_data
|
||||
|
||||
def registerStopEvent(self, stop_event):
|
||||
"""Store an orch_thread threading.Event to detect stop."""
|
||||
self._stop = stop_event
|
||||
|
@ -74,6 +79,13 @@ class BaseState(object):
|
|||
self.get_region_name(strategy_step),
|
||||
details))
|
||||
|
||||
def exception_log(self, strategy_step, details):
|
||||
LOG.exception("Stage: %s, State: %s, Subcloud: %s, Details: %s"
|
||||
% (strategy_step.stage,
|
||||
strategy_step.state,
|
||||
self.get_region_name(strategy_step),
|
||||
details))
|
||||
|
||||
@staticmethod
|
||||
def get_region_name(strategy_step):
|
||||
"""Get the region name for a strategy step"""
|
||||
|
|
|
@ -0,0 +1,20 @@
|
|||
#
|
||||
# Copyright (c) 2023 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
from dccommon.drivers.openstack import vim
|
||||
from dcmanager.common import consts
|
||||
from dcmanager.orchestrator.states.applying_vim_strategy import \
|
||||
ApplyingVIMStrategyState
|
||||
|
||||
|
||||
class ApplyingVIMPatchStrategyState(ApplyingVIMStrategyState):
|
||||
"""State for applying a VIM patch strategy."""
|
||||
|
||||
def __init__(self, region_name):
|
||||
super(ApplyingVIMPatchStrategyState, self).__init__(
|
||||
next_state=consts.STRATEGY_STATE_FINISHING_PATCH_STRATEGY,
|
||||
region_name=region_name,
|
||||
strategy_name=vim.STRATEGY_NAME_SW_PATCH)
|
|
@ -0,0 +1,45 @@
|
|||
#
|
||||
# Copyright (c) 2023 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
from dccommon.drivers.openstack import vim
|
||||
from dcmanager.common import consts
|
||||
from dcmanager.orchestrator.states.creating_vim_strategy import \
|
||||
CreatingVIMStrategyState
|
||||
|
||||
|
||||
# Max time: 2 minutes = 12 queries x 10 seconds between
|
||||
DEFAULT_MAX_QUERIES = 12
|
||||
DEFAULT_SLEEP_DURATION = 10
|
||||
|
||||
|
||||
class CreatingVIMPatchStrategyState(CreatingVIMStrategyState):
|
||||
"""State for creating a VIM patch strategy."""
|
||||
|
||||
def __init__(self, region_name):
|
||||
super(CreatingVIMPatchStrategyState, self).__init__(
|
||||
next_state=consts.STRATEGY_STATE_APPLYING_VIM_PATCH_STRATEGY,
|
||||
region_name=region_name,
|
||||
strategy_name=vim.STRATEGY_NAME_SW_PATCH)
|
||||
|
||||
self.SKIP_REASON = "no software patches need to be applied"
|
||||
self.SKIP_STATE = consts.STRATEGY_STATE_FINISHING_PATCH_STRATEGY
|
||||
|
||||
# Change CreatingVIMStrategyState default values
|
||||
self.sleep_duration = DEFAULT_SLEEP_DURATION
|
||||
self.max_queries = DEFAULT_MAX_QUERIES
|
||||
|
||||
def skip_check(self, strategy_step, subcloud_strategy):
|
||||
"""Check if the VIM stategy needs to be skipped"""
|
||||
|
||||
if (subcloud_strategy and
|
||||
(subcloud_strategy.state == vim.STATE_BUILD_FAILED) and
|
||||
(subcloud_strategy.build_phase.reason == self.SKIP_REASON)):
|
||||
self.info_log(strategy_step, "Skip forward in state machine due to:"
|
||||
" ({})".format(self.SKIP_REASON))
|
||||
return self.SKIP_STATE
|
||||
|
||||
# If we get here, there is not a reason to skip
|
||||
return None
|
|
@ -0,0 +1,71 @@
|
|||
#
|
||||
# Copyright (c) 2023 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
from dccommon.drivers.openstack import patching_v1
|
||||
from dcmanager.common import consts
|
||||
from dcmanager.common.exceptions import StrategyStoppedException
|
||||
from dcmanager.orchestrator.states.base import BaseState
|
||||
|
||||
|
||||
class FinishingPatchStrategyState(BaseState):
|
||||
"""Patch orchestration state for cleaning up patches"""
|
||||
|
||||
def __init__(self, region_name):
|
||||
super(FinishingPatchStrategyState, self).__init__(
|
||||
next_state=consts.STRATEGY_STATE_COMPLETE,
|
||||
region_name=region_name)
|
||||
self.region_one_commited_patch_ids = None
|
||||
|
||||
def set_job_data(self, job_data):
|
||||
"""Store an orch_thread job data object"""
|
||||
# This will immediately fail if these attributes are a mismatch
|
||||
self.region_one_commited_patch_ids = \
|
||||
job_data.region_one_commited_patch_ids
|
||||
|
||||
def perform_state_action(self, strategy_step):
|
||||
self.info_log(strategy_step, "Finishing subcloud patching")
|
||||
|
||||
subcloud_patches = self.get_patching_client(self.region_name).query()
|
||||
self.debug_log(strategy_step, "Patches for subcloud: %s" %
|
||||
subcloud_patches)
|
||||
|
||||
# For this subcloud, determine which patches should be committed and
|
||||
# which should be deleted. We check the patchstate here because
|
||||
# patches cannot be deleted or committed if they are in a partial
|
||||
# state (e.g. Partial-Apply or Partial-Remove).
|
||||
patches_to_commit = []
|
||||
patches_to_delete = []
|
||||
|
||||
for patch_id in subcloud_patches.keys():
|
||||
patch_state = subcloud_patches[patch_id]["patchstate"]
|
||||
|
||||
if patch_state == patching_v1.PATCH_STATE_AVAILABLE:
|
||||
self.info_log(strategy_step,
|
||||
"Patch %s will be deleted from subcloud" %
|
||||
patch_id)
|
||||
patches_to_delete.append(patch_id)
|
||||
|
||||
elif (patch_state == patching_v1.PATCH_STATE_APPLIED
|
||||
and patch_id in self.region_one_commited_patch_ids):
|
||||
self.info_log(strategy_step,
|
||||
"Patch %s will be committed in subcloud" %
|
||||
patch_id)
|
||||
patches_to_commit.append(patch_id)
|
||||
|
||||
if patches_to_delete:
|
||||
self.info_log(strategy_step, "Deleting patches %s from subcloud" %
|
||||
patches_to_delete)
|
||||
self.get_patching_client(self.region_name).delete(patches_to_delete)
|
||||
|
||||
if self.stopped():
|
||||
raise StrategyStoppedException()
|
||||
|
||||
if patches_to_commit:
|
||||
self.info_log(strategy_step, "Committing patches %s in subcloud" %
|
||||
patches_to_commit)
|
||||
self.get_patching_client(self.region_name).commit(patches_to_commit)
|
||||
|
||||
return self.next_state
|
|
@ -0,0 +1,43 @@
|
|||
#
|
||||
# Copyright (c) 2023 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
from dccommon import consts as dccommon_consts
|
||||
from dccommon.drivers.openstack import patching_v1
|
||||
from dcmanager.common import utils
|
||||
from dcmanager.orchestrator.orch_thread import OrchThread
|
||||
from oslo_log import log as logging
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PatchJobData(object):
|
||||
"""Job data initialized once and shared across state operators"""
|
||||
|
||||
def __init__(self):
|
||||
self.initialize_data()
|
||||
|
||||
def initialize_data(self):
|
||||
LOG.info("Initializing PatchOrchThread job data")
|
||||
|
||||
loads = OrchThread.get_sysinv_client(
|
||||
dccommon_consts.DEFAULT_REGION_NAME).get_loads()
|
||||
|
||||
installed_loads = utils.get_loads_for_patching(loads)
|
||||
|
||||
self.region_one_patches = OrchThread.get_patching_client(
|
||||
dccommon_consts.DEFAULT_REGION_NAME).query()
|
||||
|
||||
self.region_one_applied_patch_ids = []
|
||||
self.region_one_commited_patch_ids = []
|
||||
for patch_id, patch in self.region_one_patches.items():
|
||||
# Only the patches for the installed loads will be stored
|
||||
if patch["sw_version"] in installed_loads:
|
||||
if patch["repostate"] == patching_v1.PATCH_STATE_APPLIED:
|
||||
self.region_one_applied_patch_ids.append(patch_id)
|
||||
elif patch["repostate"] == patching_v1.PATCH_STATE_COMMITTED:
|
||||
self.region_one_commited_patch_ids.append(patch_id)
|
||||
# A commited patch is also an applied one
|
||||
self.region_one_applied_patch_ids.append(patch_id)
|
|
@ -0,0 +1,51 @@
|
|||
#
|
||||
# Copyright (c) 2023 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
from dcmanager.common import consts
|
||||
from dcmanager.orchestrator.states.base import BaseState
|
||||
|
||||
IGNORED_ALARMS_IDS = ("900.001",) # Patch in progress
|
||||
|
||||
|
||||
class PreCheckState(BaseState):
|
||||
"""Pre check patch orchestration state"""
|
||||
|
||||
def __init__(self, region_name):
|
||||
super(PreCheckState, self).__init__(
|
||||
next_state=consts.STRATEGY_STATE_UPDATING_PATCHES,
|
||||
region_name=region_name)
|
||||
|
||||
def has_mgmt_affecting_alarms(self, ignored_alarms=()):
|
||||
alarms = self.get_fm_client(self.region_name).get_alarms()
|
||||
for alarm in alarms:
|
||||
if alarm.mgmt_affecting == "True" and \
|
||||
alarm.alarm_id not in ignored_alarms:
|
||||
return True
|
||||
# No management affecting alarms
|
||||
return False
|
||||
|
||||
def perform_state_action(self, strategy_step):
|
||||
"""Pre check region status"""
|
||||
self.info_log(strategy_step, "Checking subcloud alarm status")
|
||||
|
||||
# Stop patching if the subcloud contains management affecting alarms.
|
||||
message = None
|
||||
try:
|
||||
if self.has_mgmt_affecting_alarms(ignored_alarms=IGNORED_ALARMS_IDS):
|
||||
message = ("Subcloud contains one or more management affecting"
|
||||
" alarm(s). It will not be patched. Please resolve"
|
||||
" the alarm condition(s) and try again.")
|
||||
except Exception as e:
|
||||
self.exception_log(strategy_step,
|
||||
"Failed to obtain subcloud alarm report")
|
||||
message = ("Failed to obtain subcloud alarm report due to: (%s)."
|
||||
" Please see /var/log/dcmanager/orchestrator.log for"
|
||||
" details" % str(e))
|
||||
|
||||
if message:
|
||||
raise Exception(message)
|
||||
|
||||
return self.next_state
|
|
@ -0,0 +1,159 @@
|
|||
#
|
||||
# Copyright (c) 2023 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
from dccommon.drivers.openstack import patching_v1
|
||||
from dcmanager.common import consts
|
||||
from dcmanager.common.exceptions import StrategyStoppedException
|
||||
from dcmanager.orchestrator.states.base import BaseState
|
||||
|
||||
# Max time: 1 minute = 6 queries x 10 seconds between
|
||||
DEFAULT_MAX_QUERIES = 6
|
||||
DEFAULT_SLEEP_DURATION = 10
|
||||
|
||||
|
||||
class UpdatingPatchesState(BaseState):
|
||||
"""Patch orchestration state for updating patches"""
|
||||
|
||||
def __init__(self, region_name):
|
||||
super(UpdatingPatchesState, self).__init__(
|
||||
next_state=consts.STRATEGY_STATE_CREATING_VIM_PATCH_STRATEGY,
|
||||
region_name=region_name)
|
||||
self.max_queries = DEFAULT_MAX_QUERIES
|
||||
self.sleep_duration = DEFAULT_SLEEP_DURATION
|
||||
|
||||
self.region_one_patches = None
|
||||
self.region_one_applied_patch_ids = None
|
||||
|
||||
def set_job_data(self, job_data):
|
||||
"""Store an orch_thread job data object"""
|
||||
self.region_one_patches = job_data.region_one_patches
|
||||
self.region_one_applied_patch_ids = job_data.\
|
||||
region_one_applied_patch_ids
|
||||
|
||||
def perform_state_action(self, strategy_step):
|
||||
"""Update patches in this subcloud"""
|
||||
self.info_log(strategy_step, "Updating patches")
|
||||
|
||||
# Retrieve all subcloud patches
|
||||
try:
|
||||
subcloud_patches = self.get_patching_client(self.region_name).\
|
||||
query()
|
||||
except Exception:
|
||||
message = ("Cannot retrieve subcloud patches. Please see logs for"
|
||||
" details.")
|
||||
self.exception_log(strategy_step, message)
|
||||
raise Exception(message)
|
||||
|
||||
patches_to_upload = []
|
||||
patches_to_apply = []
|
||||
patches_to_remove = []
|
||||
|
||||
subcloud_patch_ids = subcloud_patches.keys()
|
||||
|
||||
# RegionOne applied patches not present on the subcloud needs to
|
||||
# be uploaded and applied to the subcloud
|
||||
for patch_id in self.region_one_applied_patch_ids:
|
||||
if patch_id not in subcloud_patch_ids:
|
||||
self.info_log(strategy_step, "Patch %s missing from subloud" %
|
||||
patch_id)
|
||||
patches_to_upload.append(patch_id)
|
||||
patches_to_apply.append(patch_id)
|
||||
|
||||
# Check that all applied patches in subcloud match RegionOne
|
||||
for patch_id in subcloud_patch_ids:
|
||||
repostate = subcloud_patches[patch_id]["repostate"]
|
||||
if repostate == patching_v1.PATCH_STATE_APPLIED:
|
||||
if patch_id not in self.region_one_applied_patch_ids:
|
||||
self.info_log(strategy_step,
|
||||
"Patch %s will be removed from subcloud" %
|
||||
patch_id)
|
||||
patches_to_remove.append(patch_id)
|
||||
elif repostate == patching_v1.PATCH_STATE_COMMITTED:
|
||||
if patch_id not in self.region_one_applied_patch_ids:
|
||||
message = ("Patch %s is committed in subcloud but "
|
||||
"not applied in SystemController" % patch_id)
|
||||
self.warn_log(strategy_step, message)
|
||||
raise Exception(message)
|
||||
elif repostate == patching_v1.PATCH_STATE_AVAILABLE:
|
||||
if patch_id in self.region_one_applied_patch_ids:
|
||||
patches_to_apply.append(patch_id)
|
||||
|
||||
else:
|
||||
# This patch is in an invalid state
|
||||
message = ("Patch %s in subcloud is in an unexpected state: %s"
|
||||
% (patch_id, repostate))
|
||||
self.warn_log(strategy_step, message)
|
||||
raise Exception(message)
|
||||
|
||||
if patches_to_upload:
|
||||
self.info_log(strategy_step, "Uploading patches %s to subcloud" %
|
||||
patches_to_upload)
|
||||
for patch in patches_to_upload:
|
||||
patch_sw_version = self.region_one_patches[patch]["sw_version"]
|
||||
patch_file = "%s/%s/%s.patch" % (consts.PATCH_VAULT_DIR,
|
||||
patch_sw_version, patch)
|
||||
if not os.path.isfile(patch_file):
|
||||
message = "Patch file %s is missing" % patch_file
|
||||
self.error_log(strategy_step, message)
|
||||
raise Exception(message)
|
||||
|
||||
self.get_patching_client(self.region_name).upload([patch_file])
|
||||
if self.stopped():
|
||||
self.info_log(strategy_step,
|
||||
"Exiting because task is stopped")
|
||||
raise StrategyStoppedException()
|
||||
|
||||
if patches_to_remove:
|
||||
self.info_log(strategy_step, "Removing patches %s from subcloud" %
|
||||
patches_to_remove)
|
||||
self.get_patching_client(self.region_name).remove(patches_to_remove)
|
||||
|
||||
if patches_to_apply:
|
||||
self.info_log(strategy_step, "Applying patches %s to subcloud" %
|
||||
patches_to_apply)
|
||||
self.get_patching_client(self.region_name).apply(patches_to_apply)
|
||||
|
||||
# Now that we have applied/removed/uploaded patches, we need to give
|
||||
# the patch controller on this subcloud time to determine whether
|
||||
# each host on that subcloud is patch current.
|
||||
wait_count = 0
|
||||
while True:
|
||||
subcloud_hosts = self.get_patching_client(self.region_name).\
|
||||
query_hosts()
|
||||
self.debug_log(strategy_step,
|
||||
"query_hosts for subcloud returned %s" %
|
||||
subcloud_hosts)
|
||||
|
||||
for host in subcloud_hosts:
|
||||
if host["interim_state"]:
|
||||
# This host is not yet ready.
|
||||
self.debug_log(strategy_step,
|
||||
"Host %s in subcloud in interim state" %
|
||||
host["hostname"])
|
||||
break
|
||||
else:
|
||||
# All hosts in the subcloud are updated
|
||||
break
|
||||
|
||||
wait_count += 1
|
||||
if wait_count >= self.max_queries:
|
||||
# We have waited too long.
|
||||
# We log a warning but do not fail the step
|
||||
message = ("Applying patches to subcloud "
|
||||
"taking too long to recover. "
|
||||
"Continuing..")
|
||||
self.warn_log(strategy_step, message)
|
||||
break
|
||||
if self.stopped():
|
||||
self.info_log(strategy_step, "Exiting because task is stopped")
|
||||
raise StrategyStoppedException()
|
||||
# Delay between queries
|
||||
time.sleep(self.sleep_duration)
|
||||
|
||||
return self.next_state
|
|
@ -1,5 +1,5 @@
|
|||
# Copyright 2017 Ericsson AB.
|
||||
# Copyright (c) 2017-2022 Wind River Systems, Inc.
|
||||
# Copyright (c) 2017-2023 Wind River Systems, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
@ -514,19 +514,6 @@ class SwUpdateManager(manager.Manager):
|
|||
consts.SW_UPDATE_STATE_INITIAL,
|
||||
extra_args=extra_args)
|
||||
|
||||
# For 'patch', always create a strategy step for the system controller
|
||||
# A strategy step for the system controller is not added for:
|
||||
# 'upgrade', 'firmware', 'kube upgrade', 'kube rootca update'
|
||||
if strategy_type == consts.SW_UPDATE_TYPE_PATCH:
|
||||
current_stage_counter += 1
|
||||
db_api.strategy_step_create(
|
||||
context,
|
||||
None, # None means not a subcloud. ie: SystemController
|
||||
stage=current_stage_counter,
|
||||
state=consts.STRATEGY_STATE_INITIAL,
|
||||
details='')
|
||||
strategy_step_created = True
|
||||
|
||||
# Create a strategy step for each subcloud that is managed, online and
|
||||
# out of sync
|
||||
# special cases:
|
||||
|
|
|
@ -0,0 +1,18 @@
|
|||
#
|
||||
# Copyright (c) 2023 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
from dcmanager.common import consts
|
||||
from dcmanager.tests.unit.orchestrator.states.patch.test_base import \
|
||||
TestPatchState
|
||||
from dcmanager.tests.unit.orchestrator.states.test_applying_vim_strategy import \
|
||||
ApplyingVIMStrategyMixin
|
||||
|
||||
|
||||
class TestApplyingVIMPatchStrategyStage(ApplyingVIMStrategyMixin,
|
||||
TestPatchState):
|
||||
def setUp(self):
|
||||
super(TestApplyingVIMPatchStrategyStage, self).setUp()
|
||||
self.set_state(consts.STRATEGY_STATE_APPLYING_VIM_PATCH_STRATEGY,
|
||||
consts.STRATEGY_STATE_FINISHING_PATCH_STRATEGY)
|
|
@ -0,0 +1,14 @@
|
|||
#
|
||||
# Copyright (c) 2023 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
from dcmanager.common import consts
|
||||
from dcmanager.tests.unit.orchestrator.test_base import TestSwUpdate
|
||||
|
||||
|
||||
class TestPatchState(TestSwUpdate):
|
||||
DEFAULT_STRATEGY_TYPE = consts.SW_UPDATE_TYPE_PATCH
|
||||
|
||||
def setUp(self):
|
||||
super(TestPatchState, self).setUp()
|
|
@ -0,0 +1,59 @@
|
|||
#
|
||||
# Copyright (c) 2023 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
from collections import namedtuple
|
||||
|
||||
from dccommon.drivers.openstack import vim
|
||||
from dcmanager.common import consts
|
||||
from dcmanager.tests.unit.fakes import FakeVimStrategy
|
||||
from dcmanager.tests.unit.orchestrator.states.patch.test_base import \
|
||||
TestPatchState
|
||||
from dcmanager.tests.unit.orchestrator.states.test_creating_vim_strategy import \
|
||||
CreatingVIMStrategyStageMixin
|
||||
import mock
|
||||
|
||||
|
||||
BuildPhase = namedtuple("BuildPhase", "reason")
|
||||
|
||||
|
||||
REASON = "no software patches need to be applied"
|
||||
STRATEGY_BUILDING = FakeVimStrategy(state=vim.STATE_BUILDING)
|
||||
STRATEGY_FAILED_BUILDING = FakeVimStrategy(state=vim.STATE_BUILD_FAILED,
|
||||
build_phase=BuildPhase(REASON))
|
||||
|
||||
|
||||
@mock.patch("dcmanager.orchestrator.states.patch.creating_vim_patch_strategy."
|
||||
"DEFAULT_MAX_QUERIES", 3)
|
||||
@mock.patch("dcmanager.orchestrator.states.patch.creating_vim_patch_strategy."
|
||||
"DEFAULT_SLEEP_DURATION", 1)
|
||||
class TestCreatingVIMPatchStrategyStage(CreatingVIMStrategyStageMixin,
|
||||
TestPatchState):
|
||||
def setUp(self):
|
||||
super(TestCreatingVIMPatchStrategyStage, self).setUp()
|
||||
self.set_state(consts.STRATEGY_STATE_CREATING_VIM_PATCH_STRATEGY,
|
||||
consts.STRATEGY_STATE_APPLYING_VIM_PATCH_STRATEGY)
|
||||
self.skip_state = consts.STRATEGY_STATE_FINISHING_PATCH_STRATEGY
|
||||
|
||||
def test_skip_if_not_needed(self):
|
||||
"""Test creating VIM strategy when no patches need to be applied.
|
||||
|
||||
When VIM returns 'no software patches need to be applied' the state
|
||||
should skip the 'applying VIM strategy' state, returning the 'finishing'
|
||||
state instead.
|
||||
"""
|
||||
|
||||
# first api query is before the create
|
||||
self.vim_client.get_strategy.side_effect = [None,
|
||||
STRATEGY_BUILDING,
|
||||
STRATEGY_FAILED_BUILDING]
|
||||
|
||||
# API calls acts as expected
|
||||
self.vim_client.create_strategy.return_value = STRATEGY_BUILDING
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.skip_state)
|
|
@ -0,0 +1,116 @@
|
|||
#
|
||||
# Copyright (c) 2023 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
from dcmanager.common import consts
|
||||
from dcmanager.orchestrator.orch_thread import OrchThread
|
||||
from dcmanager.tests.unit.orchestrator.states.fakes import FakeLoad
|
||||
from dcmanager.tests.unit.orchestrator.states.patch.test_base import \
|
||||
TestPatchState
|
||||
import mock
|
||||
|
||||
REGION_ONE_PATCHES = {"DC.1": {"sw_version": "20.12",
|
||||
"repostate": "Applied",
|
||||
"patchstate": "Applied"},
|
||||
"DC.2": {"sw_version": "20.12",
|
||||
"repostate": "Applied",
|
||||
"patchstate": "Applied"},
|
||||
"DC.3": {"sw_version": "20.12",
|
||||
"repostate": "Committed",
|
||||
"patchstate": "Committed"},
|
||||
"DC.4": {"sw_version": "20.12",
|
||||
"repostate": "Available",
|
||||
"patchstate": "Available"},
|
||||
"DC.8": {"sw_version": "20.12",
|
||||
"repostate": "Committed",
|
||||
"patchstate": "Committed"}}
|
||||
|
||||
SUBCLOUD_PATCHES = {"DC.1": {"sw_version": "20.12",
|
||||
"repostate": "Applied",
|
||||
"patchstate": "Applied"},
|
||||
"DC.2": {"sw_version": "20.12",
|
||||
"repostate": "Applied",
|
||||
"patchstate": "Applied"},
|
||||
"DC.3": {"sw_version": "20.12",
|
||||
"repostate": "Applied",
|
||||
"patchstate": "Applied"},
|
||||
"DC.5": {"sw_version": "20.12",
|
||||
"repostate": "Available",
|
||||
"patchstate": "Available"},
|
||||
"DC.8": {"sw_version": "20.12",
|
||||
"repostate": "Committed",
|
||||
"patchstate": "Committed"}}
|
||||
|
||||
|
||||
class TestPatchFinishingStage(TestPatchState):
|
||||
def setUp(self):
|
||||
super(TestPatchFinishingStage, self).setUp()
|
||||
|
||||
self.success_state = consts.STRATEGY_STATE_COMPLETE
|
||||
|
||||
# Add the subcloud being processed by this unit test
|
||||
self.subcloud = self.setup_subcloud()
|
||||
|
||||
# Add the strategy_step state being processed by this unit test
|
||||
self.strategy_step = self.setup_strategy_step(
|
||||
self.subcloud.id, consts.STRATEGY_STATE_FINISHING_PATCH_STRATEGY)
|
||||
|
||||
# Add mock API endpoints for patching and sysinv client calls
|
||||
# invoked by this state
|
||||
self.patching_client.query = mock.MagicMock()
|
||||
self.patching_client.delete = mock.MagicMock()
|
||||
self.patching_client.commit = mock.MagicMock()
|
||||
self.sysinv_client.get_loads = mock.MagicMock()
|
||||
|
||||
# Mock OrchThread functions used by PatchJobData class
|
||||
p = mock.patch.object(OrchThread, "get_patching_client")
|
||||
self.mock_orch_patching_client = p.start()
|
||||
self.mock_orch_patching_client.return_value = self.patching_client
|
||||
self.addCleanup(p.stop)
|
||||
|
||||
p = mock.patch.object(OrchThread, "get_sysinv_client")
|
||||
self.mock_orch_sysinv_client = p.start()
|
||||
self.mock_orch_sysinv_client.return_value = self.sysinv_client
|
||||
self.addCleanup(p.stop)
|
||||
|
||||
self.fake_load = FakeLoad(1, software_version="20.12",
|
||||
state=consts.ACTIVE_LOAD_STATE)
|
||||
|
||||
def test_set_job_data(self):
|
||||
"""Test the 'set_job_data' method"""
|
||||
self.patching_client.query.side_effect = [REGION_ONE_PATCHES,
|
||||
SUBCLOUD_PATCHES]
|
||||
|
||||
self.sysinv_client.get_loads.side_effect = [[self.fake_load]]
|
||||
|
||||
# invoke the pre apply setup to create the PatchJobData object
|
||||
self.worker.pre_apply_setup()
|
||||
|
||||
# call determine_state_operator to invoke the set_job_data method
|
||||
state = self.worker.determine_state_operator(self.strategy_step)
|
||||
|
||||
# Assert that the state has the proper region_one_commited_patch_ids
|
||||
# attribute
|
||||
self.assertItemsEqual(["DC.3", "DC.8"],
|
||||
state.region_one_commited_patch_ids)
|
||||
|
||||
def test_finish(self):
|
||||
"""Test whether the 'finishing' state completes successfully"""
|
||||
self.patching_client.query.side_effect = [REGION_ONE_PATCHES,
|
||||
SUBCLOUD_PATCHES]
|
||||
|
||||
self.sysinv_client.get_loads.side_effect = [[self.fake_load]]
|
||||
|
||||
# invoke the pre apply setup to create the PatchJobData object
|
||||
self.worker.pre_apply_setup()
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
self.patching_client.delete.assert_called_with(["DC.5"])
|
||||
self.patching_client.commit.assert_called_with(["DC.3"])
|
||||
|
||||
# On success, the state should transition to the next state
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.success_state)
|
|
@ -0,0 +1,134 @@
|
|||
#
|
||||
# Copyright (c) 2023 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
from dcmanager.common import consts
|
||||
from dcmanager.orchestrator.states.patch.pre_check import IGNORED_ALARMS_IDS
|
||||
from dcmanager.tests.unit.orchestrator.states.fakes import FakeAlarm
|
||||
from dcmanager.tests.unit.orchestrator.states.patch.test_base import \
|
||||
TestPatchState
|
||||
import mock
|
||||
|
||||
|
||||
class TestPatchPreCheckStage(TestPatchState):
|
||||
def setUp(self):
|
||||
super(TestPatchPreCheckStage, self).setUp()
|
||||
|
||||
self.success_state = consts.STRATEGY_STATE_UPDATING_PATCHES
|
||||
|
||||
# Add the subcloud being processed by this unit test
|
||||
self.subcloud = self.setup_subcloud()
|
||||
|
||||
# Add the strategy_step state being processed by this unit test
|
||||
self.strategy_step = self.setup_strategy_step(
|
||||
self.subcloud.id, consts.STRATEGY_STATE_PRE_CHECK)
|
||||
|
||||
self.fm_client.get_alarms = mock.MagicMock()
|
||||
|
||||
def test_no_alarms(self):
|
||||
"""Test pre check step where there are no alarms
|
||||
|
||||
The pre-check should transition to the updating patches state
|
||||
"""
|
||||
|
||||
self.fm_client.get_alarms.return_value = []
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the get alarms API call was invoked
|
||||
self.fm_client.get_alarms.assert_called()
|
||||
|
||||
# verify the expected next state happened
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.success_state)
|
||||
|
||||
def test_no_management_affecting_alarm(self):
|
||||
"""Test pre check step where there are no management affecting alarms
|
||||
|
||||
The pre-check should transition to the updating patches state
|
||||
"""
|
||||
|
||||
self.fm_client.get_alarms.return_value = [FakeAlarm("100.114", "False")]
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the get alarms API call was invoked
|
||||
self.fm_client.get_alarms.assert_called()
|
||||
|
||||
# verify the expected next state happened
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.success_state)
|
||||
|
||||
def test_management_affected_alarm(self):
|
||||
"""Test pre check step where there is a management affecting alarm
|
||||
|
||||
The pre-check should transition to the failed state
|
||||
"""
|
||||
|
||||
alarm_list = [FakeAlarm("100.001", "True"),
|
||||
FakeAlarm("100.002", "True")]
|
||||
|
||||
# also add ignored alarms
|
||||
for alarm_str in IGNORED_ALARMS_IDS:
|
||||
alarm_list.append(FakeAlarm(alarm_str, "True"))
|
||||
|
||||
self.fm_client.get_alarms.return_value = alarm_list
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the get alarms API call was invoked
|
||||
self.fm_client.get_alarms.assert_called()
|
||||
|
||||
# verify the expected next state happened
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
consts.STRATEGY_STATE_FAILED)
|
||||
|
||||
def test_ignored_alarm(self):
|
||||
"""Test pre check step where there is only a ignored alarm
|
||||
|
||||
The pre-check should transition to the updating patches state
|
||||
"""
|
||||
# add ignored alarms
|
||||
alarm_list = []
|
||||
for alarm_str in IGNORED_ALARMS_IDS:
|
||||
alarm_list.append(FakeAlarm(alarm_str, "True"))
|
||||
|
||||
self.fm_client.get_alarms.return_value = alarm_list
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the get alarms API call was invoked
|
||||
self.fm_client.get_alarms.assert_called()
|
||||
|
||||
# verify the expected next state happened
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.success_state)
|
||||
|
||||
def test_get_alarms_unexpected_failure(self):
|
||||
"""Test pre check step where fm-client get_alarms() fails
|
||||
|
||||
The pre-check should transition to the failed state and the 'details'
|
||||
field should contain the correct message detailing the error
|
||||
"""
|
||||
|
||||
self.fm_client.get_alarms.side_effect = Exception('Test error message')
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# verify the get alarms API call was invoked
|
||||
self.fm_client.get_alarms.assert_called()
|
||||
|
||||
# verify the expected next state happened
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
consts.STRATEGY_STATE_FAILED)
|
||||
|
||||
details = ("pre check: Failed to obtain subcloud alarm report due to:"
|
||||
" (Test error message). Please see /var/log/dcmanager/orche"
|
||||
"strator.log for details")
|
||||
self.assert_step_details(self.strategy_step.subcloud_id, details)
|
|
@ -0,0 +1,228 @@
|
|||
#
|
||||
# Copyright (c) 2023 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
from os import path as os_path
|
||||
|
||||
from dcmanager.common import consts
|
||||
from dcmanager.orchestrator.orch_thread import OrchThread
|
||||
from dcmanager.tests.unit.orchestrator.states.fakes import FakeLoad
|
||||
from dcmanager.tests.unit.orchestrator.states.patch.test_base import \
|
||||
TestPatchState
|
||||
import mock
|
||||
|
||||
REGION_ONE_PATCHES = {"DC.1": {"sw_version": "20.12",
|
||||
"repostate": "Applied",
|
||||
"patchstate": "Applied"},
|
||||
"DC.2": {"sw_version": "20.12",
|
||||
"repostate": "Applied",
|
||||
"patchstate": "Applied"},
|
||||
"DC.3": {"sw_version": "20.12",
|
||||
"repostate": "Committed",
|
||||
"patchstate": "Committed"},
|
||||
"DC.4": {"sw_version": "20.12",
|
||||
"repostate": "Available",
|
||||
"patchstate": "Available"},
|
||||
"DC.8": {"sw_version": "20.12",
|
||||
"repostate": "Applied",
|
||||
"patchstate": "Applied"}}
|
||||
|
||||
SUBCLOUD_PATCHES_SUCCESS = {"DC.1": {"sw_version": "20.12",
|
||||
"repostate": "Applied",
|
||||
"patchstate": "Applied"},
|
||||
"DC.2": {"sw_version": "20.12",
|
||||
"repostate": "Available",
|
||||
"patchstate": "Available"},
|
||||
"DC.3": {"sw_version": "20.12",
|
||||
"repostate": "Available",
|
||||
"patchstate": "Partial-Remove"},
|
||||
"DC.5": {"sw_version": "20.12",
|
||||
"repostate": "Applied",
|
||||
"patchstate": "Applied"},
|
||||
"DC.6": {"sw_version": "20.12",
|
||||
"repostate": "Applied",
|
||||
"patchstate": "Partial-Apply"}}
|
||||
|
||||
SUBCLOUD_PATCHES_BAD_COMMIT = {"DC.1": {"sw_version": "20.12",
|
||||
"repostate": "Applied",
|
||||
"patchstate": "Applied"},
|
||||
"DC.2": {"sw_version": "20.12",
|
||||
"repostate": "Available",
|
||||
"patchstate": "Available"},
|
||||
"DC.3": {"sw_version": "20.12",
|
||||
"repostate": "Available",
|
||||
"patchstate": "Partial-Remove"},
|
||||
"DC.5": {"sw_version": "20.12",
|
||||
"repostate": "Committed",
|
||||
"patchstate": "Committed"},
|
||||
"DC.6": {"sw_version": "20.12",
|
||||
"repostate": "Applied",
|
||||
"patchstate": "Partial-Apply"}}
|
||||
|
||||
SUBCLOUD_PATCHES_BAD_STATE = {"DC.1": {"sw_version": "20.12",
|
||||
"repostate": "Applied",
|
||||
"patchstate": "Applied"},
|
||||
"DC.2": {"sw_version": "20.12",
|
||||
"repostate": "Available",
|
||||
"patchstate": "Available"},
|
||||
"DC.3": {"sw_version": "20.12",
|
||||
"repostate": "Available",
|
||||
"patchstate": "Partial-Remove"},
|
||||
"DC.5": {"sw_version": "20.12",
|
||||
"repostate": "Unknown",
|
||||
"patchstate": "Unknown"},
|
||||
"DC.6": {"sw_version": "20.12",
|
||||
"repostate": "Applied",
|
||||
"patchstate": "Partial-Apply"}}
|
||||
|
||||
|
||||
@mock.patch("dcmanager.orchestrator.states.patch.updating_patches."
|
||||
"DEFAULT_MAX_QUERIES", 3)
|
||||
@mock.patch("dcmanager.orchestrator.states.patch.updating_patches"
|
||||
".DEFAULT_SLEEP_DURATION", 1)
|
||||
class TestUpdatingPatchesStage(TestPatchState):
|
||||
def setUp(self):
|
||||
super(TestUpdatingPatchesStage, self).setUp()
|
||||
|
||||
self.success_state = consts.STRATEGY_STATE_CREATING_VIM_PATCH_STRATEGY
|
||||
|
||||
# Add the subcloud being processed by this unit test
|
||||
self.subcloud = self.setup_subcloud()
|
||||
|
||||
# Add the strategy_step state being processed by this unit test
|
||||
self.strategy_step = self.setup_strategy_step(
|
||||
self.subcloud.id, consts.STRATEGY_STATE_UPDATING_PATCHES)
|
||||
|
||||
# Add mock API endpoints for patching and sysinv client calls
|
||||
# invoked by this state
|
||||
self.patching_client.query = mock.MagicMock()
|
||||
self.sysinv_client.get_loads = mock.MagicMock()
|
||||
self.patching_client.remove = mock.MagicMock()
|
||||
self.patching_client.upload = mock.MagicMock()
|
||||
self.patching_client.apply = mock.MagicMock()
|
||||
self.patching_client.query_hosts = mock.MagicMock()
|
||||
|
||||
# Mock OrchThread functions used by PatchJobData class
|
||||
p = mock.patch.object(OrchThread, "get_patching_client")
|
||||
self.mock_orch_patching_client = p.start()
|
||||
self.mock_orch_patching_client.return_value = self.patching_client
|
||||
self.addCleanup(p.stop)
|
||||
|
||||
p = mock.patch.object(OrchThread, "get_sysinv_client")
|
||||
self.mock_orch_sysinv_client = p.start()
|
||||
self.mock_orch_sysinv_client.return_value = self.sysinv_client
|
||||
self.addCleanup(p.stop)
|
||||
|
||||
self.fake_load = FakeLoad(1, software_version="20.12",
|
||||
state=consts.ACTIVE_LOAD_STATE)
|
||||
|
||||
def test_set_job_data(self):
|
||||
"""Test the 'set_job_data' method"""
|
||||
self.patching_client.query.side_effect = [REGION_ONE_PATCHES,
|
||||
SUBCLOUD_PATCHES_SUCCESS]
|
||||
|
||||
self.sysinv_client.get_loads.side_effect = [[self.fake_load]]
|
||||
|
||||
# invoke the pre apply setup to create the PatchJobData object
|
||||
self.worker.pre_apply_setup()
|
||||
|
||||
# call determine_state_operator to invoke the set_job_data method
|
||||
state = self.worker.determine_state_operator(self.strategy_step)
|
||||
|
||||
# Assert that the state has the proper region_one_patches and
|
||||
# region_one_applied_patch_ids attributes
|
||||
self.assertItemsEqual(REGION_ONE_PATCHES,
|
||||
state.region_one_patches)
|
||||
self.assertItemsEqual(["DC.1", "DC.2", "DC.3", "DC.8"],
|
||||
state.region_one_applied_patch_ids)
|
||||
|
||||
@mock.patch.object(os_path, "isfile")
|
||||
def test_update_subcloud_patches_success(self, mock_os_path_isfile):
|
||||
"""Test update_patches where the API call succeeds."""
|
||||
|
||||
self.patching_client.query.side_effect = [REGION_ONE_PATCHES,
|
||||
SUBCLOUD_PATCHES_SUCCESS]
|
||||
|
||||
self.sysinv_client.get_loads.side_effect = [[self.fake_load]]
|
||||
|
||||
mock_os_path_isfile.return_value = True
|
||||
|
||||
# invoke the pre apply setup to create the PatchJobData object
|
||||
self.worker.pre_apply_setup()
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
self.patching_client.upload.assert_called_with([consts.PATCH_VAULT_DIR +
|
||||
"/20.12/DC.8.patch"])
|
||||
|
||||
call_args, _ = self.patching_client.remove.call_args_list[0]
|
||||
self.assertItemsEqual(["DC.5", "DC.6"], call_args[0])
|
||||
|
||||
call_args, _ = self.patching_client.apply.call_args_list[0]
|
||||
self.assertItemsEqual(["DC.2", "DC.3", "DC.8"], call_args[0])
|
||||
|
||||
# On success, the state should transition to the next state
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
self.success_state)
|
||||
|
||||
self.assert_step_details(self.strategy_step.subcloud_id, "")
|
||||
|
||||
@mock.patch.object(os_path, "isfile")
|
||||
def test_update_subcloud_patches_bad_committed(self, mock_os_path_isfile):
|
||||
"""Test update_patches where the API call fails.
|
||||
|
||||
The update_patches call fails because the patch is 'committed' in
|
||||
the subcloud but not 'applied' in the System Controller.
|
||||
"""
|
||||
|
||||
self.patching_client.query.side_effect = [REGION_ONE_PATCHES,
|
||||
SUBCLOUD_PATCHES_BAD_COMMIT]
|
||||
|
||||
self.sysinv_client.get_loads.side_effect = [[self.fake_load]]
|
||||
|
||||
mock_os_path_isfile.return_value = True
|
||||
|
||||
# invoke the pre apply setup to create the PatchJobData object
|
||||
self.worker.pre_apply_setup()
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# Verify it failed and moves to the next step
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
consts.STRATEGY_STATE_FAILED)
|
||||
|
||||
self.assert_step_details(self.strategy_step.subcloud_id,
|
||||
"updating patches: Patch DC.5 is committed in "
|
||||
"subcloud but not applied in SystemController")
|
||||
|
||||
@mock.patch.object(os_path, "isfile")
|
||||
def test_update_subcloud_patches_bad_state(self, mock_os_path_isfile):
|
||||
"""Test update_patches where the API call fails.
|
||||
|
||||
The update_patches call fails because the patch is 'unknown' in
|
||||
the subcloud which is not a valid state.
|
||||
"""
|
||||
|
||||
self.patching_client.query.side_effect = [REGION_ONE_PATCHES,
|
||||
SUBCLOUD_PATCHES_BAD_STATE]
|
||||
|
||||
self.sysinv_client.get_loads.side_effect = [[self.fake_load]]
|
||||
|
||||
mock_os_path_isfile.return_value = True
|
||||
|
||||
# invoke the pre apply setup to create the PatchJobData object
|
||||
self.worker.pre_apply_setup()
|
||||
|
||||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
# Verify it failed and moves to the next step
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
consts.STRATEGY_STATE_FAILED)
|
||||
|
||||
self.assert_step_details(self.strategy_step.subcloud_id,
|
||||
"updating patches: Patch DC.5 in subcloud is"
|
||||
" in an unexpected state: Unknown")
|
|
@ -1,5 +1,5 @@
|
|||
#
|
||||
# Copyright (c) 2020, 2022 Wind River Systems, Inc.
|
||||
# Copyright (c) 2020, 2023 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
@ -41,11 +41,6 @@ SUBCLOUD_PATCHES = {'DC.1': {'sw_version': '17.07',
|
|||
}
|
||||
|
||||
|
||||
def compare_call_with_unsorted_list(call, unsorted_list):
|
||||
call_args, _ = call
|
||||
return call_args[0].sort() == unsorted_list.sort()
|
||||
|
||||
|
||||
@mock.patch("dcmanager.orchestrator.states.upgrade.finishing_patch_strategy"
|
||||
".DEFAULT_MAX_QUERIES", 3)
|
||||
@mock.patch("dcmanager.orchestrator.states.upgrade.finishing_patch_strategy"
|
||||
|
@ -81,14 +76,11 @@ class TestSwUpgradeFinishingPatchStrategyStage(TestSwUpgradeState):
|
|||
# invoke the strategy state operation on the orch thread
|
||||
self.worker.perform_state_action(self.strategy_step)
|
||||
|
||||
assert(compare_call_with_unsorted_list(
|
||||
self.patching_client.delete.call_args_list[0],
|
||||
['DC.5', 'DC.6']
|
||||
))
|
||||
assert(compare_call_with_unsorted_list(
|
||||
self.patching_client.commit.call_args_list[0],
|
||||
['DC.2', 'DC.3']
|
||||
))
|
||||
call_args, _ = self.patching_client.delete.call_args_list[0]
|
||||
self.assertItemsEqual(['DC.5', 'DC.6'], call_args[0])
|
||||
|
||||
call_args, _ = self.patching_client.commit.call_args_list[0]
|
||||
self.assertItemsEqual(['DC.2', 'DC.3'], call_args[0])
|
||||
|
||||
# On success, the state should transition to the next state
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
#
|
||||
# Copyright (c) 2020, 2022 Wind River Systems, Inc.
|
||||
# Copyright (c) 2020, 2023 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
@ -81,11 +81,6 @@ SUBCLOUD_PATCHES_BAD_STATE = {'DC.1': {'sw_version': '20.12',
|
|||
}
|
||||
|
||||
|
||||
def compare_call_with_unsorted_list(call, unsorted_list):
|
||||
call_args, _ = call
|
||||
return call_args[0].sort() == unsorted_list.sort()
|
||||
|
||||
|
||||
@mock.patch("dcmanager.orchestrator.states.upgrade.updating_patches"
|
||||
".DEFAULT_MAX_QUERIES", 3)
|
||||
@mock.patch("dcmanager.orchestrator.states.upgrade.updating_patches"
|
||||
|
@ -136,14 +131,11 @@ class TestSwUpgradeUpdatingPatchesStage(TestSwUpgradeState):
|
|||
self.patching_client.upload.assert_called_with(
|
||||
[consts.PATCH_VAULT_DIR + '/20.12/DC.8.patch'])
|
||||
|
||||
assert(compare_call_with_unsorted_list(
|
||||
self.patching_client.remove.call_args_list[0],
|
||||
['DC.5', 'DC.6']
|
||||
))
|
||||
assert(compare_call_with_unsorted_list(
|
||||
self.patching_client.apply.call_args_list[0],
|
||||
['DC.2', 'DC.3', 'DC.8']
|
||||
))
|
||||
call_args, _ = self.patching_client.remove.call_args_list[0]
|
||||
self.assertItemsEqual(['DC.5', 'DC.6'], call_args[0])
|
||||
|
||||
call_args, _ = self.patching_client.apply.call_args_list[0]
|
||||
self.assertItemsEqual(['DC.2', 'DC.3', 'DC.8'], call_args[0])
|
||||
|
||||
# On success, the state should transition to the next state
|
||||
self.assert_step_updated(self.strategy_step.subcloud_id,
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
# Copyright (c) 2017-2022 Wind River Systems, Inc.
|
||||
# Copyright (c) 2017-2023 Wind River Systems, Inc.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
|
@ -201,12 +201,16 @@ class TestSwUpdate(base.DCManagerTestCase):
|
|||
|
||||
def assert_step_updated(self, subcloud_id, update_state):
|
||||
step = db_api.strategy_step_get(self.ctx, subcloud_id)
|
||||
self.assertEqual(step.state, update_state)
|
||||
self.assertEqual(update_state, step.state)
|
||||
|
||||
def assert_step_details(self, subcloud_id, details):
|
||||
step = db_api.strategy_step_get(self.ctx, subcloud_id)
|
||||
self.assertEqual(details, step.details)
|
||||
|
||||
# utility methods to help assert the value of any subcloud attribute
|
||||
def assert_subcloud_attribute(self, subcloud_id, attr_name, expected_val):
|
||||
subcloud = db_api.subcloud_get(self.ctx, subcloud_id)
|
||||
self.assertEqual(subcloud[attr_name], expected_val)
|
||||
self.assertEqual(expected_val, subcloud[attr_name])
|
||||
|
||||
def assert_subcloud_software_version(self, subcloud_id, expected_val):
|
||||
self.assert_subcloud_attribute(subcloud_id,
|
||||
|
|
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue