eb97f4c8b6
1) Remove DC manager orchestration from dcmanager-manager process 2) Create dcmanager-orchestrator process and associated files 3) Add new RPC calls for dcmanager-orchestrator process to notify dcmanager 4) Create/update unit tests, to verify the implementation changes Story: 2007267 Task: 40734 Change-Id: Ibbbae77558a8a8fd95b636fa6c3aebb1dfefb514 Signed-off-by: Jessica Castelino <jessica.castelino@windriver.com>
103 lines
4.1 KiB
Python
103 lines
4.1 KiB
Python
#
|
|
# Copyright (c) 2020 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
import time
|
|
|
|
from dcmanager.common import consts
|
|
from dcmanager.common.exceptions import StrategyStoppedException
|
|
from dcmanager.orchestrator.states.base import BaseState
|
|
|
|
|
|
ACTIVATING_COMPLETED_STATES = ['activation-complete',
|
|
'aborting']
|
|
|
|
ACTIVATING_RETRY_STATES = ['activation-failed', ]
|
|
|
|
ACTIVATING_IN_PROGRESS_STATES = ['activating', ]
|
|
|
|
# Max time: 45 minutes = 45 queries x 60 seconds sleep between queries
|
|
DEFAULT_MAX_QUERIES = 45
|
|
DEFAULT_SLEEP_DURATION = 60
|
|
MAX_FAILED_RETRIES = 10
|
|
|
|
|
|
class ActivatingUpgradeState(BaseState):
|
|
"""Upgrade state actions for activating an upgrade"""
|
|
|
|
def __init__(self, region_name):
|
|
super(ActivatingUpgradeState, self).__init__(
|
|
next_state=consts.STRATEGY_STATE_COMPLETING_UPGRADE, region_name=region_name)
|
|
# max time to wait (in seconds) is: sleep_duration * max_queries
|
|
self.sleep_duration = DEFAULT_SLEEP_DURATION
|
|
self.max_queries = DEFAULT_MAX_QUERIES
|
|
self.max_failed_retries = MAX_FAILED_RETRIES
|
|
|
|
def get_upgrade_state(self, strategy_step):
|
|
upgrades = self.get_sysinv_client(
|
|
strategy_step.subcloud.name).get_upgrades()
|
|
|
|
if len(upgrades) == 0:
|
|
raise Exception("No upgrades were found to activate")
|
|
|
|
# The list of upgrades will never contain more than one entry.
|
|
return upgrades[0].state
|
|
|
|
def perform_state_action(self, strategy_step):
|
|
"""Activate an upgrade on a subcloud
|
|
|
|
Returns the next state in the state machine on success.
|
|
Any exceptions raised by this method set the strategy to FAILED.
|
|
"""
|
|
|
|
upgrade_state = self.get_upgrade_state(strategy_step)
|
|
|
|
# Check if an existing upgrade is already activated
|
|
if upgrade_state in ACTIVATING_COMPLETED_STATES:
|
|
self.info_log(strategy_step,
|
|
"Already in an activating state:%s" % upgrade_state)
|
|
return self.next_state
|
|
|
|
# invoke the API 'upgrade-activate'.
|
|
# Throws an exception on failure (no upgrade found, bad host state)
|
|
self.get_sysinv_client(strategy_step.subcloud.name).upgrade_activate()
|
|
# Need to loop until changed to a activating completed state
|
|
counter = 0
|
|
while True:
|
|
# If event handler stop has been triggered, fail the state
|
|
if self.stopped():
|
|
raise StrategyStoppedException()
|
|
upgrade_state = self.get_upgrade_state(strategy_step)
|
|
|
|
if upgrade_state in ACTIVATING_RETRY_STATES:
|
|
if counter >= self.max_failed_retries:
|
|
raise Exception("Failed to activate upgrade. Please "
|
|
"check sysinv.log on the subcloud for "
|
|
"details.")
|
|
# We failed. Better try again
|
|
self.info_log(strategy_step,
|
|
"Activation failed, retrying... State=%s"
|
|
% upgrade_state)
|
|
self.get_sysinv_client(
|
|
strategy_step.subcloud.name).upgrade_activate()
|
|
elif upgrade_state in ACTIVATING_IN_PROGRESS_STATES:
|
|
self.info_log(strategy_step,
|
|
"Activation in progress, waiting... State=%s"
|
|
% upgrade_state)
|
|
elif upgrade_state in ACTIVATING_COMPLETED_STATES:
|
|
self.info_log(strategy_step,
|
|
"Activation completed. State=%s"
|
|
% upgrade_state)
|
|
break
|
|
counter += 1
|
|
if counter >= self.max_queries:
|
|
raise Exception("Timeout waiting for activation to complete. "
|
|
"Please check sysinv.log on the subcloud for "
|
|
"details.")
|
|
time.sleep(self.sleep_duration)
|
|
|
|
# When we return from this method without throwing an exception, the
|
|
# state machine can proceed to the next state
|
|
return self.next_state
|