starlingx
/
nfv
Code Issues Proposed changes
nfv/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py

3482 lines
125 KiB
Python
Executable File
Raw Blame History

#
# Copyright (c) 2015-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import six
from nfv_common import debug
from nfv_common.helpers import Constant
from nfv_common.helpers import Constants
from nfv_common.helpers import coroutine
from nfv_common.helpers import Singleton
from nfv_common import strategy
from nfv_common import timers
from nfv_vim import objects
from nfv_vim.strategy._strategy_defs import FW_UPDATE_LABEL
from nfv_vim.strategy._strategy_defs import STRATEGY_EVENT
from nfv_vim import tables
DLOG = debug.debug_get_logger('nfv_vim.strategy.step')
@six.add_metaclass(Singleton)
class StrategyStepNames(Constants):
"""
Strategy Step Names
"""
QUERY_HOSTS = Constant('query-hosts')
SYSTEM_STABILIZE = Constant('system-stabilize')
LOCK_HOSTS = Constant('lock-hosts')
UNLOCK_HOSTS = Constant('unlock-hosts')
REBOOT_HOSTS = Constant('reboot-hosts')
UPGRADE_HOSTS = Constant('upgrade-hosts')
START_UPGRADE = Constant('start-upgrade')
ACTIVATE_UPGRADE = Constant('activate-upgrade')
COMPLETE_UPGRADE = Constant('complete-upgrade')
SWACT_HOSTS = Constant('swact-hosts')
SW_PATCH_HOSTS = Constant('sw-patch-hosts')
FW_UPDATE_HOSTS = Constant('fw-update-hosts')
FW_UPDATE_ABORT_HOSTS = Constant('fw-update-abort-hosts')
MIGRATE_INSTANCES = Constant('migrate-instances')
STOP_INSTANCES = Constant('stop-instances')
START_INSTANCES = Constant('start-instances')
QUERY_ALARMS = Constant('query-alarms')
WAIT_DATA_SYNC = Constant('wait-data-sync')
WAIT_ALARMS_CLEAR = Constant('wait-alarms-clear')
QUERY_SW_PATCHES = Constant('query-sw-patches')
QUERY_SW_PATCH_HOSTS = Constant('query-sw-patch-hosts')
QUERY_FW_UPDATE_HOST = Constant('query-fw-update-host')
QUERY_UPGRADE = Constant('query-upgrade')
DISABLE_HOST_SERVICES = Constant('disable-host-services')
ENABLE_HOST_SERVICES = Constant('enable-host-services')
APPLY_PATCHES = Constant('apply-patches')
QUERY_KUBE_HOST_UPGRADE = Constant('query-kube-host-upgrade')
QUERY_KUBE_UPGRADE = Constant('query-kube-upgrade')
QUERY_KUBE_VERSIONS = Constant('query-kube-versions')
KUBE_UPGRADE_START = Constant('kube-upgrade-start')
KUBE_UPGRADE_CLEANUP = Constant('kube-upgrade-cleanup')
KUBE_UPGRADE_COMPLETE = Constant('kube-upgrade-complete')
KUBE_UPGRADE_DOWNLOAD_IMAGES = Constant('kube-upgrade-download-images')
KUBE_UPGRADE_NETWORKING = Constant('kube-upgrade-networking')
KUBE_HOST_UPGRADE_CONTROL_PLANE = \
Constant('kube-host-upgrade-control-plane')
KUBE_HOST_UPGRADE_KUBELET = Constant('kube-host-upgrade-kubelet')
# Constant Instantiation
STRATEGY_STEP_NAME = StrategyStepNames()
class AbstractStrategyStep(strategy.StrategyStep):
"""An abstract base class for strategy steps"""
def __init__(self, step_name, timeout_in_secs):
super(AbstractStrategyStep, self).__init__(
step_name,
timeout_in_secs=timeout_in_secs)
def from_dict(self, data):
"""
Returns the step object initialized using the given dictionary
"""
super(AbstractStrategyStep, self).from_dict(data)
return self
def as_dict(self):
"""
Represent the step as a dictionary
"""
data = super(AbstractStrategyStep, self).as_dict()
# Next 3 lines are required for all strategy steps and may be
# overridden by subclass in some cases
data['entity_type'] = ''
data['entity_names'] = list()
data['entity_uuids'] = list()
return data
class AbstractHostsStrategyStep(AbstractStrategyStep):
"""An abstract base class for strategy steps performed on list of hosts"""
def __init__(self,
step_name,
hosts,
timeout_in_secs=1800):
super(AbstractHostsStrategyStep, self).__init__(
step_name,
timeout_in_secs=timeout_in_secs)
self._hosts = hosts
self._host_names = list()
self._host_uuids = list()
for host in hosts:
self._host_names.append(host.name)
self._host_uuids.append(host.uuid)
def from_dict(self, data):
"""
Returns the step object initialized using the given dictionary
"""
super(AbstractHostsStrategyStep, self).from_dict(data)
self._hosts = list()
self._host_uuids = list()
self._host_names = data['entity_names']
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is not None:
self._hosts.append(host)
self._host_uuids.append(host.uuid)
return self
def as_dict(self):
"""
Represent the step as a dictionary
"""
data = super(AbstractHostsStrategyStep, self).as_dict()
data['entity_type'] = 'hosts'
data['entity_names'] = self._host_names
data['entity_uuids'] = self._host_uuids
return data
class UnlockHostsStep(AbstractHostsStrategyStep):
"""
Unlock Hosts - Strategy Step
"""
# During an upgrade, an unlock may need to be retried several times
# https://bugs.launchpad.net/starlingx/+bug/1914836
MAX_RETRIES = 5
RETRY_DELAY = 120
def __init__(self, hosts, retry_count=0, retry_delay=RETRY_DELAY):
"""
hosts - the list of hosts to be unlocked
retry_count - the number of times to retry per host if unlock fails
retry_delay - the amount of time to delay before retrying unlock
"""
super(UnlockHostsStep, self).__init__(STRATEGY_STEP_NAME.UNLOCK_HOSTS,
hosts,
timeout_in_secs=1800)
# step_name, hosts, timeout are serialized by parent classes
# retry_count and retry_delay must be serialized in from_dict/as_dict
self._retry_count = retry_count
self._retry_delay = retry_delay
# Do not persist: _retries, _wait_time _retrying
self._retries = dict()
for host_name in self._host_names:
self._retries[host_name] = retry_count
self._wait_time = 0
self._retry_requested = False
def from_dict(self, data):
"""
Returns unlock hosts step object initialized using the given dictionary
"""
super(UnlockHostsStep, self).from_dict(data)
# deserialize retry_delay
self._retry_count = data['retry_count']
self._retry_delay = data['retry_delay']
# Do not deserialize _retries, _wait_time and _retrying
self._wait_time = 0
self._retry_requested = False
self._retries = dict()
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is not None:
self._retries[host_name] = self._retry_count
return self
def as_dict(self):
"""
Represent the unlock hosts step as a dictionary
"""
data = super(UnlockHostsStep, self).as_dict()
# serialize retries
data['retry_count'] = self._retry_count
# serialize retry_delay
data['retry_delay'] = self._retry_delay
# Do not serialize _retries, _wait_time and _retrying
return data
def _get_hosts_to_retry(self):
hosts = []
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is None:
continue
if host.is_locked() and self._retries[host_name] > 0:
self._retries[host_name] = self._retries[host_name] - 1
hosts.append(host_name)
return hosts
def _total_hosts_unlocked_enabled(self):
"""
Returns the number of hosts that are unlocked and enabled
"""
total_hosts_enabled = 0
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is None:
return -1
if not host.is_locked() and host.is_enabled():
total_hosts_enabled += 1
return total_hosts_enabled
def _trigger_retry(self, host_name):
DLOG.info("Step (%s) retry due to failure for (%s)." % (self._name,
host_name))
# set the retry trigger
self._retry_requested = True
# reset the retry "wait" delay
self._wait_time = timers.get_monotonic_timestamp_in_ms()
# decrement the number of allowed retries for the validated host
self._retries[host_name] = self._retries[host_name] - 1
def apply(self):
"""
Unlock all hosts
"""
from nfv_vim import directors
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
self._host_names))
if len(self._host_names) == self._total_hosts_unlocked_enabled():
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
host_director = directors.get_host_director()
operation = host_director.unlock_hosts(self._host_names)
if operation.is_inprogress():
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
elif operation.is_failed():
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
def handle_event(self, event, event_data=None):
"""
Handle Host events
"""
from nfv_vim import directors
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event in [STRATEGY_EVENT.HOST_STATE_CHANGED,
STRATEGY_EVENT.HOST_AUDIT]:
total_hosts_enabled = self._total_hosts_unlocked_enabled()
if -1 == total_hosts_enabled:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "host no longer exists")
return True
if total_hosts_enabled == len(self._host_names):
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, '')
return True
# See if we have requested a retry and are not currently retrying
if self._retry_requested:
now_ms = timers.get_monotonic_timestamp_in_ms()
secs_expired = (now_ms - self._wait_time) / 1000
if self._retry_delay <= secs_expired:
self._retry_requested = False
# re-issue unlock for all hosts.
# Hosts that are already unlocked or unlocking get skipped
host_director = directors.get_host_director()
operation = host_director.unlock_hosts(self._host_names)
if operation.is_failed():
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "host unlock failed")
return True
elif event == STRATEGY_EVENT.HOST_UNLOCK_FAILED:
host = event_data
if host is not None and host.name in self._host_names:
if host.is_locked() and self._retries[host.name] > 0:
# if any unlock fails and we have retries, trigger it
# even if the last round of unlocks has not returned
self._trigger_retry(host.name)
else:
# if ANY unlock fails and we are out of retries, fail
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "host unlock failed")
return True
return False
class LockHostsStep(strategy.StrategyStep):
"""
Lock Hosts - Strategy Step
"""
def __init__(self, hosts, wait_until_disabled=True):
super(LockHostsStep, self).__init__(
STRATEGY_STEP_NAME.LOCK_HOSTS, timeout_in_secs=900)
self._hosts = hosts
self._wait_until_disabled = wait_until_disabled
self._host_names = list()
self._host_uuids = list()
for host in hosts:
self._host_names.append(host.name)
self._host_uuids.append(host.uuid)
self._wait_time = 0
def abort(self):
"""
Returns the abort step related to this step
"""
return [UnlockHostsStep(self._hosts)]
def _total_hosts_locked(self):
"""
Returns the number of hosts that are locked
"""
total_hosts_locked = 0
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is None:
return -1
if host.is_locked() and \
(not self._wait_until_disabled or host.is_disabled()):
total_hosts_locked += 1
return total_hosts_locked
def _instances_not_locked_disabled(self):
"""
Returns the instances that are not locked and disabled
"""
instances_not_locked_disabled = []
instance_table = tables.tables_get_instance_table()
for host_name in self._host_names:
for instance in instance_table.on_host(host_name):
if not (instance.is_locked() and instance.is_disabled()):
instances_not_locked_disabled.append(instance.name)
return instances_not_locked_disabled
def apply(self):
"""
Lock all hosts
"""
from nfv_vim import directors
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
self._host_names))
if len(self._host_names) == self._total_hosts_locked():
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
# Ensure that no instances are running on these hosts before locking.
# Instances should have been stopped or migrated to another host by now.
instances = self._instances_not_locked_disabled()
if instances:
reason = ("Lock of host(s) %s failed because instance(s) %s were "
"not migrated or stopped." % (','.join(self._host_names),
','.join(instances)))
return strategy.STRATEGY_STEP_RESULT.FAILED, reason
host_director = directors.get_host_director()
operation = host_director.lock_hosts(self._host_names)
if operation.is_inprogress():
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
elif operation.is_failed():
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
def handle_event(self, event, event_data=None):
"""
Handle Host events
"""
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event in [STRATEGY_EVENT.HOST_STATE_CHANGED, STRATEGY_EVENT.HOST_AUDIT]:
total_hosts_locked = self._total_hosts_locked()
if -1 == total_hosts_locked:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "host no longer exists")
return True
if not self._wait_until_disabled:
# If we are not waiting for the hosts to go disabled, then wait
# at least 15 seconds after doing the lock.
if 0 == self._wait_time:
self._wait_time = timers.get_monotonic_timestamp_in_ms()
now_ms = timers.get_monotonic_timestamp_in_ms()
secs_expired = (now_ms - self._wait_time) / 1000
if 15 >= secs_expired:
return True
if total_hosts_locked == len(self._host_names):
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, '')
return True
elif event == STRATEGY_EVENT.HOST_LOCK_FAILED:
host = event_data
if host is not None and host.name in self._host_names:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "host lock failed")
return True
return False
def from_dict(self, data):
"""
Returns the lock hosts step object initialized using the given dictionary
"""
super(LockHostsStep, self).from_dict(data)
self._hosts = list()
self._wait_until_disabled = data['wait_until_disabled']
self._host_uuids = list()
self._host_names = data['entity_names']
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is not None:
self._hosts.append(host)
self._host_uuids.append(host.uuid)
self._wait_time = 0
return self
def as_dict(self):
"""
Represent the lock hosts step as a dictionary
"""
data = super(LockHostsStep, self).as_dict()
data['entity_type'] = 'hosts'
data['entity_names'] = self._host_names
data['entity_uuids'] = self._host_uuids
data['wait_until_disabled'] = self._wait_until_disabled
return data
class RebootHostsStep(strategy.StrategyStep):
"""
Reboot Hosts - Strategy Step
"""
def __init__(self, hosts):
super(RebootHostsStep, self).__init__(
STRATEGY_STEP_NAME.REBOOT_HOSTS, timeout_in_secs=900)
self._hosts = hosts
self._host_names = list()
self._host_uuids = list()
for host in hosts:
self._host_names.append(host.name)
self._host_uuids.append(host.uuid)
self._wait_time = 0
def apply(self):
"""
Reboot all hosts
"""
from nfv_vim import directors
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
self._host_names))
host_director = directors.get_host_director()
operation = host_director.reboot_hosts(self._host_names)
if operation.is_inprogress():
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
elif operation.is_failed():
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
def handle_event(self, event, event_data=None):
"""
Handle Host events
"""
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event == STRATEGY_EVENT.HOST_REBOOT_FAILED:
host = event_data
if host is not None and host.name in self._host_names:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "host reboot failed")
return True
elif event == STRATEGY_EVENT.HOST_AUDIT:
if 0 == self._wait_time:
self._wait_time = timers.get_monotonic_timestamp_in_ms()
now_ms = timers.get_monotonic_timestamp_in_ms()
secs_expired = (now_ms - self._wait_time) / 1000
if 60 <= secs_expired:
# Wait 60 seconds, which should be enough time for the host
# to shutdown and reboot. No need to wait for the host to
# come back online since it will remain locked after the reboot.
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, '')
return True
return False
def from_dict(self, data):
"""
Returns the reboot hosts step object initialized using the given dictionary
"""
super(RebootHostsStep, self).from_dict(data)
self._hosts = list()
self._host_uuids = list()
self._host_names = data['entity_names']
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is not None:
self._hosts.append(host)
self._host_uuids.append(host.uuid)
self._wait_time = 0
return self
def as_dict(self):
"""
Represent the reboot hosts step as a dictionary
"""
data = super(RebootHostsStep, self).as_dict()
data['entity_type'] = 'hosts'
data['entity_names'] = self._host_names
data['entity_uuids'] = self._host_uuids
return data
class SwactHostsStep(strategy.StrategyStep):
"""
Swact Hosts - Strategy Step
"""
def __init__(self, hosts):
super(SwactHostsStep, self).__init__(
STRATEGY_STEP_NAME.SWACT_HOSTS, timeout_in_secs=900)
self._hosts = hosts
self._host_names = list()
self._host_uuids = list()
for host in hosts:
self._host_names.append(host.name)
self._host_uuids.append(host.uuid)
self._wait_time = 0
def apply(self):
"""
Swact hosts
"""
from nfv_vim import directors
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
self._host_names))
host_director = directors.get_host_director()
operation = host_director.swact_hosts(self._host_names)
if operation.is_inprogress():
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
elif operation.is_failed():
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
def handle_event(self, event, event_data=None):
"""
Handle Host events
"""
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event == STRATEGY_EVENT.HOST_SWACT_FAILED:
host = event_data
if host is not None and host.name in self._host_names:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "host swact failed")
return True
elif event == STRATEGY_EVENT.HOST_AUDIT:
if 0 == self._wait_time:
self._wait_time = timers.get_monotonic_timestamp_in_ms()
now_ms = timers.get_monotonic_timestamp_in_ms()
secs_expired = (now_ms - self._wait_time) / 1000
if 120 <= secs_expired:
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, '')
return True
return False
def from_dict(self, data):
"""
Returns the swact hosts step object initialized using the given dictionary
"""
super(SwactHostsStep, self).from_dict(data)
self._hosts = list()
self._host_uuids = list()
self._host_names = data['entity_names']
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is not None:
self._hosts.append(host)
self._host_uuids.append(host.uuid)
self._wait_time = 0
return self
def as_dict(self):
"""
Represent the swact hosts step as a dictionary
"""
data = super(SwactHostsStep, self).as_dict()
data['entity_type'] = 'hosts'
data['entity_names'] = self._host_names
data['entity_uuids'] = self._host_uuids
return data
class SwPatchHostsStep(strategy.StrategyStep):
"""
Software Patch Hosts - Strategy Step
"""
def __init__(self, hosts):
super(SwPatchHostsStep, self).__init__(
STRATEGY_STEP_NAME.SW_PATCH_HOSTS, timeout_in_secs=1800)
self._hosts = hosts
self._host_names = list()
self._host_uuids = list()
self._host_completed = dict()
self._query_inprogress = False
for host in hosts:
self._host_names.append(host.name)
self._host_uuids.append(host.uuid)
self._host_completed[host.name] = (False, False, '')
@coroutine
def _query_hosts_callback(self):
"""
Query Software Patch Hosts Callback
"""
response = (yield)
DLOG.debug("Query-Hosts callback response=%s." % response)
self._query_inprogress = False
if response['completed']:
for sw_patch_host in response['result-data']:
if self._host_completed.get(sw_patch_host.name, False):
if sw_patch_host.patch_current:
self._host_completed[sw_patch_host.name] = \
(True, True, '')
elif sw_patch_host.patch_failed:
self._host_completed[sw_patch_host.name] = \
(True, False, "software update failed to apply on "
"host %s" % sw_patch_host.name)
failed = False
failed_reason = ''
for host_name in self._host_completed:
completed, success, reason = self._host_completed[host_name]
if not completed:
break
if not success:
failed = True
failed_reason = reason
else:
if failed:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, failed_reason)
else:
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
@coroutine
def _sw_patch_hosts_callback(self):
"""
Software Patch Hosts Callback
"""
response = (yield)
DLOG.debug("Software-Update-Hosts callback response=%s." % response)
if not response['completed']:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
def apply(self):
"""
Software Patch Hosts
"""
from nfv_vim import nfvi
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
self._host_names))
nfvi.nfvi_sw_mgmt_update_hosts(self._host_names,
self._sw_patch_hosts_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
def handle_event(self, event, event_data=None):
"""
Handle Host events
"""
from nfv_vim import nfvi
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event == STRATEGY_EVENT.HOST_AUDIT:
if not self._query_inprogress:
self._query_inprogress = True
nfvi.nfvi_sw_mgmt_query_hosts(self._query_hosts_callback())
return True
return False
def from_dict(self, data):
"""
Returns the software patch hosts step object initialized using the given
dictionary
"""
super(SwPatchHostsStep, self).from_dict(data)
self._hosts = list()
self._host_uuids = list()
self._host_completed = dict()
self._query_inprogress = False
self._host_names = data['entity_names']
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is not None:
self._hosts.append(host)
self._host_uuids.append(host.uuid)
self._host_completed[host_name] = \
data['hosts_completed'][host_name]
return self
def as_dict(self):
"""
Represent the software patch hosts step as a dictionary
"""
data = super(SwPatchHostsStep, self).as_dict()
data['entity_type'] = 'hosts'
data['entity_names'] = self._host_names
data['entity_uuids'] = self._host_uuids
data['hosts_completed'] = self._host_completed
return data
class UpgradeHostsStep(strategy.StrategyStep):
"""
Upgrade Hosts - Strategy Step
"""
def __init__(self, hosts):
super(UpgradeHostsStep, self).__init__(
STRATEGY_STEP_NAME.UPGRADE_HOSTS, timeout_in_secs=3600)
self._hosts = hosts
self._host_names = list()
self._host_uuids = list()
for host in hosts:
self._host_names.append(host.name)
self._host_uuids.append(host.uuid)
self._wait_time = 0
def _total_hosts_upgraded(self):
"""
Returns the number of hosts that are upgraded
"""
total_hosts_upgraded = 0
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is None:
return -1
if (host.is_online() and
host.target_load == self.strategy.nfvi_upgrade.to_release and
host.software_load == self.strategy.nfvi_upgrade.to_release):
total_hosts_upgraded += 1
return total_hosts_upgraded
def apply(self):
"""
Upgrade all hosts
"""
from nfv_vim import directors
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
self._host_names))
host_director = directors.get_host_director()
operation = host_director.upgrade_hosts(self._host_names)
if operation.is_inprogress():
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
elif operation.is_failed():
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
def handle_event(self, event, event_data=None):
"""
Handle Host events
"""
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event == STRATEGY_EVENT.HOST_UPGRADE_FAILED:
host = event_data
if host is not None and host.name in self._host_names:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "host upgrade failed")
return True
elif event in [STRATEGY_EVENT.HOST_AUDIT]:
if 0 == self._wait_time:
self._wait_time = timers.get_monotonic_timestamp_in_ms()
now_ms = timers.get_monotonic_timestamp_in_ms()
secs_expired = (now_ms - self._wait_time) / 1000
# Wait at least 2 minutes for the host to go offline before
# checking whether the upgrade is complete.
if 120 <= secs_expired:
total_hosts_upgraded = self._total_hosts_upgraded()
if -1 == total_hosts_upgraded:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "host no longer exists")
return True
if total_hosts_upgraded == len(self._host_names):
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, '')
return True
return False
def from_dict(self, data):
"""
Returns the upgrade hosts step object initialized using the given
dictionary
"""
super(UpgradeHostsStep, self).from_dict(data)
self._hosts = list()
self._host_uuids = list()
self._host_names = data['entity_names']
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is not None:
self._hosts.append(host)
self._host_uuids.append(host.uuid)
self._wait_time = 0
return self
def as_dict(self):
"""
Represent the upgrade hosts step as a dictionary
"""
data = super(UpgradeHostsStep, self).as_dict()
data['entity_type'] = 'hosts'
data['entity_names'] = self._host_names
data['entity_uuids'] = self._host_uuids
return data
class UpgradeStartStep(strategy.StrategyStep):
"""
Upgrade Start - Strategy Step
"""
def __init__(self):
super(UpgradeStartStep, self).__init__(
STRATEGY_STEP_NAME.START_UPGRADE, timeout_in_secs=600)
self._wait_time = 0
self._query_inprogress = False
@coroutine
def _start_upgrade_callback(self):
"""
Start Upgrade Callback
"""
response = (yield)
DLOG.debug("Start-Upgrade callback response=%s." % response)
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_upgrade = response['result-data']
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
@coroutine
def _get_upgrade_callback(self):
"""
Get Upgrade Callback
"""
from nfv_vim import nfvi
response = (yield)
DLOG.debug("Get-Upgrade callback response=%s." % response)
self._query_inprogress = False
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_upgrade = response['result-data']
if self.strategy.nfvi_upgrade.state != \
nfvi.objects.v1.UPGRADE_STATE.STARTED:
# Keep waiting for upgrade to start
pass
else:
# Upgrade has started
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
def apply(self):
"""
Upgrade Start
"""
from nfv_vim import nfvi
DLOG.info("Step (%s) apply." % self._name)
nfvi.nfvi_upgrade_start(self._start_upgrade_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
def handle_event(self, event, event_data=None):
"""
Handle Host events
"""
from nfv_vim import nfvi
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event == STRATEGY_EVENT.HOST_AUDIT:
if 0 == self._wait_time:
self._wait_time = timers.get_monotonic_timestamp_in_ms()
now_ms = timers.get_monotonic_timestamp_in_ms()
secs_expired = (now_ms - self._wait_time) / 1000
# Wait at least 60 seconds before checking upgrade for first time
if 60 <= secs_expired and not self._query_inprogress:
self._query_inprogress = True
nfvi.nfvi_get_upgrade(self._get_upgrade_callback())
return True
return False
def from_dict(self, data):
"""
Returns the upgrade start step object initialized using the given
dictionary
"""
super(UpgradeStartStep, self).from_dict(data)
self._wait_time = 0
self._query_inprogress = False
return self
def as_dict(self):
"""
Represent the upgrade start step as a dictionary
"""
data = super(UpgradeStartStep, self).as_dict()
data['entity_type'] = ''
data['entity_names'] = list()
data['entity_uuids'] = list()
return data
class UpgradeActivateStep(strategy.StrategyStep):
"""
Upgrade Activate - Strategy Step
"""
def __init__(self):
super(UpgradeActivateStep, self).__init__(
STRATEGY_STEP_NAME.ACTIVATE_UPGRADE, timeout_in_secs=900)
self._wait_time = 0
self._query_inprogress = False
@coroutine
def _activate_upgrade_callback(self):
"""
Activate Upgrade Callback
"""
response = (yield)
DLOG.debug("Activate-Upgrade callback response=%s." % response)
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_upgrade = response['result-data']
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
@coroutine
def _get_upgrade_callback(self):
"""
Get Upgrade Callback
"""
from nfv_vim import nfvi
response = (yield)
DLOG.debug("Get-Upgrade callback response=%s." % response)
self._query_inprogress = False
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_upgrade = response['result-data']
if self.strategy.nfvi_upgrade.state != \
nfvi.objects.v1.UPGRADE_STATE.ACTIVATION_COMPLETE:
# Keep waiting for upgrade to activate
pass
else:
# Upgrade has activated
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
def apply(self):
"""
Upgrade Activate
"""
from nfv_vim import nfvi
DLOG.info("Step (%s) apply." % self._name)
nfvi.nfvi_upgrade_activate(self._activate_upgrade_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
def handle_event(self, event, event_data=None):
"""
Handle Host events
"""
from nfv_vim import nfvi
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event == STRATEGY_EVENT.HOST_AUDIT:
if 0 == self._wait_time:
self._wait_time = timers.get_monotonic_timestamp_in_ms()
now_ms = timers.get_monotonic_timestamp_in_ms()
secs_expired = (now_ms - self._wait_time) / 1000
# Wait at least 60 seconds before checking upgrade for first time
if 60 <= secs_expired and not self._query_inprogress:
self._query_inprogress = True
nfvi.nfvi_get_upgrade(self._get_upgrade_callback())
return True
return False
def from_dict(self, data):
"""
Returns the upgrade activate step object initialized using the given
dictionary
"""
super(UpgradeActivateStep, self).from_dict(data)
self._wait_time = 0
self._query_inprogress = False
return self
def as_dict(self):
"""
Represent the upgrade activate step as a dictionary
"""
data = super(UpgradeActivateStep, self).as_dict()
data['entity_type'] = ''
data['entity_names'] = list()
data['entity_uuids'] = list()
return data
class UpgradeCompleteStep(strategy.StrategyStep):
"""
Upgrade Complete - Strategy Step
"""
def __init__(self):
super(UpgradeCompleteStep, self).__init__(
STRATEGY_STEP_NAME.COMPLETE_UPGRADE, timeout_in_secs=300)
self._wait_time = 0
self._query_inprogress = False
@coroutine
def _complete_upgrade_callback(self):
"""
Complete Upgrade Callback
"""
response = (yield)
DLOG.debug("Complete-Upgrade callback response=%s." % response)
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_upgrade = response['result-data']
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
@coroutine
def _get_upgrade_callback(self):
"""
Get Upgrade Callback
"""
response = (yield)
DLOG.debug("Get-Upgrade callback response=%s." % response)
self._query_inprogress = False
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_upgrade = response['result-data']
if self.strategy.nfvi_upgrade is not None:
# Keep waiting for upgrade to complete
pass
else:
# Upgrade has been completed (upgrade record deleted)
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
def apply(self):
"""
Upgrade Complete
"""
from nfv_vim import nfvi
DLOG.info("Step (%s) apply." % self._name)
nfvi.nfvi_upgrade_complete(self._complete_upgrade_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
def handle_event(self, event, event_data=None):
"""
Handle Host events
"""
from nfv_vim import nfvi
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event == STRATEGY_EVENT.HOST_AUDIT:
if 0 == self._wait_time:
self._wait_time = timers.get_monotonic_timestamp_in_ms()
now_ms = timers.get_monotonic_timestamp_in_ms()
secs_expired = (now_ms - self._wait_time) / 1000
# Wait at least 60 seconds before checking upgrade for first time
if 60 <= secs_expired and not self._query_inprogress:
self._query_inprogress = True
nfvi.nfvi_get_upgrade(self._get_upgrade_callback())
return True
return False
def from_dict(self, data):
"""
Returns the upgrade complete step object initialized using the given
dictionary
"""
super(UpgradeCompleteStep, self).from_dict(data)
self._wait_time = 0
self._query_inprogress = False
return self
def as_dict(self):
"""
Represent the upgrade complete step as a dictionary
"""
data = super(UpgradeCompleteStep, self).as_dict()
data['entity_type'] = ''
data['entity_names'] = list()
data['entity_uuids'] = list()
return data
class MigrateInstancesStep(strategy.StrategyStep):
"""
Migrate Instances - Strategy Step
"""
def __init__(self, instances):
super(MigrateInstancesStep, self).__init__(
STRATEGY_STEP_NAME.MIGRATE_INSTANCES, timeout_in_secs=1800)
self._instances = instances
self._instance_names = list()
self._instance_uuids = list()
self._instance_host_names = dict()
for instance in instances:
self._instance_names.append(instance.name)
self._instance_uuids.append(instance.uuid)
self._instance_host_names[instance.uuid] = instance.host_name
def _all_instances_migrated(self):
"""
Returns true if all instances have migrated from the source hosts
"""
source_host_names = []
for host_name in self._instance_host_names.values():
if host_name not in source_host_names:
source_host_names.append(host_name)
instance_table = tables.tables_get_instance_table()
for host_name in source_host_names:
if instance_table.exist_on_host(host_name):
return False, ""
return True, ""
def apply(self):
"""
Migrate all instances
"""
from nfv_vim import directors
DLOG.info("Step (%s) apply for instances %s." % (self._name,
self._instance_names))
migrate_complete, reason = self._all_instances_migrated()
if migrate_complete:
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
# Ensure none of the instances have moved since the strategy step was
# created. The instance_director.migrate_instances will migrate ALL
# instances on each host containing one of the self._instance_uuids. We
# want to ensure we are only migrating instances from the host(s) they
# were originally located on..
for instance in self._instances:
if instance.host_name != self._instance_host_names[instance.uuid]:
reason = ("instance %s has moved from %s to %s after strategy "
"created" %
(instance.name, self._instance_host_names[instance.uuid],
instance.host_name))
return strategy.STRATEGY_STEP_RESULT.FAILED, reason
instance_director = directors.get_instance_director()
operation = instance_director.migrate_instances(self._instance_uuids)
if operation.is_inprogress():
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
elif operation.is_failed():
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
def handle_event(self, event, event_data=None):
"""
Handle Instance events
"""
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event in [STRATEGY_EVENT.INSTANCE_STATE_CHANGED,
STRATEGY_EVENT.INSTANCE_AUDIT]:
migrate_complete, reason = self._all_instances_migrated()
if not migrate_complete and reason:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, reason)
return True
if migrate_complete:
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, '')
return True
elif STRATEGY_EVENT.MIGRATE_INSTANCES_FAILED == event:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, event_data)
return True
return False
def from_dict(self, data):
"""
Returns the migrate instances step object initialized using the given
dictionary
"""
super(MigrateInstancesStep, self).from_dict(data)
self._instance_uuids = data['entity_uuids']
self._instances = list()
self._instance_names = list()
self._instance_host_names = dict()
instance_table = tables.tables_get_instance_table()
for instance_uuid in self._instance_uuids:
instance = instance_table.get(instance_uuid, None)
if instance is not None:
self._instances.append(instance)
self._instance_names.append(instance.name)
# Retrieve the host this instance was on when the step was
# created.
self._instance_host_names[instance.uuid] = \
data['instance_host_names'][instance.uuid]
return self
def as_dict(self):
"""
Represent the migrate instances step as a dictionary
"""
data = super(MigrateInstancesStep, self).as_dict()
data['entity_type'] = 'instances'
data['entity_names'] = self._instance_names
data['entity_uuids'] = self._instance_uuids
data['instance_host_names'] = self._instance_host_names
return data
class StartInstancesStep(strategy.StrategyStep):
"""
Start Instances - Strategy Step
"""
def __init__(self, instances):
super(StartInstancesStep, self).__init__(
STRATEGY_STEP_NAME.START_INSTANCES, timeout_in_secs=900)
self._instances = instances
self._instance_names = list()
self._instance_uuids = list()
for instance in instances:
self._instance_names.append(instance.name)
self._instance_uuids.append(instance.uuid)
def _total_instances_unlocked_enabled(self):
"""
Returns the number of instances that are unlocked and enabled
"""
total_instances_enabled = 0
instance_table = tables.tables_get_instance_table()
for instance_uuid in self._instance_uuids:
instance = instance_table.get(instance_uuid, None)
if instance is None:
return -1
if instance.is_enabled():
total_instances_enabled += 1
return total_instances_enabled
def apply(self):
"""
Start all instances
"""
from nfv_vim import directors
DLOG.info("Step (%s) apply for instances %s." % (self._name,
self._instance_names))
if len(self._instance_uuids) == self._total_instances_unlocked_enabled():
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
instance_director = directors.get_instance_director()
operation = instance_director.start_instances(self._instance_uuids)
if operation.is_inprogress():
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
elif operation.is_failed():
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
def handle_event(self, event, event_data=None):
"""
Handle Instance events
"""
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event in [STRATEGY_EVENT.INSTANCE_STATE_CHANGED,
STRATEGY_EVENT.INSTANCE_AUDIT]:
total_instances_enabled = self._total_instances_unlocked_enabled()
if -1 == total_instances_enabled:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "instance no longer exists")
return True
if total_instances_enabled == len(self._instance_uuids):
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, '')
return True
return False
def from_dict(self, data):
"""
Returns the start instances step object initialized using the given
dictionary
"""
super(StartInstancesStep, self).from_dict(data)
self._instance_uuids = data['entity_uuids']
self._instances = list()
self._instance_names = list()
instance_table = tables.tables_get_instance_table()
for instance_uuid in self._instance_uuids:
instance = instance_table.get(instance_uuid, None)
if instance is not None:
self._instances.append(instance)
self._instance_names.append(instance.name)
return self
def as_dict(self):
"""
Represent the start instances step as a dictionary
"""
data = super(StartInstancesStep, self).as_dict()
data['entity_type'] = 'instances'
data['entity_names'] = self._instance_names
data['entity_uuids'] = self._instance_uuids
return data
class StopInstancesStep(strategy.StrategyStep):
"""
Stop Instances - Strategy Step
"""
def __init__(self, instances):
super(StopInstancesStep, self).__init__(
STRATEGY_STEP_NAME.STOP_INSTANCES, timeout_in_secs=900)
self._instances = instances
self._instance_names = list()
self._instance_uuids = list()
self._instance_host_names = dict()
for instance in instances:
self._instance_names.append(instance.name)
self._instance_uuids.append(instance.uuid)
self._instance_host_names[instance.uuid] = instance.host_name
def abort(self):
"""
Returns the abort step related to this step
"""
return [StartInstancesStep(self._instances)]
def _total_instances_locked_disabled(self):
"""
Returns the number of instances that are locked and disabled
"""
total_instances_locked = 0
instance_table = tables.tables_get_instance_table()
for instance_uuid in self._instance_uuids:
instance = instance_table.get(instance_uuid, None)
if instance is None:
return -1
if instance.is_locked() and instance.is_disabled():
total_instances_locked += 1
return total_instances_locked
def apply(self):
"""
Stop all instances
"""
from nfv_vim import directors
DLOG.info("Step (%s) apply for instances %s." % (self._name,
self._instance_names))
if len(self._instance_uuids) == self._total_instances_locked_disabled():
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
# Ensure none of the instances have moved since the strategy step was
# created. There is no point stopping instances that are now on the
# wrong host.
for instance in self._instances:
if instance.host_name != self._instance_host_names[instance.uuid]:
reason = ("instance %s has moved from %s to %s after strategy "
"created" %
(instance.name, self._instance_host_names[instance.uuid],
instance.host_name))
return strategy.STRATEGY_STEP_RESULT.FAILED, reason
instance_director = directors.get_instance_director()
operation = instance_director.stop_instances(self._instance_uuids)
if operation.is_inprogress():
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
elif operation.is_failed():
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
def handle_event(self, event, event_data=None):
"""
Handle Instance events
"""
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event in [STRATEGY_EVENT.INSTANCE_STATE_CHANGED,
STRATEGY_EVENT.INSTANCE_AUDIT]:
total_instances_locked = self._total_instances_locked_disabled()
if -1 == total_instances_locked:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "instance no longer exists")
return True
if total_instances_locked == len(self._instance_uuids):
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, '')
return True
return False
def from_dict(self, data):
"""
Returns the stop instances step object initialized using the given
dictionary
"""
super(StopInstancesStep, self).from_dict(data)
self._instance_uuids = data['entity_uuids']
self._instances = list()
self._instance_names = list()
self._instance_host_names = dict()
instance_table = tables.tables_get_instance_table()
for instance_uuid in self._instance_uuids:
instance = instance_table.get(instance_uuid, None)
if instance is not None:
self._instances.append(instance)
self._instance_names.append(instance.name)
# Retrieve the host this instance was on when the step was
# created.
self._instance_host_names[instance.uuid] = \
data['instance_host_names'][instance.uuid]
return self
def as_dict(self):
"""
Represent the stop instances step as a dictionary
"""
data = super(StopInstancesStep, self).as_dict()
data['entity_type'] = 'instances'
data['entity_names'] = self._instance_names
data['entity_uuids'] = self._instance_uuids
data['instance_host_names'] = self._instance_host_names
return data
class SystemStabilizeStep(strategy.StrategyStep):
"""
System Stabilize - Strategy Step
"""
def __init__(self, timeout_in_secs=60):
super(SystemStabilizeStep, self).__init__(
STRATEGY_STEP_NAME.SYSTEM_STABILIZE, timeout_in_secs=timeout_in_secs)
def timeout(self):
"""
Timeout is expected, so override to pass
"""
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ''
def apply(self):
"""
Wait for a period of time
"""
DLOG.info("Step (%s) apply." % self._name)
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
def handle_event(self, event, event_data=None):
"""
Handle Host and Instance events
"""
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if STRATEGY_EVENT.HOST_STATE_CHANGED == event:
host = event_data
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "host %s changed state unexpectedly"
% host.name)
return True
elif STRATEGY_EVENT.INSTANCE_STATE_CHANGED == event:
instance = event_data
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "instance %s changed state "
"unexpectedly" % instance.name)
return True
return False
def as_dict(self):
"""
Represent the system stabilize step as a dictionary
"""
data = super(SystemStabilizeStep, self).as_dict()
data['entity_type'] = ''
data['entity_names'] = list()
data['entity_uuids'] = list()
return data
class QueryAlarmsStep(strategy.StrategyStep):
"""
Query Alarms - Strategy Step
"""
def __init__(self, fail_on_alarms=False, ignore_alarms=None):
super(QueryAlarmsStep, self).__init__(
STRATEGY_STEP_NAME.QUERY_ALARMS, timeout_in_secs=60)
if ignore_alarms is None:
ignore_alarms = []
self._fail_on_alarms = fail_on_alarms
self._ignore_alarms = ignore_alarms
@coroutine
def _query_alarms_callback(self, fm_service):
"""
Query Alarms Callback
"""
response = (yield)
DLOG.debug("Query-Alarms callback response=%s." % response)
if response['completed']:
if self.strategy is not None:
nfvi_alarms = self.strategy.nfvi_alarms
for nfvi_alarm in response['result-data']:
if (self.strategy._alarm_restrictions ==
strategy.STRATEGY_ALARM_RESTRICTION_TYPES.RELAXED and
nfvi_alarm.mgmt_affecting == 'False'):
DLOG.warn("Ignoring non-management affecting alarm "
"%s - uuid %s due to relaxed alarm "
"strictness" % (nfvi_alarm.alarm_id,
nfvi_alarm.alarm_uuid))
elif nfvi_alarm.alarm_id not in self._ignore_alarms:
DLOG.warn("Alarm: %s" % nfvi_alarm.alarm_id)
nfvi_alarms.append(nfvi_alarm)
else:
DLOG.warn("Ignoring alarm %s - uuid %s" %
(nfvi_alarm.alarm_id, nfvi_alarm.alarm_uuid))
self.strategy.nfvi_alarms = nfvi_alarms
if self._fail_on_alarms and self.strategy.nfvi_alarms:
result = strategy.STRATEGY_STEP_RESULT.FAILED
alarm_ids = [str(alarm.get('alarm_id')) for alarm in self.strategy.nfvi_alarms]
reason = "alarms %s from %s are present" % (alarm_ids, fm_service)
else:
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
reason = ""
self.stage.step_complete(result, reason)
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
def apply(self):
"""
Query Alarms
"""
from nfv_vim import nfvi
DLOG.info("Step (%s) apply." % self._name)
self.strategy.nfvi_alarms = list()
nfvi.nfvi_get_alarms(self._query_alarms_callback("platform"))
if not nfvi.nfvi_fault_mgmt_plugin_disabled():
nfvi.nfvi_get_openstack_alarms(self._query_alarms_callback("openstack"))
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
def from_dict(self, data):
"""
Returns the query alarms step object initialized using the given
dictionary
"""
super(QueryAlarmsStep, self).from_dict(data)
self._fail_on_alarms = data['fail_on_alarms']
self._ignore_alarms = data['ignore_alarms']
return self
def as_dict(self):
"""
Represent the query alarms step as a dictionary
"""
data = super(QueryAlarmsStep, self).as_dict()
data['entity_type'] = ''
data['entity_names'] = list()
data['entity_uuids'] = list()
data['fail_on_alarms'] = self._fail_on_alarms
data['ignore_alarms'] = self._ignore_alarms
return data
class WaitDataSyncStep(strategy.StrategyStep):
"""
Alarm Wait - Strategy Step
"""
def __init__(self, timeout_in_secs=300, ignore_alarms=None):
super(WaitDataSyncStep, self).__init__(
STRATEGY_STEP_NAME.WAIT_DATA_SYNC, timeout_in_secs=timeout_in_secs)
if ignore_alarms is None:
ignore_alarms = []
self._ignore_alarms = ignore_alarms
self._wait_time = 0
self._query_inprogress = False
@coroutine
def _query_alarms_callback(self):
"""
Query Alarms Callback
"""
response = (yield)
DLOG.debug("Query-Alarms callback response=%s." % response)
self._query_inprogress = False
if response['completed']:
if self.strategy is not None:
nfvi_alarms = list()
for nfvi_alarm in response['result-data']:
if (self.strategy._alarm_restrictions ==
strategy.STRATEGY_ALARM_RESTRICTION_TYPES.RELAXED and
nfvi_alarm.mgmt_affecting == 'False'):
DLOG.warn("Ignoring non-management affecting alarm "
"%s - uuid %s due to relaxed alarm "
"strictness" % (nfvi_alarm.alarm_id,
nfvi_alarm.alarm_uuid))
elif nfvi_alarm.alarm_id not in self._ignore_alarms:
nfvi_alarms.append(nfvi_alarm)
else:
DLOG.debug("Ignoring alarm %s - uuid %s" %
(nfvi_alarm.alarm_id, nfvi_alarm.alarm_uuid))
self.strategy.nfvi_alarms = nfvi_alarms
if self.strategy.nfvi_alarms:
# Keep waiting for alarms to clear
pass
else:
# Alarms have all cleared
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
# Unable to retrieve alarms
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
def apply(self):
"""
Alarm Wait
"""
DLOG.info("Step (%s) apply." % self._name)
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
def handle_event(self, event, event_data=None):
"""
Handle Host events
"""
from nfv_vim import nfvi
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event == STRATEGY_EVENT.HOST_AUDIT:
if 0 == self._wait_time:
self._wait_time = timers.get_monotonic_timestamp_in_ms()
now_ms = timers.get_monotonic_timestamp_in_ms()
secs_expired = (now_ms - self._wait_time) / 1000
# Wait at least 120 seconds before checking alarms for first time
if 120 <= secs_expired and not self._query_inprogress:
self._query_inprogress = True
nfvi.nfvi_get_alarms(self._query_alarms_callback())
return True
return False
def from_dict(self, data):
"""
Returns the alarm wait step object initialized using the given
dictionary
"""
super(WaitDataSyncStep, self).from_dict(data)
self._ignore_alarms = data['ignore_alarms']
self._wait_time = 0
self._query_inprogress = False
return self
def as_dict(self):
"""
Represent the alarm wait step as a dictionary
"""
data = super(WaitDataSyncStep, self).as_dict()
data['entity_type'] = ''
data['entity_names'] = list()
data['entity_uuids'] = list()
data['ignore_alarms'] = self._ignore_alarms
return data
class WaitAlarmsClearStep(strategy.StrategyStep):
"""
Alarm Wait - Strategy Step
"""
def __init__(self, timeout_in_secs=300, first_query_delay_in_secs=60, ignore_alarms=None):
super(WaitAlarmsClearStep, self).__init__(
STRATEGY_STEP_NAME.WAIT_ALARMS_CLEAR, timeout_in_secs=timeout_in_secs)
self._first_query_delay_in_secs = first_query_delay_in_secs
if ignore_alarms is None:
ignore_alarms = []
self._ignore_alarms = ignore_alarms
self._wait_time = 0
self._query_inprogress = False
@coroutine
def _query_alarms_callback(self):
"""
Query Alarms Callback
"""
response = (yield)
DLOG.debug("Query-Alarms callback response=%s." % response)
self._query_inprogress = False
if response['completed']:
if self.strategy is not None:
nfvi_alarms = list()
for nfvi_alarm in response['result-data']:
if (self.strategy._alarm_restrictions ==
strategy.STRATEGY_ALARM_RESTRICTION_TYPES.RELAXED and
nfvi_alarm.mgmt_affecting == 'False'):
DLOG.warn("Ignoring non-management affecting alarm "
"%s - uuid %s due to relaxed alarm "
"strictness" % (nfvi_alarm.alarm_id,
nfvi_alarm.alarm_uuid))
elif nfvi_alarm.alarm_id not in self._ignore_alarms:
nfvi_alarms.append(nfvi_alarm)
else:
DLOG.debug("Ignoring alarm %s - uuid %s" %
(nfvi_alarm.alarm_id, nfvi_alarm.alarm_uuid))
self.strategy.nfvi_alarms = nfvi_alarms
if self.strategy.nfvi_alarms:
# Keep waiting for alarms to clear
pass
else:
# Alarms have all cleared
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
# Unable to retrieve alarms
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
def apply(self):
"""
Alarm Wait
"""
DLOG.info("Step (%s) apply." % self._name)
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
def handle_event(self, event, event_data=None):
"""
Handle Host events
"""
from nfv_vim import nfvi
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event == STRATEGY_EVENT.HOST_AUDIT:
if 0 == self._wait_time:
self._wait_time = timers.get_monotonic_timestamp_in_ms()
now_ms = timers.get_monotonic_timestamp_in_ms()
secs_expired = (now_ms - self._wait_time) / 1000
# Wait before checking alarms for first time
if self._first_query_delay_in_secs <= secs_expired and not self._query_inprogress:
self._query_inprogress = True
nfvi.nfvi_get_alarms(self._query_alarms_callback())
return True
return False
def from_dict(self, data):
"""
Returns the alarm wait step object initialized using the given
dictionary
"""
super(WaitAlarmsClearStep, self).from_dict(data)
self._first_query_delay_in_secs = data['first_query_delay_in_secs']
self._ignore_alarms = data['ignore_alarms']
self._wait_time = 0
self._query_inprogress = False
return self
def as_dict(self):
"""
Represent the alarm wait step as a dictionary
"""
data = super(WaitAlarmsClearStep, self).as_dict()
data['entity_type'] = ''
data['entity_names'] = list()
data['entity_uuids'] = list()
data['first_query_delay_in_secs'] = self._first_query_delay_in_secs
data['ignore_alarms'] = self._ignore_alarms
return data
class QuerySwPatchesStep(strategy.StrategyStep):
"""
Query Software Patches - Strategy Step
"""
def __init__(self):
super(QuerySwPatchesStep, self).__init__(
STRATEGY_STEP_NAME.QUERY_SW_PATCHES, timeout_in_secs=60)
@coroutine
def _query_sw_patches_callback(self):
"""
Query Software Patches Callback
"""
response = (yield)
DLOG.debug("Query-Sw-Updates callback response=%s." % response)
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_sw_patches = response['result-data']
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
def apply(self):
"""
Query Software Patches
"""
from nfv_vim import nfvi
DLOG.info("Step (%s) apply." % self._name)
nfvi.nfvi_sw_mgmt_query_updates(self._query_sw_patches_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
def as_dict(self):
"""
Represent the query software update step as a dictionary
"""
data = super(QuerySwPatchesStep, self).as_dict()
data['entity_type'] = ''
data['entity_names'] = list()
data['entity_uuids'] = list()
return data
class QuerySwPatchHostsStep(strategy.StrategyStep):
"""
Query Software Patch Hosts - Strategy Step
"""
def __init__(self):
super(QuerySwPatchHostsStep, self).__init__(
STRATEGY_STEP_NAME.QUERY_SW_PATCH_HOSTS, timeout_in_secs=60)
@coroutine
def _query_hosts_callback(self):
"""
Query Software Patch Hosts Callback
"""
response = (yield)
DLOG.debug("Query-Hosts callback response=%s." % response)
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_sw_patch_hosts = response['result-data']
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
def apply(self):
"""
Query Software Patch Hosts
"""
from nfv_vim import nfvi
DLOG.info("Step (%s) apply." % self._name)
nfvi.nfvi_sw_mgmt_query_hosts(self._query_hosts_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
def as_dict(self):
"""
Represent the query software patches hosts step as a dictionary
"""
data = super(QuerySwPatchHostsStep, self).as_dict()
data['entity_type'] = ''
data['entity_names'] = list()
data['entity_uuids'] = list()
return data
class QueryFwUpdateHostStep(strategy.StrategyStep):
"""
Query Host
"""
# This step queries system inventory for the host in self._host_names
# If the host's 'device_image_update' field shows 'pending' then its
# hostname is added to the strategy's fw_update_hosts list.
def __init__(self, host):
super(QueryFwUpdateHostStep, self).__init__(
STRATEGY_STEP_NAME.QUERY_FW_UPDATE_HOST, timeout_in_secs=60)
self._host_names = list()
self._host_uuids = list()
self._host_names.append(host.name)
self._host_uuids.append(host.uuid)
@coroutine
def _get_host_callback(self):
"""
Query Host callback
"""
response = (yield)
DLOG.verbose("Get-Host %s callback response=%s." %
(self._host_names[0], response))
if response['completed']:
if self.strategy is not None:
hostname = response['result-data'].get('name')
if hostname:
device_image_update = response['result-data'].get('device_image_update')
if device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_PENDING:
self.strategy.fw_update_hosts.append(hostname)
DLOG.info("%s requires firmware update" % hostname)
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_IN_PROGRESS:
DLOG.info("%s firmware update in-progress" % hostname)
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_IN_PROGRESS_ABORTED:
DLOG.info("%s firmware update in-progress-aborted" % hostname)
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_COMPLETED:
DLOG.info("%s firmware update complete" % hostname)
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_FAILED:
DLOG.info("%s firmware update failed" % hostname)
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_NULL:
DLOG.info("%s no firmware update required" % hostname)
else:
DLOG.info("%s unknown device_image_update state; %s" %
(hostname, device_image_update))
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "firmware update query failed")
def apply(self):
"""
Query Host Apply
"""
from nfv_vim import nfvi
DLOG.info("%s %s step apply" % (self._host_names[0], self._name))
# This step is only ever called with one host name.
nfvi.nfvi_get_host(self._host_uuids[0],
self._host_names[0],
self._get_host_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
def from_dict(self, data):
"""
Load the firmware update host device list step
"""
super(QueryFwUpdateHostStep, self).from_dict(data)
self._host_names = data['entity_names']
self._host_uuids = data['entity_uuids']
return self
def as_dict(self):
"""
Represent the object as a dictionary for the strategy
"""
data = super(QueryFwUpdateHostStep, self).as_dict()
data['entity_type'] = ''
data['entity_names'] = self._host_names
data['entity_uuids'] = self._host_uuids
return data
class FwUpdateHostsStep(strategy.StrategyStep):
"""
Firmware Update Hosts - Strategy Step
"""
# This step starts the firmware update process for the passed in hosts
def __init__(self, hosts):
super(FwUpdateHostsStep, self).__init__(
STRATEGY_STEP_NAME.FW_UPDATE_HOSTS, timeout_in_secs=3600)
self._hosts = hosts
self._host_names = list()
self._host_uuids = list()
self._monitoring_fw_update = False
self._wait_time = 0
self._host_failed_device_update = dict()
self._host_completed = dict()
for host in hosts:
self._host_names.append(host.name)
self._host_uuids.append(host.uuid)
self._host_completed[host.name] = (False, False, '')
self._host_failed_device_update[host.name] = list()
@coroutine
def _get_host_callback(self):
"""
Query Host callback used for monitoring update process
"""
response = (yield)
DLOG.debug("Get-Host callback response=%s." % response)
try:
if response['completed']:
if self.strategy is not None:
hostname = response['result-data'].get('name')
if hostname:
device_image_update = response['result-data'].get('device_image_update')
if device_image_update is None:
DLOG.verbose("%s no firmware update required" % hostname)
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_PENDING:
if self._host_completed[hostname][0] is False:
DLOG.warn("%s firmware update status went pending during update" % hostname)
failed_msg = hostname + ' firmware update failed ; needs retry'
self._host_completed[hostname] = (True, False, failed_msg)
DLOG.error(failed_msg)
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_IN_PROGRESS:
DLOG.info("%s firmware update in-progress" % hostname)
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_IN_PROGRESS_ABORTED:
if self._host_completed[hostname][0] is False:
failed_msg = hostname + ' firmware update aborted while in progress'
self._host_completed[hostname] = (True, False, failed_msg)
DLOG.error(failed_msg)
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_COMPLETED or \
device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_NULL:
if self._host_completed[hostname][0] is False:
self._host_completed[hostname] = (True, True, '')
DLOG.info("%s firmware update complete" % hostname)
elif device_image_update == FW_UPDATE_LABEL.DEVICE_IMAGE_UPDATE_FAILED:
if self._host_completed[hostname][0] is False:
failed_msg = hostname + ' firmware update failed'
self._host_completed[hostname] = (True, False, failed_msg)
DLOG.error(failed_msg)
else:
if self._host_completed[hostname][0] is False:
failed_msg = hostname + \
' firmware update failed ;' \
' unknown state [' + \
device_image_update + ']'
self._host_completed[hostname] = (True, False, failed_msg)
DLOG.error(failed_msg)
# Check for firmware upgrade step complete
self._check_step_complete()
return
else:
DLOG.error("failed to get hostname or data from get host response")
else:
DLOG.error("failed to monitor firmware update ; no strategy")
else:
DLOG.error("get host request did not complete")
except Exception as e:
DLOG.exception("Caught exception interpreting host info")
DLOG.error("Response: %s" % response)
result = strategy.STRATEGY_STEP_RESULT.FAILED
fail_msg = "failed to get or parse fw update info"
self.stage.step_complete(result, fail_msg)
def _check_step_complete(self):
"""
Check for firmware upgrade step complete
"""
failed_hosts = ""
done = True
for hostname in self._host_names:
if self._host_completed[hostname][0] is False:
done = False
elif self._host_completed[hostname][1] is False:
failed_hosts += hostname + ' '
else:
DLOG.verbose("%s firmware update is complete" % hostname)
if done:
if len(failed_hosts) == 0:
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, '')
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
failed_msg = 'Firmware update failed ; %s' % failed_hosts
self.stage.step_complete(result, failed_msg)
def apply(self):
"""
Firmware Update Hosts Apply
"""
from nfv_vim import directors
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
self._host_names))
if len(self._host_names):
host_director = directors.get_host_director()
operation = host_director.fw_update_hosts(self._host_names)
if operation.is_inprogress():
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
elif operation.is_failed():
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
else:
reason = "no hosts found in firmware update step"
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, reason)
return strategy.STRATEGY_STEP_RESULT.FAILED, reason
def handle_event(self, event, event_data=None):
"""
Handle Firmware Image Update events
"""
from nfv_vim import nfvi
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event == STRATEGY_EVENT.HOST_FW_UPDATE_FAILED:
host = event_data
if host is not None and host.name in self._host_names:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "fw image update failed")
return True
elif event == STRATEGY_EVENT.HOST_AUDIT:
if not self._monitoring_fw_update:
self._monitoring_fw_update = True
DLOG.info("Start monitoring firmware update progress for %s" %
self._host_names)
if 0 == self._wait_time:
self._wait_time = timers.get_monotonic_timestamp_in_ms()
now_ms = timers.get_monotonic_timestamp_in_ms()
secs_expired = (now_ms - self._wait_time) / 1000
if 60 <= secs_expired:
# force timer reload on next audit
self._wait_time = 0
for host in self._hosts:
if self._host_completed[host.name][0] is True:
DLOG.info("%s firmware update already done ; pass=%s" %
(host.name,
self._host_completed[host.name][1]))
continue
nfvi.nfvi_get_host(host.uuid,
host.name,
self._get_host_callback())
return True
else:
DLOG.warn("Unexpected event (%s)" % event)
return False
def abort(self):
"""
Returns the abort step with applicable host list
"""
# abort all hosts that are not in the completed state
hosts = list()
for host in self._hosts:
if self._host_completed[host.name][0] is False:
hosts.append(host)
return [FwUpdateAbortHostsStep(hosts)]
def from_dict(self, data):
"""
Returns the firmware update hosts step object
initialized using the given dictionary
"""
super(FwUpdateHostsStep, self).from_dict(data)
self._hosts = list()
self._host_uuids = list()
self._host_completed = dict()
self._monitoring_fw_update = False
self._host_names = data['entity_names']
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is not None:
self._hosts.append(host)
self._host_uuids.append(host.uuid)
self._host_completed[host_name] = \
data['hosts_completed'][host_name]
return self
def as_dict(self):
"""
Represent the firmware update hosts step as a dictionary
"""
data = super(FwUpdateHostsStep, self).as_dict()
data['entity_type'] = 'hosts'
data['entity_names'] = self._host_names
data['entity_uuids'] = self._host_uuids
data['hosts_completed'] = self._host_completed
return data
class FwUpdateAbortHostsStep(strategy.StrategyStep):
"""
Firmware Update Abort Hosts Step
"""
def __init__(self, hosts):
super(FwUpdateAbortHostsStep, self).__init__(
STRATEGY_STEP_NAME.FW_UPDATE_ABORT_HOSTS, timeout_in_secs=600)
self._hosts = hosts
self._host_names = list()
self._host_uuids = list()
self._wait_time = 0
self._host_completed = dict()
for host in hosts:
self._host_names.append(host.name)
self._host_uuids.append(host.uuid)
self._host_completed[host.name] = (False, False, '')
def apply(self):
"""
Monitor Firmware Update Abort Hosts Apply
"""
from nfv_vim import directors
DLOG.info("Step (%s) apply for hosts %s." % (self._name,
self._host_names))
host_director = directors.get_host_director()
operation = host_director.fw_update_abort_hosts(self._host_names)
if operation.is_inprogress():
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
elif operation.is_failed():
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
def handle_event(self, event, event_data=None):
"""
Handle Firmware Image Update Abort events
"""
# from nfv_vim import nfvi
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event == STRATEGY_EVENT.HOST_FW_UPDATE_ABORT_FAILED:
host = event_data
if host is not None and host.name in self._host_names:
failed_msg = "device image update abort failed"
DLOG.info("%s %s" % (host.name, failed_msg))
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, failed_msg)
return True
elif event == STRATEGY_EVENT.HOST_AUDIT:
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, '')
return True
return False
def from_dict(self, data):
"""
Load the firmware update abort hosts step object
"""
super(FwUpdateAbortHostsStep, self).from_dict(data)
self._hosts = list()
self._host_uuids = list()
self._host_completed = dict()
self._host_names = data['entity_names']
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is not None:
self._hosts.append(host)
self._host_uuids.append(host.uuid)
self._host_completed[host_name] = \
data['hosts_completed'][host_name]
return self
def as_dict(self):
"""
Save the firmware update abort hosts step as a dictionary
"""
data = super(FwUpdateAbortHostsStep, self).as_dict()
data['entity_type'] = 'hosts'
data['entity_names'] = self._host_names
data['entity_uuids'] = self._host_uuids
data['hosts_completed'] = self._host_completed
return data
class QueryUpgradeStep(strategy.StrategyStep):
"""
Query Upgrade - Strategy Step
"""
def __init__(self):
super(QueryUpgradeStep, self).__init__(
STRATEGY_STEP_NAME.QUERY_UPGRADE, timeout_in_secs=60)
@coroutine
def _get_upgrade_callback(self):
"""
Get Upgrade Callback
"""
response = (yield)
DLOG.debug("Query-Upgrade callback response=%s." % response)
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_upgrade = response['result-data']
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
def apply(self):
"""
Query Software Upgrade
"""
from nfv_vim import nfvi
DLOG.info("Step (%s) apply." % self._name)
nfvi.nfvi_get_upgrade(self._get_upgrade_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
def as_dict(self):
"""
Represent the query upgrade step as a dictionary
"""
data = super(QueryUpgradeStep, self).as_dict()
data['entity_type'] = ''
data['entity_names'] = list()
data['entity_uuids'] = list()
return data
class DisableHostServicesStep(strategy.StrategyStep):
"""
Disable Host Services - Strategy Step
"""
def __init__(self, hosts, service):
super(DisableHostServicesStep, self).__init__(
"%s" % STRATEGY_STEP_NAME.DISABLE_HOST_SERVICES,
timeout_in_secs=180)
self._hosts = hosts
self._host_names = list()
self._host_uuids = list()
self._service = service
for host in hosts:
self._host_names.append(host.name)
self._host_uuids.append(host.uuid)
def abort(self):
"""
Returns the abort step related to this step
"""
return [EnableHostServicesStep(self._hosts, self._service)]
def _total_hosts_services_disabled(self):
"""
Returns the number of hosts with services disabled
"""
total_hosts_services_disabled = 0
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is None:
return -1
if (objects.HOST_SERVICE_STATE.DISABLED ==
host.host_service_state(self._service)):
total_hosts_services_disabled += 1
return total_hosts_services_disabled
def apply(self):
"""
Disable host services on specified hosts
"""
from nfv_vim import directors
DLOG.info("Step (%s) apply for hosts %s service %s." %
(self._name, self._host_names, self._service))
host_director = directors.get_host_director()
operation = host_director.disable_host_services(self._host_names,
self._service)
if operation.is_inprogress():
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
elif operation.is_failed():
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
def handle_event(self, event, event_data=None):
"""
Handle Host events
"""
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event in [STRATEGY_EVENT.HOST_STATE_CHANGED,
STRATEGY_EVENT.HOST_AUDIT]:
total_hosts_services_disabled = \
self._total_hosts_services_disabled()
if -1 == total_hosts_services_disabled:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "host no longer exists")
return True
if total_hosts_services_disabled == len(self._host_names):
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, '')
return True
elif event == STRATEGY_EVENT.DISABLE_HOST_SERVICES_FAILED:
host = event_data
if host is not None and host.name in self._host_names:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result,
"disable host services failed")
return True
return False
def from_dict(self, data):
"""
Returns the object initialized using the given dictionary
"""
super(DisableHostServicesStep, self).from_dict(data)
self._hosts = list()
self._host_uuids = list()
self._host_names = data['entity_names']
self._service = data['entity_service']
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is not None:
self._hosts.append(host)
self._host_uuids.append(host.uuid)
return self
def as_dict(self):
"""
Represent the object as a dictionary
"""
data = super(DisableHostServicesStep, self).as_dict()
data['entity_type'] = 'hosts'
data['entity_names'] = self._host_names
data['entity_uuids'] = self._host_uuids
data['entity_service'] = self._service
return data
class EnableHostServicesStep(strategy.StrategyStep):
"""
Enable Host Services - Strategy Step
"""
def __init__(self, hosts, service):
super(EnableHostServicesStep, self).__init__(
"%s" % STRATEGY_STEP_NAME.ENABLE_HOST_SERVICES,
timeout_in_secs=180)
self._hosts = hosts
self._host_names = list()
self._host_uuids = list()
self._service = service
for host in hosts:
self._host_names.append(host.name)
self._host_uuids.append(host.uuid)
def _total_hosts_services_enabled(self):
"""
Returns the number of hosts with services enabled
"""
total_hosts_services_enabled = 0
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is None:
return -1
if (objects.HOST_SERVICE_STATE.ENABLED ==
host.host_service_state(self._service)):
total_hosts_services_enabled += 1
return total_hosts_services_enabled
def apply(self):
"""
Enable host services on specified hosts
"""
from nfv_vim import directors
DLOG.info("Step (%s) apply for hosts %s service %s." %
(self._name, self._host_names, self._service))
host_director = directors.get_host_director()
operation = host_director.enable_host_services(self._host_names,
self._service)
if operation.is_inprogress():
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
elif operation.is_failed():
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
def handle_event(self, event, event_data=None):
"""
Handle Host events
"""
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event in [STRATEGY_EVENT.HOST_STATE_CHANGED,
STRATEGY_EVENT.HOST_AUDIT]:
total_hosts_services_enabled = \
self._total_hosts_services_enabled()
if -1 == total_hosts_services_enabled:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "host no longer exists")
return True
if total_hosts_services_enabled == len(self._host_names):
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, '')
return True
elif event == STRATEGY_EVENT.ENABLE_HOST_SERVICES_FAILED:
host = event_data
if host is not None and host.name in self._host_names:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "enable host services failed")
return True
return False
def from_dict(self, data):
"""
Returns the object initialized using the given dictionary
"""
super(EnableHostServicesStep, self).from_dict(data)
self._hosts = list()
self._host_uuids = list()
self._host_names = data['entity_names']
self._service = data['entity_service']
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is not None:
self._hosts.append(host)
self._host_uuids.append(host.uuid)
return self
def as_dict(self):
"""
Represent the object as a dictionary
"""
data = super(EnableHostServicesStep, self).as_dict()
data['entity_type'] = 'hosts'
data['entity_names'] = self._host_names
data['entity_uuids'] = self._host_uuids
data['entity_service'] = self._service
return data
class ApplySwPatchesStep(AbstractStrategyStep):
"""
Apply Patches using patch API
"""
def __init__(self, patches_to_apply):
super(ApplySwPatchesStep, self).__init__(
STRATEGY_STEP_NAME.APPLY_PATCHES,
timeout_in_secs=600)
self._patches_to_apply = patches_to_apply
@coroutine
def _api_callback(self):
"""
Callback for the API method invoked in apply
"""
response = (yield)
DLOG.debug("%s callback response=%s." % (self._name, response))
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_sw_patches = response['result-data']
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])
def apply(self):
"""
Apply patches
"""
from nfv_vim import nfvi
nfvi.nfvi_sw_mgmt_apply_updates(self._patches_to_apply,
self._api_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
def from_dict(self, data):
"""
Returns the step object initialized using the given dictionary
"""
super(ApplySwPatchesStep, self).from_dict(data)
# only the names are serialized
self._patches_to_apply = data['entity_names']
return self
def as_dict(self):
"""
Represent the step as a dictionary
"""
data = super(ApplySwPatchesStep, self).as_dict()
data['entity_type'] = 'patches'
data['entity_names'] = self._patches_to_apply
# there are no entity_uuids
return data
class QueryKubeUpgradeStep(AbstractStrategyStep):
"""
Query Kube Upgrade
"""
def __init__(self):
super(QueryKubeUpgradeStep, self).__init__(
STRATEGY_STEP_NAME.QUERY_KUBE_UPGRADE, timeout_in_secs=60)
@coroutine
def _get_kube_upgrade_callback(self):
"""
Get Kube Upgrade Callback
"""
response = (yield)
DLOG.debug("%s callback response=%s." % (self._name, response))
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_kube_upgrade = response['result-data']
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])
def apply(self):
"""
Query Kube Upgrade
"""
from nfv_vim import nfvi
DLOG.info("Step (%s) apply." % self._name)
nfvi.nfvi_get_kube_upgrade(self._get_kube_upgrade_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
class QueryKubeVersionsStep(AbstractStrategyStep):
"""
Query Kube Versions
This step should be used with its matching QueryKubeVersionsMixin
"""
def __init__(self):
super(QueryKubeVersionsStep, self).__init__(
STRATEGY_STEP_NAME.QUERY_KUBE_VERSIONS, timeout_in_secs=60)
@coroutine
def _query_callback(self):
"""
Get Kube Versions List Callback
"""
response = (yield)
DLOG.debug("%s callback response=%s." % (self._name, response))
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_kube_versions_list = response['result-data']
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])
def apply(self):
"""
Query Kube Versions List
"""
from nfv_vim import nfvi
nfvi.nfvi_get_kube_version_list(self._query_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
class QueryKubeHostUpgradeStep(AbstractStrategyStep):
"""
Query Kube Host Upgrade list
"""
def __init__(self):
super(QueryKubeHostUpgradeStep, self).__init__(
STRATEGY_STEP_NAME.QUERY_KUBE_HOST_UPGRADE, timeout_in_secs=60)
@coroutine
def _get_kube_host_upgrade_list_callback(self):
"""
Get Kube Host Upgrade List Callback
"""
response = (yield)
DLOG.debug("%s callback response=%s." % (self._name, response))
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_kube_host_upgrade_list = \
response['result-data']
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])
def apply(self):
"""
Query Kube Host Upgrade List
"""
from nfv_vim import nfvi
nfvi.nfvi_get_kube_host_upgrade_list(
self._get_kube_host_upgrade_list_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
class AbstractKubeUpgradeStep(AbstractStrategyStep):
def __init__(self,
step_name,
success_state,
fail_state,
timeout_in_secs=600):
super(AbstractKubeUpgradeStep, self).__init__(step_name,
timeout_in_secs)
# These two attributes are not persisted
self._wait_time = 0
self._query_inprogress = False
# success and fail state validators are persisted
self._success_state = success_state
self._fail_state = fail_state
@coroutine
def _get_kube_upgrade_callback(self):
"""Get Upgrade Callback"""
response = (yield)
DLOG.debug("(%s) callback response=%s." % (self._name, response))
self._query_inprogress = False
if response['completed']:
if self.strategy is None:
# there is no longer a strategy. abort.
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, 'strategy no longer exists')
kube_upgrade_obj = response['result-data']
# replace the object in the strategy with the most recent object
self.strategy.nfvi_kube_upgrade = kube_upgrade_obj
# break out of the loop if fail or success states match
if kube_upgrade_obj.state == self._success_state:
DLOG.debug("(%s) successfully reached (%s)."
% (self._name, self._success_state))
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
elif (self._fail_state is not None
and kube_upgrade_obj.state == self._fail_state):
DLOG.warn("(%s) encountered failure state(%s)."
% (self._name, self._fail_state))
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(
result,
'(%s) failed:(%s)' % (self._name, self._fail_state)
)
else:
# Keep waiting for upgrade to reach success or fail state
# timeout will occur if it is never reached.
DLOG.debug("(%s) in state (%s) waiting for (%s) or (%s)."
% (self._name,
kube_upgrade_obj.state,
self._success_state,
self._fail_state))
pass
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])
def handle_event(self, event, event_data=None):
"""Handle Host events"""
from nfv_vim import nfvi
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event == STRATEGY_EVENT.HOST_AUDIT:
if 0 == self._wait_time:
self._wait_time = timers.get_monotonic_timestamp_in_ms()
now_ms = timers.get_monotonic_timestamp_in_ms()
secs_expired = (now_ms - self._wait_time) / 1000
# Wait at least 60 seconds before checking upgrade for first time
if 60 <= secs_expired and not self._query_inprogress:
self._query_inprogress = True
nfvi.nfvi_get_kube_upgrade(self._get_kube_upgrade_callback())
return True
return False
def from_dict(self, data):
"""
Returns the step object initialized using the given dictionary
"""
super(AbstractKubeUpgradeStep, self).from_dict(data)
# these two attributes are not persisted
self._wait_time = 0
self._query_inprogress = False
# validation states are persisted
self._success_state = data['success_state']
self._fail_state = data['fail_state']
return self
def as_dict(self):
"""
Represent the kube upgrade step as a dictionary
"""
data = super(AbstractKubeUpgradeStep, self).as_dict()
data['success_state'] = self._success_state
data['fail_state'] = self._fail_state
return data
class KubeUpgradeStartStep(AbstractKubeUpgradeStep):
"""Kube Upgrade Start - Strategy Step"""
def __init__(self, to_version, force=False):
from nfv_vim import nfvi
super(KubeUpgradeStartStep, self).__init__(
STRATEGY_STEP_NAME.KUBE_UPGRADE_START,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTED,
None) # there is no failure state if upgrade-start fails
# next 2 attributes must be persisted through from_dict/as_dict
self._to_version = to_version
self._force = force
def from_dict(self, data):
"""
Returns the step object initialized using the given dictionary
"""
super(KubeUpgradeStartStep, self).from_dict(data)
self._to_version = data['to_version']
self._force = data['force']
return self
def as_dict(self):
"""
Represent the kube upgrade step as a dictionary
"""
data = super(KubeUpgradeStartStep, self).as_dict()
data['to_version'] = self._to_version
data['force'] = self._force
return data
@coroutine
def _response_callback(self):
"""Kube Upgrade Start - Callback"""
response = (yield)
DLOG.debug("%s callback response=%s." % (self._name, response))
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_kube_upgrade = response['result-data']
# We do not set 'success' here, let the handle_event do this
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])
def apply(self):
"""Kube Upgrade Start"""
from nfv_vim import nfvi
alarm_ignore_list = ["900.401", ] # ignore the auto apply alarm
nfvi.nfvi_kube_upgrade_start(self._to_version,
self._force,
alarm_ignore_list,
self._response_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
class KubeUpgradeCleanupStep(AbstractKubeUpgradeStep):
"""Kube Upgrade Cleanup - Strategy Step"""
def __init__(self):
super(KubeUpgradeCleanupStep, self).__init__(
STRATEGY_STEP_NAME.KUBE_UPGRADE_CLEANUP,
None, # there is no success state for this cleanup activity
None) # there is no failure state for this cleanup activity
@coroutine
def _response_callback(self):
"""Kube Upgrade Cleanup - Callback"""
response = (yield)
DLOG.debug("%s callback response=%s." % (self._name, response))
if response['completed']:
if self.strategy is not None:
# cleanup deletes the kube upgrade, clear it from the strategy
self.strategy.nfvi_kube_upgrade = None
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])
def apply(self):
"""Kube Upgrade Cleanup"""
from nfv_vim import nfvi
nfvi.nfvi_kube_upgrade_cleanup(self._response_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
class KubeUpgradeCompleteStep(AbstractKubeUpgradeStep):
"""Kube Upgrade Complete - Strategy Step"""
def __init__(self):
from nfv_vim import nfvi
super(KubeUpgradeCompleteStep, self).__init__(
STRATEGY_STEP_NAME.KUBE_UPGRADE_COMPLETE,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_COMPLETE,
None) # there is no failure state for upgrade-complete
@coroutine
def _response_callback(self):
"""Kube Upgrade Complete - Callback"""
response = (yield)
DLOG.debug("%s callback response=%s." % (self._name, response))
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_kube_upgrade = response['result-data']
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])
def apply(self):
"""Kube Upgrade Complete """
from nfv_vim import nfvi
nfvi.nfvi_kube_upgrade_complete(self._response_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
class KubeUpgradeDownloadImagesStep(AbstractKubeUpgradeStep):
"""Kube Upgrade Download Images - Strategy Step"""
def __init__(self):
from nfv_vim import nfvi
super(KubeUpgradeDownloadImagesStep, self).__init__(
STRATEGY_STEP_NAME.KUBE_UPGRADE_DOWNLOAD_IMAGES,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADED_IMAGES,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED)
@coroutine
def _response_callback(self):
"""Kube Upgrade Download Images - Callback"""
response = (yield)
DLOG.debug("%s callback response=%s." % (self._name, response))
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_kube_upgrade = response['result-data']
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])
def apply(self):
"""Kube Upgrade Download Images """
from nfv_vim import nfvi
nfvi.nfvi_kube_upgrade_download_images(self._response_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
class KubeUpgradeNetworkingStep(AbstractKubeUpgradeStep):
"""Kube Upgrade Networking - Strategy Step"""
def __init__(self):
from nfv_vim import nfvi
super(KubeUpgradeNetworkingStep, self).__init__(
STRATEGY_STEP_NAME.KUBE_UPGRADE_NETWORKING,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADED_NETWORKING,
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADING_NETWORKING_FAILED)
@coroutine
def _response_callback(self):
"""Kube Upgrade Networking - Callback"""
response = (yield)
DLOG.debug("%s callback response=%s." % (self._name, response))
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_kube_upgrade = response['result-data']
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])
def apply(self):
"""Kube Upgrade Networking"""
from nfv_vim import nfvi
nfvi.nfvi_kube_upgrade_networking(self._response_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
class AbstractKubeHostUpgradeStep(AbstractKubeUpgradeStep):
"""Kube Upgrade Host - Abstract Strategy Step
This operation issues a host command, which updates the kube upgrade object
"""
def __init__(self,
host,
force,
step_name,
success_state,
fail_state,
timeout_in_secs=600):
super(AbstractKubeHostUpgradeStep, self).__init__(
step_name,
success_state,
fail_state,
timeout_in_secs=timeout_in_secs)
self._force = force
# This class accepts only a single host
# but serializes as a list of hosts (list size of one)
self._hosts = list()
self._host_names = list()
self._host_uuids = list()
self._hosts.append(host)
self._host_names.append(host.name)
self._host_uuids.append(host.uuid)
def from_dict(self, data):
"""
Returns the step object initialized using the given dictionary
"""
super(AbstractKubeHostUpgradeStep, self).from_dict(data)
self._force = data['force']
self._hosts = list()
self._host_uuids = list()
self._host_names = data['entity_names']
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is not None:
self._hosts.append(host)
self._host_uuids.append(host.uuid)
return self
def as_dict(self):
"""
Represent the step as a dictionary
"""
data = super(AbstractKubeHostUpgradeStep, self).as_dict()
data['force'] = self._force
data['entity_type'] = 'hosts'
data['entity_names'] = self._host_names
data['entity_uuids'] = self._host_uuids
return data
class KubeHostUpgradeControlPlaneStep(AbstractKubeHostUpgradeStep):
"""Kube Host Upgrade Control Plane - Strategy Step
This operation issues a host command, which updates the kube upgrade object
"""
def __init__(self, host, force, target_state, target_failure_state):
super(KubeHostUpgradeControlPlaneStep, self).__init__(
host,
force,
STRATEGY_STEP_NAME.KUBE_HOST_UPGRADE_CONTROL_PLANE,
target_state,
target_failure_state,
timeout_in_secs=600)
def handle_event(self, event, event_data=None):
"""
Handle Host events - does not query kube host upgrade list but
instead queries kube host upgrade directly.
"""
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event == STRATEGY_EVENT.KUBE_HOST_UPGRADE_CONTROL_PLANE_FAILED:
host = event_data
if host is not None and host.name in self._host_names:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(
result,
"kube host upgrade control plane (%s) failed" % host.name)
return True
# return handle_event of parent class
return super(KubeHostUpgradeControlPlaneStep, self).handle_event(
event, event_data=event_data)
def apply(self):
"""Kube Host Upgrade Control Plane"""
from nfv_vim import directors
DLOG.info("Step (%s) apply to hostnames (%s)."
% (self._name, self._host_names))
host_director = directors.get_host_director()
operation = \
host_director.kube_upgrade_hosts_control_plane(self._host_names,
self._force)
if operation.is_inprogress():
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
elif operation.is_failed():
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
class KubeHostUpgradeKubeletStep(AbstractKubeHostUpgradeStep):
"""Kube Host Upgrade Kubelet - Strategy Step
This operation issues a host command, which indirectly updates the kube
upgrade object, however additional calls to other hosts do not change it.
This step should only be invoked on a locked host.
"""
def __init__(self, host, force):
super(KubeHostUpgradeKubeletStep, self).__init__(
host,
force,
STRATEGY_STEP_NAME.KUBE_HOST_UPGRADE_KUBELET,
None, # there is no kube upgrade success state for kubelets
None, # there is no kube upgrade failure state for kubelets
timeout_in_secs=900) # kubelet takes longer than control plane
@coroutine
def _get_kube_host_upgrade_list_callback(self):
"""Get Kube Host Upgrade List Callback"""
response = (yield)
DLOG.debug("(%s) callback response=%s." % (self._name, response))
self._query_inprogress = False
if response['completed']:
self.strategy.nfvi_kube_host_upgrade_list = response['result-data']
host_count = 0
match_count = 0
for host_uuid in self._host_uuids:
for k_host in self.strategy.nfvi_kube_host_upgrade_list:
if k_host.host_uuid == host_uuid:
if k_host.kubelet_version == self.strategy.to_version:
match_count += 1
host_count += 1
# break out of inner loop, since uuids match
break
if host_count == len(self._host_uuids):
# this is a pointless break
break
if match_count == len(self._host_uuids):
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
# keep waiting for kubelet state to change
pass
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])
def handle_event(self, event, event_data=None):
"""
Handle Host events - queries kube host upgrade list
Override to bypass checking for kube upgrade state.
"""
from nfv_vim import nfvi
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event == STRATEGY_EVENT.KUBE_HOST_UPGRADE_KUBELET_FAILED:
host = event_data
if host is not None and host.name in self._host_names:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(
result,
"kube host upgrade kubelet (%s) failed" % host.name)
return True
elif event == STRATEGY_EVENT.HOST_AUDIT:
if 0 == self._wait_time:
self._wait_time = timers.get_monotonic_timestamp_in_ms()
now_ms = timers.get_monotonic_timestamp_in_ms()
secs_expired = (now_ms - self._wait_time) / 1000
# Wait at least 60 seconds before checking upgrade for first time
if 60 <= secs_expired and not self._query_inprogress:
self._query_inprogress = True
nfvi.nfvi_get_kube_host_upgrade_list(
self._get_kube_host_upgrade_list_callback())
return True
return False
def apply(self):
"""Kube Upgrade Kubelet"""
from nfv_vim import directors
DLOG.info("Step (%s) apply to hostnames (%s)."
% (self._name, self._host_names))
host_director = directors.get_host_director()
operation = \
host_director.kube_upgrade_hosts_kubelet(self._host_names,
self._force)
if operation.is_inprogress():
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
elif operation.is_failed():
return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason
return strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
def strategy_step_rebuild_from_dict(data):
"""
Returns the strategy step object initialized using the given dictionary
"""
rebuild_map = {
STRATEGY_STEP_NAME.APPLY_PATCHES: ApplySwPatchesStep,
STRATEGY_STEP_NAME.KUBE_HOST_UPGRADE_CONTROL_PLANE:
KubeHostUpgradeControlPlaneStep,
STRATEGY_STEP_NAME.KUBE_HOST_UPGRADE_KUBELET:
KubeHostUpgradeKubeletStep,
STRATEGY_STEP_NAME.KUBE_UPGRADE_CLEANUP: KubeUpgradeCleanupStep,
STRATEGY_STEP_NAME.KUBE_UPGRADE_COMPLETE: KubeUpgradeCompleteStep,
STRATEGY_STEP_NAME.KUBE_UPGRADE_DOWNLOAD_IMAGES:
KubeUpgradeDownloadImagesStep,
STRATEGY_STEP_NAME.KUBE_UPGRADE_NETWORKING: KubeUpgradeNetworkingStep,
STRATEGY_STEP_NAME.KUBE_UPGRADE_START: KubeUpgradeStartStep,
STRATEGY_STEP_NAME.QUERY_KUBE_HOST_UPGRADE: QueryKubeHostUpgradeStep,
STRATEGY_STEP_NAME.QUERY_KUBE_UPGRADE: QueryKubeUpgradeStep,
STRATEGY_STEP_NAME.QUERY_KUBE_VERSIONS: QueryKubeVersionsStep,
}
obj_type = rebuild_map.get(data['name'])
if obj_type is not None:
step_obj = object.__new__(obj_type)
elif STRATEGY_STEP_NAME.SYSTEM_STABILIZE == data['name']:
step_obj = object.__new__(SystemStabilizeStep)
elif STRATEGY_STEP_NAME.UNLOCK_HOSTS == data['name']:
step_obj = object.__new__(UnlockHostsStep)
elif STRATEGY_STEP_NAME.REBOOT_HOSTS == data['name']:
step_obj = object.__new__(RebootHostsStep)
elif STRATEGY_STEP_NAME.LOCK_HOSTS == data['name']:
step_obj = object.__new__(LockHostsStep)
elif STRATEGY_STEP_NAME.SWACT_HOSTS == data['name']:
step_obj = object.__new__(SwactHostsStep)
elif STRATEGY_STEP_NAME.UPGRADE_HOSTS == data['name']:
step_obj = object.__new__(UpgradeHostsStep)
elif STRATEGY_STEP_NAME.START_UPGRADE == data['name']:
step_obj = object.__new__(UpgradeStartStep)
elif STRATEGY_STEP_NAME.ACTIVATE_UPGRADE == data['name']:
step_obj = object.__new__(UpgradeActivateStep)
elif STRATEGY_STEP_NAME.COMPLETE_UPGRADE == data['name']:
step_obj = object.__new__(UpgradeCompleteStep)
elif STRATEGY_STEP_NAME.SW_PATCH_HOSTS == data['name']:
step_obj = object.__new__(SwPatchHostsStep)
elif STRATEGY_STEP_NAME.MIGRATE_INSTANCES == data['name']:
step_obj = object.__new__(MigrateInstancesStep)
elif STRATEGY_STEP_NAME.START_INSTANCES == data['name']:
step_obj = object.__new__(StartInstancesStep)
elif STRATEGY_STEP_NAME.STOP_INSTANCES == data['name']:
step_obj = object.__new__(StopInstancesStep)
elif STRATEGY_STEP_NAME.QUERY_ALARMS == data['name']:
step_obj = object.__new__(QueryAlarmsStep)
elif STRATEGY_STEP_NAME.WAIT_DATA_SYNC == data['name']:
step_obj = object.__new__(WaitDataSyncStep)
elif STRATEGY_STEP_NAME.WAIT_ALARMS_CLEAR == data['name']:
step_obj = object.__new__(WaitAlarmsClearStep)
elif STRATEGY_STEP_NAME.QUERY_SW_PATCHES == data['name']:
step_obj = object.__new__(QuerySwPatchesStep)
elif STRATEGY_STEP_NAME.QUERY_SW_PATCH_HOSTS == data['name']:
step_obj = object.__new__(QuerySwPatchHostsStep)
elif STRATEGY_STEP_NAME.QUERY_UPGRADE == data['name']:
step_obj = object.__new__(QueryUpgradeStep)
elif STRATEGY_STEP_NAME.DISABLE_HOST_SERVICES == data['name']:
step_obj = object.__new__(DisableHostServicesStep)
elif STRATEGY_STEP_NAME.ENABLE_HOST_SERVICES == data['name']:
step_obj = object.__new__(EnableHostServicesStep)
elif STRATEGY_STEP_NAME.FW_UPDATE_HOSTS == data['name']:
step_obj = object.__new__(FwUpdateHostsStep)
elif STRATEGY_STEP_NAME.FW_UPDATE_ABORT_HOSTS == data['name']:
step_obj = object.__new__(FwUpdateAbortHostsStep)
elif STRATEGY_STEP_NAME.QUERY_FW_UPDATE_HOST == data['name']:
step_obj = object.__new__(QueryFwUpdateHostStep)
else:
step_obj = object.__new__(strategy.StrategyStep)
step_obj.from_dict(data)
return step_obj
View Git Blame Copy Permalink