1621 lines
68 KiB
Python
Executable File
1621 lines
68 KiB
Python
Executable File
#
|
|
# Copyright (c) 2015-2016 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
import six
|
|
import weakref
|
|
|
|
from nfv_common import debug
|
|
from nfv_common import strategy
|
|
from nfv_common.helpers import Constant, Constants, Singleton, get_local_host_name
|
|
|
|
from nfv_vim.objects import SW_UPDATE_APPLY_TYPE, SW_UPDATE_INSTANCE_ACTION
|
|
from nfv_vim.objects import HOST_PERSONALITY, HOST_GROUP_POLICY, HOST_NAME
|
|
from nfv_vim.objects import INSTANCE_GROUP_POLICY
|
|
|
|
from nfv_vim.nfvi.objects.v1 import UPGRADE_STATE
|
|
|
|
DLOG = debug.debug_get_logger('nfv_vim.strategy')
|
|
|
|
|
|
@six.add_metaclass(Singleton)
|
|
class StrategyNames(Constants):
|
|
"""
|
|
Strategy Names
|
|
"""
|
|
SW_PATCH = Constant('sw-patch')
|
|
SW_UPGRADE = Constant('sw-upgrade')
|
|
|
|
|
|
# Constant Instantiation
|
|
STRATEGY_NAME = StrategyNames()
|
|
|
|
# SystemStabilize timeout constants:
|
|
# After a reboot patch is applied, we need to wait for maintenance to detect
|
|
# that the host is patch current
|
|
MTCE_DELAY = 15
|
|
# a no-reboot patch can stabilize in 30 seconds
|
|
NO_REBOOT_DELAY = 30
|
|
|
|
|
|
class SwUpdateStrategy(strategy.Strategy):
|
|
"""
|
|
Software Update - Strategy
|
|
"""
|
|
def __init__(self, uuid, strategy_name, controller_apply_type,
|
|
storage_apply_type,
|
|
swift_apply_type, compute_apply_type,
|
|
max_parallel_compute_hosts, default_instance_action,
|
|
alarm_restrictions,
|
|
ignore_alarms):
|
|
super(SwUpdateStrategy, self).__init__(uuid, strategy_name)
|
|
self._controller_apply_type = controller_apply_type
|
|
self._storage_apply_type = storage_apply_type
|
|
self._swift_apply_type = swift_apply_type
|
|
self._compute_apply_type = compute_apply_type
|
|
self._max_parallel_compute_hosts = max_parallel_compute_hosts
|
|
self._default_instance_action = default_instance_action
|
|
self._alarm_restrictions = alarm_restrictions
|
|
self._ignore_alarms = ignore_alarms
|
|
self._sw_update_obj_reference = None
|
|
|
|
self._nfvi_alarms = list()
|
|
|
|
@property
|
|
def sw_update_obj(self):
|
|
"""
|
|
Returns the software update object this strategy is a part of
|
|
"""
|
|
return self._sw_update_obj_reference()
|
|
|
|
@sw_update_obj.setter
|
|
def sw_update_obj(self, sw_update_obj_value):
|
|
"""
|
|
Set the software update object this strategy is a part of
|
|
"""
|
|
self._sw_update_obj_reference = weakref.ref(sw_update_obj_value)
|
|
|
|
@property
|
|
def nfvi_alarms(self):
|
|
"""
|
|
Returns the alarms raised in the NFVI layer
|
|
"""
|
|
return self._nfvi_alarms
|
|
|
|
@nfvi_alarms.setter
|
|
def nfvi_alarms(self, nfvi_alarms):
|
|
"""
|
|
Save the alarms raised in the NFVI Layer
|
|
"""
|
|
self._nfvi_alarms = nfvi_alarms
|
|
|
|
def save(self):
|
|
"""
|
|
Save the software update strategy object information
|
|
"""
|
|
if self.sw_update_obj is not None:
|
|
self.sw_update_obj.save()
|
|
|
|
def build(self):
|
|
"""
|
|
Build the strategy (expected to be overridden by child class)
|
|
"""
|
|
super(SwUpdateStrategy, self).build()
|
|
|
|
def _create_storage_host_lists(self, storage_hosts):
|
|
"""
|
|
Create host lists for updating storage hosts
|
|
"""
|
|
from nfv_vim import tables
|
|
|
|
if SW_UPDATE_APPLY_TYPE.IGNORE != self._storage_apply_type:
|
|
host_table = tables.tables_get_host_table()
|
|
|
|
for host in storage_hosts:
|
|
if HOST_PERSONALITY.STORAGE not in host.personality:
|
|
DLOG.error("Host inventory personality storage mismatch "
|
|
"detected for host %s." % host.name)
|
|
reason = 'host inventory personality storage mismatch detected'
|
|
return None, reason
|
|
|
|
if 2 > host_table.total_by_personality(HOST_PERSONALITY.STORAGE):
|
|
DLOG.warn("Not enough storage hosts to apply software updates.")
|
|
reason = 'not enough storage hosts to apply software updates'
|
|
return None, reason
|
|
|
|
host_lists = list()
|
|
|
|
if SW_UPDATE_APPLY_TYPE.SERIAL == self._storage_apply_type:
|
|
for host in storage_hosts:
|
|
host_lists.append([host])
|
|
|
|
elif SW_UPDATE_APPLY_TYPE.PARALLEL == self._storage_apply_type:
|
|
policy = HOST_GROUP_POLICY.STORAGE_REPLICATION
|
|
host_group_table = tables.tables_get_host_group_table()
|
|
|
|
for host in storage_hosts:
|
|
# find the first list that can add this host
|
|
# else create a new list
|
|
for host_list in host_lists:
|
|
for peer_host in host_list:
|
|
if host_group_table.same_group(policy, host.name,
|
|
peer_host.name):
|
|
break
|
|
else:
|
|
host_list.append(host)
|
|
break
|
|
else:
|
|
host_lists.append([host])
|
|
else:
|
|
DLOG.verbose("Storage apply type set to ignore.")
|
|
|
|
return host_lists, ''
|
|
|
|
def _create_compute_host_lists(self, compute_hosts, reboot):
|
|
"""
|
|
Create host lists for updating compute hosts
|
|
"""
|
|
from nfv_vim import tables
|
|
|
|
def has_policy_conflict():
|
|
for instance in instance_table.on_host(host.name):
|
|
for peer_instance in instance_table.on_host(peer_host.name):
|
|
for policy in policies:
|
|
if instance_group_table.same_group(policy, instance.uuid,
|
|
peer_instance.uuid):
|
|
return True
|
|
DLOG.debug("No instance group policy conflict between host %s and "
|
|
"host %s." % (host.name, peer_host.name))
|
|
return False
|
|
|
|
def calculate_host_aggregate_limits():
|
|
"""
|
|
Calculate limit for each host aggregate
|
|
"""
|
|
# Use the ratio of the max parallel compute hosts to the total
|
|
# number of compute hosts to limit the number of hosts in each
|
|
# aggregate that will be patched at the same time. If there
|
|
# are multiple aggregates, that will help us select hosts
|
|
# from more than one aggregate for each stage.
|
|
host_table = tables.tables_get_host_table()
|
|
num_compute_hosts = host_table.total_by_personality(
|
|
HOST_PERSONALITY.COMPUTE)
|
|
aggregate_ratio = \
|
|
float(self._max_parallel_compute_hosts) / num_compute_hosts
|
|
# Limit the ratio to half the compute hosts in an aggregate
|
|
if aggregate_ratio > 0.5:
|
|
aggregate_ratio = 0.5
|
|
|
|
for host_aggregate in host_aggregate_table:
|
|
aggregate_count = len(
|
|
host_aggregate_table[host_aggregate].host_names)
|
|
if aggregate_count == 1:
|
|
# only one host in this aggregate
|
|
host_aggregate_limit[host_aggregate] = 1
|
|
else:
|
|
# multiple hosts in the aggregate - use the ratio,
|
|
# rounding down, but no lower than 1.
|
|
host_aggregate_limit[host_aggregate] = max(
|
|
1, int(aggregate_count * aggregate_ratio))
|
|
|
|
def aggregate_limit_reached():
|
|
"""
|
|
Determine whether adding this host to a host_list would exceed
|
|
the number of hosts to be updated in the same aggregate
|
|
Note: This isn't efficient, because we will be calling the
|
|
host_aggregate_table.get_by_host many times, which will traverse
|
|
all the aggregates each time. It would be more efficient to
|
|
create a dictionary mapping host names to a list of aggregates
|
|
for that host. We could do this once and then use it to more
|
|
quickly calculate the host_aggregate_count here.
|
|
"""
|
|
|
|
# count the number of hosts from the current host_list in each aggregate
|
|
host_aggregate_count = {}
|
|
for existing_host in host_list:
|
|
for aggregate in host_aggregate_table.get_by_host(
|
|
existing_host.name):
|
|
if aggregate.name in host_aggregate_count:
|
|
host_aggregate_count[aggregate.name] += 1
|
|
else:
|
|
host_aggregate_count[aggregate.name] = 1
|
|
|
|
# now check whether adding the current host will exceed the limit
|
|
# for any aggregate
|
|
for aggregate in host_aggregate_table.get_by_host(host.name):
|
|
if aggregate.name in host_aggregate_count:
|
|
if host_aggregate_count[aggregate.name] == \
|
|
host_aggregate_limit[aggregate.name]:
|
|
return True
|
|
|
|
DLOG.debug("No host aggregate limit reached for host %s." % (host.name))
|
|
return False
|
|
|
|
instance_table = tables.tables_get_instance_table()
|
|
instance_group_table = tables.tables_get_instance_group_table()
|
|
|
|
if SW_UPDATE_APPLY_TYPE.IGNORE != self._compute_apply_type:
|
|
for host in compute_hosts:
|
|
if HOST_PERSONALITY.COMPUTE not in host.personality:
|
|
DLOG.error("Host inventory personality compute mismatch "
|
|
"detected for host %s." % host.name)
|
|
reason = 'host inventory personality compute mismatch detected'
|
|
return None, reason
|
|
|
|
# Do not allow reboots if there are locked instances that
|
|
# that are members of an instance group. This could result in a
|
|
# service disruption when the remaining instances are stopped or
|
|
# migrated.
|
|
if reboot:
|
|
for instance in instance_table.itervalues():
|
|
if instance.is_locked():
|
|
for instance_group in instance_group_table.get_by_instance(
|
|
instance.uuid):
|
|
DLOG.warn(
|
|
"Instance %s in group %s must not be shut down"
|
|
% (instance.name, instance_group.name))
|
|
reason = (
|
|
'instance %s in group %s must not be shut down'
|
|
% (instance.name, instance_group.name))
|
|
return None, reason
|
|
|
|
host_lists = list()
|
|
|
|
if SW_UPDATE_APPLY_TYPE.SERIAL == self._compute_apply_type:
|
|
host_with_instances_lists = list()
|
|
|
|
# handle the computes with no instances first
|
|
for host in compute_hosts:
|
|
if not instance_table.exist_on_host(host.name):
|
|
host_lists.append([host])
|
|
else:
|
|
host_with_instances_lists.append([host])
|
|
|
|
# then add computes with instances
|
|
if host_with_instances_lists:
|
|
host_lists += host_with_instances_lists
|
|
|
|
elif SW_UPDATE_APPLY_TYPE.PARALLEL == self._compute_apply_type:
|
|
policies = [INSTANCE_GROUP_POLICY.ANTI_AFFINITY,
|
|
INSTANCE_GROUP_POLICY.ANTI_AFFINITY_BEST_EFFORT]
|
|
|
|
host_aggregate_table = tables.tables_get_host_aggregate_table()
|
|
host_aggregate_limit = {}
|
|
calculate_host_aggregate_limits()
|
|
controller_list = list()
|
|
host_lists.append([]) # start with empty list of computes
|
|
|
|
for host in compute_hosts:
|
|
if HOST_PERSONALITY.CONTROLLER in host.personality:
|
|
# have to swact the controller so put it in its own list
|
|
controller_list.append([host])
|
|
continue
|
|
elif not reboot:
|
|
# parallel no-reboot can group all computes together
|
|
host_lists[0].append(host)
|
|
continue
|
|
elif not instance_table.exist_on_host(host.name):
|
|
# group the computes with no instances together
|
|
host_lists[0].append(host)
|
|
continue
|
|
|
|
# find the first list that can add this host else create a new list
|
|
for idx in range(1, len(host_lists), 1):
|
|
host_list = host_lists[idx]
|
|
if len(host_list) >= self._max_parallel_compute_hosts:
|
|
# this list is full - don't add the host
|
|
continue
|
|
|
|
for peer_host in host_list:
|
|
if has_policy_conflict():
|
|
# don't add host to the current list
|
|
break
|
|
else:
|
|
if aggregate_limit_reached():
|
|
# don't add host to the current list
|
|
continue
|
|
|
|
# add host to the current list
|
|
host_list.append(host)
|
|
break
|
|
else:
|
|
# create a new list with this host
|
|
host_lists.append([host])
|
|
|
|
if controller_list:
|
|
host_lists += controller_list
|
|
|
|
else:
|
|
DLOG.verbose("Compute apply type set to ignore.")
|
|
|
|
# Drop empty lists and enforce a maximum number of hosts to be updated
|
|
# at once (only required list of computes with no instances, as we
|
|
# enforced the limit for compute hosts with instances above).
|
|
sized_host_lists = list()
|
|
for host_list in host_lists:
|
|
# drop empty host lists
|
|
if not host_list:
|
|
continue
|
|
|
|
if self._max_parallel_compute_hosts < len(host_list):
|
|
start = 0
|
|
end = self._max_parallel_compute_hosts
|
|
while start < len(host_list):
|
|
sized_host_lists.append(host_list[start:end])
|
|
start = end
|
|
end += self._max_parallel_compute_hosts
|
|
else:
|
|
sized_host_lists.append(host_list)
|
|
|
|
return sized_host_lists, ''
|
|
|
|
def build_complete(self, result, result_reason):
|
|
"""
|
|
Strategy Build Complete
|
|
"""
|
|
result, result_reason = \
|
|
super(SwUpdateStrategy, self).build_complete(result, result_reason)
|
|
return result, result_reason
|
|
|
|
def apply(self, stage_id):
|
|
"""
|
|
Apply the strategy
|
|
"""
|
|
success, reason = super(SwUpdateStrategy, self).apply(stage_id)
|
|
return success, reason
|
|
|
|
def apply_complete(self, result, result_reason):
|
|
"""
|
|
Strategy Apply Complete
|
|
"""
|
|
result, result_reason = \
|
|
super(SwUpdateStrategy, self).apply_complete(result, result_reason)
|
|
|
|
DLOG.info("Apply Complete Callback, result=%s, reason=%s."
|
|
% (result, result_reason))
|
|
|
|
if result in [strategy.STRATEGY_RESULT.SUCCESS,
|
|
strategy.STRATEGY_RESULT.DEGRADED]:
|
|
self.sw_update_obj.strategy_apply_complete(True, '')
|
|
else:
|
|
self.sw_update_obj.strategy_apply_complete(
|
|
False, self.apply_phase.result_reason)
|
|
|
|
def abort(self, stage_id):
|
|
"""
|
|
Abort the strategy
|
|
"""
|
|
success, reason = super(SwUpdateStrategy, self).abort(stage_id)
|
|
return success, reason
|
|
|
|
def abort_complete(self, result, result_reason):
|
|
"""
|
|
Strategy Abort Complete
|
|
"""
|
|
result, result_reason = \
|
|
super(SwUpdateStrategy, self).abort_complete(result, result_reason)
|
|
|
|
DLOG.info("Abort Complete Callback, result=%s, reason=%s."
|
|
% (result, result_reason))
|
|
|
|
if result in [strategy.STRATEGY_RESULT.SUCCESS,
|
|
strategy.STRATEGY_RESULT.DEGRADED]:
|
|
self.sw_update_obj.strategy_abort_complete(True, '')
|
|
else:
|
|
self.sw_update_obj.strategy_abort_complete(
|
|
False, self.abort_phase.result_reason)
|
|
|
|
def from_dict(self, data, build_phase=None, apply_phase=None, abort_phase=None):
|
|
"""
|
|
Initializes a software update strategy object using the given dictionary
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
super(SwUpdateStrategy, self).from_dict(data, build_phase, apply_phase,
|
|
abort_phase)
|
|
self._controller_apply_type = data['controller_apply_type']
|
|
self._storage_apply_type = data['storage_apply_type']
|
|
self._swift_apply_type = data['swift_apply_type']
|
|
self._compute_apply_type = data['compute_apply_type']
|
|
self._max_parallel_compute_hosts = data['max_parallel_compute_hosts']
|
|
self._default_instance_action = data['default_instance_action']
|
|
self._alarm_restrictions = data['alarm_restrictions']
|
|
self._ignore_alarms = data['ignore_alarms']
|
|
|
|
nfvi_alarms = list()
|
|
for alarm_data in data['nfvi_alarms_data']:
|
|
alarm = nfvi.objects.v1.Alarm(
|
|
alarm_data['alarm_uuid'], alarm_data['alarm_id'],
|
|
alarm_data['entity_instance_id'], alarm_data['severity'],
|
|
alarm_data['reason_text'], alarm_data['timestamp'],
|
|
alarm_data['mgmt_affecting'])
|
|
nfvi_alarms.append(alarm)
|
|
self._nfvi_alarms = nfvi_alarms
|
|
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the software update strategy as a dictionary
|
|
"""
|
|
data = super(SwUpdateStrategy, self).as_dict()
|
|
data['controller_apply_type'] = self._controller_apply_type
|
|
data['storage_apply_type'] = self._storage_apply_type
|
|
data['swift_apply_type'] = self._swift_apply_type
|
|
data['compute_apply_type'] = self._compute_apply_type
|
|
data['max_parallel_compute_hosts'] = self._max_parallel_compute_hosts
|
|
data['default_instance_action'] = self._default_instance_action
|
|
data['alarm_restrictions'] = self._alarm_restrictions
|
|
data['ignore_alarms'] = self._ignore_alarms
|
|
|
|
nfvi_alarms_data = list()
|
|
for alarm in self._nfvi_alarms:
|
|
nfvi_alarms_data.append(alarm.as_dict())
|
|
data['nfvi_alarms_data'] = nfvi_alarms_data
|
|
|
|
return data
|
|
|
|
|
|
class SwPatchStrategy(SwUpdateStrategy):
|
|
"""
|
|
Software Patch - Strategy
|
|
"""
|
|
def __init__(self, uuid, controller_apply_type, storage_apply_type,
|
|
swift_apply_type, compute_apply_type,
|
|
max_parallel_compute_hosts, default_instance_action,
|
|
alarm_restrictions,
|
|
ignore_alarms,
|
|
single_controller):
|
|
super(SwPatchStrategy, self).__init__(
|
|
uuid,
|
|
STRATEGY_NAME.SW_PATCH,
|
|
controller_apply_type,
|
|
storage_apply_type,
|
|
swift_apply_type,
|
|
compute_apply_type,
|
|
max_parallel_compute_hosts,
|
|
default_instance_action,
|
|
alarm_restrictions,
|
|
ignore_alarms)
|
|
|
|
# The following alarms will not prevent a software patch operation
|
|
IGNORE_ALARMS = ['900.001', # Patch in progress
|
|
'900.005', # Upgrade in progress
|
|
'900.101', # Software patch auto apply in progress
|
|
'200.001', # Maintenance host lock alarm
|
|
'700.004', # VM stopped
|
|
'280.002', # Subcloud resource out-of-sync
|
|
]
|
|
self._ignore_alarms += IGNORE_ALARMS
|
|
self._single_controller = single_controller
|
|
|
|
self._nfvi_sw_patches = list()
|
|
self._nfvi_sw_patch_hosts = list()
|
|
|
|
@property
|
|
def nfvi_sw_patches(self):
|
|
"""
|
|
Returns the software patches from the NFVI layer
|
|
"""
|
|
return self._nfvi_sw_patches
|
|
|
|
@nfvi_sw_patches.setter
|
|
def nfvi_sw_patches(self, nfvi_sw_patches):
|
|
"""
|
|
Save the software patches from the NFVI Layer
|
|
"""
|
|
self._nfvi_sw_patches = nfvi_sw_patches
|
|
|
|
@property
|
|
def nfvi_sw_patch_hosts(self):
|
|
"""
|
|
Returns the software patch hosts from the NFVI layer
|
|
"""
|
|
return self._nfvi_sw_patch_hosts
|
|
|
|
@nfvi_sw_patch_hosts.setter
|
|
def nfvi_sw_patch_hosts(self, nfvi_sw_patch_hosts):
|
|
"""
|
|
Save the software patch hosts from the NFVI Layer
|
|
"""
|
|
self._nfvi_sw_patch_hosts = nfvi_sw_patch_hosts
|
|
|
|
def build(self):
|
|
"""
|
|
Build the strategy
|
|
"""
|
|
from nfv_vim import strategy
|
|
|
|
stage = strategy.StrategyStage(
|
|
strategy.STRATEGY_STAGE_NAME.SW_PATCH_QUERY)
|
|
stage.add_step(strategy.QueryAlarmsStep(ignore_alarms=self._ignore_alarms))
|
|
stage.add_step(strategy.QuerySwPatchesStep())
|
|
stage.add_step(strategy.QuerySwPatchHostsStep())
|
|
self.build_phase.add_stage(stage)
|
|
super(SwPatchStrategy, self).build()
|
|
|
|
def _add_controller_strategy_stages(self, controllers, reboot):
|
|
"""
|
|
Add controller software patch strategy stages
|
|
"""
|
|
from nfv_vim import tables
|
|
from nfv_vim import strategy
|
|
|
|
if SW_UPDATE_APPLY_TYPE.IGNORE != self._controller_apply_type:
|
|
host_table = tables.tables_get_host_table()
|
|
|
|
for host in controllers:
|
|
if HOST_PERSONALITY.CONTROLLER not in host.personality:
|
|
DLOG.error("Host inventory personality controller mismatch "
|
|
"detected for host %s." % host.name)
|
|
reason = ('host inventory personality controller mismatch '
|
|
'detected')
|
|
return False, reason
|
|
|
|
if (not self._single_controller and
|
|
2 > host_table.total_by_personality(
|
|
HOST_PERSONALITY.CONTROLLER)):
|
|
DLOG.warn("Not enough controllers to apply software patches.")
|
|
reason = 'not enough controllers to apply software patches'
|
|
return False, reason
|
|
|
|
if self._controller_apply_type == SW_UPDATE_APPLY_TYPE.SERIAL:
|
|
local_host = None
|
|
local_host_name = get_local_host_name()
|
|
|
|
for host in controllers:
|
|
if HOST_PERSONALITY.COMPUTE not in host.personality:
|
|
if local_host_name == host.name:
|
|
local_host = host
|
|
else:
|
|
host_list = [host]
|
|
stage = strategy.StrategyStage(
|
|
strategy.STRATEGY_STAGE_NAME.SW_PATCH_CONTROLLERS)
|
|
stage.add_step(strategy.QueryAlarmsStep(
|
|
True, ignore_alarms=self._ignore_alarms))
|
|
if reboot:
|
|
stage.add_step(strategy.SwactHostsStep(host_list))
|
|
stage.add_step(strategy.LockHostsStep(host_list))
|
|
stage.add_step(strategy.SwPatchHostsStep(host_list))
|
|
if reboot:
|
|
# Cannot unlock right away after SwPatchHostsStep
|
|
stage.add_step(strategy.SystemStabilizeStep(
|
|
timeout_in_secs=MTCE_DELAY))
|
|
stage.add_step(strategy.UnlockHostsStep(host_list))
|
|
stage.add_step(strategy.SystemStabilizeStep())
|
|
else:
|
|
# Less time required if host is not rebooting
|
|
stage.add_step(strategy.SystemStabilizeStep(
|
|
timeout_in_secs=NO_REBOOT_DELAY))
|
|
self.apply_phase.add_stage(stage)
|
|
|
|
if local_host is not None:
|
|
host_list = [local_host]
|
|
stage = strategy.StrategyStage(
|
|
strategy.STRATEGY_STAGE_NAME.SW_PATCH_CONTROLLERS)
|
|
stage.add_step(strategy.QueryAlarmsStep(
|
|
True, ignore_alarms=self._ignore_alarms))
|
|
if reboot:
|
|
stage.add_step(strategy.SwactHostsStep(host_list))
|
|
stage.add_step(strategy.LockHostsStep(host_list))
|
|
stage.add_step(strategy.SwPatchHostsStep(host_list))
|
|
if reboot:
|
|
# Cannot unlock right away after SwPatchHostsStep
|
|
stage.add_step(strategy.SystemStabilizeStep(
|
|
timeout_in_secs=MTCE_DELAY))
|
|
stage.add_step(strategy.UnlockHostsStep(host_list))
|
|
stage.add_step(strategy.SystemStabilizeStep())
|
|
else:
|
|
# Less time required if host is not rebooting
|
|
stage.add_step(strategy.SystemStabilizeStep(
|
|
timeout_in_secs=NO_REBOOT_DELAY))
|
|
|
|
self.apply_phase.add_stage(stage)
|
|
|
|
elif self._controller_apply_type == SW_UPDATE_APPLY_TYPE.PARALLEL:
|
|
DLOG.warn("Parallel apply type cannot be used for controllers.")
|
|
reason = 'parallel apply type not allowed for controllers'
|
|
return False, reason
|
|
else:
|
|
DLOG.verbose("Controller apply type set to ignore.")
|
|
|
|
return True, ''
|
|
|
|
def _add_storage_strategy_stages(self, storage_hosts, reboot):
|
|
"""
|
|
Add storage software patch strategy stages
|
|
"""
|
|
from nfv_vim import strategy
|
|
|
|
host_lists, reason = self._create_storage_host_lists(storage_hosts)
|
|
if host_lists is None:
|
|
return False, reason
|
|
|
|
for host_list in host_lists:
|
|
stage = strategy.StrategyStage(
|
|
strategy.STRATEGY_STAGE_NAME.SW_PATCH_STORAGE_HOSTS)
|
|
stage.add_step(strategy.QueryAlarmsStep(
|
|
True, ignore_alarms=self._ignore_alarms))
|
|
if reboot:
|
|
stage.add_step(strategy.LockHostsStep(host_list))
|
|
stage.add_step(strategy.SwPatchHostsStep(host_list))
|
|
if reboot:
|
|
# Cannot unlock right away after SwPatchHostsStep
|
|
stage.add_step(strategy.SystemStabilizeStep(
|
|
timeout_in_secs=MTCE_DELAY))
|
|
stage.add_step(strategy.UnlockHostsStep(host_list))
|
|
# After storage node(s) are unlocked, we need extra time to
|
|
# allow the OSDs to go back in sync and the storage related
|
|
# alarms to clear.
|
|
stage.add_step(strategy.WaitDataSyncStep(
|
|
timeout_in_secs=30 * 60,
|
|
ignore_alarms=self._ignore_alarms))
|
|
else:
|
|
stage.add_step(strategy.SystemStabilizeStep(
|
|
timeout_in_secs=NO_REBOOT_DELAY))
|
|
self.apply_phase.add_stage(stage)
|
|
|
|
return True, ''
|
|
|
|
def _add_swift_strategy_stages(self, swift_hosts, reboot):
|
|
"""
|
|
Add swift software patch strategy stages
|
|
"""
|
|
from nfv_vim import tables
|
|
from nfv_vim import strategy
|
|
|
|
if SW_UPDATE_APPLY_TYPE.IGNORE != self._swift_apply_type:
|
|
host_table = tables.tables_get_host_table()
|
|
|
|
for host in swift_hosts:
|
|
if HOST_PERSONALITY.SWIFT not in host.personality:
|
|
DLOG.error("Host inventory personality swift mismatch "
|
|
"detected for host %s." % host.name)
|
|
reason = 'host inventory personality swift mismatch detected'
|
|
return False, reason
|
|
|
|
if 2 > host_table.total_by_personality(HOST_PERSONALITY.SWIFT):
|
|
DLOG.warn("Not enough swift hosts to apply software patches.")
|
|
reason = 'not enough swift hosts to apply software patches'
|
|
return False, reason
|
|
|
|
if self._swift_apply_type in [SW_UPDATE_APPLY_TYPE.SERIAL,
|
|
SW_UPDATE_APPLY_TYPE.PARALLEL]:
|
|
for host in swift_hosts:
|
|
host_list = [host]
|
|
stage = strategy.StrategyStage(
|
|
strategy.STRATEGY_STAGE_NAME.SW_PATCH_SWIFT_HOSTS)
|
|
stage.add_step(strategy.QueryAlarmsStep(
|
|
True, ignore_alarms=self._ignore_alarms))
|
|
if reboot:
|
|
stage.add_step(strategy.LockHostsStep(host_list))
|
|
stage.add_step(strategy.SwPatchHostsStep(host_list))
|
|
if reboot:
|
|
# Cannot unlock right away after SwPatchHostsStep
|
|
stage.add_step(strategy.SystemStabilizeStep(
|
|
timeout_in_secs=MTCE_DELAY))
|
|
stage.add_step(strategy.UnlockHostsStep(host_list))
|
|
stage.add_step(strategy.SystemStabilizeStep())
|
|
else:
|
|
stage.add_step(strategy.SystemStabilizeStep(
|
|
timeout_in_secs=NO_REBOOT_DELAY))
|
|
self.apply_phase.add_stage(stage)
|
|
else:
|
|
DLOG.verbose("Swift apply type set to ignore.")
|
|
|
|
return True, ''
|
|
|
|
def _add_compute_strategy_stages(self, compute_hosts, reboot):
|
|
"""
|
|
Add compute software patch strategy stages
|
|
"""
|
|
from nfv_vim import tables
|
|
from nfv_vim import strategy
|
|
|
|
if SW_UPDATE_APPLY_TYPE.IGNORE != self._compute_apply_type:
|
|
# When using a single controller/compute host, only allow the
|
|
# stop/start instance action.
|
|
if self._single_controller:
|
|
for host in compute_hosts:
|
|
if HOST_PERSONALITY.CONTROLLER in host.personality and \
|
|
SW_UPDATE_INSTANCE_ACTION.STOP_START != \
|
|
self._default_instance_action:
|
|
DLOG.error("Cannot migrate instances in a single "
|
|
"controller configuration")
|
|
reason = 'cannot migrate instances in a single ' \
|
|
'controller configuration'
|
|
return False, reason
|
|
|
|
host_lists, reason = self._create_compute_host_lists(compute_hosts, reboot)
|
|
if host_lists is None:
|
|
return False, reason
|
|
|
|
instance_table = tables.tables_get_instance_table()
|
|
|
|
for host_list in host_lists:
|
|
instance_list = list()
|
|
|
|
for host in host_list:
|
|
for instance in instance_table.on_host(host.name):
|
|
# Do not take action (migrate or stop-start) on an instance
|
|
# if it is locked (i.e. stopped).
|
|
if not instance.is_locked():
|
|
instance_list.append(instance)
|
|
|
|
hosts_to_lock = list()
|
|
hosts_to_reboot = list()
|
|
if reboot:
|
|
hosts_to_lock = [x for x in host_list if not x.is_locked()]
|
|
hosts_to_reboot = [x for x in host_list if x.is_locked()]
|
|
|
|
stage = strategy.StrategyStage(
|
|
strategy.STRATEGY_STAGE_NAME.SW_PATCH_COMPUTE_HOSTS)
|
|
|
|
stage.add_step(strategy.QueryAlarmsStep(
|
|
True, ignore_alarms=self._ignore_alarms))
|
|
|
|
if reboot:
|
|
if 1 == len(host_list):
|
|
if HOST_PERSONALITY.CONTROLLER in host_list[0].personality:
|
|
if not self._single_controller:
|
|
# Swact controller before locking
|
|
stage.add_step(strategy.SwactHostsStep(host_list))
|
|
|
|
if 0 != len(instance_list):
|
|
# Migrate or stop instances as necessary
|
|
if SW_UPDATE_INSTANCE_ACTION.MIGRATE == \
|
|
self._default_instance_action:
|
|
if SW_UPDATE_APPLY_TYPE.PARALLEL == \
|
|
self._compute_apply_type:
|
|
# Disable host services before migrating to ensure
|
|
# instances do not migrate to compute hosts in the
|
|
# same set of hosts.
|
|
stage.add_step(strategy.DisableHostServicesStep(
|
|
host_list))
|
|
stage.add_step(strategy.MigrateInstancesStep(
|
|
instance_list))
|
|
else:
|
|
stage.add_step(strategy.StopInstancesStep(
|
|
instance_list))
|
|
|
|
if hosts_to_lock:
|
|
wait_until_disabled = True
|
|
if 1 == len(hosts_to_lock):
|
|
if HOST_PERSONALITY.CONTROLLER in \
|
|
hosts_to_lock[0].personality:
|
|
if self._single_controller:
|
|
# A single controller will not go disabled when
|
|
# it is locked.
|
|
wait_until_disabled = False
|
|
# Lock hosts
|
|
stage.add_step(strategy.LockHostsStep(
|
|
hosts_to_lock, wait_until_disabled=wait_until_disabled))
|
|
|
|
# Patch hosts
|
|
stage.add_step(strategy.SwPatchHostsStep(host_list))
|
|
|
|
if reboot:
|
|
# Cannot unlock right away after SwPatchHostsStep
|
|
stage.add_step(strategy.SystemStabilizeStep(
|
|
timeout_in_secs=MTCE_DELAY))
|
|
if hosts_to_lock:
|
|
# Unlock hosts that were locked
|
|
stage.add_step(strategy.UnlockHostsStep(hosts_to_lock))
|
|
if hosts_to_reboot:
|
|
# Reboot hosts that were already locked
|
|
stage.add_step(strategy.RebootHostsStep(hosts_to_reboot))
|
|
|
|
if 0 != len(instance_list):
|
|
# Start any instances that were stopped
|
|
if SW_UPDATE_INSTANCE_ACTION.MIGRATE != \
|
|
self._default_instance_action:
|
|
stage.add_step(strategy.StartInstancesStep(
|
|
instance_list))
|
|
|
|
stage.add_step(strategy.SystemStabilizeStep())
|
|
else:
|
|
# Less time required if host is not rebooting
|
|
stage.add_step(strategy.SystemStabilizeStep(
|
|
timeout_in_secs=NO_REBOOT_DELAY))
|
|
self.apply_phase.add_stage(stage)
|
|
|
|
return True, ''
|
|
|
|
def build_complete(self, result, result_reason):
|
|
"""
|
|
Strategy Build Complete
|
|
"""
|
|
from nfv_vim import tables
|
|
from nfv_vim import strategy
|
|
|
|
result, result_reason = \
|
|
super(SwPatchStrategy, self).build_complete(result, result_reason)
|
|
|
|
DLOG.info("Build Complete Callback, result=%s, reason=%s."
|
|
% (result, result_reason))
|
|
|
|
if result in [strategy.STRATEGY_RESULT.SUCCESS,
|
|
strategy.STRATEGY_RESULT.DEGRADED]:
|
|
|
|
host_table = tables.tables_get_host_table()
|
|
|
|
if not self.nfvi_sw_patches:
|
|
DLOG.warn("No software patches found.")
|
|
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
|
|
self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED
|
|
self.build_phase.result_reason = 'no software patches found'
|
|
self.sw_update_obj.strategy_build_complete(
|
|
False, self.build_phase.result_reason)
|
|
self.save()
|
|
return
|
|
|
|
if self._nfvi_alarms:
|
|
DLOG.warn("Active alarms found, can't apply software patches.")
|
|
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
|
|
self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED
|
|
self.build_phase.result_reason = 'active alarms present'
|
|
self.sw_update_obj.strategy_build_complete(
|
|
False, self.build_phase.result_reason)
|
|
self.save()
|
|
return
|
|
|
|
for host in host_table.itervalues():
|
|
if HOST_PERSONALITY.COMPUTE in host.personality and \
|
|
HOST_PERSONALITY.CONTROLLER not in host.personality:
|
|
# Allow patch orchestration when compute hosts are available,
|
|
# locked or powered down.
|
|
if not ((host.is_unlocked() and host.is_enabled() and
|
|
host.is_available()) or
|
|
(host.is_locked() and host.is_disabled() and
|
|
host.is_offline()) or
|
|
(host.is_locked() and host.is_disabled() and
|
|
host.is_online())):
|
|
DLOG.warn(
|
|
"All compute hosts must be unlocked-enabled-available, "
|
|
"locked-disabled-online or locked-disabled-offline, "
|
|
"can't apply software patches.")
|
|
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
|
|
self.build_phase.result = \
|
|
strategy.STRATEGY_PHASE_RESULT.FAILED
|
|
self.build_phase.result_reason = (
|
|
'all compute hosts must be unlocked-enabled-available, '
|
|
'locked-disabled-online or locked-disabled-offline')
|
|
self.sw_update_obj.strategy_build_complete(
|
|
False, self.build_phase.result_reason)
|
|
self.save()
|
|
return
|
|
else:
|
|
# Only allow patch orchestration when all controller,
|
|
# storage and swift hosts are available. It is not safe to
|
|
# automate patch application when we do not have full
|
|
# redundancy.
|
|
if not (host.is_unlocked() and host.is_enabled() and
|
|
host.is_available()):
|
|
DLOG.warn(
|
|
"All %s hosts must be unlocked-enabled-available, "
|
|
"can't apply software patches." % host.personality)
|
|
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
|
|
self.build_phase.result = \
|
|
strategy.STRATEGY_PHASE_RESULT.FAILED
|
|
self.build_phase.result_reason = (
|
|
'all %s hosts must be unlocked-enabled-available' %
|
|
host.personality)
|
|
self.sw_update_obj.strategy_build_complete(
|
|
False, self.build_phase.result_reason)
|
|
self.save()
|
|
return
|
|
|
|
controllers = list()
|
|
controllers_no_reboot = list()
|
|
storage_hosts = list()
|
|
storage_hosts_no_reboot = list()
|
|
swift_hosts = list()
|
|
swift_hosts_no_reboot = list()
|
|
compute_hosts = list()
|
|
compute_hosts_no_reboot = list()
|
|
|
|
for sw_patch_host in self.nfvi_sw_patch_hosts:
|
|
host = host_table.get(sw_patch_host.name, None)
|
|
if host is None:
|
|
DLOG.error("Host inventory mismatch detected for host %s."
|
|
% sw_patch_host.name)
|
|
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
|
|
self.build_phase.result = \
|
|
strategy.STRATEGY_PHASE_RESULT.FAILED
|
|
self.build_phase.result_reason = \
|
|
'host inventory mismatch detected'
|
|
self.sw_update_obj.strategy_build_complete(
|
|
False, self.build_phase.result_reason)
|
|
self.save()
|
|
return
|
|
|
|
if sw_patch_host.interim_state:
|
|
# A patch operation has been done recently and we don't
|
|
# have an up-to-date state for this host.
|
|
DLOG.warn("Host %s is in pending patch current state."
|
|
% sw_patch_host.name)
|
|
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
|
|
self.build_phase.result = \
|
|
strategy.STRATEGY_PHASE_RESULT.FAILED
|
|
self.build_phase.result_reason = (
|
|
'at least one host is in pending patch current state')
|
|
self.sw_update_obj.strategy_build_complete(
|
|
False, self.build_phase.result_reason)
|
|
self.save()
|
|
return
|
|
|
|
if sw_patch_host.patch_current:
|
|
# No need to patch this host
|
|
continue
|
|
|
|
if HOST_PERSONALITY.CONTROLLER in sw_patch_host.personality:
|
|
if sw_patch_host.requires_reboot:
|
|
controllers.append(host)
|
|
else:
|
|
controllers_no_reboot.append(host)
|
|
|
|
elif HOST_PERSONALITY.STORAGE in sw_patch_host.personality:
|
|
if sw_patch_host.requires_reboot:
|
|
storage_hosts.append(host)
|
|
else:
|
|
storage_hosts_no_reboot.append(host)
|
|
|
|
elif HOST_PERSONALITY.SWIFT in sw_patch_host.personality:
|
|
if sw_patch_host.requires_reboot:
|
|
swift_hosts.append(host)
|
|
else:
|
|
swift_hosts_no_reboot.append(host)
|
|
|
|
# Separate if check to handle CPE where host has multiple
|
|
# personality disorder.
|
|
if HOST_PERSONALITY.COMPUTE in sw_patch_host.personality:
|
|
# Ignore compute hosts that are powered down
|
|
if not host.is_offline():
|
|
if sw_patch_host.requires_reboot:
|
|
compute_hosts.append(host)
|
|
else:
|
|
compute_hosts_no_reboot.append(host)
|
|
|
|
STRATEGY_CREATION_COMMANDS = [
|
|
(self._add_controller_strategy_stages,
|
|
controllers_no_reboot, False),
|
|
(self._add_controller_strategy_stages,
|
|
controllers, True),
|
|
(self._add_storage_strategy_stages,
|
|
storage_hosts_no_reboot, False),
|
|
(self._add_storage_strategy_stages,
|
|
storage_hosts, True),
|
|
(self._add_swift_strategy_stages,
|
|
swift_hosts_no_reboot, False),
|
|
(self._add_swift_strategy_stages,
|
|
swift_hosts, True),
|
|
(self._add_compute_strategy_stages,
|
|
compute_hosts_no_reboot, False),
|
|
(self._add_compute_strategy_stages,
|
|
compute_hosts, True)
|
|
]
|
|
|
|
for add_strategy_stages_function, host_list, reboot in \
|
|
STRATEGY_CREATION_COMMANDS:
|
|
if host_list:
|
|
success, reason = add_strategy_stages_function(
|
|
host_list, reboot)
|
|
if not success:
|
|
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
|
|
self.build_phase.result = \
|
|
strategy.STRATEGY_PHASE_RESULT.FAILED
|
|
self.build_phase.result_reason = reason
|
|
self.sw_update_obj.strategy_build_complete(
|
|
False, self.build_phase.result_reason)
|
|
self.save()
|
|
return
|
|
|
|
if 0 == len(self.apply_phase.stages):
|
|
DLOG.warn("No software patches need to be applied.")
|
|
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
|
|
self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED
|
|
self.build_phase.result_reason = ('no software patches need to be '
|
|
'applied')
|
|
self.sw_update_obj.strategy_build_complete(
|
|
False, self.build_phase.result_reason)
|
|
self.save()
|
|
return
|
|
else:
|
|
self.sw_update_obj.strategy_build_complete(
|
|
False, self.build_phase.result_reason)
|
|
|
|
self.sw_update_obj.strategy_build_complete(True, '')
|
|
self.save()
|
|
|
|
def from_dict(self, data, build_phase=None, apply_phase=None, abort_phase=None):
|
|
"""
|
|
Initializes a software patch strategy object using the given dictionary
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
super(SwPatchStrategy, self).from_dict(data, build_phase, apply_phase,
|
|
abort_phase)
|
|
|
|
self._single_controller = data['single_controller']
|
|
|
|
nfvi_sw_patches = list()
|
|
for sw_patch_data in data['nfvi_sw_patches_data']:
|
|
sw_patch = nfvi.objects.v1.SwPatch(
|
|
sw_patch_data['name'], sw_patch_data['sw_version'],
|
|
sw_patch_data['repo_state'], sw_patch_data['patch_state'])
|
|
nfvi_sw_patches.append(sw_patch)
|
|
self._nfvi_sw_patches = nfvi_sw_patches
|
|
|
|
nfvi_sw_patch_hosts = list()
|
|
for host_data in data['nfvi_sw_patch_hosts_data']:
|
|
host = nfvi.objects.v1.HostSwPatch(
|
|
host_data['name'], host_data['personality'],
|
|
host_data['sw_version'], host_data['requires_reboot'],
|
|
host_data['patch_current'], host_data['state'],
|
|
host_data['patch_failed'], host_data['interim_state'])
|
|
nfvi_sw_patch_hosts.append(host)
|
|
self._nfvi_sw_patch_hosts = nfvi_sw_patch_hosts
|
|
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the software patch strategy as a dictionary
|
|
"""
|
|
data = super(SwPatchStrategy, self).as_dict()
|
|
|
|
data['single_controller'] = self._single_controller
|
|
|
|
nfvi_sw_patches_data = list()
|
|
for sw_patch in self._nfvi_sw_patches:
|
|
nfvi_sw_patches_data.append(sw_patch.as_dict())
|
|
data['nfvi_sw_patches_data'] = nfvi_sw_patches_data
|
|
|
|
nfvi_sw_patch_hosts_data = list()
|
|
for host in self._nfvi_sw_patch_hosts:
|
|
nfvi_sw_patch_hosts_data.append(host.as_dict())
|
|
data['nfvi_sw_patch_hosts_data'] = nfvi_sw_patch_hosts_data
|
|
|
|
return data
|
|
|
|
|
|
class SwUpgradeStrategy(SwUpdateStrategy):
|
|
"""
|
|
Software Upgrade - Strategy
|
|
"""
|
|
def __init__(self, uuid, storage_apply_type, compute_apply_type,
|
|
max_parallel_compute_hosts,
|
|
alarm_restrictions, start_upgrade, complete_upgrade,
|
|
ignore_alarms):
|
|
super(SwUpgradeStrategy, self).__init__(
|
|
uuid,
|
|
STRATEGY_NAME.SW_UPGRADE,
|
|
SW_UPDATE_APPLY_TYPE.SERIAL,
|
|
storage_apply_type,
|
|
SW_UPDATE_APPLY_TYPE.IGNORE,
|
|
compute_apply_type,
|
|
max_parallel_compute_hosts,
|
|
SW_UPDATE_INSTANCE_ACTION.MIGRATE,
|
|
alarm_restrictions,
|
|
ignore_alarms)
|
|
|
|
# Note: The support for start_upgrade was implemented and (mostly)
|
|
# tested, but there is a problem. When the sw-upgrade-start stage
|
|
# runs, it will start the upgrade, upgrade controller-1 and swact to
|
|
# it. However, when controller-1 becomes active, it will be using the
|
|
# snapshot of the VIM database that was created when the upgrade was
|
|
# started, so the strategy object created from the database will be
|
|
# long out of date (it thinks the upgrade start step is still in
|
|
# progress) and the strategy apply will fail. Fixing this would be
|
|
# complex, so we will not support the start_upgrade option for now,
|
|
# which would only have been for lab use.
|
|
if start_upgrade:
|
|
raise Exception("No support for start_upgrade")
|
|
self._start_upgrade = start_upgrade
|
|
self._complete_upgrade = complete_upgrade
|
|
# The following alarms will not prevent a software upgrade operation
|
|
IGNORE_ALARMS = ['900.005', # Upgrade in progress
|
|
'900.201', # Software upgrade auto apply in progress
|
|
]
|
|
self._ignore_alarms += IGNORE_ALARMS
|
|
|
|
self._nfvi_upgrade = None
|
|
|
|
@property
|
|
def nfvi_upgrade(self):
|
|
"""
|
|
Returns the upgrade from the NFVI layer
|
|
"""
|
|
return self._nfvi_upgrade
|
|
|
|
@nfvi_upgrade.setter
|
|
def nfvi_upgrade(self, nfvi_upgrade):
|
|
"""
|
|
Save the upgrade from the NFVI Layer
|
|
"""
|
|
self._nfvi_upgrade = nfvi_upgrade
|
|
|
|
def build(self):
|
|
"""
|
|
Build the strategy
|
|
"""
|
|
from nfv_vim import strategy
|
|
|
|
stage = strategy.StrategyStage(
|
|
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_QUERY)
|
|
stage.add_step(strategy.QueryAlarmsStep(
|
|
ignore_alarms=self._ignore_alarms))
|
|
stage.add_step(strategy.QueryUpgradeStep())
|
|
self.build_phase.add_stage(stage)
|
|
super(SwUpgradeStrategy, self).build()
|
|
|
|
def _add_upgrade_start_stage(self):
|
|
"""
|
|
Add upgrade start strategy stage
|
|
"""
|
|
from nfv_vim import tables
|
|
from nfv_vim import strategy
|
|
|
|
host_table = tables.tables_get_host_table()
|
|
controller_1_host = None
|
|
for host in host_table.get_by_personality(HOST_PERSONALITY.CONTROLLER):
|
|
if HOST_NAME.CONTROLLER_1 == host.name:
|
|
controller_1_host = host
|
|
break
|
|
host_list = [controller_1_host]
|
|
|
|
stage = strategy.StrategyStage(
|
|
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_START)
|
|
# Do not ignore any alarms when starting an upgrade
|
|
stage.add_step(strategy.QueryAlarmsStep(True))
|
|
# Upgrade start can only be done from controller-0
|
|
stage.add_step(strategy.SwactHostsStep(host_list))
|
|
stage.add_step(strategy.UpgradeStartStep())
|
|
stage.add_step(strategy.SystemStabilizeStep())
|
|
self.apply_phase.add_stage(stage)
|
|
|
|
def _add_upgrade_complete_stage(self):
|
|
"""
|
|
Add upgrade complete strategy stage
|
|
"""
|
|
from nfv_vim import tables
|
|
from nfv_vim import strategy
|
|
|
|
host_table = tables.tables_get_host_table()
|
|
controller_1_host = None
|
|
for host in host_table.get_by_personality(HOST_PERSONALITY.CONTROLLER):
|
|
if HOST_NAME.CONTROLLER_1 == host.name:
|
|
controller_1_host = host
|
|
break
|
|
host_list = [controller_1_host]
|
|
|
|
stage = strategy.StrategyStage(
|
|
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_COMPLETE)
|
|
stage.add_step(strategy.QueryAlarmsStep(
|
|
True, ignore_alarms=self._ignore_alarms))
|
|
# Upgrade complete can only be done from controller-0
|
|
stage.add_step(strategy.SwactHostsStep(host_list))
|
|
stage.add_step(strategy.UpgradeActivateStep())
|
|
stage.add_step(strategy.UpgradeCompleteStep())
|
|
stage.add_step(strategy.SystemStabilizeStep())
|
|
self.apply_phase.add_stage(stage)
|
|
|
|
def _add_controller_strategy_stages(self, controllers, reboot):
|
|
"""
|
|
Add controller software upgrade strategy stages
|
|
"""
|
|
from nfv_vim import tables
|
|
from nfv_vim import strategy
|
|
|
|
host_table = tables.tables_get_host_table()
|
|
|
|
if 2 > host_table.total_by_personality(HOST_PERSONALITY.CONTROLLER):
|
|
DLOG.warn("Not enough controllers to apply software upgrades.")
|
|
reason = 'not enough controllers to apply software upgrades'
|
|
return False, reason
|
|
|
|
controller_0_host = None
|
|
controller_1_host = None
|
|
|
|
for host in controllers:
|
|
if HOST_PERSONALITY.COMPUTE in host.personality:
|
|
DLOG.warn("Cannot apply software upgrades to CPE configuration.")
|
|
reason = 'cannot apply software upgrades to CPE configuration'
|
|
return False, reason
|
|
elif HOST_NAME.CONTROLLER_1 == host.name:
|
|
controller_1_host = host
|
|
elif HOST_NAME.CONTROLLER_0 == host.name:
|
|
controller_0_host = host
|
|
|
|
if controller_1_host is not None:
|
|
host_list = [controller_1_host]
|
|
stage = strategy.StrategyStage(
|
|
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_CONTROLLERS)
|
|
stage.add_step(strategy.QueryAlarmsStep(
|
|
True, ignore_alarms=self._ignore_alarms))
|
|
stage.add_step(strategy.LockHostsStep(host_list))
|
|
stage.add_step(strategy.UpgradeHostsStep(host_list))
|
|
stage.add_step(strategy.UnlockHostsStep(host_list))
|
|
# Allow up to four hours for controller disks to synchronize
|
|
stage.add_step(strategy.WaitDataSyncStep(
|
|
timeout_in_secs=4 * 60 * 60,
|
|
ignore_alarms=self._ignore_alarms))
|
|
self.apply_phase.add_stage(stage)
|
|
|
|
if controller_0_host is not None:
|
|
host_list = [controller_0_host]
|
|
stage = strategy.StrategyStage(
|
|
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_CONTROLLERS)
|
|
stage.add_step(strategy.QueryAlarmsStep(
|
|
True, ignore_alarms=self._ignore_alarms))
|
|
if controller_1_host is not None:
|
|
# Only swact to controller-1 if it was upgraded. If we are only
|
|
# upgrading controller-0, then controller-1 needs to be
|
|
# active already.
|
|
stage.add_step(strategy.SwactHostsStep(host_list))
|
|
stage.add_step(strategy.LockHostsStep(host_list))
|
|
stage.add_step(strategy.UpgradeHostsStep(host_list))
|
|
stage.add_step(strategy.UnlockHostsStep(host_list))
|
|
# Allow up to four hours for controller disks to synchronize
|
|
stage.add_step(strategy.WaitDataSyncStep(
|
|
timeout_in_secs=4 * 60 * 60,
|
|
ignore_alarms=self._ignore_alarms))
|
|
self.apply_phase.add_stage(stage)
|
|
|
|
return True, ''
|
|
|
|
def _add_storage_strategy_stages(self, storage_hosts, reboot):
|
|
"""
|
|
Add storage software upgrade strategy stages
|
|
"""
|
|
from nfv_vim import strategy
|
|
|
|
storage_0_host_list = list()
|
|
storage_0_host_lists = list()
|
|
other_storage_host_list = list()
|
|
|
|
for host in storage_hosts:
|
|
if HOST_NAME.STORAGE_0 == host.name:
|
|
storage_0_host_list.append(host)
|
|
else:
|
|
other_storage_host_list.append(host)
|
|
|
|
if len(storage_0_host_list) == 1:
|
|
storage_0_host_lists, reason = self._create_storage_host_lists(
|
|
storage_0_host_list)
|
|
if storage_0_host_lists is None:
|
|
return False, reason
|
|
|
|
other_storage_host_lists, reason = self._create_storage_host_lists(
|
|
other_storage_host_list)
|
|
if other_storage_host_lists is None:
|
|
return False, reason
|
|
|
|
# Upgrade storage-0 first and on its own since it has a ceph monitor
|
|
if len(storage_0_host_lists) == 1:
|
|
combined_host_lists = storage_0_host_lists + other_storage_host_lists
|
|
else:
|
|
combined_host_lists = other_storage_host_lists
|
|
|
|
for host_list in combined_host_lists:
|
|
stage = strategy.StrategyStage(
|
|
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_STORAGE_HOSTS)
|
|
stage.add_step(strategy.QueryAlarmsStep(
|
|
True, ignore_alarms=self._ignore_alarms))
|
|
stage.add_step(strategy.LockHostsStep(host_list))
|
|
stage.add_step(strategy.UpgradeHostsStep(host_list))
|
|
stage.add_step(strategy.UnlockHostsStep(host_list))
|
|
# After storage node(s) are unlocked, we need extra time to
|
|
# allow the OSDs to go back in sync and the storage related
|
|
# alarms to clear. We no longer wipe the OSD disks when upgrading
|
|
# a storage node, so they should only be syncing data that changed
|
|
# while they were being upgraded.
|
|
stage.add_step(strategy.WaitDataSyncStep(
|
|
timeout_in_secs=2 * 60 * 60,
|
|
ignore_alarms=self._ignore_alarms))
|
|
self.apply_phase.add_stage(stage)
|
|
|
|
return True, ''
|
|
|
|
def _add_compute_strategy_stages(self, compute_hosts, reboot):
|
|
"""
|
|
Add compute software upgrade strategy stages
|
|
"""
|
|
from nfv_vim import tables
|
|
from nfv_vim import strategy
|
|
|
|
host_lists, reason = self._create_compute_host_lists(compute_hosts, reboot)
|
|
if host_lists is None:
|
|
return False, reason
|
|
|
|
instance_table = tables.tables_get_instance_table()
|
|
|
|
for host_list in host_lists:
|
|
instance_list = list()
|
|
|
|
for host in host_list:
|
|
for instance in instance_table.on_host(host.name):
|
|
if not instance.is_locked():
|
|
instance_list.append(instance)
|
|
else:
|
|
DLOG.warn("Instance %s must not be shut down" %
|
|
instance.name)
|
|
reason = ('instance %s must not be shut down' %
|
|
instance.name)
|
|
return False, reason
|
|
|
|
# Computes with no instances
|
|
if 0 == len(instance_list):
|
|
stage = strategy.StrategyStage(
|
|
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_COMPUTE_HOSTS)
|
|
stage.add_step(strategy.QueryAlarmsStep(
|
|
True, ignore_alarms=self._ignore_alarms))
|
|
stage.add_step(strategy.LockHostsStep(host_list))
|
|
stage.add_step(strategy.UpgradeHostsStep(host_list))
|
|
stage.add_step(strategy.UnlockHostsStep(host_list))
|
|
stage.add_step(strategy.SystemStabilizeStep())
|
|
self.apply_phase.add_stage(stage)
|
|
continue
|
|
|
|
# Computes with instances
|
|
stage = strategy.StrategyStage(
|
|
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_COMPUTE_HOSTS)
|
|
|
|
stage.add_step(strategy.QueryAlarmsStep(
|
|
True, ignore_alarms=self._ignore_alarms))
|
|
|
|
if SW_UPDATE_APPLY_TYPE.PARALLEL == self._compute_apply_type:
|
|
# Disable host services before migrating to ensure
|
|
# instances do not migrate to compute hosts in the
|
|
# same set of hosts.
|
|
stage.add_step(strategy.DisableHostServicesStep(host_list))
|
|
stage.add_step(strategy.MigrateInstancesStep(instance_list))
|
|
stage.add_step(strategy.LockHostsStep(host_list))
|
|
stage.add_step(strategy.UpgradeHostsStep(host_list))
|
|
stage.add_step(strategy.UnlockHostsStep(host_list))
|
|
stage.add_step(strategy.SystemStabilizeStep())
|
|
self.apply_phase.add_stage(stage)
|
|
|
|
return True, ''
|
|
|
|
def build_complete(self, result, result_reason):
|
|
"""
|
|
Strategy Build Complete
|
|
"""
|
|
from nfv_vim import tables
|
|
from nfv_vim import strategy
|
|
|
|
result, result_reason = \
|
|
super(SwUpgradeStrategy, self).build_complete(result, result_reason)
|
|
|
|
DLOG.info("Build Complete Callback, result=%s, reason=%s."
|
|
% (result, result_reason))
|
|
|
|
if result in [strategy.STRATEGY_RESULT.SUCCESS,
|
|
strategy.STRATEGY_RESULT.DEGRADED]:
|
|
|
|
# Check whether the upgrade is in a valid state for orchestration
|
|
if self.nfvi_upgrade is None:
|
|
if not self._start_upgrade:
|
|
DLOG.warn("No upgrade in progress.")
|
|
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
|
|
self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED
|
|
self.build_phase.result_reason = 'no upgrade in progress'
|
|
self.sw_update_obj.strategy_build_complete(
|
|
False, self.build_phase.result_reason)
|
|
self.save()
|
|
return
|
|
else:
|
|
if self._start_upgrade:
|
|
valid_states = [UPGRADE_STATE.STARTED,
|
|
UPGRADE_STATE.DATA_MIGRATION_COMPLETE,
|
|
UPGRADE_STATE.UPGRADING_CONTROLLERS,
|
|
UPGRADE_STATE.UPGRADING_HOSTS]
|
|
else:
|
|
valid_states = [UPGRADE_STATE.UPGRADING_CONTROLLERS,
|
|
UPGRADE_STATE.UPGRADING_HOSTS]
|
|
|
|
if self.nfvi_upgrade.state not in valid_states:
|
|
DLOG.warn("Invalid upgrade state for orchestration: %s." %
|
|
self.nfvi_upgrade.state)
|
|
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
|
|
self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED
|
|
self.build_phase.result_reason = (
|
|
'invalid upgrade state for orchestration: %s' %
|
|
self.nfvi_upgrade.state)
|
|
self.sw_update_obj.strategy_build_complete(
|
|
False, self.build_phase.result_reason)
|
|
self.save()
|
|
return
|
|
|
|
# If controller-1 has been upgraded and we have yet to upgrade
|
|
# controller-0, then controller-1 must be active.
|
|
if UPGRADE_STATE.UPGRADING_CONTROLLERS == self.nfvi_upgrade.state:
|
|
if HOST_NAME.CONTROLLER_1 != get_local_host_name():
|
|
DLOG.warn(
|
|
"Controller-1 must be active for orchestration to "
|
|
"upgrade controller-0.")
|
|
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
|
|
self.build_phase.result = \
|
|
strategy.STRATEGY_PHASE_RESULT.FAILED
|
|
self.build_phase.result_reason = (
|
|
'controller-1 must be active for orchestration to '
|
|
'upgrade controller-0')
|
|
self.sw_update_obj.strategy_build_complete(
|
|
False, self.build_phase.result_reason)
|
|
self.save()
|
|
return
|
|
|
|
if self._nfvi_alarms:
|
|
DLOG.warn(
|
|
"Active alarms found, can't apply software upgrade.")
|
|
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
|
|
self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED
|
|
self.build_phase.result_reason = 'active alarms present'
|
|
self.sw_update_obj.strategy_build_complete(
|
|
False, self.build_phase.result_reason)
|
|
self.save()
|
|
return
|
|
|
|
host_table = tables.tables_get_host_table()
|
|
for host in host_table.itervalues():
|
|
# Only allow upgrade orchestration when all hosts are
|
|
# available. It is not safe to automate upgrade application
|
|
# when we do not have full redundancy.
|
|
if not (host.is_unlocked() and host.is_enabled() and
|
|
host.is_available()):
|
|
DLOG.warn(
|
|
"All %s hosts must be unlocked-enabled-available, "
|
|
"can't apply software upgrades." % host.personality)
|
|
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
|
|
self.build_phase.result = \
|
|
strategy.STRATEGY_PHASE_RESULT.FAILED
|
|
self.build_phase.result_reason = (
|
|
'all %s hosts must be unlocked-enabled-available' %
|
|
host.personality)
|
|
self.sw_update_obj.strategy_build_complete(
|
|
False, self.build_phase.result_reason)
|
|
self.save()
|
|
return
|
|
|
|
controller_hosts = list()
|
|
storage_hosts = list()
|
|
compute_hosts = list()
|
|
|
|
if self.nfvi_upgrade is None:
|
|
# Start upgrade
|
|
self._add_upgrade_start_stage()
|
|
|
|
# All hosts will be upgraded
|
|
for host in host_table.itervalues():
|
|
if HOST_PERSONALITY.CONTROLLER in host.personality:
|
|
controller_hosts.append(host)
|
|
|
|
elif HOST_PERSONALITY.STORAGE in host.personality:
|
|
storage_hosts.append(host)
|
|
|
|
elif HOST_PERSONALITY.COMPUTE in host.personality:
|
|
compute_hosts.append(host)
|
|
else:
|
|
# Only hosts not yet upgraded will be upgraded
|
|
to_load = self.nfvi_upgrade.to_release
|
|
for host in host_table.itervalues():
|
|
if host.software_load == to_load:
|
|
# No need to upgrade this host
|
|
continue
|
|
|
|
if HOST_PERSONALITY.CONTROLLER in host.personality:
|
|
controller_hosts.append(host)
|
|
|
|
elif HOST_PERSONALITY.STORAGE in host.personality:
|
|
storage_hosts.append(host)
|
|
|
|
elif HOST_PERSONALITY.COMPUTE in host.personality:
|
|
compute_hosts.append(host)
|
|
|
|
STRATEGY_CREATION_COMMANDS = [
|
|
(self._add_controller_strategy_stages,
|
|
controller_hosts, True),
|
|
(self._add_storage_strategy_stages,
|
|
storage_hosts, True),
|
|
(self._add_compute_strategy_stages,
|
|
compute_hosts, True)
|
|
]
|
|
|
|
for add_strategy_stages_function, host_list, reboot in \
|
|
STRATEGY_CREATION_COMMANDS:
|
|
if host_list:
|
|
success, reason = add_strategy_stages_function(
|
|
host_list, reboot)
|
|
if not success:
|
|
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
|
|
self.build_phase.result = \
|
|
strategy.STRATEGY_PHASE_RESULT.FAILED
|
|
self.build_phase.result_reason = reason
|
|
self.sw_update_obj.strategy_build_complete(
|
|
False, self.build_phase.result_reason)
|
|
self.save()
|
|
return
|
|
|
|
if self._complete_upgrade:
|
|
self._add_upgrade_complete_stage()
|
|
|
|
if 0 == len(self.apply_phase.stages):
|
|
DLOG.warn("No software upgrades need to be applied.")
|
|
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
|
|
self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED
|
|
self.build_phase.result_reason = ('no software upgrades need to be '
|
|
'applied')
|
|
self.sw_update_obj.strategy_build_complete(
|
|
False, self.build_phase.result_reason)
|
|
self.save()
|
|
return
|
|
else:
|
|
self.sw_update_obj.strategy_build_complete(
|
|
False, self.build_phase.result_reason)
|
|
|
|
self.sw_update_obj.strategy_build_complete(True, '')
|
|
self.save()
|
|
|
|
def from_dict(self, data, build_phase=None, apply_phase=None, abort_phase=None):
|
|
"""
|
|
Initializes a software upgrade strategy object using the given dictionary
|
|
"""
|
|
from nfv_vim import nfvi
|
|
|
|
super(SwUpgradeStrategy, self).from_dict(data, build_phase, apply_phase,
|
|
abort_phase)
|
|
self._start_upgrade = data['start_upgrade']
|
|
self._complete_upgrade = data['complete_upgrade']
|
|
nfvi_upgrade_data = data['nfvi_upgrade_data']
|
|
if nfvi_upgrade_data:
|
|
self._nfvi_upgrade = nfvi.objects.v1.Upgrade(
|
|
nfvi_upgrade_data['state'],
|
|
nfvi_upgrade_data['from_release'],
|
|
nfvi_upgrade_data['to_release'])
|
|
else:
|
|
self._nfvi_upgrade = None
|
|
|
|
return self
|
|
|
|
def as_dict(self):
|
|
"""
|
|
Represent the software upgrade strategy as a dictionary
|
|
"""
|
|
data = super(SwUpgradeStrategy, self).as_dict()
|
|
|
|
data['start_upgrade'] = self._start_upgrade
|
|
data['complete_upgrade'] = self._complete_upgrade
|
|
if self._nfvi_upgrade:
|
|
nfvi_upgrade_data = self._nfvi_upgrade.as_dict()
|
|
else:
|
|
nfvi_upgrade_data = None
|
|
data['nfvi_upgrade_data'] = nfvi_upgrade_data
|
|
|
|
return data
|
|
|
|
|
|
def strategy_rebuild_from_dict(data):
|
|
"""
|
|
Returns the strategy object initialized using the given dictionary
|
|
"""
|
|
from _strategy_phases import strategy_phase_rebuild_from_dict
|
|
|
|
if not data:
|
|
return None
|
|
|
|
build_phase = strategy_phase_rebuild_from_dict(data['build_phase'])
|
|
apply_phase = strategy_phase_rebuild_from_dict(data['apply_phase'])
|
|
abort_phase = strategy_phase_rebuild_from_dict(data['abort_phase'])
|
|
|
|
if STRATEGY_NAME.SW_PATCH == data['name']:
|
|
strategy_obj = object.__new__(SwPatchStrategy)
|
|
elif STRATEGY_NAME.SW_UPGRADE == data['name']:
|
|
strategy_obj = object.__new__(SwUpgradeStrategy)
|
|
else:
|
|
strategy_obj = object.__new__(strategy.StrategyStage)
|
|
|
|
strategy_obj.from_dict(data, build_phase, apply_phase, abort_phase)
|
|
return strategy_obj
|