starlingx
/
config
Code Issues Proposed changes

Allow Hardware Replacement During Upgrades 

Prevent host-upgrade/unlock/swact controller-1 if there is new hardware
detected (hosts, disks, stors, interfaces) since the upgrade start.
Allow force option to allow for user override.

Allow a new host to be added by allowing installation
of the host with the same load as the active controller.

Change-Id: I1752dbb8648d3cffa77e1040169cee0e16730f22
Story: 2002886
Task: 22847
Signed-off-by: Jack Ding <jack.ding@windriver.com>
This commit is contained in:
John Kung 2018-04-12 16:44:53 -04:00 committed by Jack Ding
parent 0e68f13e27
commit ba353a6dda
5 changed files with 141 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py
View File

@ -2314,6 +2314,31 @@ class HostController(rest.RestController):
pecan.request.dbapi.ihost_destroy(ihost_id)
def _check_upgrade_provision_order(self, personality, hostname):
LOG.info("_check_upgrade_provision_order personality=%s, hostname=%s" %
(personality, hostname))
# If this is a simplex system skip this check; there's no other nodes
simplex = (utils.get_system_mode() == constants.SYSTEM_MODE_SIMPLEX)
if simplex:
return
try:
upgrade = pecan.request.dbapi.software_upgrade_get_one()
except exception.NotFound:
return
loads = pecan.request.dbapi.load_get_list()
target_load = cutils.get_imported_load(loads)
if personality == constants.STORAGE:
if hostname == constants.STORAGE_0_HOSTNAME:
LOG.warn("Allow storage-0 add during upgrade")
else:
LOG.info("Adding storage, ensure controllers upgraded")
self._check_personality_load(constants.CONTROLLER,
target_load)
@cutils.synchronized(LOCK_NAME)
@wsme_pecan.wsexpose(Host, unicode, body=unicode)
def upgrade(self, uuid, body):
@ -2945,8 +2970,7 @@ class HostController(rest.RestController):
'count': count})
raise wsme.exc.ClientSideError(msg)
@staticmethod
def _semantic_check_unlock_upgrade(ihost):
def _semantic_check_unlock_upgrade(self, ihost, force_unlock=False):
"""
Perform semantic checks related to upgrades prior to unlocking host.
"""
@ -2971,6 +2995,9 @@ class HostController(rest.RestController):
ihost['hostname'])
raise wsme.exc.ClientSideError(msg)
# Check for new hardware since upgrade-start
self._semantic_check_upgrade_refresh(upgrade, ihost, force_unlock)
@staticmethod
def _semantic_check_oam_interface(ihost):
"""
@ -3676,6 +3703,81 @@ class HostController(rest.RestController):
'ok': constants.SB_STATE_CONFIGURED}
raise wsme.exc.ClientSideError(msg)
@staticmethod
def _new_host_hardware_since_upgrade(host, upgrade_created_at):
"""
Determines the new hardware on the host since the upgrade started.
:param host host object
:param upgrade_created_at upgrade start timestamp
returns: new_hw tuple of new hardware on host
"""
new_hw = []
disks = pecan.request.dbapi.idisk_get_by_ihost(host.id)
new_disks = [x.uuid for x in disks
if x.created_at > upgrade_created_at]
if new_disks:
new_hw.append(('disks', host.hostname, new_disks))
interfaces = pecan.request.dbapi.iinterface_get_by_ihost(host.id)
new_interfaces = [x.uuid for x in interfaces
if x.created_at > upgrade_created_at]
if new_interfaces:
new_hw.append(('interfaces', host.hostname, new_interfaces))
stors = pecan.request.dbapi.istor_get_by_ihost(host.id)
new_stors = [x.uuid for x in stors
if x.created_at > upgrade_created_at]
if new_stors:
new_hw.append(('stors', host.hostname, new_stors))
return new_hw
def _semantic_check_upgrade_refresh(self, upgrade, ihost, force):
"""
Determine whether upgrade should be aborted/refreshed due to
new hardware since upgrade start
"""
if force:
LOG.info("_semantic_check_upgrade_refresh check force")
return
if ihost['hostname'] != constants.CONTROLLER_1_HOSTNAME:
return
if upgrade.state not in [constants.UPGRADE_STARTED,
constants.UPGRADE_DATA_MIGRATION,
constants.UPGRADE_DATA_MIGRATION_COMPLETE,
constants.UPGRADE_UPGRADING_CONTROLLERS]:
LOG.info("_semantic_check_upgrade_refresh allow upgrade state=%s" %
upgrade.state)
return
upgrade_created_at = upgrade.created_at
# check for new host hardware since upgrade started
hosts = pecan.request.dbapi.ihost_get_list()
new_hw = []
for h in hosts:
if not h.personality:
continue
if h.created_at > upgrade_created_at:
new_hw.append(('host', h.hostname, h.uuid))
break
new_hw_h = self._new_host_hardware_since_upgrade(
h, upgrade_created_at)
if new_hw_h:
new_hw.append(new_hw_h)
if new_hw:
msg = _("New hardware %s detected after upgrade started at %s. "
"Upgrade should be aborted."
% (new_hw, upgrade_created_at))
raise wsme.exc.ClientSideError(msg)
@staticmethod
def _semantic_check_nova_local_storage(ihost_uuid, personality):
"""
@ -4232,6 +4334,13 @@ class HostController(rest.RestController):
LOG.error("Unexpected personality: %s" %
hostupdate.ihost_patch['personality'])
hostname = (hostupdate.ihost_val.get('hostname') or
hostupdate.ihost_patch['hostname'])
# Check host personality provisioning order during upgrades
self._check_upgrade_provision_order(
hostupdate.ihost_patch['personality'],
hostname)
# Always configure when the personality has been set - this will
# set up the PXE boot information so the software can be installed
hostupdate.configure_required = True
@ -4467,7 +4576,7 @@ class HostController(rest.RestController):
personality = hostupdate.ihost_patch.get('personality')
if personality == constants.CONTROLLER:
self.check_unlock_controller(hostupdate)
self.check_unlock_controller(hostupdate, force_unlock)
if cutils.host_has_function(hostupdate.ihost_patch, constants.COMPUTE):
self.check_unlock_compute(hostupdate)
@ -4650,10 +4759,10 @@ class HostController(rest.RestController):
raise wsme.exc.ClientSideError(
_("%s" % response['error_details']))
def check_unlock_controller(self, hostupdate):
def check_unlock_controller(self, hostupdate, force_unlock=False):
"""Pre unlock semantic checks for controller"""
LOG.info("%s ihost check_unlock_controller" % hostupdate.displayid)
self._semantic_check_unlock_upgrade(hostupdate.ihost_orig)
self._semantic_check_unlock_upgrade(hostupdate.ihost_orig, force_unlock)
self._semantic_check_oam_interface(hostupdate.ihost_orig)
self._semantic_check_cinder_volumes(hostupdate.ihost_orig)
self._semantic_check_storage_backend(hostupdate.ihost_orig)
@ -4933,8 +5042,7 @@ class HostController(rest.RestController):
"Please run system certificate-install -m tpm_mode "
"before re-attempting." % ihost['hostname']))
@staticmethod
def _semantic_check_swact_upgrade(from_host, to_host):
def _semantic_check_swact_upgrade(self, from_host, to_host, force_swact=False):
"""
Perform semantic checks related to upgrades prior to swacting host.
"""
@ -4984,7 +5092,10 @@ class HostController(rest.RestController):
"operation can proceed. Currently using load %s.") %
(to_host['hostname'], to_sw_version, to_host_sw_version))
def check_swact(self, hostupdate):
# Check for new hardware since upgrade-start
self._semantic_check_upgrade_refresh(upgrade, to_host, force_swact)
def check_swact(self, hostupdate, force_swact=False):
"""Pre swact semantic checks for controller"""
if hostupdate.ihost_orig['personality'] != constants.CONTROLLER:
@ -5065,7 +5176,8 @@ class HostController(rest.RestController):
(ihost_ctr.hostname, ihost_ctr.config_target))
self._semantic_check_swact_upgrade(hostupdate.ihost_orig,
ihost_ctr)
ihost_ctr,
force_swact)
# If HTTPS is enabled then we may be in TPM mode
if utils.get_https_enabled():

4
sysinv/sysinv/sysinv/sysinv/api/controllers/v1/load.py
View File

@ -238,7 +238,9 @@ class LoadController(rest.RestController):
values['forihostid'] = host.id
values['software_load'] = new_load.id
values['target_load'] = new_load.id
pecan.request.dbapi.host_upgrade_create(host.id, values)
pecan.request.dbapi.host_upgrade_create(host.id,
new_load.software_version,
values)
except exception.SysinvException as e:
LOG.exception(e)

5
sysinv/sysinv/sysinv/sysinv/conductor/manager.py
View File

@ -696,12 +696,15 @@ class ConductorManager(service.PeriodicService):
if utils.is_host_active_controller(h):
active_controller = h
break
software_load = None
if active_controller is not None:
tboot_value = active_controller.get('tboot')
if tboot_value is not None:
values.update({'tboot': tboot_value})
software_load = active_controller.software_load
LOG.info("create_ihost software_load=%s" % software_load)
ihost = self.dbapi.ihost_create(values)
ihost = self.dbapi.ihost_create(values, software_load=software_load)
# A host is being created, generate discovery log.
self._log_host_create(ihost, reason)

3
sysinv/sysinv/sysinv/sysinv/db/api.py
View File

@ -130,7 +130,7 @@ class Connection(object):
"""
@abc.abstractmethod
def ihost_create(self, values, session=None):
def ihost_create(self, values, software_load=None):
"""Create a new ihost.
:param values: A dict containing several items used to identify
@ -148,6 +148,7 @@ class Connection(object):
'availability': 'offduty',
'extra': { ... },
}
:param: software_load. The load software_version.
:returns: A ihost.
"""

19
sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py
View File

@ -1240,7 +1240,7 @@ class Connection(api.Connection):
raise exception.ServerNotFound(server=server)
@objects.objectify(objects.host)
def ihost_create(self, values):
def ihost_create(self, values, software_load=None):
if not values.get('uuid'):
values['uuid'] = uuidutils.generate_uuid()
host = models.ihost()
@ -1251,7 +1251,7 @@ class Connection(api.Connection):
session.flush()
except db_exc.DBDuplicateEntry:
raise exception.NodeAlreadyExists(uuid=values['uuid'])
self._host_upgrade_create(host.id)
self._host_upgrade_create(host.id, software_load)
return self._host_get(values['uuid'])
@objects.objectify(objects.host)
@ -6264,12 +6264,15 @@ class Connection(api.Connection):
query.delete()
def _host_upgrade_create(self, host_id, values=None, session=None):
def _host_upgrade_create(self, host_id, version, values=None):
if values is None:
values = dict()
systems = self.isystem_get_list()
if systems is not None:
version = systems[0].software_version
if not version:
systems = self.isystem_get_list()
if systems is not None:
version = systems[0].software_version
LOG.info("_host_upgrade_create system version=%s" % version)
if version:
# get the load_id from the loads table
query = model_query(models.Load)
query = query.filter_by(software_version=version)
@ -6295,8 +6298,8 @@ class Connection(api.Connection):
return upgrade
@objects.objectify(objects.host_upgrade)
def host_upgrade_create(self, host_id, values):
return self._host_upgrade_create(host_id, values)
def host_upgrade_create(self, host_id, version, values):
return self._host_upgrade_create(host_id, version, values)
@objects.objectify(objects.host_upgrade)
def host_upgrade_get(self, id):