starlingx/config
Code Issues Proposed changes

Merge "Enhance app updates during Kubernetes upgrades"

Browse Source
This commit is contained in:
Zuul 2024-04-30 15:58:21 +00:00 committed by Gerrit Code Review
commit 79c94ed7b2
10 changed files with 874 additions and 309 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
sysinv/cgts-client/cgts-client/cgtsclient/tests/v1/test_kube_upgrade_shell.py
View File

@ -85,6 +85,33 @@ class KubeUpgradeTest(test_shell.ShellTest):
self.assertIn(fake_kube_upgrade['created_at'], results)
self.assertIn(fake_kube_upgrade['updated_at'], results)
@mock.patch('cgtsclient.v1.kube_upgrade.KubeUpgradeManager.update')
def test_kube_pre_application_update(self, mock_update):
fake_kube_upgrade = {'from_version': 'v1.42.1',
'to_version': 'v1.42.2',
'state': 'pre-updating-apps',
'uuid': 'cb737aba-1820-4184-b0dc-9b073822af48',
'created_at': 'fake-created-time',
'updated_at': 'fake-updated-time',
}
mock_update.return_value = KubeUpgrade(None, fake_kube_upgrade, True)
self.make_env()
results = self.shell("kube-pre-application-update")
patch = {'op': 'replace',
'path': '/state',
'value': 'pre-updating-apps'
}
mock_update.assert_called_once_with([patch])
self.assertIn(fake_kube_upgrade['from_version'], results)
self.assertIn(fake_kube_upgrade['to_version'], results)
self.assertIn(fake_kube_upgrade['state'], results)
self.assertIn(fake_kube_upgrade['uuid'], results)
self.assertIn(fake_kube_upgrade['created_at'], results)
self.assertIn(fake_kube_upgrade['updated_at'], results)
@mock.patch('cgtsclient.v1.kube_upgrade.KubeUpgradeManager.update')
def test_kube_upgrade_download_images(self, mock_update):
fake_kube_upgrade = {'from_version': 'v1.42.1',
@ -125,6 +152,33 @@ class KubeUpgradeTest(test_shell.ShellTest):
self.assertIn(fake_kube_upgrade['created_at'], results)
self.assertIn(fake_kube_upgrade['updated_at'], results)
@mock.patch('cgtsclient.v1.kube_upgrade.KubeUpgradeManager.update')
def test_kube_post_application_update(self, mock_update):
fake_kube_upgrade = {'from_version': 'v1.42.1',
'to_version': 'v1.42.2',
'state': 'post-updating-apps',
'uuid': 'cb737aba-1820-4184-b0dc-9b073822af48',
'created_at': 'fake-created-time',
'updated_at': 'fake-updated-time',
}
mock_update.return_value = KubeUpgrade(None, fake_kube_upgrade, True)
self.make_env()
results = self.shell("kube-post-application-update")
patch = {'op': 'replace',
'path': '/state',
'value': 'post-updating-apps'
}
mock_update.assert_called_once_with([patch])
self.assertIn(fake_kube_upgrade['from_version'], results)
self.assertIn(fake_kube_upgrade['to_version'], results)
self.assertIn(fake_kube_upgrade['state'], results)
self.assertIn(fake_kube_upgrade['uuid'], results)
self.assertIn(fake_kube_upgrade['created_at'], results)
self.assertIn(fake_kube_upgrade['updated_at'], results)
@mock.patch('cgtsclient.v1.kube_upgrade.KubeUpgradeManager.update')
def test_kube_upgrade_complete(self, mock_update):
fake_kube_upgrade = {'from_version': 'v1.42.1',

94
sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_upgrade_shell.py
View File

@ -9,6 +9,7 @@ from cgtsclient import exc
from cgtsclient.v1 import ihost as ihost_utils
# Kubernetes constants
KUBE_UPGRADE_STATE_PRE_UPDATING_APPS = 'pre-updating-apps'
KUBE_UPGRADE_STATE_DOWNLOADING_IMAGES = 'downloading-images'
KUBE_UPGRADE_STATE_UPGRADING_NETWORKING = 'upgrading-networking'
KUBE_UPGRADE_STATE_UPGRADING_STORAGE = 'upgrading-storage'
@ -18,6 +19,7 @@ KUBE_UPGRADE_STATE_UPGRADING_SECOND_MASTER = 'upgrading-second-master'
KUBE_UPGRADE_STATE_ABORTING = 'upgrade-aborting'
KUBE_UPGRADE_STATE_CORDON = 'cordon-started'
KUBE_UPGRADE_STATE_UNCORDON = 'uncordon-started'
KUBE_UPGRADE_STATE_POST_UPDATING_APPS = 'post-updating-apps'
def _print_kube_upgrade_show(obj):
@ -56,11 +58,8 @@ def do_kube_upgrade_start(cc, args):
_print_kube_upgrade_show(kube_upgrade)
def do_kube_upgrade_download_images(cc, args):
"""Download kubernetes images."""
data = dict()
data['state'] = KUBE_UPGRADE_STATE_DOWNLOADING_IMAGES
def patch_kube_upgrade(cc, data):
"""" Call patch HTTP method for kube upgrades"""
patch = []
for (k, v) in data.items():
@ -73,6 +72,24 @@ def do_kube_upgrade_download_images(cc, args):
_print_kube_upgrade_show(kube_upgrade)
def do_kube_upgrade_download_images(cc, args):
"""Download kubernetes images."""
data = dict()
data['state'] = KUBE_UPGRADE_STATE_DOWNLOADING_IMAGES
patch_kube_upgrade(cc, data)
def do_kube_pre_application_update(cc, args):
"""Update applications before Kubernetes is upgraded."""
data = dict()
data['state'] = KUBE_UPGRADE_STATE_PRE_UPDATING_APPS
patch_kube_upgrade(cc, data)
@utils.arg('hostid', metavar='<hostname or id>',
help="Name or ID of host")
def do_kube_host_cordon(cc, args):
@ -83,15 +100,7 @@ def do_kube_host_cordon(cc, args):
data['hostname'] = ihost.hostname
data['state'] = KUBE_UPGRADE_STATE_CORDON
patch = []
for (k, v) in data.items():
patch.append({'op': 'replace', 'path': '/' + k, 'value': v})
try:
kube_upgrade = cc.kube_upgrade.update(patch)
except exc.HTTPNotFound:
raise exc.CommandError('Kubernetes upgrade UUID not found')
_print_kube_upgrade_show(kube_upgrade)
patch_kube_upgrade(cc, data)
@utils.arg('hostid', metavar='<hostname or id>',
@ -104,15 +113,7 @@ def do_kube_host_uncordon(cc, args):
data['hostname'] = ihost.hostname
data['state'] = KUBE_UPGRADE_STATE_UNCORDON
patch = []
for (k, v) in data.items():
patch.append({'op': 'replace', 'path': '/' + k, 'value': v})
try:
kube_upgrade = cc.kube_upgrade.update(patch)
except exc.HTTPNotFound:
raise exc.CommandError('Kubernetes upgrade UUID not found')
_print_kube_upgrade_show(kube_upgrade)
patch_kube_upgrade(cc, data)
def do_kube_upgrade_networking(cc, args):
@ -121,15 +122,7 @@ def do_kube_upgrade_networking(cc, args):
data = dict()
data['state'] = KUBE_UPGRADE_STATE_UPGRADING_NETWORKING
patch = []
for (k, v) in data.items():
patch.append({'op': 'replace', 'path': '/' + k, 'value': v})
try:
kube_upgrade = cc.kube_upgrade.update(patch)
except exc.HTTPNotFound:
raise exc.CommandError('Kubernetes upgrade UUID not found')
_print_kube_upgrade_show(kube_upgrade)
patch_kube_upgrade(cc, data)
def do_kube_upgrade_storage(cc, args):
@ -138,15 +131,16 @@ def do_kube_upgrade_storage(cc, args):
data = dict()
data['state'] = KUBE_UPGRADE_STATE_UPGRADING_STORAGE
patch = []
for (k, v) in data.items():
patch.append({'op': 'replace', 'path': '/' + k, 'value': v})
try:
kube_upgrade = cc.kube_upgrade.update(patch)
except exc.HTTPNotFound:
raise exc.CommandError('Kubernetes upgrade UUID not found')
patch_kube_upgrade(cc, data)
_print_kube_upgrade_show(kube_upgrade)
def do_kube_post_application_update(cc, args):
"""Update applications after Kubernetes is upgraded."""
data = dict()
data['state'] = KUBE_UPGRADE_STATE_POST_UPDATING_APPS
patch_kube_upgrade(cc, data)
def do_kube_upgrade_abort(cc, args):
@ -155,15 +149,7 @@ def do_kube_upgrade_abort(cc, args):
data = dict()
data['state'] = KUBE_UPGRADE_STATE_ABORTING
patch = []
for (k, v) in data.items():
patch.append({'op': 'replace', 'path': '/' + k, 'value': v})
try:
kube_upgrade = cc.kube_upgrade.update(patch)
except exc.HTTPNotFound:
raise exc.CommandError('Kubernetes upgrade not found')
_print_kube_upgrade_show(kube_upgrade)
patch_kube_upgrade(cc, data)
def do_kube_upgrade_complete(cc, args):
@ -172,15 +158,7 @@ def do_kube_upgrade_complete(cc, args):
data = dict()
data['state'] = KUBE_UPGRADE_STATE_COMPLETE
patch = []
for (k, v) in data.items():
patch.append({'op': 'replace', 'path': '/' + k, 'value': v})
try:
kube_upgrade = cc.kube_upgrade.update(patch)
except exc.HTTPNotFound:
raise exc.CommandError('Kubernetes upgrade UUID not found')
_print_kube_upgrade_show(kube_upgrade)
patch_kube_upgrade(cc, data)
def do_kube_upgrade_delete(cc, args):

301
sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py
View File

@ -7,6 +7,7 @@
from fm_api import constants as fm_constants
from fm_api import fm_api
from distutils.version import LooseVersion
import pecan
from pecan import rest
import os
@ -152,6 +153,116 @@ class KubeUpgradeController(rest.RestController):
"the kubernetes upgrade: %s" %
available_patches))
def _check_applied_apps_compatibility(self, from_version, to_version):
"""Ensure that applied applications are compatible with
Kubernetes versions across the upgrade process
:param from_version: Initial Kubernetes version
:param to_version: Target Kubernetes version
"""
system = pecan.request.dbapi.isystem_get_one()
if system.system_mode == constants.SYSTEM_MODE_SIMPLEX:
next_versions = self._kube_operator.kube_get_higher_patch_version(from_version,
to_version)
else:
next_versions = [to_version]
if not next_versions:
raise wsme.exc.ClientSideError(_("Error while retrieving Kubernetes intermediate "
"versions"))
incompatible_apps = set()
lower_k8s_version_from_incompatible_apps = None
from_version = from_version.lstrip('v')
to_version = to_version.lstrip('v')
next_versions = [x.lstrip('v') for x in next_versions]
apps = pecan.request.dbapi.kube_app_get_all()
for app in apps:
if app.status != constants.APP_APPLY_SUCCESS:
continue
# Applications with timing=pre need to be compatible with the current,
# intermediate and target k8s versions:
pre_update_compatible = pecan.request.dbapi.kube_app_bundle_is_k8s_compatible(
name=app.name,
k8s_timing=constants.APP_METADATA_TIMING_PRE,
current_k8s_version=from_version,
target_k8s_version=to_version)
if not pre_update_compatible:
LOG.debug("Unable to find a version of application {} to be pre updated."
.format(app.name))
# Applications with timing=post should be compatible with the target k8s version.
# Compatibility with current or intermediate versions is not required:
post_update_compatible = \
pecan.request.dbapi.kube_app_bundle_is_k8s_compatible(
name=app.name,
k8s_timing=constants.APP_METADATA_TIMING_POST,
target_k8s_version=to_version)
if not post_update_compatible:
LOG.debug("Unable to find a version of application {} to be post updated."
.format(app.name))
if not pre_update_compatible and not post_update_compatible:
# If the app cannot be pre or post updated, check if we can proceed with
# the current applied version.
applied_app_kube_min_version, applied_app_kube_max_version = \
cutils.get_app_supported_kube_version(app.name, app.app_version)
if kubernetes.is_kube_version_supported(
to_version, applied_app_kube_min_version, applied_app_kube_max_version):
LOG.info("No updates found for application {} during Kubernetes upgrade "
"to {} but current applied version {} is supported.".format(
app.name,
to_version,
app.app_version))
continue
max_compatible_version = \
pecan.request.dbapi.kube_app_bundle_max_k8s_compatible_by_name(
name=app.name,
current_k8s_version=from_version,
target_k8s_version=to_version)
if max_compatible_version is None:
max_compatible_version = from_version
if (LooseVersion(applied_app_kube_max_version) >
LooseVersion(max_compatible_version)):
max_compatible_version = applied_app_kube_max_version
incompatible_versions = [x for x in next_versions if LooseVersion(x) >
LooseVersion(max_compatible_version)]
LOG.error("Unable to find a suitable version of application {} "
"compatible with the following Kubernetes versions: {}."
.format(app.name, ', '.join(str(s) for s in incompatible_versions)))
incompatible_apps.add(app.name)
if (lower_k8s_version_from_incompatible_apps is None or
LooseVersion(max_compatible_version) <
LooseVersion(lower_k8s_version_from_incompatible_apps)):
lower_k8s_version_from_incompatible_apps = max_compatible_version
# If the lowest compatible version found amongst all apps is out of the version range we
# support then there is no upgrade path available.
# If the lowest compatible version found amongst all apps is within the range we support
# then inform the highest supported version.
if (lower_k8s_version_from_incompatible_apps and
LooseVersion(lower_k8s_version_from_incompatible_apps) <
LooseVersion(next_versions[0])):
raise wsme.exc.ClientSideError(_(
"The following apps are incompatible with intermediate/target Kubernetes "
"versions: {}. No upgrade path available to Kubernetes version {}."
.format(', '.join(str(s) for s in incompatible_apps), to_version)))
elif lower_k8s_version_from_incompatible_apps:
highest_supported_version = next(
x for x in list(reversed(next_versions))
if (LooseVersion(x) <=
LooseVersion(lower_k8s_version_from_incompatible_apps)))
raise wsme.exc.ClientSideError(_(
"The following apps are incompatible with intermediate/target Kubernetes "
"versions: {}. The system can be upgraded up to Kubernetes {}."
.format(', '.join(str(s) for s in incompatible_apps), highest_supported_version)))
@wsme_pecan.wsexpose(KubeUpgradeCollection)
def get_all(self):
"""Retrieve a list of kubernetes upgrades."""
@ -175,7 +286,6 @@ class KubeUpgradeController(rest.RestController):
force = body.get('force', False) is True
alarm_ignore_list = body.get('alarm_ignore_list')
system = pecan.request.dbapi.isystem_get_one()
retry = False
# There must not be a platform upgrade in progress
try:
@ -189,21 +299,12 @@ class KubeUpgradeController(rest.RestController):
# There must not already be a kubernetes upgrade in progress
try:
kube_upgrade_obj = objects.kube_upgrade.get_one(pecan.request.context)
pecan.request.dbapi.kube_upgrade_get_one()
except exception.NotFound:
pass
else:
# Allow retrying the new Kubernetes upgrade if the current
# state is 'upgrade-starting-failed'.
if (kube_upgrade_obj.state == kubernetes.KUBE_UPGRADE_STARTING_FAILED and
kube_upgrade_obj.to_version == to_version):
retry = True
if alarm_ignore_list is None:
alarm_ignore_list = []
alarm_ignore_list.append(fm_constants.FM_ALARM_ID_KUBE_UPGRADE_IN_PROGRESS)
else:
raise wsme.exc.ClientSideError(_(
"A kubernetes upgrade is already in progress"))
raise wsme.exc.ClientSideError(_(
"A kubernetes upgrade is already in progress"))
# Check whether target version is available or not
try:
@ -260,63 +361,51 @@ class KubeUpgradeController(rest.RestController):
"System is not in a valid state for kubernetes upgrade. "
"Run system health-query-kube-upgrade for more details."))
if retry:
# Update upgrade record
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_STARTING
kube_upgrade_obj.save()
else:
# Create upgrade record.
create_values = {'from_version': current_kube_version,
'to_version': to_version,
'state': kubernetes.KUBE_UPGRADE_STARTING}
kube_upgrade_obj = pecan.request.dbapi.kube_upgrade_create(create_values)
# Check app compatibility
self._check_applied_apps_compatibility(current_kube_version, to_version)
try:
# Set the target version for each host to the current version
update_values = {'target_version': current_kube_version}
kube_host_upgrades = pecan.request.dbapi.kube_host_upgrade_get_list()
for kube_host_upgrade in kube_host_upgrades:
pecan.request.dbapi.kube_host_upgrade_update(kube_host_upgrade.id,
update_values)
# Raise alarm to show a kubernetes upgrade is in progress
entity_instance_id = "%s=%s" % (fm_constants.FM_ENTITY_TYPE_HOST,
constants.CONTROLLER_HOSTNAME)
fault = fm_api.Fault(
alarm_id=fm_constants.FM_ALARM_ID_KUBE_UPGRADE_IN_PROGRESS,
alarm_state=fm_constants.FM_ALARM_STATE_SET,
entity_type_id=fm_constants.FM_ENTITY_TYPE_HOST,
entity_instance_id=entity_instance_id,
severity=fm_constants.FM_ALARM_SEVERITY_MINOR,
reason_text="Kubernetes upgrade in progress.",
# operational
alarm_type=fm_constants.FM_ALARM_TYPE_7,
# congestion
probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_8,
proposed_repair_action="No action required.",
service_affecting=False)
fm_api.FaultAPIs().set_fault(fault)
# Create upgrade record.
create_values = {'from_version': current_kube_version,
'to_version': to_version,
'state': kubernetes.KUBE_UPGRADE_STARTED}
new_upgrade = pecan.request.dbapi.kube_upgrade_create(create_values)
# Set the new kubeadm version in the DB.
# This will not actually change the bind mounts until we apply a
# puppet manifest that makes use of it.
kube_cmd_versions = objects.kube_cmd_version.get(
pecan.request.context)
kube_cmd_versions.kubeadm_version = to_version.lstrip('v')
kube_cmd_versions.save()
# Set the target version for each host to the current version
update_values = {'target_version': current_kube_version}
kube_host_upgrades = pecan.request.dbapi.kube_host_upgrade_get_list()
for kube_host_upgrade in kube_host_upgrades:
pecan.request.dbapi.kube_host_upgrade_update(kube_host_upgrade.id,
update_values)
# Raise alarm to show a kubernetes upgrade is in progress
entity_instance_id = "%s=%s" % (fm_constants.FM_ENTITY_TYPE_HOST,
constants.CONTROLLER_HOSTNAME)
fault = fm_api.Fault(
alarm_id=fm_constants.FM_ALARM_ID_KUBE_UPGRADE_IN_PROGRESS,
alarm_state=fm_constants.FM_ALARM_STATE_SET,
entity_type_id=fm_constants.FM_ENTITY_TYPE_HOST,
entity_instance_id=entity_instance_id,
severity=fm_constants.FM_ALARM_SEVERITY_MINOR,
reason_text="Kubernetes upgrade in progress.",
# operational
alarm_type=fm_constants.FM_ALARM_TYPE_7,
# congestion
probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_8,
proposed_repair_action="No action required.",
service_affecting=False)
fm_api.FaultAPIs().set_fault(fault)
LOG.info("Starting kubernetes upgrade from version: %s to version: %s"
% (current_kube_version, to_version))
# Set the new kubeadm version in the DB.
# This will not actually change the executable version until we apply a
# puppet manifest that makes use of it.
kube_cmd_versions = objects.kube_cmd_version.get(
pecan.request.context)
kube_cmd_versions.kubeadm_version = to_version.lstrip('v')
kube_cmd_versions.save()
# Tell the conductor to update the required apps and mark the upgrade as started
pecan.request.rpcapi.kube_upgrade_start(
pecan.request.context,
to_version)
except Exception as e:
LOG.exception("Failed to start Kubernetes upgrade: %s" % e)
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_STARTING_FAILED
kube_upgrade_obj.save()
LOG.info("Started kubernetes upgrade from version: %s to version: %s"
% (current_kube_version, to_version))
return KubeUpgrade.convert_with_links(kube_upgrade_obj)
return KubeUpgrade.convert_with_links(new_upgrade)
@cutils.synchronized(LOCK_NAME)
@wsme.validate([KubeUpgradePatchType])
@ -337,10 +426,12 @@ class KubeUpgradeController(rest.RestController):
if updates['state'] and updates['state'].split('-')[-1] == 'failed':
if kube_upgrade_obj.state in [
kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES,
kubernetes.KUBE_PRE_UPDATING_APPS,
kubernetes.KUBE_UPGRADING_FIRST_MASTER,
kubernetes.KUBE_UPGRADING_SECOND_MASTER,
kubernetes.KUBE_UPGRADING_STORAGE,
kubernetes.KUBE_UPGRADING_NETWORKING]:
kubernetes.KUBE_UPGRADING_NETWORKING,
kubernetes.KUBE_POST_UPDATING_APPS]:
kube_upgrade_obj.state = updates['state']
kube_upgrade_obj.save()
LOG.info("Kubernetes upgrade state is changed to %s" % updates['state'])
@ -383,15 +474,37 @@ class KubeUpgradeController(rest.RestController):
kube_upgrade_obj.to_version)
return KubeUpgrade.convert_with_links(kube_upgrade_obj)
elif updates['state'] == kubernetes.KUBE_PRE_UPDATING_APPS:
# Make sure upgrade is in the correct state to update apps
if kube_upgrade_obj.state not in [
kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
kubernetes.KUBE_PRE_UPDATING_APPS_FAILED]:
raise wsme.exc.ClientSideError(_(
"Kubernetes upgrade must be in %s or %s state to "
"update applications" %
(kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
kubernetes.KUBE_PRE_UPDATING_APPS_FAILED)))
# Update the upgrade state
kube_upgrade_obj.state = kubernetes.KUBE_PRE_UPDATING_APPS
kube_upgrade_obj.save()
# Tell the conductor to update the required apps
pecan.request.rpcapi.kube_pre_application_update(pecan.request.context)
LOG.info("Updating applications to match target Kubernetes version %s" %
kube_upgrade_obj.to_version)
return KubeUpgrade.convert_with_links(kube_upgrade_obj)
elif updates['state'] == kubernetes.KUBE_UPGRADING_NETWORKING:
# Make sure upgrade is in the correct state to upgrade networking
if kube_upgrade_obj.state not in [
kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
kubernetes.KUBE_PRE_UPDATED_APPS,
kubernetes.KUBE_UPGRADING_NETWORKING_FAILED]:
raise wsme.exc.ClientSideError(_(
"Kubernetes upgrade must be in %s or %s state to "
"upgrade networking" %
(kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
(kubernetes.KUBE_PRE_UPDATED_APPS,
kubernetes.KUBE_UPGRADING_NETWORKING_FAILED)))
# Update the upgrade state
@ -438,7 +551,11 @@ class KubeUpgradeController(rest.RestController):
"in %s" % system.system_mode))
if kube_upgrade_obj.state in [kubernetes.KUBE_UPGRADE_ABORTING,
kubernetes.KUBE_UPGRADE_ABORTED,
kubernetes.KUBE_UPGRADE_COMPLETE]:
kubernetes.KUBE_UPGRADE_COMPLETE,
kubernetes.KUBE_POST_UPDATING_APPS,
kubernetes.KUBE_POST_UPDATING_APPS_FAILED,
kubernetes.KUBE_POST_UPDATED_APPS
]:
raise wsme.exc.ClientSideError(_(
"Cannot abort the kubernetes upgrade it is in %s state" %
(kube_upgrade_obj.state)))
@ -461,12 +578,13 @@ class KubeUpgradeController(rest.RestController):
# Update the state as aborted for these states since no actual k8s changes done
# so we don't need to do anything more to complete the abort.
if kube_upgrade_obj.state in [kubernetes.KUBE_UPGRADE_STARTING,
kubernetes.KUBE_UPGRADE_STARTING_FAILED,
kubernetes.KUBE_UPGRADE_STARTED,
if kube_upgrade_obj.state in [kubernetes.KUBE_UPGRADE_STARTED,
kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES,
kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED,
kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES]:
kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
kubernetes.KUBE_PRE_UPDATING_APPS,
kubernetes.KUBE_PRE_UPDATING_APPS_FAILED,
kubernetes.KUBE_PRE_UPDATED_APPS]:
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_ABORTED
kube_upgrade_obj.save()
else:
@ -627,12 +745,6 @@ class KubeUpgradeController(rest.RestController):
if role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER:
dc_api.notify_dcmanager_kubernetes_upgrade_completed()
# Update apps that contain 'k8s_upgrade.timing = post' metadata
pecan.request.rpcapi.update_apps_based_on_k8s_version_async(
pecan.request.context,
kube_upgrade_obj.to_version,
constants.APP_METADATA_TIMING_POST)
# Check if apps need to be reapplied
pecan.request.rpcapi.evaluate_apps_reapply(
pecan.request.context,
@ -640,6 +752,27 @@ class KubeUpgradeController(rest.RestController):
return KubeUpgrade.convert_with_links(kube_upgrade_obj)
elif updates['state'] == kubernetes.KUBE_POST_UPDATING_APPS:
# Make sure upgrade is in the correct state to update apps
if kube_upgrade_obj.state not in [
kubernetes.KUBE_UPGRADE_COMPLETE,
kubernetes.KUBE_POST_UPDATING_APPS_FAILED]:
raise wsme.exc.ClientSideError(_(
"Kubernetes upgrade must be in %s or %s state to update applications" %
(kubernetes.KUBE_UPGRADE_COMPLETE,
kubernetes.KUBE_POST_UPDATING_APPS_FAILED)))
# Update the upgrade state
kube_upgrade_obj.state = kubernetes.KUBE_POST_UPDATING_APPS
kube_upgrade_obj.save()
# Update apps that contain 'k8s_upgrade.timing = post' metadata
pecan.request.rpcapi.kube_post_application_update(pecan.request.context,
kube_upgrade_obj.to_version)
LOG.info("Updating applications to match current Kubernetes version %s" %
kube_upgrade_obj.to_version)
return KubeUpgrade.convert_with_links(kube_upgrade_obj)
else:
raise wsme.exc.ClientSideError(_(
"Invalid state %s supplied" % updates['state']))
@ -656,12 +789,16 @@ class KubeUpgradeController(rest.RestController):
raise wsme.exc.ClientSideError(_(
"A kubernetes upgrade is not in progress"))
if kube_upgrade_obj.state not in [kubernetes.KUBE_UPGRADE_COMPLETE,
kubernetes.KUBE_UPGRADE_ABORTED]:
kubernetes.KUBE_POST_UPDATING_APPS_FAILED,
kubernetes.KUBE_POST_UPDATED_APPS,
kubernetes.KUBE_UPGRADE_ABORTED]:
# The upgrade must be in complete or abort state to delete
raise wsme.exc.ClientSideError(_(
"Kubernetes upgrade must be in %s or %s state to delete" %
"Kubernetes upgrade must be in %s, %s, %s or %s state to delete" %
(kubernetes.KUBE_UPGRADE_COMPLETE,
kubernetes.KUBE_UPGRADE_ABORTED)))
kubernetes.KUBE_POST_UPDATING_APPS_FAILED,
kubernetes.KUBE_POST_UPDATED_APPS,
kubernetes.KUBE_UPGRADE_ABORTED)))
# Clean up k8s control-plane backup
pecan.request.rpcapi.remove_kube_control_plane_backup(

8
sysinv/sysinv/sysinv/sysinv/common/kubernetes.py
View File

@ -78,12 +78,16 @@ KUBE_CONTROLLER_MANAGER = 'kube-controller-manager'
KUBE_SCHEDULER = 'kube-scheduler'
# Kubernetes upgrade states
KUBE_UPGRADE_STARTING = 'upgrade-starting'
KUBE_UPGRADE_STARTING_FAILED = 'upgrade-starting-failed'
KUBE_UPGRADE_STARTED = 'upgrade-started'
KUBE_UPGRADE_DOWNLOADING_IMAGES = 'downloading-images'
KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED = 'downloading-images-failed'
KUBE_UPGRADE_DOWNLOADED_IMAGES = 'downloaded-images'
KUBE_PRE_UPDATING_APPS = 'pre-updating-apps'
KUBE_PRE_UPDATING_APPS_FAILED = 'pre-updating-apps-failed'
KUBE_PRE_UPDATED_APPS = 'pre-updated-apps'
KUBE_POST_UPDATING_APPS = 'post-updating-apps'
KUBE_POST_UPDATING_APPS_FAILED = 'post-updating-apps-failed'
KUBE_POST_UPDATED_APPS = 'post-updated-apps'
KUBE_UPGRADING_NETWORKING = 'upgrading-networking'
KUBE_UPGRADING_NETWORKING_FAILED = 'upgrading-networking-failed'
KUBE_UPGRADED_NETWORKING = 'upgraded-networking'

309
sysinv/sysinv/sysinv/sysinv/conductor/manager.py
View File

@ -7618,9 +7618,45 @@ class ConductorManager(service.PeriodicService):
self._update_image_conversion_alarm(fm_constants.FM_ALARM_STATE_CLEAR,
constants.FILESYSTEM_NAME_IMAGE_CONVERSION)
def _auto_upload_managed_app(self, context, app_name):
def _auto_upload_managed_app(self,
context,
app_name,
k8s_version=None,
k8s_upgrade_timing=None,
async_upload=True):
""" Automatically upload managed applications.
:param context: Context of the request.
:param app_name: Name of the application to be uploaded.
:param k8s_version: Kubernetes target version.
:param k8s_upgrade_timing: When applications should be uploaded.
:param async_upload: Upload asynchronously if True. Upload synchronously if False.
:return: True if the upload successfully started when running asynchronously.
True if the app was successfully uploaded when running synchronously.
False if an error has occurred.
None if there is not an upload version available for the given app.
"""
if self._patching_operation_is_occurring():
return
return False
# Delete current uploaded version if a newer one is available
try:
existing_app = kubeapp_obj.get_by_name(context, app_name)
app_bundle = self._get_app_bundle_for_update(existing_app, k8s_version, k8s_upgrade_timing)
if app_bundle:
hook_info_delete = LifecycleHookInfo()
hook_info_delete.mode = constants.APP_LIFECYCLE_MODE_AUTO
self.perform_app_delete(context, existing_app, hook_info_delete)
else:
LOG.debug("No bundle found for uploading a new version of %s" % app_name)
return
except exception.KubeAppNotFound:
pass
except Exception as e:
LOG.exception("Failed to delete app {} during automatic upload: {}"
.format(app_name, e))
return False
LOG.info("Platform managed application %s: Creating..." % app_name)
app_data = {'name': app_name,
@ -7628,46 +7664,65 @@ class ConductorManager(service.PeriodicService):
'manifest_name': constants.APP_MANIFEST_NAME_PLACEHOLDER,
'manifest_file': constants.APP_TARFILE_NAME_PLACEHOLDER,
'status': constants.APP_UPLOAD_IN_PROGRESS}
try:
self.dbapi.kube_app_create(app_data)
app = kubeapp_obj.get_by_name(context, app_name)
app_bundle = self._get_app_bundle_for_update(app, k8s_version, k8s_upgrade_timing)
if app_bundle is None:
# Skip if no bundles are found
LOG.debug("No bundle found for uploading %s" % app_name)
return
tarball = self._check_tarfile(app_name, app_bundle.file_path)
if ((tarball.manifest_name is None) or
(tarball.manifest_file is None)):
app.status = constants.APP_UPLOAD_FAILURE
app.save()
return False
app.name = tarball.app_name
app.app_version = tarball.app_version
app.manifest_name = tarball.manifest_name
app.manifest_file = os.path.basename(tarball.manifest_file)
app.save()
# Action: Upload.
# Do not block this audit task or any other periodic task. This
# could be long running. The next audit cycle will pick up the
# latest status.
LOG.info("Platform managed application %s: "
"Uploading..." % app_name)
hook_info = LifecycleHookInfo()
hook_info.mode = constants.APP_LIFECYCLE_MODE_AUTO
if async_upload:
greenthread.spawn(self.perform_app_upload,
context,
app,
tarball.tarball_name,
hook_info)
else:
self.perform_app_upload(context,
app,
tarball.tarball_name,
hook_info)
except exception.KubeAppAlreadyExists as e:
LOG.exception(e)
return
return False
except exception.KubeAppNotFound as e:
LOG.exception(e)
return
tarfile = self._search_tarfile(app_name, managed_app=True)
if tarfile is None:
# Skip if no tarball or multiple tarballs found
return
tarball = self._check_tarfile(app_name, tarfile)
if ((tarball.manifest_name is None) or
(tarball.manifest_file is None)):
app.status = constants.APP_UPLOAD_FAILURE
app.save()
return
app.name = tarball.app_name
app.app_version = tarball.app_version
app.manifest_name = tarball.manifest_name
app.manifest_file = os.path.basename(tarball.manifest_file)
app.save()
# Action: Upload.
# Do not block this audit task or any other periodic task. This
# could be long running. The next audit cycle will pick up the
# latest status.
LOG.info("Platform managed application %s: "
"Uploading..." % app_name)
hook_info = LifecycleHookInfo()
hook_info.mode = constants.APP_LIFECYCLE_MODE_AUTO
greenthread.spawn(self.perform_app_upload, context,
app, tarball.tarball_name, hook_info)
return False
except Exception as e:
if k8s_version:
LOG.exception("App {} automatic upload to match Kubernetes version {} "
"failed with: {}".format(app.name, k8s_version, e))
else:
LOG.exception("App {} automatic upload {} failed with: {}"
.format(app.name, k8s_version, e))
return False
def _auto_apply_managed_app(self, context, app_name):
try:
@ -7702,7 +7757,7 @@ class ConductorManager(service.PeriodicService):
self._inner_sync_auto_apply(context, app_name)
def update_apps_based_on_k8s_version_sync(self, context, k8s_version, k8s_upgrade_timing):
def update_apps_based_on_k8s_version(self, context, k8s_version, k8s_upgrade_timing):
""" Update applications based on a given Kubernetes version (blocking).
:param context: Context of the request
@ -7730,10 +7785,12 @@ class ConductorManager(service.PeriodicService):
except exception.KubeAppNotFound:
continue
# Apps should be either in 'applied' or 'apply-failure' state.
# Apps should be either in 'applied' or 'apply-failure' state to be updated.
# Applied apps are selected to be updated since they are currently in use.
# If the app is in 'apply-failure' state we give it a chance to be
# successfully applied via the update process.
# If a newer compatible version of an app in 'uploaded' or 'uploaded-failed' state
# is available then the current version is removed and the new one is uploaded.
if (app.status == constants.APP_APPLY_SUCCESS or
app.status == constants.APP_APPLY_FAILURE):
threads[app.name] = threadpool.spawn(self._auto_update_app,
@ -7742,6 +7799,14 @@ class ConductorManager(service.PeriodicService):
k8s_version,
k8s_upgrade_timing,
async_update=False)
elif (app.status == constants.APP_UPLOAD_SUCCESS or
app.status == constants.APP_UPLOAD_FAILURE):
threads[app.name] = threadpool.spawn(self._auto_upload_managed_app,
context,
app_name,
k8s_version,
k8s_upgrade_timing,
async_upload=False)
# Wait for all updates to finish
threadpool.waitall()
@ -7755,38 +7820,6 @@ class ConductorManager(service.PeriodicService):
return result
def update_apps_based_on_k8s_version_async(self, context, k8s_version, k8s_upgrade_timing):
""" Update applications based on a given Kubernetes version (non-blocking).
:param context: Context of the request
:param k8s_version: Kubernetes target version.
:param k8s_upgrade_timing: When applications should be updated.
"""
update_candidates = [app_name for app_name in
self.apps_metadata[constants.APP_METADATA_APPS].keys()]
LOG.info("Checking available application updates for Kubernetes version {}."
.format(k8s_version))
for app_name in update_candidates:
try:
app = kubeapp_obj.get_by_name(context, app_name)
except exception.KubeAppNotFound:
continue
# Apps should be either in 'applied' or 'apply-failure' state.
# Applied apps are selected to be updated since they are currently in use.
# If the app is in 'apply-failure' state we give it a chance to be
# successfully applied via the update process.
if (app.status == constants.APP_APPLY_SUCCESS or
app.status == constants.APP_APPLY_FAILURE):
if self._auto_update_app(context,
app_name,
k8s_version,
k8s_upgrade_timing) is False:
LOG.error("Failed to update {} to match Kubernetes version {}"
.format(app_name, k8s_version))
def _get_app_bundle_for_update(self, app, k8s_version=None, k8s_upgrade_timing=None):
""" Retrieve metadata from the most updated application bundle
that can be used to update the given app.
@ -7833,11 +7866,13 @@ class ConductorManager(service.PeriodicService):
.format(k8s_version,
bundle_metadata.k8s_maximum_version,
bundle_metadata.file_path))
elif LooseVersion(bundle_metadata.version) == LooseVersion(app.app_version):
elif (app.app_version != constants.APP_VERSION_PLACEHOLDER and
LooseVersion(bundle_metadata.version) == LooseVersion(app.app_version)):
LOG.debug("Bundle {} version and installed app version are the same ({})"
.format(bundle_metadata.file_path,
app.app_version))
elif LooseVersion(bundle_metadata.version) < LooseVersion(app.app_version):
elif (app.app_version != constants.APP_VERSION_PLACEHOLDER and
LooseVersion(bundle_metadata.version) < LooseVersion(app.app_version)):
LOG.debug("Bundle {} version {} is lower than installed app version ({})"
.format(bundle_metadata.file_path,
bundle_metadata.version,
@ -7859,7 +7894,8 @@ class ConductorManager(service.PeriodicService):
# bundle is available instead.
if (auto_downgrade and
app.app_version not in available_versions and
latest_downgrade_bundle is not None):
latest_downgrade_bundle is not None and
k8s_upgrade_timing is None):
LOG.info("Application {} will be downgraded from version {} to {}"
.format(app.name, app.app_version, latest_downgrade_bundle.version))
return latest_downgrade_bundle
@ -8462,11 +8498,12 @@ class ConductorManager(service.PeriodicService):
# Skip kubernetes labels audit when K8S upgrade is in progress.
# The kube-apiserver will not be available during kube-upgrade-abort operation.
# Kubernetes upgrade may be completed but apps still need to be post updated.
try:
self.verify_k8s_upgrade_not_in_progress()
except Exception:
LOG.info("k8s Upgrade in progress - _k8s_application_audit skip "
"activity")
self.verify_k8s_app_upgrade_is_completed()
except Exception as e:
LOG.info("_k8s_application_audit skip activity: {}".format(str(e)))
return
if self._verify_restore_in_progress():
@ -10621,6 +10658,22 @@ class ConductorManager(service.PeriodicService):
raise exception.SysinvException(_(
"Kubernetes upgrade is in progress and not completed."))
def verify_k8s_app_upgrade_is_completed(self):
""" Check if application update steps have finished during a k8s upgrade
Raise an exception if the final update step (post-updated-apps) hasn't
been reached.
"""
try:
kube_upgrade = self.dbapi.kube_upgrade_get_one()
if kube_upgrade.state == kubernetes.KUBE_POST_UPDATED_APPS:
return
except exception.NotFound:
pass
else:
raise exception.SysinvException(_(
"Application post update not completed for the existing k8s upgrade"))
def verify_upgrade_not_in_progress(self):
""" Check if there is an upgrade in progress.
@ -16234,8 +16287,10 @@ class ConductorManager(service.PeriodicService):
"""
# Defer apps reapply evaluation if Kubernetes upgrades are in progress
# or if apps are still post updating.
try:
self.verify_k8s_upgrade_not_in_progress()
self.verify_k8s_app_upgrade_is_completed()
except Exception as e:
LOG.info("Deferring apps reapply evaluation. {}".format(str(e)))
return
@ -16615,6 +16670,7 @@ class ConductorManager(service.PeriodicService):
lifecycle_hook_info_app_upload)
except Exception as e:
LOG.error("Error performing app_lifecycle_actions %s" % str(e))
return False
def perform_app_apply(self, context, rpc_app, mode, lifecycle_hook_info_app_apply):
"""Handling of application install request (via AppOperator)
@ -16939,6 +16995,18 @@ class ConductorManager(service.PeriodicService):
LOG.info("Successfully completed k8s control plane backup.")
def _check_app_kube_compatibility(self, app, kube_version):
"""Checks if an application is compatible with a kubernetes version
:param app: Application object
:param kube_version: Kubernetes version
"""
kube_min_version, kube_max_version = \
cutils.get_app_supported_kube_version(app.name, app.app_version)
return kubernetes.is_kube_version_supported(
kube_version, kube_min_version, kube_max_version)
def _check_installed_apps_compatibility(self, kube_version):
"""Checks whether all installed applications are compatible
with the new k8s version
@ -16956,11 +17024,7 @@ class ConductorManager(service.PeriodicService):
if app.status != constants.APP_APPLY_SUCCESS:
continue
kube_min_version, kube_max_version = \
cutils.get_app_supported_kube_version(app.name, app.app_version)
if not kubernetes.is_kube_version_supported(
kube_version, kube_min_version, kube_max_version):
if not self._check_app_kube_compatibility(app, kube_version):
LOG.error("The installed Application {} ({}) is incompatible with the "
"new Kubernetes version {}.".format(app.name,
app.app_version,
@ -16969,30 +17033,6 @@ class ConductorManager(service.PeriodicService):
return success
def kube_upgrade_start(self, context, k8s_version):
""" Start a Kubernetes upgrade by updating all required apps.
:param context: Context of the request.
:param k8s_version: Kubernetes target version.
:param k8s_upgrade_timing: When apps should be updated.
"""
kube_upgrade_obj = objects.kube_upgrade.get_one(context)
if (self.update_apps_based_on_k8s_version_sync(context,
k8s_version,
constants.APP_METADATA_TIMING_PRE) and
self._check_installed_apps_compatibility(k8s_version)):
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_STARTED
LOG.info("Started kubernetes upgrade from version: %s to version: %s"
% (kube_upgrade_obj.from_version, kube_upgrade_obj.to_version))
else:
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_STARTING_FAILED
LOG.info("Failed to start kubernetes upgrade from version: %s to version: %s"
% (kube_upgrade_obj.from_version, kube_upgrade_obj.to_version))
kube_upgrade_obj.save()
def kube_download_images(self, context, kube_version):
"""Download the kubernetes images for this version"""
@ -17082,6 +17122,42 @@ class ConductorManager(service.PeriodicService):
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES
kube_upgrade_obj.save()
def kube_application_update(self, context, timing, success_state, failure_state):
""" Generic method to update applications during Kubernetes upgrade
:param context: Context of the request.
"""
kube_upgrade_obj = objects.kube_upgrade.get_one(context)
# Update all apps that are compatible with the target k8s version.
# Check for compatibility after updating since an app update may fail
# and be reverted to a previous incompatible version.
if (self.update_apps_based_on_k8s_version(context,
kube_upgrade_obj.to_version,
timing) and
self._check_installed_apps_compatibility(kube_upgrade_obj.to_version)):
kube_upgrade_obj.state = success_state
LOG.info("Applications updated to match Kubernetes version %s."
% (kube_upgrade_obj.to_version))
else:
kube_upgrade_obj.state = failure_state
LOG.info("Failed to update applications to match Kubernetes version %s."
% (kube_upgrade_obj.to_version))
kube_upgrade_obj.save()
def kube_pre_application_update(self, context):
""" Update applications before Kubernetes is upgraded.
:param context: Context of the request.
"""
self.kube_application_update(context,
constants.APP_METADATA_TIMING_PRE,
kubernetes.KUBE_PRE_UPDATED_APPS,
kubernetes.KUBE_PRE_UPDATING_APPS_FAILED)
def kube_host_cordon(self, context, host_name):
"""Cordon the pods to evict on this host"""
@ -17400,6 +17476,31 @@ class ConductorManager(service.PeriodicService):
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADED_STORAGE
kube_upgrade_obj.save()
def kube_post_application_update(self, context, k8s_version):
""" Update applications after Kubernetes is upgraded.
:param context: Context of the request.
:param k8s_version: Target Kubernetes version
"""
self.kube_application_update(context,
constants.APP_METADATA_TIMING_POST,
kubernetes.KUBE_POST_UPDATED_APPS,
kubernetes.KUBE_POST_UPDATING_APPS_FAILED)
# Remove remaining uploaded apps that are not compatible with the new
# Kubernetes version
apps = self.dbapi.kube_app_get_all()
for app in apps:
if app.status != constants.APP_UPLOAD_SUCCESS and \
app.status != constants.APP_UPLOAD_FAILURE:
continue
if not self._check_app_kube_compatibility(app, k8s_version):
hook_info_delete = LifecycleHookInfo()
hook_info_delete.mode = constants.APP_LIFECYCLE_MODE_AUTO
self.perform_app_delete(context, app, hook_info_delete)
def kube_upgrade_abort(self, context, kube_state):
"""
This is an abort procedure we call via 'system kube-upgrade-abort'

34
sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py
View File

@ -1707,32 +1707,24 @@ class ConductorAPI(sysinv.openstack.common.rpc.proxy.RpcProxy):
return self.call(context, self.make_msg('get_fernet_keys',
key_id=key_id))
def kube_upgrade_start(self, context, k8s_version):
"""Asynchronously, start a Kubernetes upgrade to the given version.
def kube_pre_application_update(self, context):
"""Asynchronously, update applications before Kubernetes is upgraded.
:param context: Context of the request
:param k8s_version: Kubernetes target version
:param k8s_upgrade_timing: When applications should be updated
:param context: Context of the request.
"""
return self.cast(context, self.make_msg('kube_upgrade_start',
return self.cast(context, self.make_msg('kube_pre_application_update'))
def kube_post_application_update(self, context, k8s_version):
"""Asynchronously, update applications after Kubernetes is upgraded.
:param context: Context of the request.
:param k8s_version: Target Kubernetes version.
"""
return self.cast(context, self.make_msg('kube_post_application_update',
k8s_version=k8s_version))
def update_apps_based_on_k8s_version_async(self,
context,
k8s_version,
k8s_upgrade_timing):
"""Asynchronously, update all applications based on a given Kubernetes version.
:param context: Context of the request
:param k8s_version: Kubernetes target version
:param k8s_upgrade_timing: When applications should be updated
"""
return self.cast(context, self.make_msg('update_apps_based_on_k8s_version_async',
k8s_version=k8s_version,
k8s_upgrade_timing=k8s_upgrade_timing))
def evaluate_apps_reapply(self, context, trigger):
"""Synchronously, determine whether an application
re-apply is needed, and if so, raise the re-apply flag.

35
sysinv/sysinv/sysinv/sysinv/db/api.py
View File

@ -5134,9 +5134,9 @@ class Connection(object):
:param name: Application name.
:param k8s_auto_update: Whether automatically updating the application
is enabled when upgrading Kubernetes.
:param timing: Application update timing during Kubernetes upgrade
"pre": during kube-upgrade-start.
"post": during kube-upgrade-complete.
:param timing: Application update timing during Kubernetes upgrade.
"pre": before upgrading Kubernetes.
"post": after upgrading Kubernetes.
:param limit: Maximum number of entries to return.
:param marker: The last item of the previous page; we return the next
result set.
@ -5155,6 +5155,35 @@ class Connection(object):
def kube_app_bundle_destroy_by_file_path(self, file_path):
"""Delete records from kube_app_bundle that match a file path"""
@abc.abstractmethod
def kube_app_bundle_is_k8s_compatible(self,
name, k8s_timing,
target_k8s_version, current_k8s_version=None):
"""Check if a given application has bundles compatible with current
and target Kubernetes versions.
:param name: Application name.
:param timing: Application update timing during Kubernetes upgrade
"pre": before upgrading Kubernetes.
"post": after upgrading Kubernetes.
:param target_k8s_version: Kubernetes version that is going to be installed.
:param current_k8s_version: Kubernetes version that is currently running (optional).
:returns: True if app is compatible. False otherwise.
"""
@abc.abstractmethod
def kube_app_bundle_max_k8s_compatible_by_name(self,
name,
current_k8s_version,
target_k8s_version):
""" Get the maximum compatible Kubernetes version for a given app
:param name: Application name.
:param current_k8s_version: Kubernetes version that is currently running.
:param target_k8s_version: Kubernetes version that is going to be installed.
:returns: maximum compatible Kubernetes version
"""
@abc.abstractmethod
def address_get_by_name_and_family(self, name, family):
""" Search database address using name and family

47
sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py
View File

@ -27,6 +27,7 @@ from oslo_db import exception as db_exc
from oslo_db.sqlalchemy import enginefacade
from oslo_db.sqlalchemy import utils as db_utils
from sqlalchemy import func
from sqlalchemy import insert
from sqlalchemy import inspect
from sqlalchemy import or_
@ -9620,3 +9621,49 @@ class Connection(api.Connection):
def kube_app_bundle_destroy_by_file_path(self, file_path):
self.kube_app_bundle_destroy_all(file_path)
def kube_app_bundle_is_k8s_compatible(self,
name, k8s_timing,
target_k8s_version, current_k8s_version=None):
# Stacking filters for better readability
target_k8s_version = target_k8s_version.strip().lstrip('v')
query = model_query(models.KubeAppBundle)
query = query.filter_by(name=name,
auto_update=True,
k8s_auto_update=True,
k8s_timing=k8s_timing)
query = query.filter(models.KubeAppBundle.k8s_minimum_version <= target_k8s_version)
query = query.filter(or_(models.KubeAppBundle.k8s_maximum_version >= target_k8s_version,
models.KubeAppBundle.k8s_maximum_version.is_(None)))
if current_k8s_version:
current_k8s_version = current_k8s_version.strip().lstrip('v')
query = query.filter(models.KubeAppBundle.k8s_minimum_version <= current_k8s_version)
query = query.filter(or_(models.KubeAppBundle.k8s_maximum_version >= current_k8s_version,
models.KubeAppBundle.k8s_maximum_version.is_(None)))
return query.count() > 0
def kube_app_bundle_max_k8s_compatible_by_name(self,
name,
current_k8s_version,
target_k8s_version):
# Stacking filters for better readability
current_k8s_version = current_k8s_version.strip().lstrip('v')
target_k8s_version = target_k8s_version.strip().lstrip('v')
query = model_query(func.max(models.KubeAppBundle.k8s_maximum_version))
query = query.filter_by(name=name, auto_update=True, k8s_auto_update=True)
query = query.filter(or_(
models.KubeAppBundle.k8s_timing == constants.APP_METADATA_TIMING_POST,
models.KubeAppBundle.k8s_minimum_version == current_k8s_version))
query = query.filter(models.KubeAppBundle.k8s_maximum_version <= target_k8s_version)
try:
result = query.one()[0]
except NoResultFound:
result = None
except MultipleResultsFound:
raise exception.MultipleResults()
return result

222
sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py
View File

@ -72,11 +72,11 @@ class FakeFmClient(object):
class FakeConductorAPI(object):
def __init__(self):
self.kube_upgrade_start = mock.MagicMock()
self.kube_pre_application_update = mock.MagicMock()
self.kube_download_images = mock.MagicMock()
self.kube_upgrade_networking = mock.MagicMock()
self.kube_post_application_update = mock.MagicMock()
self.kube_upgrade_abort = mock.MagicMock()
self.update_apps_based_on_k8s_version_async = mock.MagicMock()
self.evaluate_apps_reapply = mock.MagicMock()
self.remove_kube_control_plane_backup = mock.MagicMock()
self.kube_delete_container_images = mock.MagicMock()
@ -274,15 +274,11 @@ class TestPostKubeUpgradeSimplex(TestKubeUpgrade,
result = self.post_json('/kube_upgrade', create_dict,
headers={'User-Agent': 'sysinv-test'})
# Verify that the upgrade was started
self.fake_conductor_api.kube_upgrade_start.\
assert_called_with(mock.ANY, 'v1.43.3')
# Verify that the upgrade has the expected attributes
self.assertEqual(result.json['from_version'], 'v1.42.1')
self.assertEqual(result.json['to_version'], 'v1.43.3')
self.assertEqual(result.json['state'],
kubernetes.KUBE_UPGRADE_STARTING)
kubernetes.KUBE_UPGRADE_STARTED)
# see if kubeadm_version was changed in DB
kube_cmd_version = self.dbapi.kube_cmd_version_get()
@ -380,15 +376,11 @@ class TestPostKubeUpgrade(TestKubeUpgrade,
result = self.post_json('/kube_upgrade', create_dict,
headers={'User-Agent': 'sysinv-test'})
# Verify that the upgrade was started
self.fake_conductor_api.kube_upgrade_start.\
assert_called_with(mock.ANY, 'v1.43.2')
# Verify that the upgrade has the expected attributes
self.assertEqual(result.json['from_version'], 'v1.43.1')
self.assertEqual(result.json['to_version