Allow recovery from failures during kubernetes upgrade
Allow the user to recover from failures at any stage of a kubernetes upgrade. This includes the following changes: - Add new failure states to denote the action that has failed. - Add failure handling code to allow recovery from the new failure states. - Add a new CLI and state for downloading images to allow recovery if this step fails. - Add a function that is called each time the sysinv conductor starts and fails any kubernetes upgrade operation that is in a transitory state and will not complete. Change-Id: Ib48f84a66dc9260c63fdb427c0b8fed877370474 Story: 2006781 Task: 37582 Signed-off-by: Bart Wensley <barton.wensley@windriver.com>
This commit is contained in:
Bart Wensley
parent
5af9f695c3
commit
7787aaf399
|
|
@ -97,6 +97,31 @@ class KubeUpgradeTest(test_shell.ShellTest):
|
|||
self.assertIn(fake_kube_upgrade['created_at'], results)
|
||||
self.assertIn(fake_kube_upgrade['updated_at'], results)
|
||||
|
||||
@mock.patch('cgtsclient.v1.kube_upgrade.KubeUpgradeManager.update')
|
||||
@mock.patch('cgtsclient.client._get_ksclient')
|
||||
@mock.patch('cgtsclient.client._get_endpoint')
|
||||
def test_kube_upgrade_download_images(self, mock_get_endpoint,
|
||||
mock_get_client,
|
||||
mock_update):
|
||||
mock_get_endpoint.return_value = 'http://fakelocalhost:6385/v1'
|
||||
fake_kube_upgrade = {'from_version': 'v1.42.1',
|
||||
'to_version': 'v1.42.2',
|
||||
'state': 'downloading-images',
|
||||
'uuid': 'cb737aba-1820-4184-b0dc-9b073822af48',
|
||||
'created_at': 'fake-created-time',
|
||||
'updated_at': 'fake-updated-time',
|
||||
}
|
||||
mock_update.return_value = KubeUpgrade(None, fake_kube_upgrade, True)
|
||||
|
||||
self.make_env()
|
||||
results = self.shell("kube-upgrade-download-images")
|
||||
self.assertIn(fake_kube_upgrade['from_version'], results)
|
||||
self.assertIn(fake_kube_upgrade['to_version'], results)
|
||||
self.assertIn(fake_kube_upgrade['state'], results)
|
||||
self.assertIn(fake_kube_upgrade['uuid'], results)
|
||||
self.assertIn(fake_kube_upgrade['created_at'], results)
|
||||
self.assertIn(fake_kube_upgrade['updated_at'], results)
|
||||
|
||||
@mock.patch('cgtsclient.v1.kube_upgrade.KubeUpgradeManager.update')
|
||||
@mock.patch('cgtsclient.client._get_ksclient')
|
||||
@mock.patch('cgtsclient.client._get_endpoint')
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from cgtsclient.common import utils
|
|||
from cgtsclient import exc
|
||||
|
||||
# Kubernetes constants
|
||||
KUBE_UPGRADE_STATE_DOWNLOADING_IMAGES = 'downloading-images'
|
||||
KUBE_UPGRADE_STATE_UPGRADING_NETWORKING = 'upgrading-networking'
|
||||
KUBE_UPGRADE_STATE_COMPLETE = 'upgrade-complete'
|
||||
|
||||
|
|
@ -48,6 +49,23 @@ def do_kube_upgrade_start(cc, args):
|
|||
_print_kube_upgrade_show(kube_upgrade)
|
||||
|
||||
|
||||
def do_kube_upgrade_download_images(cc, args):
|
||||
"""Download kubernetes images."""
|
||||
|
||||
data = dict()
|
||||
data['state'] = KUBE_UPGRADE_STATE_DOWNLOADING_IMAGES
|
||||
|
||||
patch = []
|
||||
for (k, v) in data.items():
|
||||
patch.append({'op': 'replace', 'path': '/' + k, 'value': v})
|
||||
try:
|
||||
kube_upgrade = cc.kube_upgrade.update(patch)
|
||||
except exc.HTTPNotFound:
|
||||
raise exc.CommandError('Kubernetes upgrade UUID not found')
|
||||
|
||||
_print_kube_upgrade_show(kube_upgrade)
|
||||
|
||||
|
||||
def do_kube_upgrade_networking(cc, args):
|
||||
"""Upgrade kubernetes networking."""
|
||||
|
||||
|
|
|
|||
|
|
@ -6480,12 +6480,45 @@ class HostController(rest.RestController):
|
|||
hostupdate.ihost_val_prenotify_update(val)
|
||||
hostupdate.ihost_val.update(val)
|
||||
|
||||
@staticmethod
|
||||
def _check_patch_requirements(region_name,
|
||||
applied_patches=None,
|
||||
available_patches=None):
|
||||
"""Checks whether specified patches are applied or available"""
|
||||
|
||||
api_token = None
|
||||
if applied_patches is not None:
|
||||
patches_applied = patch_api.patch_is_applied(
|
||||
token=api_token,
|
||||
timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS,
|
||||
region_name=region_name,
|
||||
patches=applied_patches
|
||||
)
|
||||
if not patches_applied:
|
||||
raise wsme.exc.ClientSideError(_(
|
||||
"The following patches must be applied before doing "
|
||||
"this action: %s" % applied_patches))
|
||||
|
||||
if available_patches is not None:
|
||||
patches_available = patch_api.patch_is_available(
|
||||
token=api_token,
|
||||
timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS,
|
||||
region_name=region_name,
|
||||
patches=available_patches
|
||||
)
|
||||
if not patches_available:
|
||||
raise wsme.exc.ClientSideError(_(
|
||||
"The following patches must be available before doing "
|
||||
"this action: %s" % available_patches))
|
||||
|
||||
@cutils.synchronized(LOCK_NAME)
|
||||
@wsme_pecan.wsexpose(Host, six.text_type, body=six.text_type)
|
||||
def kube_upgrade_control_plane(self, uuid, body):
|
||||
"""Upgrade the kubernetes control plane on this host"""
|
||||
|
||||
host_obj = objects.host.get_by_uuid(pecan.request.context, uuid)
|
||||
kube_host_upgrade_obj = objects.kube_host_upgrade.get_by_host_id(
|
||||
pecan.request.context, host_obj.id)
|
||||
|
||||
# The kubernetes upgrade must have been started
|
||||
try:
|
||||
|
|
@ -6495,27 +6528,54 @@ class HostController(rest.RestController):
|
|||
raise wsme.exc.ClientSideError(_(
|
||||
"A kubernetes upgrade is not in progress."))
|
||||
|
||||
LOG.info("Attempting to upgrade control plane on host %s while in "
|
||||
"state %s" % (host_obj.hostname, kube_upgrade_obj.state))
|
||||
|
||||
# Either controller can be upgraded
|
||||
if host_obj.personality != constants.CONTROLLER:
|
||||
raise wsme.exc.ClientSideError(_(
|
||||
"This host does not have a kubernetes control plane."))
|
||||
|
||||
# Verify the upgrade is in the correct state
|
||||
if kube_upgrade_obj.state not in [kubernetes.KUBE_UPGRADE_STARTED,
|
||||
kubernetes.KUBE_UPGRADED_NETWORKING]:
|
||||
if kube_upgrade_obj.state in [
|
||||
kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
|
||||
kubernetes.KUBE_UPGRADED_NETWORKING,
|
||||
kubernetes.KUBE_UPGRADED_FIRST_MASTER]:
|
||||
# We are upgrading a control plane
|
||||
pass
|
||||
elif kube_upgrade_obj.state in [
|
||||
kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED,
|
||||
kubernetes.KUBE_UPGRADING_SECOND_MASTER_FAILED]:
|
||||
# We are re-attempting the upgrade of a control plane. Make sure
|
||||
# this really is a re-attempt.
|
||||
if kube_host_upgrade_obj.target_version != \
|
||||
kube_upgrade_obj.to_version:
|
||||
raise wsme.exc.ClientSideError(_(
|
||||
"The first control plane upgrade must be completed before "
|
||||
"upgrading the second control plane."))
|
||||
else:
|
||||
raise wsme.exc.ClientSideError(_(
|
||||
"The kubernetes upgrade must be in the %s or %s state to "
|
||||
"upgrade the control plane." % (
|
||||
kubernetes.KUBE_UPGRADE_STARTED,
|
||||
kubernetes.KUBE_UPGRADED_NETWORKING)))
|
||||
"The kubernetes upgrade is not in a valid state to "
|
||||
"upgrade the control plane."))
|
||||
|
||||
# Verify patching requirements (since the api server may not be
|
||||
# upgraded yet, patches could have been removed)
|
||||
system = pecan.request.dbapi.isystem_get_one()
|
||||
target_version_obj = objects.kube_version.get_by_version(
|
||||
kube_upgrade_obj.to_version)
|
||||
self._check_patch_requirements(
|
||||
system.region_name,
|
||||
applied_patches=target_version_obj.applied_patches,
|
||||
available_patches=target_version_obj.available_patches)
|
||||
|
||||
# Check the existing control plane version
|
||||
cp_versions = self._kube_operator.kube_get_control_plane_versions()
|
||||
current_cp_version = cp_versions.get(host_obj.hostname)
|
||||
if current_cp_version == kube_upgrade_obj.to_version:
|
||||
raise wsme.exc.ClientSideError(_(
|
||||
"The kubernetes control plane on this host was already "
|
||||
"upgraded."))
|
||||
# The control plane was already upgraded, but we didn't progress
|
||||
# to the next state, so something failed along the way.
|
||||
LOG.info("Redoing kubernetes control plane upgrade for %s" %
|
||||
host_obj.hostname)
|
||||
elif current_cp_version is None:
|
||||
raise wsme.exc.ClientSideError(_(
|
||||
"Unable to determine the version of the kubernetes control "
|
||||
|
|
@ -6528,29 +6588,39 @@ class HostController(rest.RestController):
|
|||
"The host must be unlocked and available to upgrade the "
|
||||
"control plane."))
|
||||
|
||||
# Verify the control plane is not being upgraded
|
||||
if kube_host_upgrade_obj.status == \
|
||||
kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE:
|
||||
raise wsme.exc.ClientSideError(_(
|
||||
"The control plane on this host is already being upgraded."))
|
||||
|
||||
# Update the target version for this host
|
||||
kube_host_upgrade_obj = objects.kube_host_upgrade.get_by_host_id(
|
||||
pecan.request.context, host_obj.id)
|
||||
kube_host_upgrade_obj.target_version = kube_upgrade_obj.to_version
|
||||
|
||||
# Set the status
|
||||
kube_host_upgrade_obj.status = \
|
||||
kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE
|
||||
kube_host_upgrade_obj.save()
|
||||
|
||||
if kube_upgrade_obj.state == kubernetes.KUBE_UPGRADE_STARTED:
|
||||
# Tell the conductor to upgrade the control plane
|
||||
pecan.request.rpcapi.kube_upgrade_control_plane(
|
||||
pecan.request.context, host_obj.uuid)
|
||||
|
||||
if kube_upgrade_obj.state in [
|
||||
kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
|
||||
kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED]:
|
||||
# Update the upgrade state
|
||||
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADING_FIRST_MASTER
|
||||
kube_upgrade_obj.save()
|
||||
else:
|
||||
|
||||
# Tell the conductor to upgrade the control plane
|
||||
pecan.request.rpcapi.kube_upgrade_control_plane(
|
||||
pecan.request.context, host_obj.uuid)
|
||||
|
||||
else:
|
||||
# Update the upgrade state
|
||||
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADING_SECOND_MASTER
|
||||
kube_upgrade_obj.save()
|
||||
|
||||
# Tell the conductor to upgrade the control plane
|
||||
pecan.request.rpcapi.kube_upgrade_control_plane(
|
||||
pecan.request.context, host_obj.uuid)
|
||||
|
||||
LOG.info("Upgrading kubernetes control plane on host %s" %
|
||||
host_obj.hostname)
|
||||
return Host.convert_with_links(host_obj)
|
||||
|
|
@ -6571,6 +6641,9 @@ class HostController(rest.RestController):
|
|||
raise wsme.exc.ClientSideError(_(
|
||||
"A kubernetes upgrade is not in progress."))
|
||||
|
||||
LOG.info("Attempting to upgrade kubelet on host %s while in "
|
||||
"state %s" % (host_obj.hostname, kube_upgrade_obj.state))
|
||||
|
||||
# Verify the host has a kubelet
|
||||
if host_obj.personality not in [constants.CONTROLLER,
|
||||
constants.WORKER]:
|
||||
|
|
@ -6579,11 +6652,14 @@ class HostController(rest.RestController):
|
|||
|
||||
# Verify the upgrade is in the correct state
|
||||
if utils.get_system_mode() == constants.SYSTEM_MODE_SIMPLEX:
|
||||
if kube_upgrade_obj.state != kubernetes.KUBE_UPGRADED_NETWORKING:
|
||||
if kube_upgrade_obj.state not in [
|
||||
kubernetes.KUBE_UPGRADED_NETWORKING,
|
||||
kubernetes.KUBE_UPGRADING_KUBELETS]:
|
||||
raise wsme.exc.ClientSideError(_(
|
||||
"The kubernetes upgrade must be in the %s state to "
|
||||
"upgrade the kubelet." %
|
||||
kubernetes.KUBE_UPGRADED_NETWORKING))
|
||||
"The kubernetes upgrade must be in the %s or %s state to "
|
||||
"upgrade the kubelet." % (
|
||||
kubernetes.KUBE_UPGRADED_NETWORKING,
|
||||
kubernetes.KUBE_UPGRADING_KUBELETS)))
|
||||
elif kube_upgrade_obj.state not in [
|
||||
kubernetes.KUBE_UPGRADED_SECOND_MASTER,
|
||||
kubernetes.KUBE_UPGRADING_KUBELETS]:
|
||||
|
|
@ -6593,22 +6669,13 @@ class HostController(rest.RestController):
|
|||
kubernetes.KUBE_UPGRADED_SECOND_MASTER,
|
||||
kubernetes.KUBE_UPGRADING_KUBELETS)))
|
||||
|
||||
# The necessary patches must be applied
|
||||
api_token = None
|
||||
# Verify patching requirements
|
||||
system = pecan.request.dbapi.isystem_get_one()
|
||||
target_version_obj = objects.kube_version.get_by_version(
|
||||
kube_upgrade_obj.to_version)
|
||||
if target_version_obj.available_patches:
|
||||
patches_applied = patch_api.patch_is_applied(
|
||||
token=api_token,
|
||||
timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS,
|
||||
region_name=system.region_name,
|
||||
patches=target_version_obj.available_patches
|
||||
)
|
||||
if not patches_applied:
|
||||
raise wsme.exc.ClientSideError(_(
|
||||
"The following patches must be applied before upgrading "
|
||||
"the kubelets: %s" % target_version_obj.available_patches))
|
||||
self._check_patch_requirements(
|
||||
system.region_name,
|
||||
applied_patches=target_version_obj.available_patches)
|
||||
|
||||
# Enforce the ordering of controllers first
|
||||
kubelet_versions = self._kube_operator.kube_get_kubelet_versions()
|
||||
|
|
@ -6630,7 +6697,7 @@ class HostController(rest.RestController):
|
|||
if current_kubelet_version == kube_upgrade_obj.to_version:
|
||||
# If the force option was used, we will redo the upgrade
|
||||
if force:
|
||||
LOG.info("Redoing kubernetes upgrade for %s" %
|
||||
LOG.info("Redoing kubelet upgrade for %s" %
|
||||
host_obj.hostname)
|
||||
else:
|
||||
raise wsme.exc.ClientSideError(_(
|
||||
|
|
@ -6652,10 +6719,19 @@ class HostController(rest.RestController):
|
|||
raise wsme.exc.ClientSideError(_(
|
||||
"The host must be locked and online to upgrade the kubelet."))
|
||||
|
||||
# Verify the kubelet is not being upgraded
|
||||
if kube_host_upgrade_obj.status == \
|
||||
kubernetes.KUBE_HOST_UPGRADING_KUBELET:
|
||||
raise wsme.exc.ClientSideError(_(
|
||||
"The kubelet on this host is already being upgraded."))
|
||||
|
||||
# Set the target version if this is a worker host
|
||||
if host_obj.personality == constants.WORKER:
|
||||
kube_host_upgrade_obj.target_version = kube_upgrade_obj.to_version
|
||||
kube_host_upgrade_obj.save()
|
||||
|
||||
# Set the status
|
||||
kube_host_upgrade_obj.status = kubernetes.KUBE_HOST_UPGRADING_KUBELET
|
||||
kube_host_upgrade_obj.save()
|
||||
|
||||
# Tell the conductor to upgrade the kubelet
|
||||
pecan.request.rpcapi.kube_upgrade_kubelet(pecan.request.context,
|
||||
|
|
|
|||
|
|
@ -116,6 +116,38 @@ class KubeUpgradeController(rest.RestController):
|
|||
updates[attribute] = p['value']
|
||||
return updates
|
||||
|
||||
@staticmethod
|
||||
def _check_patch_requirements(region_name,
|
||||
applied_patches=None,
|
||||
available_patches=None):
|
||||
"""Checks whether specified patches are applied or available"""
|
||||
|
||||
api_token = None
|
||||
if applied_patches:
|
||||
patches_applied = patch_api.patch_is_applied(
|
||||
token=api_token,
|
||||
timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS,
|
||||
region_name=region_name,
|
||||
patches=applied_patches
|
||||
)
|
||||
if not patches_applied:
|
||||
raise wsme.exc.ClientSideError(_(
|
||||
"The following patches must be applied before doing "
|
||||
"the kubernetes upgrade: %s" % applied_patches))
|
||||
|
||||
if available_patches:
|
||||
patches_available = patch_api.patch_is_available(
|
||||
token=api_token,
|
||||
timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS,
|
||||
region_name=region_name,
|
||||
patches=available_patches
|
||||
)
|
||||
if not patches_available:
|
||||
raise wsme.exc.ClientSideError(_(
|
||||
"The following patches must be available before doing "
|
||||
"the kubernetes upgrade: %s" %
|
||||
available_patches))
|
||||
|
||||
@wsme_pecan.wsexpose(KubeUpgradeCollection)
|
||||
def get_all(self):
|
||||
"""Retrieve a list of kubernetes upgrades."""
|
||||
|
|
@ -162,7 +194,7 @@ class KubeUpgradeController(rest.RestController):
|
|||
raise wsme.exc.ClientSideError(_(
|
||||
"The installed Kubernetes version %s cannot upgrade to "
|
||||
"version %s" % (current_kube_version,
|
||||
target_version_obj.version)))
|
||||
target_version_obj.version)))
|
||||
|
||||
# The current kubernetes version must be active
|
||||
version_states = self._kube_operator.kube_get_version_states()
|
||||
|
|
@ -172,35 +204,12 @@ class KubeUpgradeController(rest.RestController):
|
|||
"The installed Kubernetes version %s is not active on all "
|
||||
"hosts" % current_kube_version))
|
||||
|
||||
# The necessary patches must be applied
|
||||
api_token = None
|
||||
# Verify patching requirements
|
||||
system = pecan.request.dbapi.isystem_get_one()
|
||||
if target_version_obj.applied_patches:
|
||||
patches_applied = patch_api.patch_is_applied(
|
||||
token=api_token,
|
||||
timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS,
|
||||
region_name=system.region_name,
|
||||
patches=target_version_obj.applied_patches
|
||||
)
|
||||
if not patches_applied:
|
||||
raise wsme.exc.ClientSideError(_(
|
||||
"The following patches must be applied before starting "
|
||||
"the kubernetes upgrade: %s" %
|
||||
target_version_obj.applied_patches))
|
||||
|
||||
# The necessary patches must be available
|
||||
if target_version_obj.available_patches:
|
||||
patches_available = patch_api.patch_is_available(
|
||||
token=api_token,
|
||||
timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS,
|
||||
region_name=system.region_name,
|
||||
patches=target_version_obj.available_patches
|
||||
)
|
||||
if not patches_available:
|
||||
raise wsme.exc.ClientSideError(_(
|
||||
"The following patches must be available before starting "
|
||||
"the kubernetes upgrade: %s" %
|
||||
target_version_obj.available_patches))
|
||||
self._check_patch_requirements(
|
||||
system.region_name,
|
||||
applied_patches=target_version_obj.applied_patches,
|
||||
available_patches=target_version_obj.available_patches)
|
||||
|
||||
# TODO: check that all installed applications support new k8s version
|
||||
# TODO: check that tiller/armada support new k8s version
|
||||
|
|
@ -220,14 +229,10 @@ class KubeUpgradeController(rest.RestController):
|
|||
|
||||
# TODO: kubernetes related health checks...
|
||||
|
||||
# Tell the conductor to download the images for the new version
|
||||
pecan.request.rpcapi.kube_download_images(
|
||||
pecan.request.context, to_version)
|
||||
|
||||
# Create upgrade record.
|
||||
create_values = {'from_version': current_kube_version,
|
||||
'to_version': to_version,
|
||||
'state': kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES}
|
||||
'state': kubernetes.KUBE_UPGRADE_STARTED}
|
||||
new_upgrade = pecan.request.dbapi.kube_upgrade_create(create_values)
|
||||
|
||||
# Set the target version for each host to the current version
|
||||
|
|
@ -237,7 +242,7 @@ class KubeUpgradeController(rest.RestController):
|
|||
pecan.request.dbapi.kube_host_upgrade_update(kube_host_upgrade.id,
|
||||
update_values)
|
||||
|
||||
LOG.info("Starting kubernetes upgrade from version: %s to version: %s"
|
||||
LOG.info("Started kubernetes upgrade from version: %s to version: %s"
|
||||
% (current_kube_version, to_version))
|
||||
|
||||
return KubeUpgrade.convert_with_links(new_upgrade)
|
||||
|
|
@ -258,14 +263,49 @@ class KubeUpgradeController(rest.RestController):
|
|||
raise wsme.exc.ClientSideError(_(
|
||||
"A kubernetes upgrade is not in progress"))
|
||||
|
||||
if updates['state'] == kubernetes.KUBE_UPGRADING_NETWORKING:
|
||||
# Make sure upgrade is in the correct state to upgrade networking
|
||||
if kube_upgrade_obj.state != \
|
||||
kubernetes.KUBE_UPGRADED_FIRST_MASTER:
|
||||
if updates['state'] == kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES:
|
||||
# Make sure upgrade is in the correct state to download images
|
||||
if kube_upgrade_obj.state not in [
|
||||
kubernetes.KUBE_UPGRADE_STARTED,
|
||||
kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED]:
|
||||
raise wsme.exc.ClientSideError(_(
|
||||
"Kubernetes upgrade must be in %s state to upgrade "
|
||||
"Kubernetes upgrade must be in %s or %s state to download "
|
||||
"images" %
|
||||
(kubernetes.KUBE_UPGRADE_STARTED,
|
||||
kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED)))
|
||||
|
||||
# Verify patching requirements (since the api server is not
|
||||
# upgraded yet, patches could have been removed)
|
||||
system = pecan.request.dbapi.isystem_get_one()
|
||||
target_version_obj = objects.kube_version.get_by_version(
|
||||
kube_upgrade_obj.to_version)
|
||||
self._check_patch_requirements(
|
||||
system.region_name,
|
||||
applied_patches=target_version_obj.applied_patches,
|
||||
available_patches=target_version_obj.available_patches)
|
||||
|
||||
# Update the upgrade state
|
||||
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES
|
||||
kube_upgrade_obj.save()
|
||||
|
||||
# Tell the conductor to download the images for the new version
|
||||
pecan.request.rpcapi.kube_download_images(
|
||||
pecan.request.context, kube_upgrade_obj.to_version)
|
||||
|
||||
LOG.info("Downloading kubernetes images for version: %s" %
|
||||
kube_upgrade_obj.to_version)
|
||||
return KubeUpgrade.convert_with_links(kube_upgrade_obj)
|
||||
|
||||
elif updates['state'] == kubernetes.KUBE_UPGRADING_NETWORKING:
|
||||
# Make sure upgrade is in the correct state to upgrade networking
|
||||
if kube_upgrade_obj.state not in [
|
||||
kubernetes.KUBE_UPGRADED_FIRST_MASTER,
|
||||
kubernetes.KUBE_UPGRADING_NETWORKING_FAILED]:
|
||||
raise wsme.exc.ClientSideError(_(
|
||||
"Kubernetes upgrade must be in %s or %s state to upgrade "
|
||||
"networking" %
|
||||
kubernetes.KUBE_UPGRADED_FIRST_MASTER))
|
||||
(kubernetes.KUBE_UPGRADED_FIRST_MASTER,
|
||||
kubernetes.KUBE_UPGRADING_NETWORKING_FAILED)))
|
||||
|
||||
# Update the upgrade state
|
||||
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADING_NETWORKING
|
||||
|
|
@ -275,6 +315,8 @@ class KubeUpgradeController(rest.RestController):
|
|||
pecan.request.rpcapi.kube_upgrade_networking(
|
||||
pecan.request.context, kube_upgrade_obj.to_version)
|
||||
|
||||
LOG.info("Upgrading kubernetes networking to version: %s" %
|
||||
kube_upgrade_obj.to_version)
|
||||
return KubeUpgrade.convert_with_links(kube_upgrade_obj)
|
||||
|
||||
elif updates['state'] == kubernetes.KUBE_UPGRADE_COMPLETE:
|
||||
|
|
@ -295,6 +337,9 @@ class KubeUpgradeController(rest.RestController):
|
|||
# All is well, mark the upgrade as complete
|
||||
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_COMPLETE
|
||||
kube_upgrade_obj.save()
|
||||
|
||||
LOG.info("Completed kubernetes upgrade to version: %s" %
|
||||
kube_upgrade_obj.to_version)
|
||||
return KubeUpgrade.convert_with_links(kube_upgrade_obj)
|
||||
|
||||
else:
|
||||
|
|
@ -302,7 +347,7 @@ class KubeUpgradeController(rest.RestController):
|
|||
"Invalid state %s supplied" % updates['state']))
|
||||
|
||||
@cutils.synchronized(LOCK_NAME)
|
||||
@wsme_pecan.wsexpose(KubeUpgrade)
|
||||
@wsme_pecan.wsexpose(None)
|
||||
def delete(self):
|
||||
"""Delete Kubernetes Upgrade."""
|
||||
|
||||
|
|
|
|||
|
|
@ -41,17 +41,27 @@ KUBE_CONTROLLER_MANAGER = 'kube-controller-manager'
|
|||
KUBE_SCHEDULER = 'kube-scheduler'
|
||||
|
||||
# Kubernetes upgrade states
|
||||
KUBE_UPGRADE_DOWNLOADING_IMAGES = 'downloading-images'
|
||||
KUBE_UPGRADE_STARTED = 'upgrade-started'
|
||||
KUBE_UPGRADE_DOWNLOADING_IMAGES = 'downloading-images'
|
||||
KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED = 'downloading-images-failed'
|
||||
KUBE_UPGRADE_DOWNLOADED_IMAGES = 'downloaded-images'
|
||||
KUBE_UPGRADING_FIRST_MASTER = 'upgrading-first-master'
|
||||
KUBE_UPGRADING_FIRST_MASTER_FAILED = 'upgrading-first-master-failed'
|
||||
KUBE_UPGRADED_FIRST_MASTER = 'upgraded-first-master'
|
||||
KUBE_UPGRADING_NETWORKING = 'upgrading-networking'
|
||||
KUBE_UPGRADING_NETWORKING_FAILED = 'upgrading-networking-failed'
|
||||
KUBE_UPGRADED_NETWORKING = 'upgraded-networking'
|
||||
KUBE_UPGRADING_SECOND_MASTER = 'upgrading-second-master'
|
||||
KUBE_UPGRADING_SECOND_MASTER_FAILED = 'upgrading-second-master-failed'
|
||||
KUBE_UPGRADED_SECOND_MASTER = 'upgraded-second-master'
|
||||
KUBE_UPGRADING_KUBELETS = 'upgrading-kubelets'
|
||||
KUBE_UPGRADE_COMPLETE = 'upgrade-complete'
|
||||
KUBE_UPGRADE_FAILED = 'upgrade-failed'
|
||||
|
||||
# Kubernetes host upgrade statuses
|
||||
KUBE_HOST_UPGRADING_CONTROL_PLANE = 'upgrading-control-plane'
|
||||
KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED = 'upgrading-control-plane-failed'
|
||||
KUBE_HOST_UPGRADING_KUBELET = 'upgrading-kubelet'
|
||||
KUBE_HOST_UPGRADING_KUBELET_FAILED = 'upgrading-kubelet-failed'
|
||||
|
||||
# Kubernetes constants
|
||||
MANIFEST_APPLY_TIMEOUT = 60 * 15
|
||||
|
|
@ -391,7 +401,7 @@ class KubeOperator(object):
|
|||
|
||||
node_versions = dict()
|
||||
for node_name in master_nodes:
|
||||
versions = dict()
|
||||
versions = list()
|
||||
for component in [KUBE_APISERVER,
|
||||
KUBE_CONTROLLER_MANAGER,
|
||||
KUBE_SCHEDULER]:
|
||||
|
|
@ -400,14 +410,11 @@ class KubeOperator(object):
|
|||
pod_name = component + '-' + node_name
|
||||
image = self.kube_get_image_by_pod_name(
|
||||
pod_name, NAMESPACE_KUBE_SYSTEM, component)
|
||||
versions[component] = image.rsplit(':')[-1]
|
||||
if image is not None:
|
||||
versions.append(LooseVersion(image.rsplit(':')[-1]))
|
||||
|
||||
# Calculate the lowest version
|
||||
lowest_version = min(
|
||||
LooseVersion(versions[KUBE_APISERVER]),
|
||||
LooseVersion(versions[KUBE_CONTROLLER_MANAGER]),
|
||||
LooseVersion(versions[KUBE_SCHEDULER]))
|
||||
node_versions[node_name] = str(lowest_version)
|
||||
node_versions[node_name] = str(min(versions))
|
||||
|
||||
return node_versions
|
||||
|
||||
|
|
|
|||
|
|
@ -234,6 +234,7 @@ class ConductorManager(service.PeriodicService):
|
|||
|
||||
# Upgrade start tasks
|
||||
self._upgrade_init_actions()
|
||||
self._kube_upgrade_init_actions()
|
||||
|
||||
self._handle_restore_in_progress()
|
||||
|
||||
|
|
@ -10682,6 +10683,51 @@ class ConductorManager(service.PeriodicService):
|
|||
|
||||
os.chmod(constants.SYSINV_CONF_DEFAULT_PATH, 0o400)
|
||||
|
||||
def _kube_upgrade_init_actions(self):
|
||||
""" Perform any kubernetes upgrade related startup actions"""
|
||||
try:
|
||||
kube_upgrade = self.dbapi.kube_upgrade_get_one()
|
||||
except exception.NotFound:
|
||||
# Not upgrading kubernetes
|
||||
return
|
||||
|
||||
# Fail any upgrade operation that is in a transitory state. This
|
||||
# service is responsible for monitoring these operations and since
|
||||
# we were just restarted, the operation will never progress.
|
||||
fail_state = None
|
||||
if kube_upgrade.state == kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES:
|
||||
fail_state = kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED
|
||||
elif kube_upgrade.state == kubernetes.KUBE_UPGRADING_FIRST_MASTER:
|
||||
fail_state = kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED
|
||||
elif kube_upgrade.state == kubernetes.KUBE_UPGRADING_NETWORKING:
|
||||
fail_state = kubernetes.KUBE_UPGRADING_NETWORKING_FAILED
|
||||
elif kube_upgrade.state == kubernetes.KUBE_UPGRADING_SECOND_MASTER:
|
||||
fail_state = kubernetes.KUBE_UPGRADING_SECOND_MASTER_FAILED
|
||||
|
||||
if fail_state is not None:
|
||||
LOG.warning("Failing upgrade in %s state due to service restart" %
|
||||
kube_upgrade.state)
|
||||
self.dbapi.kube_upgrade_update(kube_upgrade.id,
|
||||
{'state': fail_state})
|
||||
|
||||
# Fail any host upgrade operation that is in a transitory state.
|
||||
kube_host_upgrades = self.dbapi.kube_host_upgrade_get_list()
|
||||
for kube_host_upgrade in kube_host_upgrades:
|
||||
fail_status = None
|
||||
if kube_host_upgrade.status == \
|
||||
kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE:
|
||||
fail_status = \
|
||||
kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED
|
||||
elif kube_host_upgrade.status == \
|
||||
kubernetes.KUBE_HOST_UPGRADING_KUBELET:
|
||||
fail_status = kubernetes.KUBE_HOST_UPGRADING_KUBELET_FAILED
|
||||
|
||||
if fail_status is not None:
|
||||
LOG.warning("Failing host upgrade with %s status due to "
|
||||
"service restart" % kube_host_upgrade.status)
|
||||
self.dbapi.kube_host_upgrade_update(kube_host_upgrade.id,
|
||||
{'status': fail_status})
|
||||
|
||||
def kube_download_images(self, context, kube_version):
|
||||
"""Download the kubernetes images for this version"""
|
||||
|
||||
|
|
@ -10703,7 +10749,8 @@ class ConductorManager(service.PeriodicService):
|
|||
proc.returncode)
|
||||
# Update the upgrade state
|
||||
kube_upgrade_obj = objects.kube_upgrade.get_one(context)
|
||||
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_FAILED
|
||||
kube_upgrade_obj.state = \
|
||||
kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED
|
||||
kube_upgrade_obj.save()
|
||||
return
|
||||
|
||||
|
|
@ -10738,13 +10785,14 @@ class ConductorManager(service.PeriodicService):
|
|||
else:
|
||||
LOG.warning("Manifest apply failed for a controller host")
|
||||
kube_upgrade_obj = objects.kube_upgrade.get_one(context)
|
||||
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_FAILED
|
||||
kube_upgrade_obj.state = \
|
||||
kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED
|
||||
kube_upgrade_obj.save()
|
||||
return
|
||||
|
||||
# Update the upgrade state
|
||||
kube_upgrade_obj = objects.kube_upgrade.get_one(context)
|
||||
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_STARTED
|
||||
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES
|
||||
kube_upgrade_obj.save()
|
||||
|
||||
def kube_upgrade_control_plane(self, context, host_uuid):
|
||||
|
|
@ -10760,17 +10808,15 @@ class ConductorManager(service.PeriodicService):
|
|||
if kube_upgrade_obj.state == kubernetes.KUBE_UPGRADING_FIRST_MASTER:
|
||||
puppet_class = 'platform::kubernetes::upgrade_first_control_plane'
|
||||
new_state = kubernetes.KUBE_UPGRADED_FIRST_MASTER
|
||||
fail_state = kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED
|
||||
elif kube_upgrade_obj.state == kubernetes.KUBE_UPGRADING_SECOND_MASTER:
|
||||
puppet_class = 'platform::kubernetes::upgrade_control_plane'
|
||||
new_state = kubernetes.KUBE_UPGRADED_SECOND_MASTER
|
||||
fail_state = kubernetes.KUBE_UPGRADING_SECOND_MASTER_FAILED
|
||||
else:
|
||||
LOG.error("Invalid state %s to upgrade control plane." %
|
||||
kube_upgrade_obj.state)
|
||||
return
|
||||
|
||||
# Update status
|
||||
kube_host_upgrade_obj.status = "Upgrading control plane"
|
||||
kube_host_upgrade_obj.save()
|
||||
raise exception.SysinvException(_(
|
||||
"Invalid state %s to upgrade control plane." %
|
||||
kube_upgrade_obj.state))
|
||||
|
||||
# Update the config for this host
|
||||
personalities = [host_obj.personality]
|
||||
|
|
@ -10799,10 +10845,11 @@ class ConductorManager(service.PeriodicService):
|
|||
LOG.warning("Manifest apply failed for host %s" % host_name)
|
||||
kube_host_upgrade_obj = objects.kube_host_upgrade.get_by_host_id(
|
||||
context, host_obj.id)
|
||||
kube_host_upgrade_obj.status = "Control plane upgrade failed"
|
||||
kube_host_upgrade_obj.status = \
|
||||
kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED
|
||||
kube_host_upgrade_obj.save()
|
||||
kube_upgrade_obj = objects.kube_upgrade.get_one(context)
|
||||
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_FAILED
|
||||
kube_upgrade_obj.state = fail_state
|
||||
kube_upgrade_obj.save()
|
||||
return
|
||||
|
||||
|
|
@ -10822,10 +10869,11 @@ class ConductorManager(service.PeriodicService):
|
|||
host_name)
|
||||
kube_host_upgrade_obj = objects.kube_host_upgrade.get_by_host_id(
|
||||
context, host_obj.id)
|
||||
kube_host_upgrade_obj.status = "Control plane upgrade failed"
|
||||
kube_host_upgrade_obj.status = \
|
||||
kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED
|
||||
kube_host_upgrade_obj.save()
|
||||
kube_upgrade_obj = objects.kube_upgrade.get_one(context)
|
||||
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_FAILED
|
||||
kube_upgrade_obj.state = fail_state
|
||||
kube_upgrade_obj.save()
|
||||
return
|
||||
|
||||
|
|
@ -10852,13 +10900,9 @@ class ConductorManager(service.PeriodicService):
|
|||
elif host_obj.personality == constants.WORKER:
|
||||
puppet_class = 'platform::kubernetes::worker::upgrade_kubelet'
|
||||
else:
|
||||
LOG.error("Invalid personality %s to upgrade kubelet." %
|
||||
host_obj.personality)
|
||||
return
|
||||
|
||||
# Update status
|
||||
kube_host_upgrade_obj.status = "Upgrading kubelet"
|
||||
kube_host_upgrade_obj.save()
|
||||
raise exception.SysinvException(_(
|
||||
"Invalid personality %s to upgrade kubelet." %
|
||||
host_obj.personality))
|
||||
|
||||
# Update the config for this host
|
||||
personalities = [host_obj.personality]
|
||||
|
|
@ -10887,11 +10931,9 @@ class ConductorManager(service.PeriodicService):
|
|||
LOG.warning("Manifest apply failed for host %s" % host_name)
|
||||
kube_host_upgrade_obj = objects.kube_host_upgrade.get_by_host_id(
|
||||
context, host_obj.id)
|
||||
kube_host_upgrade_obj.status = "Control plane upgrade failed"
|
||||
kube_host_upgrade_obj.status = \
|
||||
kubernetes.KUBE_HOST_UPGRADING_KUBELET_FAILED
|
||||
kube_host_upgrade_obj.save()
|
||||
kube_upgrade_obj = objects.kube_upgrade.get_one(context)
|
||||
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_FAILED
|
||||
kube_upgrade_obj.save()
|
||||
return
|
||||
|
||||
# Wait for the kubelet to start with the new version
|
||||
|
|
@ -10909,11 +10951,9 @@ class ConductorManager(service.PeriodicService):
|
|||
LOG.warning("Kubelet upgrade failed for host %s" % host_name)
|
||||
kube_host_upgrade_obj = objects.kube_host_upgrade.get_by_host_id(
|
||||
context, host_obj.id)
|
||||
kube_host_upgrade_obj.status = "Kubelet upgrade failed"
|
||||
kube_host_upgrade_obj.status = \
|
||||
kubernetes.KUBE_HOST_UPGRADING_KUBELET_FAILED
|
||||
kube_host_upgrade_obj.save()
|
||||
kube_upgrade_obj = objects.kube_upgrade.get_one(context)
|
||||
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_FAILED
|
||||
kube_upgrade_obj.save()
|
||||
return
|
||||
|
||||
# The kubelet update was successful
|
||||
|
|
|
|||
|
|
@ -346,6 +346,16 @@ class TestPostKubeUpgrades(TestHost):
|
|||
self.mocked_patch_is_applied.start()
|
||||
self.addCleanup(self.mocked_patch_is_applied.stop)
|
||||
|
||||
self.mock_patch_is_available_result = True
|
||||
|
||||
def mock_patch_is_available(token, timeout, region_name, patches):
|
||||
return self.mock_patch_is_available_result
|
||||
self.mocked_patch_is_available = mock.patch(
|
||||
'sysinv.api.controllers.v1.patch_api.patch_is_available',
|
||||
mock_patch_is_available)
|
||||
self.mocked_patch_is_available.start()
|
||||
self.addCleanup(self.mocked_patch_is_available.stop)
|
||||
|
||||
def test_kube_upgrade_control_plane_controller_0(self):
|
||||
# Test upgrading kubernetes control plane on controller-0
|
||||
|
||||
|
|
@ -360,7 +370,7 @@ class TestPostKubeUpgrades(TestHost):
|
|||
kube_upgrade = dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.42.1',
|
||||
to_version='v1.42.2',
|
||||
state=kubernetes.KUBE_UPGRADE_STARTED,
|
||||
state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
|
||||
)
|
||||
|
||||
# Upgrade the control plane
|
||||
|
|
@ -376,9 +386,58 @@ class TestPostKubeUpgrades(TestHost):
|
|||
self.fake_conductor_api.kube_upgrade_control_plane.\
|
||||
assert_called_with(mock.ANY, c0.uuid)
|
||||
|
||||
# Verify that the target version was updated
|
||||
# Verify that the target version and status was updated
|
||||
result = self.get_json('/kube_host_upgrades/1')
|
||||
self.assertEqual(result['target_version'], 'v1.42.2')
|
||||
self.assertEqual(result['status'],
|
||||
kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE)
|
||||
|
||||
# Verify that the upgrade state was updated
|
||||
result = self.get_json('/kube_upgrade/%s' % kube_upgrade.uuid)
|
||||
self.assertEqual(result['state'],
|
||||
kubernetes.KUBE_UPGRADING_FIRST_MASTER)
|
||||
|
||||
def test_kube_upgrade_control_plane_controller_0_after_failure(self):
|
||||
# Test upgrading kubernetes control plane on controller-0 after a
|
||||
# failure
|
||||
|
||||
# Create controller-0
|
||||
c0 = self._create_controller_0(
|
||||
invprovision=constants.PROVISIONED,
|
||||
administrative=constants.ADMIN_UNLOCKED,
|
||||
operational=constants.OPERATIONAL_ENABLED,
|
||||
availability=constants.AVAILABILITY_ONLINE)
|
||||
|
||||
# Create the upgrade
|
||||
kube_upgrade = dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.42.1',
|
||||
to_version='v1.42.2',
|
||||
state=kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED,
|
||||
)
|
||||
|
||||
# Mark the kube host upgrade as failed
|
||||
values = {'target_version': 'v1.42.2',
|
||||
'status': kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED}
|
||||
self.dbapi.kube_host_upgrade_update(1, values)
|
||||
|
||||
# Upgrade the control plane
|
||||
body = {}
|
||||
result = self.post_json(
|
||||
'/ihosts/controller-0/kube_upgrade_control_plane',
|
||||
body, headers={'User-Agent': 'sysinv-test'})
|
||||
|
||||
# Verify the host was returned
|
||||
self.assertEqual(result.json['hostname'], 'controller-0')
|
||||
|
||||
# Verify the control plane was upgraded
|
||||
self.fake_conductor_api.kube_upgrade_control_plane.\
|
||||
assert_called_with(mock.ANY, c0.uuid)
|
||||
|
||||
# Verify that the target version and status was updated
|
||||
result = self.get_json('/kube_host_upgrades/1')
|
||||
self.assertEqual(result['target_version'], 'v1.42.2')
|
||||
self.assertEqual(result['status'],
|
||||
kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE)
|
||||
|
||||
# Verify that the upgrade state was updated
|
||||
result = self.get_json('/kube_upgrade/%s' % kube_upgrade.uuid)
|
||||
|
|
@ -404,7 +463,7 @@ class TestPostKubeUpgrades(TestHost):
|
|||
kube_upgrade = dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.42.1',
|
||||
to_version='v1.42.2',
|
||||
state=kubernetes.KUBE_UPGRADE_STARTED,
|
||||
state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
|
||||
)
|
||||
|
||||
# Upgrade the control plane
|
||||
|
|
@ -420,9 +479,11 @@ class TestPostKubeUpgrades(TestHost):
|
|||
self.fake_conductor_api.kube_upgrade_control_plane.\
|
||||
assert_called_with(mock.ANY, c1.uuid)
|
||||
|
||||
# Verify that the target version was updated
|
||||
# Verify that the target version and status was updated
|
||||
result = self.get_json('/kube_host_upgrades/2')
|
||||
self.assertEqual(result['target_version'], 'v1.42.2')
|
||||
self.assertEqual(result['status'],
|
||||
kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE)
|
||||
|
||||
# Verify that the upgrade state was updated
|
||||
result = self.get_json('/kube_upgrade/%s' % kube_upgrade.uuid)
|
||||
|
|
@ -448,9 +509,13 @@ class TestPostKubeUpgrades(TestHost):
|
|||
kube_upgrade = dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.42.1',
|
||||
to_version='v1.42.2',
|
||||
state=kubernetes.KUBE_UPGRADED_NETWORKING,
|
||||
state=kubernetes.KUBE_UPGRADED_FIRST_MASTER,
|
||||
)
|
||||
|
||||
# Mark the first kube host upgrade as OK
|
||||
values = {'target_version': 'v1.42.2'}
|
||||
self.dbapi.kube_host_upgrade_update(1, values)
|
||||
|
||||
# Upgrade the control plane
|
||||
body = {}
|
||||
result = self.post_json(
|
||||
|
|
@ -464,15 +529,114 @@ class TestPostKubeUpgrades(TestHost):
|
|||
self.fake_conductor_api.kube_upgrade_control_plane.\
|
||||
assert_called_with(mock.ANY, c1.uuid)
|
||||
|
||||
# Verify that the target version was updated
|
||||
# Verify that the target version and status was updated
|
||||
result = self.get_json('/kube_host_upgrades/2')
|
||||
self.assertEqual(result['target_version'], 'v1.42.2')
|
||||
self.assertEqual(result['status'],
|
||||
kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE)
|
||||
|
||||
# Verify that the upgrade state was updated
|
||||
result = self.get_json('/kube_upgrade/%s' % kube_upgrade.uuid)
|
||||
self.assertEqual(result['state'],
|
||||
kubernetes.KUBE_UPGRADING_SECOND_MASTER)
|
||||
|
||||
def test_kube_upgrade_control_plane_second_controller_after_failure(self):
|
||||
# Test upgrading kubernetes control plane on the second controller
|
||||
# after a failure
|
||||
|
||||
# Create controllers
|
||||
self._create_controller_0(
|
||||
invprovision=constants.PROVISIONED,
|
||||
administrative=constants.ADMIN_UNLOCKED,
|
||||
operational=constants.OPERATIONAL_ENABLED,
|
||||
availability=constants.AVAILABILITY_ONLINE)
|
||||
c1 = self._create_controller_1(
|
||||
invprovision=constants.PROVISIONED,
|
||||
administrative=constants.ADMIN_UNLOCKED,
|
||||
operational=constants.OPERATIONAL_ENABLED,
|
||||
availability=constants.AVAILABILITY_ONLINE)
|
||||
|
||||
# Create the upgrade
|
||||
kube_upgrade = dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.42.1',
|
||||
to_version='v1.42.2',
|
||||
state=kubernetes.KUBE_UPGRADING_SECOND_MASTER_FAILED,
|
||||
)
|
||||
|
||||
# Mark the first kube host upgrade as OK
|
||||
values = {'target_version': 'v1.42.2'}
|
||||
self.dbapi.kube_host_upgrade_update(1, values)
|
||||
|
||||
# Mark the second kube host upgrade as failed
|
||||
values = {'target_version': 'v1.42.2',
|
||||
'status': kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED}
|
||||
self.dbapi.kube_host_upgrade_update(2, values)
|
||||
|
||||
# Upgrade the control plane
|
||||
body = {}
|
||||
result = self.post_json(
|
||||
'/ihosts/controller-1/kube_upgrade_control_plane',
|
||||
body, headers={'User-Agent': 'sysinv-test'})
|
||||
|
||||
# Verify the host was returned
|
||||
self.assertEqual(result.json['hostname'], 'controller-1')
|
||||
|
||||
# Verify the control plane was upgraded
|
||||
self.fake_conductor_api.kube_upgrade_control_plane.\
|
||||
assert_called_with(mock.ANY, c1.uuid)
|
||||
|
||||
# Verify that the target version and status was updated
|
||||
result = self.get_json('/kube_host_upgrades/2')
|
||||
self.assertEqual(result['target_version'], 'v1.42.2')
|
||||
self.assertEqual(result['status'],
|
||||
kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE)
|
||||
|
||||
# Verify that the upgrade state was updated
|
||||
result = self.get_json('/kube_upgrade/%s' % kube_upgrade.uuid)
|
||||
self.assertEqual(result['state'],
|
||||
kubernetes.KUBE_UPGRADING_SECOND_MASTER)
|
||||
|
||||
def test_kube_upgrade_control_plane_wrong_controller_after_failure(self):
|
||||
# Test upgrading kubernetes control plane on the wrong controller
|
||||
# after a failure
|
||||
|
||||
# Create controllers
|
||||
self._create_controller_0(
|
||||
invprovision=constants.PROVISIONED,
|
||||
administrative=constants.ADMIN_UNLOCKED,
|
||||
operational=constants.OPERATIONAL_ENABLED,
|
||||
availability=constants.AVAILABILITY_ONLINE)
|
||||
self._create_controller_1(
|
||||
invprovision=constants.PROVISIONED,
|
||||
administrative=constants.ADMIN_UNLOCKED,
|
||||
operational=constants.OPERATIONAL_ENABLED,
|
||||
availability=constants.AVAILABILITY_ONLINE)
|
||||
|
||||
# Create the upgrade
|
||||
dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.42.1',
|
||||
to_version='v1.42.2',
|
||||
state=kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED,
|
||||
)
|
||||
|
||||
# Mark the first kube host upgrade as failed
|
||||
values = {'target_version': 'v1.42.2',
|
||||
'status': kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED}
|
||||
self.dbapi.kube_host_upgrade_update(1, values)
|
||||
|
||||
# Upgrade the second control plane
|
||||
result = self.post_json(
|
||||
'/ihosts/controller-1/kube_upgrade_control_plane',
|
||||
{}, headers={'User-Agent': 'sysinv-test'},
|
||||
expect_errors=True)
|
||||
|
||||
# Verify the failure
|
||||
self.assertEqual(result.content_type, 'application/json')
|
||||
self.assertEqual(http_client.BAD_REQUEST, result.status_int)
|
||||
self.assertTrue(result.json['error_message'])
|
||||
self.assertIn("The first control plane upgrade must be completed",
|
||||
result.json['error_message'])
|
||||
|
||||
def test_kube_upgrade_control_plane_no_upgrade(self):
|
||||
# Test upgrading kubernetes control plane with no upgrade
|
||||
|
||||
|
|
@ -523,7 +687,104 @@ class TestPostKubeUpgrades(TestHost):
|
|||
self.assertEqual(result.content_type, 'application/json')
|
||||
self.assertEqual(http_client.BAD_REQUEST, result.status_int)
|
||||
self.assertTrue(result.json['error_message'])
|
||||
self.assertIn("upgrade must be in the",
|
||||
self.assertIn("The kubernetes upgrade is not in a valid state",
|
||||
result.json['error_message'])
|
||||
|
||||
def test_kube_upgrade_control_plane_controller_0_missing_applied_patches(
|
||||
self):
|
||||
# Test upgrading kubernetes control plane with missing applied patches
|
||||
|
||||
# Create controller-0
|
||||
self._create_controller_0(
|
||||
invprovision=constants.PROVISIONED,
|
||||
administrative=constants.ADMIN_UNLOCKED,
|
||||
operational=constants.OPERATIONAL_ENABLED,
|
||||
availability=constants.AVAILABILITY_ONLINE)
|
||||
|
||||
# Create the upgrade
|
||||
dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.42.1',
|
||||
to_version='v1.42.2',
|
||||
state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
|
||||
)
|
||||
|
||||
# Fake the missing patches
|
||||
self.mock_patch_is_applied_result = False
|
||||
self.get_kube_versions_result = [
|
||||
{'version': 'v1.42.1',
|
||||
'upgrade_from': [],
|
||||
'downgrade_to': [],
|
||||
'applied_patches': [],
|
||||
'available_patches': [],
|
||||
},
|
||||
{'version': 'v1.42.2',
|
||||
'upgrade_from': ['v1.42.1'],
|
||||
'downgrade_to': [],
|
||||
'applied_patches': ['MISSING_PATCH.1'],
|
||||
'available_patches': ['MISSING_PATCH.2'],
|
||||
},
|
||||
]
|
||||
|
||||
# Upgrade the control plane
|
||||
result = self.post_json(
|
||||
'/ihosts/controller-0/kube_upgrade_control_plane',
|
||||
{}, headers={'User-Agent': 'sysinv-test'},
|
||||
expect_errors=True)
|
||||
|
||||
# Verify the failure
|
||||
self.assertEqual(result.content_type, 'application/json')
|
||||
self.assertEqual(http_client.BAD_REQUEST, result.status_int)
|
||||
self.assertTrue(result.json['error_message'])
|
||||
self.assertIn("The following patches must be applied",
|
||||
result.json['error_message'])
|
||||
|
||||
def test_kube_upgrade_control_plane_controller_0_missing_available_patches(
|
||||
self):
|
||||
# Test upgrading kubernetes control plane with missing available
|
||||
# patches
|
||||
|
||||
# Create controller-0
|
||||
self._create_controller_0(
|
||||
invprovision=constants.PROVISIONED,
|
||||
administrative=constants.ADMIN_UNLOCKED,
|
||||
operational=constants.OPERATIONAL_ENABLED,
|
||||
availability=constants.AVAILABILITY_ONLINE)
|
||||
|
||||
# Create the upgrade
|
||||
dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.42.1',
|
||||
to_version='v1.42.2',
|
||||
state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
|
||||
)
|
||||
|
||||
# Fake the missing patches
|
||||
self.mock_patch_is_available_result = False
|
||||
self.get_kube_versions_result = [
|
||||
{'version': 'v1.42.1',
|
||||
'upgrade_from': [],
|
||||
'downgrade_to': [],
|
||||
'applied_patches': [],
|
||||
'available_patches': [],
|
||||
},
|
||||
{'version': 'v1.42.2',
|
||||
'upgrade_from': ['v1.42.1'],
|
||||
'downgrade_to': [],
|
||||
'applied_patches': [],
|
||||
'available_patches': ['MISSING_PATCH.2'],
|
||||
},
|
||||
]
|
||||
|
||||
# Upgrade the control plane
|
||||
result = self.post_json(
|
||||
'/ihosts/controller-0/kube_upgrade_control_plane',
|
||||
{}, headers={'User-Agent': 'sysinv-test'},
|
||||
expect_errors=True)
|
||||
|
||||
# Verify the failure
|
||||
self.assertEqual(result.content_type, 'application/json')
|
||||
self.assertEqual(http_client.BAD_REQUEST, result.status_int)
|
||||
self.assertTrue(result.json['error_message'])
|
||||
self.assertIn("The following patches must be available",
|
||||
result.json['error_message'])
|
||||
|
||||
def test_kube_upgrade_control_plane_wrong_personality(self):
|
||||
|
|
@ -546,7 +807,7 @@ class TestPostKubeUpgrades(TestHost):
|
|||
dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.42.1',
|
||||
to_version='v1.42.2',
|
||||
state=kubernetes.KUBE_UPGRADE_STARTED,
|
||||
state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
|
||||
)
|
||||
|
||||
# Upgrade the control plane
|
||||
|
|
@ -576,7 +837,7 @@ class TestPostKubeUpgrades(TestHost):
|
|||
dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.42.1',
|
||||
to_version='v1.42.2',
|
||||
state=kubernetes.KUBE_UPGRADE_STARTED,
|
||||
state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
|
||||
)
|
||||
|
||||
# No control plane version for this controller
|
||||
|
|
@ -611,7 +872,7 @@ class TestPostKubeUpgrades(TestHost):
|
|||
dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.42.1',
|
||||
to_version='v1.42.2',
|
||||
state=kubernetes.KUBE_UPGRADE_STARTED,
|
||||
state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
|
||||
)
|
||||
|
||||
# Upgrade the control plane
|
||||
|
|
@ -627,6 +888,40 @@ class TestPostKubeUpgrades(TestHost):
|
|||
self.assertIn("must be unlocked and available",
|
||||
result.json['error_message'])
|
||||
|
||||
def test_kube_upgrade_control_plane_already_in_progress(self):
|
||||
# Test upgrading kubernetes control plane with upgrade in progress
|
||||
|
||||
# Create controller-0
|
||||
self._create_controller_0(
|
||||
invprovision=constants.PROVISIONED,
|
||||
administrative=constants.ADMIN_UNLOCKED,
|
||||
operational=constants.OPERATIONAL_ENABLED,
|
||||
availability=constants.AVAILABILITY_ONLINE)
|
||||
|
||||
# Create the upgrade
|
||||
dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.42.1',
|
||||
to_version='v1.42.2',
|
||||
state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
|
||||
)
|
||||
|
||||
# Mark the kube host as already upgrading
|
||||
values = {'status': kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE}
|
||||
self.dbapi.kube_host_upgrade_update(1, values)
|
||||
|
||||
# Upgrade the control plane
|
||||
result = self.post_json(
|
||||
'/ihosts/controller-0/kube_upgrade_control_plane',
|
||||
{}, headers={'User-Agent': 'sysinv-test'},
|
||||
expect_errors=True)
|
||||
|
||||
# Verify the failure
|
||||
self.assertEqual(result.content_type, 'application/json')
|
||||
self.assertEqual(http_client.BAD_REQUEST, result.status_int)
|
||||
self.assertTrue(result.json['error_message'])
|
||||
self.assertIn("control plane on this host is already being upgraded",
|
||||
result.json['error_message'])
|
||||
|
||||
def test_kube_upgrade_kubelet_controller_0(self):
|
||||
# Test upgrading kubernetes kubelet on controller-0
|
||||
|
||||
|
|
@ -657,6 +952,11 @@ class TestPostKubeUpgrades(TestHost):
|
|||
self.fake_conductor_api.kube_upgrade_kubelet.\
|
||||
assert_called_with(mock.ANY, c0.uuid)
|
||||
|
||||
# Verify that the status was updated
|
||||
result = self.get_json('/kube_host_upgrades/1')
|
||||
self.assertEqual(result['status'],
|
||||
kubernetes.KUBE_HOST_UPGRADING_KUBELET)
|
||||
|
||||
# Verify that the upgrade state was updated
|
||||
result = self.get_json('/kube_upgrade/%s' % kube_upgrade.uuid)
|
||||
self.assertEqual(result['state'],
|
||||
|
|
@ -697,6 +997,11 @@ class TestPostKubeUpgrades(TestHost):
|
|||
self.fake_conductor_api.kube_upgrade_kubelet.\
|
||||
assert_called_with(mock.ANY, c1.uuid)
|
||||
|
||||
# Verify that the status was updated
|
||||
result = self.get_json('/kube_host_upgrades/2')
|
||||
self.assertEqual(result['status'],
|
||||
kubernetes.KUBE_HOST_UPGRADING_KUBELET)
|
||||
|
||||
# Verify that the upgrade state was updated
|
||||
result = self.get_json('/kube_upgrade/%s' % kube_upgrade.uuid)
|
||||
self.assertEqual(result['state'],
|
||||
|
|
@ -745,9 +1050,11 @@ class TestPostKubeUpgrades(TestHost):
|
|||
# Verify the host was returned
|
||||
self.assertEqual(result.json['hostname'], 'worker-0')
|
||||
|
||||
# Verify that the target version was updated
|
||||
# Verify that the target version and status was updated
|
||||
result = self.get_json('/kube_host_upgrades/3')
|
||||
self.assertEqual(result['target_version'], 'v1.42.2')
|
||||
self.assertEqual(result['status'],
|
||||
kubernetes.KUBE_HOST_UPGRADING_KUBELET)
|
||||
|
||||
# Verify the kubelet was upgraded
|
||||
self.fake_conductor_api.kube_upgrade_kubelet.\
|
||||
|
|
@ -887,6 +1194,11 @@ class TestPostKubeUpgrades(TestHost):
|
|||
self.fake_conductor_api.kube_upgrade_kubelet.\
|
||||
assert_called_with(mock.ANY, c0.uuid)
|
||||
|
||||
# Verify that the status was updated
|
||||
result = self.get_json('/kube_host_upgrades/1')
|
||||
self.assertEqual(result['status'],
|
||||
kubernetes.KUBE_HOST_UPGRADING_KUBELET)
|
||||
|
||||
# Verify that the upgrade state was updated
|
||||
result = self.get_json('/kube_upgrade/%s' % kube_upgrade.uuid)
|
||||
self.assertEqual(result['state'],
|
||||
|
|
@ -907,7 +1219,7 @@ class TestPostKubeUpgrades(TestHost):
|
|||
dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.42.1',
|
||||
to_version='v1.42.2',
|
||||
state=kubernetes.KUBE_UPGRADE_STARTED,
|
||||
state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
|
||||
)
|
||||
|
||||
# Upgrade the kubelet
|
||||
|
|
@ -1094,6 +1406,41 @@ class TestPostKubeUpgrades(TestHost):
|
|||
self.assertIn("The host must be locked and online",
|
||||
result.json['error_message'])
|
||||
|
||||
def test_kube_upgrade_kubelet_already_in_progress(self):
|
||||
# Test upgrading kubernetes kubelet with upgrade in progress
|
||||
|
||||
# Create controller-0
|
||||
self._create_controller_0(
|
||||
invprovision=constants.PROVISIONED,
|
||||
administrative=constants.ADMIN_LOCKED,
|
||||
operational=constants.OPERATIONAL_DISABLED,
|
||||
availability=constants.AVAILABILITY_ONLINE)
|
||||
|
||||
# Create the upgrade
|
||||
dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.42.1',
|
||||
to_version='v1.42.2',
|
||||
state=kubernetes.KUBE_UPGRADED_SECOND_MASTER,
|
||||
)
|
||||
|
||||
# Mark the kube host as already upgrading
|
||||
values = {'status': kubernetes.KUBE_HOST_UPGRADING_KUBELET}
|
||||
self.dbapi.kube_host_upgrade_update(1, values)
|
||||
|
||||
# Upgrade the kubelet
|
||||
body = {}
|
||||
result = self.post_json(
|
||||
'/ihosts/controller-0/kube_upgrade_kubelet',
|
||||
body, headers={'User-Agent': 'sysinv-test'},
|
||||
expect_errors=True)
|
||||
|
||||
# Verify the failure
|
||||
self.assertEqual(result.content_type, 'application/json')
|
||||
self.assertEqual(http_client.BAD_REQUEST, result.status_int)
|
||||
self.assertTrue(result.json['error_message'])
|
||||
self.assertIn("The kubelet on this host is already being upgraded",
|
||||
result.json['error_message'])
|
||||
|
||||
|
||||
class TestDelete(TestHost):
|
||||
|
||||
|
|
|
|||
|
|
@ -173,14 +173,11 @@ class TestPostKubeUpgrade(TestKubeUpgrade, dbbase.ControllerHostTestCase):
|
|||
result = self.post_json('/kube_upgrade', create_dict,
|
||||
headers={'User-Agent': 'sysinv-test'})
|
||||
|
||||
# Verify that the images were downloaded
|
||||
self.fake_conductor_api.kube_download_images.\
|
||||
assert_called_with(mock.ANY, 'v1.43.2')
|
||||
# Verify that the upgrade has the expected attributes
|
||||
self.assertEqual(result.json['from_version'], 'v1.43.1')
|
||||
self.assertEqual(result.json['to_version'], 'v1.43.2')
|
||||
self.assertEqual(result.json['state'],
|
||||
kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES)
|
||||
kubernetes.KUBE_UPGRADE_STARTED)
|
||||
# Verify that the target version for the host was updated
|
||||
kube_host_upgrade = self.dbapi.kube_host_upgrade_get_by_host(
|
||||
self.host.id)
|
||||
|
|
@ -278,7 +275,7 @@ class TestPostKubeUpgrade(TestKubeUpgrade, dbbase.ControllerHostTestCase):
|
|||
self.assertEqual(result.json['from_version'], 'v1.43.1')
|
||||
self.assertEqual(result.json['to_version'], 'v1.43.2')
|
||||
self.assertEqual(result.json['state'],
|
||||
kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES)
|
||||
kubernetes.KUBE_UPGRADE_STARTED)
|
||||
|
||||
def test_create_no_patches_required(self):
|
||||
# Test creation of upgrade when no applied patches are required
|
||||
|
|
@ -298,7 +295,7 @@ class TestPostKubeUpgrade(TestKubeUpgrade, dbbase.ControllerHostTestCase):
|
|||
self.assertEqual(result.json['from_version'], 'v1.43.2')
|
||||
self.assertEqual(result.json['to_version'], 'v1.43.3')
|
||||
self.assertEqual(result.json['state'],
|
||||
kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES)
|
||||
kubernetes.KUBE_UPGRADE_STARTED)
|
||||
|
||||
def test_create_applied_patch_missing(self):
|
||||
# Test creation of upgrade when applied patch is missing
|
||||
|
|
@ -331,6 +328,98 @@ class TestPostKubeUpgrade(TestKubeUpgrade, dbbase.ControllerHostTestCase):
|
|||
|
||||
class TestPatch(TestKubeUpgrade):
|
||||
|
||||
def test_update_state_download_images(self):
|
||||
# Test updating the state of an upgrade to download images
|
||||
|
||||
# Create the upgrade
|
||||
kube_upgrade = dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.43.1',
|
||||
to_version='v1.43.2',
|
||||
state=kubernetes.KUBE_UPGRADE_STARTED)
|
||||
uuid = kube_upgrade.uuid
|
||||
|
||||
# Update state
|
||||
new_state = kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES
|
||||
response = self.patch_json('/kube_upgrade',
|
||||
[{'path': '/state',
|
||||
'value': new_state,
|
||||
'op': 'replace'}],
|
||||
headers={'User-Agent': 'sysinv-test'})
|
||||
self.assertEqual(response.content_type, 'application/json')
|
||||
self.assertEqual(response.status_code, http_client.OK)
|
||||
self.assertEqual(response.json['from_version'], 'v1.43.1')
|
||||
self.assertEqual(response.json['to_version'], 'v1.43.2')
|
||||
self.assertEqual(response.json['state'], new_state)
|
||||
|
||||
# Verify that the images were downloaded
|
||||
self.fake_conductor_api.kube_download_images.\
|
||||
assert_called_with(mock.ANY, 'v1.43.2')
|
||||
|
||||
# Verify that the upgrade was updated with the new state
|
||||
result = self.get_json('/kube_upgrade/%s' % uuid)
|
||||
self.assertEqual(result['from_version'], 'v1.43.1')
|
||||
self.assertEqual(result['to_version'], 'v1.43.2')
|
||||
self.assertEqual(result['state'], new_state)
|
||||
|
||||
def test_update_state_download_images_after_failure(self):
|
||||
# Test updating the state of an upgrade to download images after a
|
||||
# failure
|
||||
|
||||
# Create the upgrade
|
||||
kube_upgrade = dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.43.1',
|
||||
to_version='v1.43.2',
|
||||
state=kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED)
|
||||
uuid = kube_upgrade.uuid
|
||||
|
||||
# Update state
|
||||
new_state = kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES
|
||||
response = self.patch_json('/kube_upgrade',
|
||||
[{'path': '/state',
|
||||
'value': new_state,
|
||||
'op': 'replace'}],
|
||||
headers={'User-Agent': 'sysinv-test'})
|
||||
self.assertEqual(response.content_type, 'application/json')
|
||||
self.assertEqual(response.status_code, http_client.OK)
|
||||
self.assertEqual(response.json['from_version'], 'v1.43.1')
|
||||
self.assertEqual(response.json['to_version'], 'v1.43.2')
|
||||
self.assertEqual(response.json['state'], new_state)
|
||||
|
||||
# Verify that the images were downloaded
|
||||
self.fake_conductor_api.kube_download_images.\
|
||||
assert_called_with(mock.ANY, 'v1.43.2')
|
||||
|
||||
# Verify that the upgrade was updated with the new state
|
||||
result = self.get_json('/kube_upgrade/%s' % uuid)
|
||||
self.assertEqual(result['from_version'], 'v1.43.1')
|
||||
self.assertEqual(result['to_version'], 'v1.43.2')
|
||||
self.assertEqual(result['state'], new_state)
|
||||
|
||||
def test_update_state_download_images_invalid_state(self):
|
||||
# Test updating the state of an upgrade to download images in an
|
||||
# invalid state
|
||||
|
||||
# Create the upgrade
|
||||
dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.43.1',
|
||||
to_version='v1.43.2',
|
||||
state=kubernetes.KUBE_UPGRADING_KUBELETS)
|
||||
|
||||
# Update state
|
||||
new_state = kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES
|
||||
result = self.patch_json('/kube_upgrade',
|
||||
[{'path': '/state',
|
||||
'value': new_state,
|
||||
'op': 'replace'}],
|
||||
headers={'User-Agent': 'sysinv-test'},
|
||||
expect_errors=True)
|
||||
|
||||
# Verify the failure
|
||||
self.assertEqual(result.content_type, 'application/json')
|
||||
self.assertEqual(http_client.BAD_REQUEST, result.status_int)
|
||||
self.assertIn("Kubernetes upgrade must be in",
|
||||
result.json['error_message'])
|
||||
|
||||
def test_update_state_upgrade_networking(self):
|
||||
# Test updating the state of an upgrade to upgrade networking
|
||||
|
||||
|
|
@ -364,6 +453,65 @@ class TestPatch(TestKubeUpgrade):
|
|||
self.assertEqual(result['to_version'], 'v1.43.2')
|
||||
self.assertEqual(result['state'], new_state)
|
||||
|
||||
def test_update_state_upgrade_networking_after_failure(self):
|
||||
# Test updating the state of an upgrade to upgrade networking after a
|
||||
# failure
|
||||
|
||||
# Create the upgrade
|
||||
kube_upgrade = dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.43.1',
|
||||
to_version='v1.43.2',
|
||||
state=kubernetes.KUBE_UPGRADING_NETWORKING_FAILED)
|
||||
uuid = kube_upgrade.uuid
|
||||
|
||||
# Update state
|
||||
new_state = kubernetes.KUBE_UPGRADING_NETWORKING
|
||||
response = self.patch_json('/kube_upgrade',
|
||||
[{'path': '/state',
|
||||
'value': new_state,
|
||||
'op': 'replace'}],
|
||||
headers={'User-Agent': 'sysinv-test'})
|
||||
self.assertEqual(response.content_type, 'application/json')
|
||||
self.assertEqual(response.status_code, http_client.OK)
|
||||
self.assertEqual(response.json['from_version'], 'v1.43.1')
|
||||
self.assertEqual(response.json['to_version'], 'v1.43.2')
|
||||
self.assertEqual(response.json['state'], new_state)
|
||||
|
||||
# Verify that networking was upgraded
|
||||
self.fake_conductor_api.kube_upgrade_networking.\
|
||||
assert_called_with(mock.ANY, 'v1.43.2')
|
||||
|
||||
# Verify that the upgrade was updated with the new state
|
||||
result = self.get_json('/kube_upgrade/%s' % uuid)
|
||||
self.assertEqual(result['from_version'], 'v1.43.1')
|
||||
self.assertEqual(result['to_version'], 'v1.43.2')
|
||||
self.assertEqual(result['state'], new_state)
|
||||
|
||||
def test_update_state_upgrade_networking_invalid_state(self):
|
||||
# Test updating the state of an upgrade to upgrade networking in an
|
||||
# invalid state
|
||||
|
||||
# Create the upgrade
|
||||
dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.43.1',
|
||||
to_version='v1.43.2',
|
||||
state=kubernetes.KUBE_UPGRADING_KUBELETS)
|
||||
|
||||
# Update state
|
||||
new_state = kubernetes.KUBE_UPGRADING_NETWORKING
|
||||
result = self.patch_json('/kube_upgrade',
|
||||
[{'path': '/state',
|
||||
'value': new_state,
|
||||
'op': 'replace'}],
|
||||
headers={'User-Agent': 'sysinv-test'},
|
||||
expect_errors=True)
|
||||
|
||||
# Verify the failure
|
||||
self.assertEqual(result.content_type, 'application/json')
|
||||
self.assertEqual(http_client.BAD_REQUEST, result.status_int)
|
||||
self.assertIn("Kubernetes upgrade must be in",
|
||||
result.json['error_message'])
|
||||
|
||||
def test_update_state_complete(self):
|
||||
# Test updating the state of an upgrade to complete
|
||||
self.kube_get_version_states_result = {'v1.42.1': 'available',
|
||||
|
|
|
|||
|
|
@ -114,7 +114,7 @@ class TestListKubeVersions(TestKubeVersion):
|
|||
dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.43.1',
|
||||
to_version='v1.43.2',
|
||||
state=kubernetes.KUBE_UPGRADE_STARTED,
|
||||
state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
|
||||
)
|
||||
|
||||
result = self.get_json('/kube_versions/v1.43.2')
|
||||
|
|
@ -157,7 +157,7 @@ class TestListKubeVersions(TestKubeVersion):
|
|||
dbutils.create_test_kube_upgrade(
|
||||
from_version='v1.43.1',
|
||||
to_version='v1.43.2',
|
||||
state=kubernetes.KUBE_UPGRADE_STARTED,
|
||||
state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
|
||||
)
|
||||
|
||||
data = self.get_json('/kube_versions')
|
||||
|
|
|
|||
|
|
@ -305,6 +305,99 @@ class TestKubeOperator(base.TestCase):
|
|||
),
|
||||
}
|
||||
|
||||
self.cp_pods_missing_result = {
|
||||
'kube-apiserver-test-node-1':
|
||||
kubernetes.client.V1PodList(
|
||||
api_version="v1",
|
||||
items=[
|
||||
kubernetes.client.V1Pod(
|
||||
api_version="v1",
|
||||
kind="Pod",
|
||||
metadata=kubernetes.client.V1ObjectMeta(
|
||||
name="kube-apiserver-test-node-1",
|
||||
namespace="kube-system"),
|
||||
spec=kubernetes.client.V1PodSpec(
|
||||
containers=[
|
||||
kubernetes.client.V1Container(
|
||||
name="kube-apiserver",
|
||||
image="test-image-1:v1.42.1"),
|
||||
],
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
'kube-controller-manager-test-node-1':
|
||||
kubernetes.client.V1PodList(
|
||||
api_version="v1",
|
||||
items=[],
|
||||
),
|
||||
'kube-scheduler-test-node-1':
|
||||
kubernetes.client.V1PodList(
|
||||
api_version="v1",
|
||||
items=[
|
||||
kubernetes.client.V1Pod(
|
||||
api_version="v1",
|
||||
kind="Pod",
|
||||
metadata=kubernetes.client.V1ObjectMeta(
|
||||
name="kube-scheduler-test-node-1",
|
||||
namespace="kube-system"),
|
||||
spec=kubernetes.client.V1PodSpec(
|
||||
containers=[
|
||||
kubernetes.client.V1Container(
|
||||
name="kube-scheduler",
|
||||
image="test-image-3:v1.42.1"),
|
||||
],
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
'kube-apiserver-test-node-2':
|
||||
kubernetes.client.V1PodList(
|
||||
api_version="v1",
|
||||
items=[
|
||||
kubernetes.client.V1Pod(
|
||||
api_version="v1",
|
||||
kind="Pod",
|
||||
metadata=kubernetes.client.V1ObjectMeta(
|
||||
name="kube-apiserver-test-node-2",
|
||||
namespace="kube-system"),
|
||||
spec=kubernetes.client.V1PodSpec(
|
||||
containers=[
|
||||
kubernetes.client.V1Container(
|
||||
name="kube-apiserver",
|
||||
image="test-image-1:v1.42.1"),
|
||||
],
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
'kube-controller-manager-test-node-2':
|
||||
kubernetes.client.V1PodList(
|
||||
api_version="v1",
|
||||
items=[
|
||||
kubernetes.client.V1Pod(
|
||||
api_version="v1",
|
||||
kind="Pod",
|
||||
metadata=kubernetes.client.V1ObjectMeta(
|
||||
name="kube-controller-manager-test-node-2",
|
||||
namespace="kube-system"),
|
||||
spec=kubernetes.client.V1PodSpec(
|
||||
containers=[
|
||||
kubernetes.client.V1Container(
|
||||
name="kube-controller-manager",
|
||||
image="test-image-2:v1.42.1"),
|
||||
],
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
'kube-scheduler-test-node-2':
|
||||
kubernetes.client.V1PodList(
|
||||
api_version="v1",
|
||||
items=[],
|
||||
),
|
||||
}
|
||||
|
||||
self.single_node_result = kubernetes.client.V1NodeList(
|
||||
api_version="v1",
|
||||
items=[
|
||||
|
|
@ -489,6 +582,20 @@ class TestKubeOperator(base.TestCase):
|
|||
result = self.kube_operator.kube_get_control_plane_versions()
|
||||
assert result == {'test-node-1': 'v1.42.0'}
|
||||
|
||||
def test_kube_get_control_plane_versions_missing_component(self):
|
||||
|
||||
self.list_namespaced_pod_result = self.cp_pods_missing_result
|
||||
self.list_node_result = self.multi_node_result
|
||||
|
||||
self.cp_pods_missing_result['kube-apiserver-test-node-1'].\
|
||||
items[0].spec.containers[0].image = "test-image-1:v1.42.0"
|
||||
self.cp_pods_missing_result['kube-controller-manager-test-node-2'].\
|
||||
items[0].spec.containers[0].image = "test-image-3:v1.42.3"
|
||||
|
||||
result = self.kube_operator.kube_get_control_plane_versions()
|
||||
assert result == {'test-node-1': 'v1.42.0',
|
||||
'test-node-2': 'v1.42.1'}
|
||||
|
||||
def test_kube_get_control_plane_versions_multi_node(self):
|
||||
|
||||
self.list_namespaced_pod_result = self.cp_pods_result
|
||||
|
|
|
|||
|
|
@ -426,7 +426,8 @@ class ManagerTestCase(base.DbTestCase):
|
|||
|
||||
# Verify that the upgrade state was updated
|
||||
updated_upgrade = self.dbapi.kube_upgrade_get_one()
|
||||
self.assertEqual(updated_upgrade.state, kubernetes.KUBE_UPGRADE_STARTED)
|
||||
self.assertEqual(updated_upgrade.state,
|
||||
kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES)
|
||||
|
||||
def test_kube_download_images_ansible_fail(self):
|
||||
# Create an upgrade
|
||||
|
|
@ -443,7 +444,67 @@ class ManagerTestCase(base.DbTestCase):
|
|||
|
||||
# Verify that the upgrade state was updated
|
||||
updated_upgrade = self.dbapi.kube_upgrade_get_one()
|
||||
self.assertEqual(updated_upgrade.state, kubernetes.KUBE_UPGRADE_FAILED)
|
||||
self.assertEqual(updated_upgrade.state,
|
||||
kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED)
|
||||
|
||||
def test_kube_upgrade_init_actions(self):
|
||||
# Create controller-0
|
||||
config_uuid = str(uuid.uuid4())
|
||||
self._create_test_ihost(
|
||||
personality=constants.CONTROLLER,
|
||||
hostname='controller-0',
|
||||
uuid=str(uuid.uuid4()),
|
||||
config_status=None,
|
||||
config_applied=config_uuid,
|
||||
config_target=config_uuid,
|
||||
invprovision=constants.PROVISIONED,
|
||||
administrative=constants.ADMIN_UNLOCKED,
|
||||
operational=constants.OPERATIONAL_ENABLED,
|
||||
availability=constants.AVAILABILITY_ONLINE,
|
||||
)
|
||||
|
||||
# Test the handling of transitory upgrade states
|
||||
expected_fail_results = [
|
||||
(kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES,
|
||||
kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED),
|
||||
(kubernetes.KUBE_UPGRADING_FIRST_MASTER,
|
||||
kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED),
|
||||
(kubernetes.KUBE_UPGRADING_NETWORKING,
|
||||
kubernetes.KUBE_UPGRADING_NETWORKING_FAILED),
|
||||
(kubernetes.KUBE_UPGRADING_SECOND_MASTER,
|
||||
kubernetes.KUBE_UPGRADING_SECOND_MASTER_FAILED),
|
||||
]
|
||||
|
||||
for current_state, fail_state in expected_fail_results:
|
||||
utils.create_test_kube_upgrade(
|
||||
from_version='v1.42.1',
|
||||
to_version='v1.42.2',
|
||||
state=current_state,
|
||||
)
|
||||
self.service._kube_upgrade_init_actions()
|
||||
updated_upgrade = self.dbapi.kube_upgrade_get_one()
|
||||
self.assertEqual(updated_upgrade.state, fail_state)
|
||||
self.dbapi.kube_upgrade_destroy(updated_upgrade.id)
|
||||
|
||||
# Test the handling of transitory host upgrade states
|
||||
expected_fail_results = [
|
||||
(kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE,
|
||||
kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED),
|
||||
(kubernetes.KUBE_HOST_UPGRADING_KUBELET,
|
||||
kubernetes.KUBE_HOST_UPGRADING_KUBELET_FAILED),
|
||||
]
|
||||
|
||||
utils.create_test_kube_upgrade(
|
||||
from_version='v1.42.1',
|
||||
to_version='v1.42.2',
|
||||
state=kubernetes.KUBE_UPGRADING_KUBELETS,
|
||||
)
|
||||
|
||||
for current_status, fail_status in expected_fail_results:
|
||||
self.dbapi.kube_host_upgrade_update(1, {'status': current_status})
|
||||
self.service._kube_upgrade_init_actions()
|
||||
updated_host_upgrade = self.dbapi.kube_host_upgrade_get(1)
|
||||
self.assertEqual(updated_host_upgrade.status, fail_status)
|
||||
|
||||
def test_kube_download_images_one_controller(self):
|
||||
# Create an upgrade
|
||||
|
|
@ -476,7 +537,8 @@ class ManagerTestCase(base.DbTestCase):
|
|||
|
||||
# Verify that the upgrade state was updated
|
||||
updated_upgrade = self.dbapi.kube_upgrade_get_one()
|
||||
self.assertEqual(updated_upgrade.state, kubernetes.KUBE_UPGRADE_STARTED)
|
||||
self.assertEqual(updated_upgrade.state,
|
||||
kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES)
|
||||
|
||||
def test_kube_download_images_one_controller_manifest_timeout(self):
|
||||
# Create an upgrade
|
||||
|
|
@ -513,7 +575,7 @@ class ManagerTestCase(base.DbTestCase):
|
|||
# Verify that the upgrade state was updated
|
||||
updated_upgrade = self.dbapi.kube_upgrade_get_one()
|
||||
self.assertEqual(updated_upgrade.state,
|
||||
kubernetes.KUBE_UPGRADE_FAILED)
|
||||
kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED)
|
||||
|
||||
def test_kube_download_images_two_controllers(self):
|
||||
# Create an upgrade
|
||||
|
|
@ -562,7 +624,8 @@ class ManagerTestCase(base.DbTestCase):
|
|||
|
||||
# Verify that the upgrade state was updated
|
||||
updated_upgrade = self.dbapi.kube_upgrade_get_one()
|
||||
self.assertEqual(updated_upgrade.state, kubernetes.KUBE_UPGRADE_STARTED)
|
||||
self.assertEqual(updated_upgrade.state,
|
||||
kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES)
|
||||
|
||||
def test_kube_upgrade_control_plane_first_master(self):
|
||||
# Create an upgrade
|
||||
|
|
@ -645,11 +708,12 @@ class ManagerTestCase(base.DbTestCase):
|
|||
# Verify that the upgrade state was updated
|
||||
updated_upgrade = self.dbapi.kube_upgrade_get_one()
|
||||
self.assertEqual(updated_upgrade.state,
|
||||
kubernetes.KUBE_UPGRADE_FAILED)
|
||||
kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED)
|
||||
|
||||
# Verify that the host upgrade status was set
|
||||
updated_host_upgrade = self.dbapi.kube_host_upgrade_get(1)
|
||||
self.assertIsNotNone(updated_host_upgrade.status)
|
||||
self.assertEqual(updated_host_upgrade.status,
|
||||
kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED)
|
||||
|
||||
def test_kube_upgrade_control_plane_first_master_upgrade_fail(self):
|
||||
# Create an upgrade
|
||||
|
|
@ -686,7 +750,7 @@ class ManagerTestCase(base.DbTestCase):
|
|||
# Verify that the upgrade state was updated
|
||||
updated_upgrade = self.dbapi.kube_upgrade_get_one()
|
||||
self.assertEqual(updated_upgrade.state,
|
||||
kubernetes.KUBE_UPGRADE_FAILED)
|
||||
kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED)
|
||||
|
||||
# Verify that the host upgrade status was cleared
|
||||
updated_host_upgrade = self.dbapi.kube_host_upgrade_get(1)
|
||||
|
|
@ -870,7 +934,7 @@ class ManagerTestCase(base.DbTestCase):
|
|||
utils.create_test_kube_upgrade(
|
||||
from_version='v1.42.1',
|
||||
to_version='v1.42.2',
|
||||
state=kubernetes.KUBE_UPGRADING_FIRST_MASTER,
|
||||
state=kubernetes.KUBE_UPGRADING_KUBELETS,
|
||||
)
|
||||
# Create controller-0
|
||||
config_uuid = str(uuid.uuid4())
|
||||
|
|
@ -898,21 +962,22 @@ class ManagerTestCase(base.DbTestCase):
|
|||
# Upgrade the kubelet
|
||||
self.service.kube_upgrade_kubelet(self.context, c0.uuid)
|
||||
|
||||
# Verify that the upgrade state was updated
|
||||
# Verify that the upgrade state was not updated
|
||||
updated_upgrade = self.dbapi.kube_upgrade_get_one()
|
||||
self.assertEqual(updated_upgrade.state,
|
||||
kubernetes.KUBE_UPGRADE_FAILED)
|
||||
kubernetes.KUBE_UPGRADING_KUBELETS)
|
||||
|
||||
# Verify that the host upgrade status was set
|
||||
updated_host_upgrade = self.dbapi.kube_host_upgrade_get(1)
|
||||
self.assertIsNotNone(updated_host_upgrade.status)
|
||||
self.assertEqual(updated_host_upgrade.status,
|
||||
kubernetes.KUBE_HOST_UPGRADING_KUBELET_FAILED)
|
||||
|
||||
def test_kube_upgrade_kubelet_controller_upgrade_fail(self):
|
||||
# Create an upgrade
|
||||
utils.create_test_kube_upgrade(
|
||||
from_version='v1.42.1',
|
||||
to_version='v1.42.2',
|
||||
state=kubernetes.KUBE_UPGRADING_FIRST_MASTER,
|
||||
state=kubernetes.KUBE_UPGRADING_KUBELETS,
|
||||
)
|
||||
# Create controller-0
|
||||
config_uuid = str(uuid.uuid4())
|
||||
|
|
@ -939,10 +1004,10 @@ class ManagerTestCase(base.DbTestCase):
|
|||
# Upgrade the kubelet
|
||||
self.service.kube_upgrade_kubelet(self.context, c0.uuid)
|
||||
|
||||
# Verify that the upgrade state was updated
|
||||
# Verify that the upgrade state was not updated
|
||||
updated_upgrade = self.dbapi.kube_upgrade_get_one()
|
||||
self.assertEqual(updated_upgrade.state,
|
||||
kubernetes.KUBE_UPGRADE_FAILED)
|
||||
kubernetes.KUBE_UPGRADING_KUBELETS)
|
||||
|
||||
# Verify that the host upgrade status was cleared
|
||||
updated_host_upgrade = self.dbapi.kube_host_upgrade_get(1)
|
||||
|
|
|
|||
Loading…
Reference in New Issue