Merge "AIO-SX K8s upgrade optimization: "cordon" and "uncordon" as optional"

This commit is contained in:
Zuul 2023-05-19 21:56:09 +00:00 committed by Gerrit Code Review
commit 19c5d1df21
4 changed files with 58 additions and 17 deletions

View File

@ -7114,6 +7114,7 @@ class HostController(rest.RestController):
if utils.get_system_mode() == constants.SYSTEM_MODE_SIMPLEX:
check_upgraded_state = [
kubernetes.KUBE_UPGRADED_NETWORKING,
kubernetes.KUBE_UPGRADED_FIRST_MASTER,
kubernetes.KUBE_UPGRADE_CORDON_COMPLETE]
else:

View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2019 Wind River Systems, Inc.
# Copyright (c) 2019-2023 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -468,14 +468,31 @@ class KubeUpgradeController(rest.RestController):
# Make sure upgrade is in the correct state to complete
system = pecan.request.dbapi.isystem_get_one()
if system.system_mode == constants.SYSTEM_MODE_SIMPLEX:
if kube_upgrade_obj.state not in [
kubernetes.KUBE_UPGRADE_UNCORDON_COMPLETE]:
raise wsme.exc.ClientSideError(_(
"Kubernetes upgrade must be in %s state to complete" %
kubernetes.KUBE_UPGRADE_UNCORDON_COMPLETE))
# If the node is unschedulable=True then the cordon command
# executed already and some of the pods are in pending status.
# The uncordon command needs to be triggered.
# If the node is unschedulable=None then the pods are in the
# running status.
unschedulable = None
node_status = self._kube_operator.kube_get_node_status(constants.CONTROLLER_0_HOSTNAME)
LOG.debug("Node status: %s" % node_status)
if node_status:
unschedulable = node_status.spec.unschedulable
kube_host_upgrades = \
pecan.request.dbapi.kube_host_upgrade_get_list()
if unschedulable:
if kube_upgrade_obj.state not in [
kubernetes.KUBE_UPGRADE_UNCORDON_COMPLETE]:
raise wsme.exc.ClientSideError(_(
"Kubernetes upgrade must be in %s state to complete" %
kubernetes.KUBE_UPGRADE_UNCORDON_COMPLETE))
else:
if kube_upgrade_obj.state not in [
kubernetes.KUBE_UPGRADING_KUBELETS,
kubernetes.KUBE_UPGRADE_UNCORDON_COMPLETE]:
raise wsme.exc.ClientSideError(_(
"Kubernetes upgrade must be in %s or %s state to complete" %
(kubernetes.KUBE_UPGRADING_KUBELETS,
kubernetes.KUBE_UPGRADE_UNCORDON_COMPLETE)))
else:
if kube_upgrade_obj.state not in [
kubernetes.KUBE_UPGRADING_KUBELETS]:
@ -483,15 +500,15 @@ class KubeUpgradeController(rest.RestController):
"Kubernetes upgrade must be in %s state to complete" %
kubernetes.KUBE_UPGRADING_KUBELETS))
# Make sure no hosts are in a transitory or failed state
kube_host_upgrades = \
pecan.request.dbapi.kube_host_upgrade_get_list()
for kube_host_upgrade in kube_host_upgrades:
if kube_host_upgrade.status != \
kubernetes.KUBE_HOST_UPGRADED_KUBELET:
raise wsme.exc.ClientSideError(_(
"At least one host has not completed the kubernetes "
"upgrade"))
# Make sure no hosts are in a transitory or failed state
kube_host_upgrades = \
pecan.request.dbapi.kube_host_upgrade_get_list()
for kube_host_upgrade in kube_host_upgrades:
if kube_host_upgrade.status != \
kubernetes.KUBE_HOST_UPGRADED_KUBELET:
raise wsme.exc.ClientSideError(_(
"At least one host has not completed the kubernetes "
"upgrade"))
# Make sure the target version is active
version_states = self._kube_operator.kube_get_version_states()

View File

@ -392,6 +392,18 @@ class KubeOperator(object):
LOG.warn("Kubernetes exception in kube_get_nodes: %s" % e)
raise
@retry(stop_max_attempt_number=API_RETRY_ATTEMPT_NUMBER,
wait_fixed=API_RETRY_INTERVAL,
retry_on_exception=_retry_on_urllibs3_MaxRetryError)
def kube_get_node_status(self, name):
try:
api_response = self._get_kubernetesclient_core().read_node_status(name)
LOG.debug("Response: %s" % api_response)
return api_response
except Exception as e:
LOG.warn("Kubernetes exception in kube_get_node_status: %s" % e)
raise
def kube_namespaced_pods_exist(self, namespace):
LOG.debug("kube_namespaced_pods_exist, namespace=%s" %
(namespace))

View File

@ -132,6 +132,17 @@ class TestKubeUpgrade(base.FunctionalTest):
self.mocked_get_kube_versions.start()
self.addCleanup(self.mocked_get_kube_versions.stop)
# Mock the KubeOperator
self.kube_get_node_status_result = None
def mock_kube_get_node_status(obj):
return self.kube_get_node_status_result
self.mocked_kube_get_node_status = mock.patch(
'sysinv.common.kubernetes.KubeOperator.kube_get_node_status',
mock_kube_get_node_status)
self.mocked_kube_get_node_status.start()
self.addCleanup(self.mocked_kube_get_node_status.stop)
# Mock the KubeOperator
self.kube_get_kubernetes_version_result = 'v1.43.1'