diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/tests/v1/test_kube_upgrade_shell.py b/sysinv/cgts-client/cgts-client/cgtsclient/tests/v1/test_kube_upgrade_shell.py index df07e32651..4a25383f15 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/tests/v1/test_kube_upgrade_shell.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/tests/v1/test_kube_upgrade_shell.py @@ -97,6 +97,31 @@ class KubeUpgradeTest(test_shell.ShellTest): self.assertIn(fake_kube_upgrade['created_at'], results) self.assertIn(fake_kube_upgrade['updated_at'], results) + @mock.patch('cgtsclient.v1.kube_upgrade.KubeUpgradeManager.update') + @mock.patch('cgtsclient.client._get_ksclient') + @mock.patch('cgtsclient.client._get_endpoint') + def test_kube_upgrade_download_images(self, mock_get_endpoint, + mock_get_client, + mock_update): + mock_get_endpoint.return_value = 'http://fakelocalhost:6385/v1' + fake_kube_upgrade = {'from_version': 'v1.42.1', + 'to_version': 'v1.42.2', + 'state': 'downloading-images', + 'uuid': 'cb737aba-1820-4184-b0dc-9b073822af48', + 'created_at': 'fake-created-time', + 'updated_at': 'fake-updated-time', + } + mock_update.return_value = KubeUpgrade(None, fake_kube_upgrade, True) + + self.make_env() + results = self.shell("kube-upgrade-download-images") + self.assertIn(fake_kube_upgrade['from_version'], results) + self.assertIn(fake_kube_upgrade['to_version'], results) + self.assertIn(fake_kube_upgrade['state'], results) + self.assertIn(fake_kube_upgrade['uuid'], results) + self.assertIn(fake_kube_upgrade['created_at'], results) + self.assertIn(fake_kube_upgrade['updated_at'], results) + @mock.patch('cgtsclient.v1.kube_upgrade.KubeUpgradeManager.update') @mock.patch('cgtsclient.client._get_ksclient') @mock.patch('cgtsclient.client._get_endpoint') diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_upgrade_shell.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_upgrade_shell.py index 07c0cc1794..06bd93b5b9 100755 --- a/sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_upgrade_shell.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_upgrade_shell.py @@ -8,6 +8,7 @@ from cgtsclient.common import utils from cgtsclient import exc # Kubernetes constants +KUBE_UPGRADE_STATE_DOWNLOADING_IMAGES = 'downloading-images' KUBE_UPGRADE_STATE_UPGRADING_NETWORKING = 'upgrading-networking' KUBE_UPGRADE_STATE_COMPLETE = 'upgrade-complete' @@ -48,6 +49,23 @@ def do_kube_upgrade_start(cc, args): _print_kube_upgrade_show(kube_upgrade) +def do_kube_upgrade_download_images(cc, args): + """Download kubernetes images.""" + + data = dict() + data['state'] = KUBE_UPGRADE_STATE_DOWNLOADING_IMAGES + + patch = [] + for (k, v) in data.items(): + patch.append({'op': 'replace', 'path': '/' + k, 'value': v}) + try: + kube_upgrade = cc.kube_upgrade.update(patch) + except exc.HTTPNotFound: + raise exc.CommandError('Kubernetes upgrade UUID not found') + + _print_kube_upgrade_show(kube_upgrade) + + def do_kube_upgrade_networking(cc, args): """Upgrade kubernetes networking.""" diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py index 1901d14c24..fdbbcf6604 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py @@ -6480,12 +6480,45 @@ class HostController(rest.RestController): hostupdate.ihost_val_prenotify_update(val) hostupdate.ihost_val.update(val) + @staticmethod + def _check_patch_requirements(region_name, + applied_patches=None, + available_patches=None): + """Checks whether specified patches are applied or available""" + + api_token = None + if applied_patches is not None: + patches_applied = patch_api.patch_is_applied( + token=api_token, + timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS, + region_name=region_name, + patches=applied_patches + ) + if not patches_applied: + raise wsme.exc.ClientSideError(_( + "The following patches must be applied before doing " + "this action: %s" % applied_patches)) + + if available_patches is not None: + patches_available = patch_api.patch_is_available( + token=api_token, + timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS, + region_name=region_name, + patches=available_patches + ) + if not patches_available: + raise wsme.exc.ClientSideError(_( + "The following patches must be available before doing " + "this action: %s" % available_patches)) + @cutils.synchronized(LOCK_NAME) @wsme_pecan.wsexpose(Host, six.text_type, body=six.text_type) def kube_upgrade_control_plane(self, uuid, body): """Upgrade the kubernetes control plane on this host""" host_obj = objects.host.get_by_uuid(pecan.request.context, uuid) + kube_host_upgrade_obj = objects.kube_host_upgrade.get_by_host_id( + pecan.request.context, host_obj.id) # The kubernetes upgrade must have been started try: @@ -6495,27 +6528,54 @@ class HostController(rest.RestController): raise wsme.exc.ClientSideError(_( "A kubernetes upgrade is not in progress.")) + LOG.info("Attempting to upgrade control plane on host %s while in " + "state %s" % (host_obj.hostname, kube_upgrade_obj.state)) + # Either controller can be upgraded if host_obj.personality != constants.CONTROLLER: raise wsme.exc.ClientSideError(_( "This host does not have a kubernetes control plane.")) # Verify the upgrade is in the correct state - if kube_upgrade_obj.state not in [kubernetes.KUBE_UPGRADE_STARTED, - kubernetes.KUBE_UPGRADED_NETWORKING]: + if kube_upgrade_obj.state in [ + kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES, + kubernetes.KUBE_UPGRADED_NETWORKING, + kubernetes.KUBE_UPGRADED_FIRST_MASTER]: + # We are upgrading a control plane + pass + elif kube_upgrade_obj.state in [ + kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED, + kubernetes.KUBE_UPGRADING_SECOND_MASTER_FAILED]: + # We are re-attempting the upgrade of a control plane. Make sure + # this really is a re-attempt. + if kube_host_upgrade_obj.target_version != \ + kube_upgrade_obj.to_version: + raise wsme.exc.ClientSideError(_( + "The first control plane upgrade must be completed before " + "upgrading the second control plane.")) + else: raise wsme.exc.ClientSideError(_( - "The kubernetes upgrade must be in the %s or %s state to " - "upgrade the control plane." % ( - kubernetes.KUBE_UPGRADE_STARTED, - kubernetes.KUBE_UPGRADED_NETWORKING))) + "The kubernetes upgrade is not in a valid state to " + "upgrade the control plane.")) + + # Verify patching requirements (since the api server may not be + # upgraded yet, patches could have been removed) + system = pecan.request.dbapi.isystem_get_one() + target_version_obj = objects.kube_version.get_by_version( + kube_upgrade_obj.to_version) + self._check_patch_requirements( + system.region_name, + applied_patches=target_version_obj.applied_patches, + available_patches=target_version_obj.available_patches) # Check the existing control plane version cp_versions = self._kube_operator.kube_get_control_plane_versions() current_cp_version = cp_versions.get(host_obj.hostname) if current_cp_version == kube_upgrade_obj.to_version: - raise wsme.exc.ClientSideError(_( - "The kubernetes control plane on this host was already " - "upgraded.")) + # The control plane was already upgraded, but we didn't progress + # to the next state, so something failed along the way. + LOG.info("Redoing kubernetes control plane upgrade for %s" % + host_obj.hostname) elif current_cp_version is None: raise wsme.exc.ClientSideError(_( "Unable to determine the version of the kubernetes control " @@ -6528,29 +6588,39 @@ class HostController(rest.RestController): "The host must be unlocked and available to upgrade the " "control plane.")) + # Verify the control plane is not being upgraded + if kube_host_upgrade_obj.status == \ + kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE: + raise wsme.exc.ClientSideError(_( + "The control plane on this host is already being upgraded.")) + # Update the target version for this host - kube_host_upgrade_obj = objects.kube_host_upgrade.get_by_host_id( - pecan.request.context, host_obj.id) kube_host_upgrade_obj.target_version = kube_upgrade_obj.to_version + + # Set the status + kube_host_upgrade_obj.status = \ + kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE kube_host_upgrade_obj.save() - if kube_upgrade_obj.state == kubernetes.KUBE_UPGRADE_STARTED: - # Tell the conductor to upgrade the control plane - pecan.request.rpcapi.kube_upgrade_control_plane( - pecan.request.context, host_obj.uuid) - + if kube_upgrade_obj.state in [ + kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES, + kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED]: # Update the upgrade state kube_upgrade_obj.state = kubernetes.KUBE_UPGRADING_FIRST_MASTER kube_upgrade_obj.save() - else: + # Tell the conductor to upgrade the control plane pecan.request.rpcapi.kube_upgrade_control_plane( pecan.request.context, host_obj.uuid) - + else: # Update the upgrade state kube_upgrade_obj.state = kubernetes.KUBE_UPGRADING_SECOND_MASTER kube_upgrade_obj.save() + # Tell the conductor to upgrade the control plane + pecan.request.rpcapi.kube_upgrade_control_plane( + pecan.request.context, host_obj.uuid) + LOG.info("Upgrading kubernetes control plane on host %s" % host_obj.hostname) return Host.convert_with_links(host_obj) @@ -6571,6 +6641,9 @@ class HostController(rest.RestController): raise wsme.exc.ClientSideError(_( "A kubernetes upgrade is not in progress.")) + LOG.info("Attempting to upgrade kubelet on host %s while in " + "state %s" % (host_obj.hostname, kube_upgrade_obj.state)) + # Verify the host has a kubelet if host_obj.personality not in [constants.CONTROLLER, constants.WORKER]: @@ -6579,11 +6652,14 @@ class HostController(rest.RestController): # Verify the upgrade is in the correct state if utils.get_system_mode() == constants.SYSTEM_MODE_SIMPLEX: - if kube_upgrade_obj.state != kubernetes.KUBE_UPGRADED_NETWORKING: + if kube_upgrade_obj.state not in [ + kubernetes.KUBE_UPGRADED_NETWORKING, + kubernetes.KUBE_UPGRADING_KUBELETS]: raise wsme.exc.ClientSideError(_( - "The kubernetes upgrade must be in the %s state to " - "upgrade the kubelet." % - kubernetes.KUBE_UPGRADED_NETWORKING)) + "The kubernetes upgrade must be in the %s or %s state to " + "upgrade the kubelet." % ( + kubernetes.KUBE_UPGRADED_NETWORKING, + kubernetes.KUBE_UPGRADING_KUBELETS))) elif kube_upgrade_obj.state not in [ kubernetes.KUBE_UPGRADED_SECOND_MASTER, kubernetes.KUBE_UPGRADING_KUBELETS]: @@ -6593,22 +6669,13 @@ class HostController(rest.RestController): kubernetes.KUBE_UPGRADED_SECOND_MASTER, kubernetes.KUBE_UPGRADING_KUBELETS))) - # The necessary patches must be applied - api_token = None + # Verify patching requirements system = pecan.request.dbapi.isystem_get_one() target_version_obj = objects.kube_version.get_by_version( kube_upgrade_obj.to_version) - if target_version_obj.available_patches: - patches_applied = patch_api.patch_is_applied( - token=api_token, - timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS, - region_name=system.region_name, - patches=target_version_obj.available_patches - ) - if not patches_applied: - raise wsme.exc.ClientSideError(_( - "The following patches must be applied before upgrading " - "the kubelets: %s" % target_version_obj.available_patches)) + self._check_patch_requirements( + system.region_name, + applied_patches=target_version_obj.available_patches) # Enforce the ordering of controllers first kubelet_versions = self._kube_operator.kube_get_kubelet_versions() @@ -6630,7 +6697,7 @@ class HostController(rest.RestController): if current_kubelet_version == kube_upgrade_obj.to_version: # If the force option was used, we will redo the upgrade if force: - LOG.info("Redoing kubernetes upgrade for %s" % + LOG.info("Redoing kubelet upgrade for %s" % host_obj.hostname) else: raise wsme.exc.ClientSideError(_( @@ -6652,10 +6719,19 @@ class HostController(rest.RestController): raise wsme.exc.ClientSideError(_( "The host must be locked and online to upgrade the kubelet.")) + # Verify the kubelet is not being upgraded + if kube_host_upgrade_obj.status == \ + kubernetes.KUBE_HOST_UPGRADING_KUBELET: + raise wsme.exc.ClientSideError(_( + "The kubelet on this host is already being upgraded.")) + # Set the target version if this is a worker host if host_obj.personality == constants.WORKER: kube_host_upgrade_obj.target_version = kube_upgrade_obj.to_version - kube_host_upgrade_obj.save() + + # Set the status + kube_host_upgrade_obj.status = kubernetes.KUBE_HOST_UPGRADING_KUBELET + kube_host_upgrade_obj.save() # Tell the conductor to upgrade the kubelet pecan.request.rpcapi.kube_upgrade_kubelet(pecan.request.context, diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py index 920a549885..8d16a5d31b 100755 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py @@ -116,6 +116,38 @@ class KubeUpgradeController(rest.RestController): updates[attribute] = p['value'] return updates + @staticmethod + def _check_patch_requirements(region_name, + applied_patches=None, + available_patches=None): + """Checks whether specified patches are applied or available""" + + api_token = None + if applied_patches: + patches_applied = patch_api.patch_is_applied( + token=api_token, + timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS, + region_name=region_name, + patches=applied_patches + ) + if not patches_applied: + raise wsme.exc.ClientSideError(_( + "The following patches must be applied before doing " + "the kubernetes upgrade: %s" % applied_patches)) + + if available_patches: + patches_available = patch_api.patch_is_available( + token=api_token, + timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS, + region_name=region_name, + patches=available_patches + ) + if not patches_available: + raise wsme.exc.ClientSideError(_( + "The following patches must be available before doing " + "the kubernetes upgrade: %s" % + available_patches)) + @wsme_pecan.wsexpose(KubeUpgradeCollection) def get_all(self): """Retrieve a list of kubernetes upgrades.""" @@ -162,7 +194,7 @@ class KubeUpgradeController(rest.RestController): raise wsme.exc.ClientSideError(_( "The installed Kubernetes version %s cannot upgrade to " "version %s" % (current_kube_version, - target_version_obj.version))) + target_version_obj.version))) # The current kubernetes version must be active version_states = self._kube_operator.kube_get_version_states() @@ -172,35 +204,12 @@ class KubeUpgradeController(rest.RestController): "The installed Kubernetes version %s is not active on all " "hosts" % current_kube_version)) - # The necessary patches must be applied - api_token = None + # Verify patching requirements system = pecan.request.dbapi.isystem_get_one() - if target_version_obj.applied_patches: - patches_applied = patch_api.patch_is_applied( - token=api_token, - timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS, - region_name=system.region_name, - patches=target_version_obj.applied_patches - ) - if not patches_applied: - raise wsme.exc.ClientSideError(_( - "The following patches must be applied before starting " - "the kubernetes upgrade: %s" % - target_version_obj.applied_patches)) - - # The necessary patches must be available - if target_version_obj.available_patches: - patches_available = patch_api.patch_is_available( - token=api_token, - timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS, - region_name=system.region_name, - patches=target_version_obj.available_patches - ) - if not patches_available: - raise wsme.exc.ClientSideError(_( - "The following patches must be available before starting " - "the kubernetes upgrade: %s" % - target_version_obj.available_patches)) + self._check_patch_requirements( + system.region_name, + applied_patches=target_version_obj.applied_patches, + available_patches=target_version_obj.available_patches) # TODO: check that all installed applications support new k8s version # TODO: check that tiller/armada support new k8s version @@ -220,14 +229,10 @@ class KubeUpgradeController(rest.RestController): # TODO: kubernetes related health checks... - # Tell the conductor to download the images for the new version - pecan.request.rpcapi.kube_download_images( - pecan.request.context, to_version) - # Create upgrade record. create_values = {'from_version': current_kube_version, 'to_version': to_version, - 'state': kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES} + 'state': kubernetes.KUBE_UPGRADE_STARTED} new_upgrade = pecan.request.dbapi.kube_upgrade_create(create_values) # Set the target version for each host to the current version @@ -237,7 +242,7 @@ class KubeUpgradeController(rest.RestController): pecan.request.dbapi.kube_host_upgrade_update(kube_host_upgrade.id, update_values) - LOG.info("Starting kubernetes upgrade from version: %s to version: %s" + LOG.info("Started kubernetes upgrade from version: %s to version: %s" % (current_kube_version, to_version)) return KubeUpgrade.convert_with_links(new_upgrade) @@ -258,14 +263,49 @@ class KubeUpgradeController(rest.RestController): raise wsme.exc.ClientSideError(_( "A kubernetes upgrade is not in progress")) - if updates['state'] == kubernetes.KUBE_UPGRADING_NETWORKING: - # Make sure upgrade is in the correct state to upgrade networking - if kube_upgrade_obj.state != \ - kubernetes.KUBE_UPGRADED_FIRST_MASTER: + if updates['state'] == kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES: + # Make sure upgrade is in the correct state to download images + if kube_upgrade_obj.state not in [ + kubernetes.KUBE_UPGRADE_STARTED, + kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED]: raise wsme.exc.ClientSideError(_( - "Kubernetes upgrade must be in %s state to upgrade " + "Kubernetes upgrade must be in %s or %s state to download " + "images" % + (kubernetes.KUBE_UPGRADE_STARTED, + kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED))) + + # Verify patching requirements (since the api server is not + # upgraded yet, patches could have been removed) + system = pecan.request.dbapi.isystem_get_one() + target_version_obj = objects.kube_version.get_by_version( + kube_upgrade_obj.to_version) + self._check_patch_requirements( + system.region_name, + applied_patches=target_version_obj.applied_patches, + available_patches=target_version_obj.available_patches) + + # Update the upgrade state + kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES + kube_upgrade_obj.save() + + # Tell the conductor to download the images for the new version + pecan.request.rpcapi.kube_download_images( + pecan.request.context, kube_upgrade_obj.to_version) + + LOG.info("Downloading kubernetes images for version: %s" % + kube_upgrade_obj.to_version) + return KubeUpgrade.convert_with_links(kube_upgrade_obj) + + elif updates['state'] == kubernetes.KUBE_UPGRADING_NETWORKING: + # Make sure upgrade is in the correct state to upgrade networking + if kube_upgrade_obj.state not in [ + kubernetes.KUBE_UPGRADED_FIRST_MASTER, + kubernetes.KUBE_UPGRADING_NETWORKING_FAILED]: + raise wsme.exc.ClientSideError(_( + "Kubernetes upgrade must be in %s or %s state to upgrade " "networking" % - kubernetes.KUBE_UPGRADED_FIRST_MASTER)) + (kubernetes.KUBE_UPGRADED_FIRST_MASTER, + kubernetes.KUBE_UPGRADING_NETWORKING_FAILED))) # Update the upgrade state kube_upgrade_obj.state = kubernetes.KUBE_UPGRADING_NETWORKING @@ -275,6 +315,8 @@ class KubeUpgradeController(rest.RestController): pecan.request.rpcapi.kube_upgrade_networking( pecan.request.context, kube_upgrade_obj.to_version) + LOG.info("Upgrading kubernetes networking to version: %s" % + kube_upgrade_obj.to_version) return KubeUpgrade.convert_with_links(kube_upgrade_obj) elif updates['state'] == kubernetes.KUBE_UPGRADE_COMPLETE: @@ -295,6 +337,9 @@ class KubeUpgradeController(rest.RestController): # All is well, mark the upgrade as complete kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_COMPLETE kube_upgrade_obj.save() + + LOG.info("Completed kubernetes upgrade to version: %s" % + kube_upgrade_obj.to_version) return KubeUpgrade.convert_with_links(kube_upgrade_obj) else: @@ -302,7 +347,7 @@ class KubeUpgradeController(rest.RestController): "Invalid state %s supplied" % updates['state'])) @cutils.synchronized(LOCK_NAME) - @wsme_pecan.wsexpose(KubeUpgrade) + @wsme_pecan.wsexpose(None) def delete(self): """Delete Kubernetes Upgrade.""" diff --git a/sysinv/sysinv/sysinv/sysinv/common/kubernetes.py b/sysinv/sysinv/sysinv/sysinv/common/kubernetes.py index fed6f2ceb2..ad12f4e4ff 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/kubernetes.py +++ b/sysinv/sysinv/sysinv/sysinv/common/kubernetes.py @@ -41,17 +41,27 @@ KUBE_CONTROLLER_MANAGER = 'kube-controller-manager' KUBE_SCHEDULER = 'kube-scheduler' # Kubernetes upgrade states -KUBE_UPGRADE_DOWNLOADING_IMAGES = 'downloading-images' KUBE_UPGRADE_STARTED = 'upgrade-started' +KUBE_UPGRADE_DOWNLOADING_IMAGES = 'downloading-images' +KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED = 'downloading-images-failed' +KUBE_UPGRADE_DOWNLOADED_IMAGES = 'downloaded-images' KUBE_UPGRADING_FIRST_MASTER = 'upgrading-first-master' +KUBE_UPGRADING_FIRST_MASTER_FAILED = 'upgrading-first-master-failed' KUBE_UPGRADED_FIRST_MASTER = 'upgraded-first-master' KUBE_UPGRADING_NETWORKING = 'upgrading-networking' +KUBE_UPGRADING_NETWORKING_FAILED = 'upgrading-networking-failed' KUBE_UPGRADED_NETWORKING = 'upgraded-networking' KUBE_UPGRADING_SECOND_MASTER = 'upgrading-second-master' +KUBE_UPGRADING_SECOND_MASTER_FAILED = 'upgrading-second-master-failed' KUBE_UPGRADED_SECOND_MASTER = 'upgraded-second-master' KUBE_UPGRADING_KUBELETS = 'upgrading-kubelets' KUBE_UPGRADE_COMPLETE = 'upgrade-complete' -KUBE_UPGRADE_FAILED = 'upgrade-failed' + +# Kubernetes host upgrade statuses +KUBE_HOST_UPGRADING_CONTROL_PLANE = 'upgrading-control-plane' +KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED = 'upgrading-control-plane-failed' +KUBE_HOST_UPGRADING_KUBELET = 'upgrading-kubelet' +KUBE_HOST_UPGRADING_KUBELET_FAILED = 'upgrading-kubelet-failed' # Kubernetes constants MANIFEST_APPLY_TIMEOUT = 60 * 15 @@ -391,7 +401,7 @@ class KubeOperator(object): node_versions = dict() for node_name in master_nodes: - versions = dict() + versions = list() for component in [KUBE_APISERVER, KUBE_CONTROLLER_MANAGER, KUBE_SCHEDULER]: @@ -400,14 +410,11 @@ class KubeOperator(object): pod_name = component + '-' + node_name image = self.kube_get_image_by_pod_name( pod_name, NAMESPACE_KUBE_SYSTEM, component) - versions[component] = image.rsplit(':')[-1] + if image is not None: + versions.append(LooseVersion(image.rsplit(':')[-1])) # Calculate the lowest version - lowest_version = min( - LooseVersion(versions[KUBE_APISERVER]), - LooseVersion(versions[KUBE_CONTROLLER_MANAGER]), - LooseVersion(versions[KUBE_SCHEDULER])) - node_versions[node_name] = str(lowest_version) + node_versions[node_name] = str(min(versions)) return node_versions diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index 682bae6519..24ee8dc342 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -234,6 +234,7 @@ class ConductorManager(service.PeriodicService): # Upgrade start tasks self._upgrade_init_actions() + self._kube_upgrade_init_actions() self._handle_restore_in_progress() @@ -10682,6 +10683,51 @@ class ConductorManager(service.PeriodicService): os.chmod(constants.SYSINV_CONF_DEFAULT_PATH, 0o400) + def _kube_upgrade_init_actions(self): + """ Perform any kubernetes upgrade related startup actions""" + try: + kube_upgrade = self.dbapi.kube_upgrade_get_one() + except exception.NotFound: + # Not upgrading kubernetes + return + + # Fail any upgrade operation that is in a transitory state. This + # service is responsible for monitoring these operations and since + # we were just restarted, the operation will never progress. + fail_state = None + if kube_upgrade.state == kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES: + fail_state = kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED + elif kube_upgrade.state == kubernetes.KUBE_UPGRADING_FIRST_MASTER: + fail_state = kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED + elif kube_upgrade.state == kubernetes.KUBE_UPGRADING_NETWORKING: + fail_state = kubernetes.KUBE_UPGRADING_NETWORKING_FAILED + elif kube_upgrade.state == kubernetes.KUBE_UPGRADING_SECOND_MASTER: + fail_state = kubernetes.KUBE_UPGRADING_SECOND_MASTER_FAILED + + if fail_state is not None: + LOG.warning("Failing upgrade in %s state due to service restart" % + kube_upgrade.state) + self.dbapi.kube_upgrade_update(kube_upgrade.id, + {'state': fail_state}) + + # Fail any host upgrade operation that is in a transitory state. + kube_host_upgrades = self.dbapi.kube_host_upgrade_get_list() + for kube_host_upgrade in kube_host_upgrades: + fail_status = None + if kube_host_upgrade.status == \ + kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE: + fail_status = \ + kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED + elif kube_host_upgrade.status == \ + kubernetes.KUBE_HOST_UPGRADING_KUBELET: + fail_status = kubernetes.KUBE_HOST_UPGRADING_KUBELET_FAILED + + if fail_status is not None: + LOG.warning("Failing host upgrade with %s status due to " + "service restart" % kube_host_upgrade.status) + self.dbapi.kube_host_upgrade_update(kube_host_upgrade.id, + {'status': fail_status}) + def kube_download_images(self, context, kube_version): """Download the kubernetes images for this version""" @@ -10703,7 +10749,8 @@ class ConductorManager(service.PeriodicService): proc.returncode) # Update the upgrade state kube_upgrade_obj = objects.kube_upgrade.get_one(context) - kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_FAILED + kube_upgrade_obj.state = \ + kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED kube_upgrade_obj.save() return @@ -10738,13 +10785,14 @@ class ConductorManager(service.PeriodicService): else: LOG.warning("Manifest apply failed for a controller host") kube_upgrade_obj = objects.kube_upgrade.get_one(context) - kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_FAILED + kube_upgrade_obj.state = \ + kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED kube_upgrade_obj.save() return # Update the upgrade state kube_upgrade_obj = objects.kube_upgrade.get_one(context) - kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_STARTED + kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES kube_upgrade_obj.save() def kube_upgrade_control_plane(self, context, host_uuid): @@ -10760,17 +10808,15 @@ class ConductorManager(service.PeriodicService): if kube_upgrade_obj.state == kubernetes.KUBE_UPGRADING_FIRST_MASTER: puppet_class = 'platform::kubernetes::upgrade_first_control_plane' new_state = kubernetes.KUBE_UPGRADED_FIRST_MASTER + fail_state = kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED elif kube_upgrade_obj.state == kubernetes.KUBE_UPGRADING_SECOND_MASTER: puppet_class = 'platform::kubernetes::upgrade_control_plane' new_state = kubernetes.KUBE_UPGRADED_SECOND_MASTER + fail_state = kubernetes.KUBE_UPGRADING_SECOND_MASTER_FAILED else: - LOG.error("Invalid state %s to upgrade control plane." % - kube_upgrade_obj.state) - return - - # Update status - kube_host_upgrade_obj.status = "Upgrading control plane" - kube_host_upgrade_obj.save() + raise exception.SysinvException(_( + "Invalid state %s to upgrade control plane." % + kube_upgrade_obj.state)) # Update the config for this host personalities = [host_obj.personality] @@ -10799,10 +10845,11 @@ class ConductorManager(service.PeriodicService): LOG.warning("Manifest apply failed for host %s" % host_name) kube_host_upgrade_obj = objects.kube_host_upgrade.get_by_host_id( context, host_obj.id) - kube_host_upgrade_obj.status = "Control plane upgrade failed" + kube_host_upgrade_obj.status = \ + kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED kube_host_upgrade_obj.save() kube_upgrade_obj = objects.kube_upgrade.get_one(context) - kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_FAILED + kube_upgrade_obj.state = fail_state kube_upgrade_obj.save() return @@ -10822,10 +10869,11 @@ class ConductorManager(service.PeriodicService): host_name) kube_host_upgrade_obj = objects.kube_host_upgrade.get_by_host_id( context, host_obj.id) - kube_host_upgrade_obj.status = "Control plane upgrade failed" + kube_host_upgrade_obj.status = \ + kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED kube_host_upgrade_obj.save() kube_upgrade_obj = objects.kube_upgrade.get_one(context) - kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_FAILED + kube_upgrade_obj.state = fail_state kube_upgrade_obj.save() return @@ -10852,13 +10900,9 @@ class ConductorManager(service.PeriodicService): elif host_obj.personality == constants.WORKER: puppet_class = 'platform::kubernetes::worker::upgrade_kubelet' else: - LOG.error("Invalid personality %s to upgrade kubelet." % - host_obj.personality) - return - - # Update status - kube_host_upgrade_obj.status = "Upgrading kubelet" - kube_host_upgrade_obj.save() + raise exception.SysinvException(_( + "Invalid personality %s to upgrade kubelet." % + host_obj.personality)) # Update the config for this host personalities = [host_obj.personality] @@ -10887,11 +10931,9 @@ class ConductorManager(service.PeriodicService): LOG.warning("Manifest apply failed for host %s" % host_name) kube_host_upgrade_obj = objects.kube_host_upgrade.get_by_host_id( context, host_obj.id) - kube_host_upgrade_obj.status = "Control plane upgrade failed" + kube_host_upgrade_obj.status = \ + kubernetes.KUBE_HOST_UPGRADING_KUBELET_FAILED kube_host_upgrade_obj.save() - kube_upgrade_obj = objects.kube_upgrade.get_one(context) - kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_FAILED - kube_upgrade_obj.save() return # Wait for the kubelet to start with the new version @@ -10909,11 +10951,9 @@ class ConductorManager(service.PeriodicService): LOG.warning("Kubelet upgrade failed for host %s" % host_name) kube_host_upgrade_obj = objects.kube_host_upgrade.get_by_host_id( context, host_obj.id) - kube_host_upgrade_obj.status = "Kubelet upgrade failed" + kube_host_upgrade_obj.status = \ + kubernetes.KUBE_HOST_UPGRADING_KUBELET_FAILED kube_host_upgrade_obj.save() - kube_upgrade_obj = objects.kube_upgrade.get_one(context) - kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_FAILED - kube_upgrade_obj.save() return # The kubelet update was successful diff --git a/sysinv/sysinv/sysinv/sysinv/tests/api/test_host.py b/sysinv/sysinv/sysinv/sysinv/tests/api/test_host.py index bc241ed18b..5dc1fa958b 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/api/test_host.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/api/test_host.py @@ -346,6 +346,16 @@ class TestPostKubeUpgrades(TestHost): self.mocked_patch_is_applied.start() self.addCleanup(self.mocked_patch_is_applied.stop) + self.mock_patch_is_available_result = True + + def mock_patch_is_available(token, timeout, region_name, patches): + return self.mock_patch_is_available_result + self.mocked_patch_is_available = mock.patch( + 'sysinv.api.controllers.v1.patch_api.patch_is_available', + mock_patch_is_available) + self.mocked_patch_is_available.start() + self.addCleanup(self.mocked_patch_is_available.stop) + def test_kube_upgrade_control_plane_controller_0(self): # Test upgrading kubernetes control plane on controller-0 @@ -360,7 +370,7 @@ class TestPostKubeUpgrades(TestHost): kube_upgrade = dbutils.create_test_kube_upgrade( from_version='v1.42.1', to_version='v1.42.2', - state=kubernetes.KUBE_UPGRADE_STARTED, + state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES, ) # Upgrade the control plane @@ -376,9 +386,58 @@ class TestPostKubeUpgrades(TestHost): self.fake_conductor_api.kube_upgrade_control_plane.\ assert_called_with(mock.ANY, c0.uuid) - # Verify that the target version was updated + # Verify that the target version and status was updated result = self.get_json('/kube_host_upgrades/1') self.assertEqual(result['target_version'], 'v1.42.2') + self.assertEqual(result['status'], + kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE) + + # Verify that the upgrade state was updated + result = self.get_json('/kube_upgrade/%s' % kube_upgrade.uuid) + self.assertEqual(result['state'], + kubernetes.KUBE_UPGRADING_FIRST_MASTER) + + def test_kube_upgrade_control_plane_controller_0_after_failure(self): + # Test upgrading kubernetes control plane on controller-0 after a + # failure + + # Create controller-0 + c0 = self._create_controller_0( + invprovision=constants.PROVISIONED, + administrative=constants.ADMIN_UNLOCKED, + operational=constants.OPERATIONAL_ENABLED, + availability=constants.AVAILABILITY_ONLINE) + + # Create the upgrade + kube_upgrade = dbutils.create_test_kube_upgrade( + from_version='v1.42.1', + to_version='v1.42.2', + state=kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED, + ) + + # Mark the kube host upgrade as failed + values = {'target_version': 'v1.42.2', + 'status': kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED} + self.dbapi.kube_host_upgrade_update(1, values) + + # Upgrade the control plane + body = {} + result = self.post_json( + '/ihosts/controller-0/kube_upgrade_control_plane', + body, headers={'User-Agent': 'sysinv-test'}) + + # Verify the host was returned + self.assertEqual(result.json['hostname'], 'controller-0') + + # Verify the control plane was upgraded + self.fake_conductor_api.kube_upgrade_control_plane.\ + assert_called_with(mock.ANY, c0.uuid) + + # Verify that the target version and status was updated + result = self.get_json('/kube_host_upgrades/1') + self.assertEqual(result['target_version'], 'v1.42.2') + self.assertEqual(result['status'], + kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE) # Verify that the upgrade state was updated result = self.get_json('/kube_upgrade/%s' % kube_upgrade.uuid) @@ -404,7 +463,7 @@ class TestPostKubeUpgrades(TestHost): kube_upgrade = dbutils.create_test_kube_upgrade( from_version='v1.42.1', to_version='v1.42.2', - state=kubernetes.KUBE_UPGRADE_STARTED, + state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES, ) # Upgrade the control plane @@ -420,9 +479,11 @@ class TestPostKubeUpgrades(TestHost): self.fake_conductor_api.kube_upgrade_control_plane.\ assert_called_with(mock.ANY, c1.uuid) - # Verify that the target version was updated + # Verify that the target version and status was updated result = self.get_json('/kube_host_upgrades/2') self.assertEqual(result['target_version'], 'v1.42.2') + self.assertEqual(result['status'], + kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE) # Verify that the upgrade state was updated result = self.get_json('/kube_upgrade/%s' % kube_upgrade.uuid) @@ -448,9 +509,13 @@ class TestPostKubeUpgrades(TestHost): kube_upgrade = dbutils.create_test_kube_upgrade( from_version='v1.42.1', to_version='v1.42.2', - state=kubernetes.KUBE_UPGRADED_NETWORKING, + state=kubernetes.KUBE_UPGRADED_FIRST_MASTER, ) + # Mark the first kube host upgrade as OK + values = {'target_version': 'v1.42.2'} + self.dbapi.kube_host_upgrade_update(1, values) + # Upgrade the control plane body = {} result = self.post_json( @@ -464,15 +529,114 @@ class TestPostKubeUpgrades(TestHost): self.fake_conductor_api.kube_upgrade_control_plane.\ assert_called_with(mock.ANY, c1.uuid) - # Verify that the target version was updated + # Verify that the target version and status was updated result = self.get_json('/kube_host_upgrades/2') self.assertEqual(result['target_version'], 'v1.42.2') + self.assertEqual(result['status'], + kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE) # Verify that the upgrade state was updated result = self.get_json('/kube_upgrade/%s' % kube_upgrade.uuid) self.assertEqual(result['state'], kubernetes.KUBE_UPGRADING_SECOND_MASTER) + def test_kube_upgrade_control_plane_second_controller_after_failure(self): + # Test upgrading kubernetes control plane on the second controller + # after a failure + + # Create controllers + self._create_controller_0( + invprovision=constants.PROVISIONED, + administrative=constants.ADMIN_UNLOCKED, + operational=constants.OPERATIONAL_ENABLED, + availability=constants.AVAILABILITY_ONLINE) + c1 = self._create_controller_1( + invprovision=constants.PROVISIONED, + administrative=constants.ADMIN_UNLOCKED, + operational=constants.OPERATIONAL_ENABLED, + availability=constants.AVAILABILITY_ONLINE) + + # Create the upgrade + kube_upgrade = dbutils.create_test_kube_upgrade( + from_version='v1.42.1', + to_version='v1.42.2', + state=kubernetes.KUBE_UPGRADING_SECOND_MASTER_FAILED, + ) + + # Mark the first kube host upgrade as OK + values = {'target_version': 'v1.42.2'} + self.dbapi.kube_host_upgrade_update(1, values) + + # Mark the second kube host upgrade as failed + values = {'target_version': 'v1.42.2', + 'status': kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED} + self.dbapi.kube_host_upgrade_update(2, values) + + # Upgrade the control plane + body = {} + result = self.post_json( + '/ihosts/controller-1/kube_upgrade_control_plane', + body, headers={'User-Agent': 'sysinv-test'}) + + # Verify the host was returned + self.assertEqual(result.json['hostname'], 'controller-1') + + # Verify the control plane was upgraded + self.fake_conductor_api.kube_upgrade_control_plane.\ + assert_called_with(mock.ANY, c1.uuid) + + # Verify that the target version and status was updated + result = self.get_json('/kube_host_upgrades/2') + self.assertEqual(result['target_version'], 'v1.42.2') + self.assertEqual(result['status'], + kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE) + + # Verify that the upgrade state was updated + result = self.get_json('/kube_upgrade/%s' % kube_upgrade.uuid) + self.assertEqual(result['state'], + kubernetes.KUBE_UPGRADING_SECOND_MASTER) + + def test_kube_upgrade_control_plane_wrong_controller_after_failure(self): + # Test upgrading kubernetes control plane on the wrong controller + # after a failure + + # Create controllers + self._create_controller_0( + invprovision=constants.PROVISIONED, + administrative=constants.ADMIN_UNLOCKED, + operational=constants.OPERATIONAL_ENABLED, + availability=constants.AVAILABILITY_ONLINE) + self._create_controller_1( + invprovision=constants.PROVISIONED, + administrative=constants.ADMIN_UNLOCKED, + operational=constants.OPERATIONAL_ENABLED, + availability=constants.AVAILABILITY_ONLINE) + + # Create the upgrade + dbutils.create_test_kube_upgrade( + from_version='v1.42.1', + to_version='v1.42.2', + state=kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED, + ) + + # Mark the first kube host upgrade as failed + values = {'target_version': 'v1.42.2', + 'status': kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED} + self.dbapi.kube_host_upgrade_update(1, values) + + # Upgrade the second control plane + result = self.post_json( + '/ihosts/controller-1/kube_upgrade_control_plane', + {}, headers={'User-Agent': 'sysinv-test'}, + expect_errors=True) + + # Verify the failure + self.assertEqual(result.content_type, 'application/json') + self.assertEqual(http_client.BAD_REQUEST, result.status_int) + self.assertTrue(result.json['error_message']) + self.assertIn("The first control plane upgrade must be completed", + result.json['error_message']) + def test_kube_upgrade_control_plane_no_upgrade(self): # Test upgrading kubernetes control plane with no upgrade @@ -523,7 +687,104 @@ class TestPostKubeUpgrades(TestHost): self.assertEqual(result.content_type, 'application/json') self.assertEqual(http_client.BAD_REQUEST, result.status_int) self.assertTrue(result.json['error_message']) - self.assertIn("upgrade must be in the", + self.assertIn("The kubernetes upgrade is not in a valid state", + result.json['error_message']) + + def test_kube_upgrade_control_plane_controller_0_missing_applied_patches( + self): + # Test upgrading kubernetes control plane with missing applied patches + + # Create controller-0 + self._create_controller_0( + invprovision=constants.PROVISIONED, + administrative=constants.ADMIN_UNLOCKED, + operational=constants.OPERATIONAL_ENABLED, + availability=constants.AVAILABILITY_ONLINE) + + # Create the upgrade + dbutils.create_test_kube_upgrade( + from_version='v1.42.1', + to_version='v1.42.2', + state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES, + ) + + # Fake the missing patches + self.mock_patch_is_applied_result = False + self.get_kube_versions_result = [ + {'version': 'v1.42.1', + 'upgrade_from': [], + 'downgrade_to': [], + 'applied_patches': [], + 'available_patches': [], + }, + {'version': 'v1.42.2', + 'upgrade_from': ['v1.42.1'], + 'downgrade_to': [], + 'applied_patches': ['MISSING_PATCH.1'], + 'available_patches': ['MISSING_PATCH.2'], + }, + ] + + # Upgrade the control plane + result = self.post_json( + '/ihosts/controller-0/kube_upgrade_control_plane', + {}, headers={'User-Agent': 'sysinv-test'}, + expect_errors=True) + + # Verify the failure + self.assertEqual(result.content_type, 'application/json') + self.assertEqual(http_client.BAD_REQUEST, result.status_int) + self.assertTrue(result.json['error_message']) + self.assertIn("The following patches must be applied", + result.json['error_message']) + + def test_kube_upgrade_control_plane_controller_0_missing_available_patches( + self): + # Test upgrading kubernetes control plane with missing available + # patches + + # Create controller-0 + self._create_controller_0( + invprovision=constants.PROVISIONED, + administrative=constants.ADMIN_UNLOCKED, + operational=constants.OPERATIONAL_ENABLED, + availability=constants.AVAILABILITY_ONLINE) + + # Create the upgrade + dbutils.create_test_kube_upgrade( + from_version='v1.42.1', + to_version='v1.42.2', + state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES, + ) + + # Fake the missing patches + self.mock_patch_is_available_result = False + self.get_kube_versions_result = [ + {'version': 'v1.42.1', + 'upgrade_from': [], + 'downgrade_to': [], + 'applied_patches': [], + 'available_patches': [], + }, + {'version': 'v1.42.2', + 'upgrade_from': ['v1.42.1'], + 'downgrade_to': [], + 'applied_patches': [], + 'available_patches': ['MISSING_PATCH.2'], + }, + ] + + # Upgrade the control plane + result = self.post_json( + '/ihosts/controller-0/kube_upgrade_control_plane', + {}, headers={'User-Agent': 'sysinv-test'}, + expect_errors=True) + + # Verify the failure + self.assertEqual(result.content_type, 'application/json') + self.assertEqual(http_client.BAD_REQUEST, result.status_int) + self.assertTrue(result.json['error_message']) + self.assertIn("The following patches must be available", result.json['error_message']) def test_kube_upgrade_control_plane_wrong_personality(self): @@ -546,7 +807,7 @@ class TestPostKubeUpgrades(TestHost): dbutils.create_test_kube_upgrade( from_version='v1.42.1', to_version='v1.42.2', - state=kubernetes.KUBE_UPGRADE_STARTED, + state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES, ) # Upgrade the control plane @@ -576,7 +837,7 @@ class TestPostKubeUpgrades(TestHost): dbutils.create_test_kube_upgrade( from_version='v1.42.1', to_version='v1.42.2', - state=kubernetes.KUBE_UPGRADE_STARTED, + state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES, ) # No control plane version for this controller @@ -611,7 +872,7 @@ class TestPostKubeUpgrades(TestHost): dbutils.create_test_kube_upgrade( from_version='v1.42.1', to_version='v1.42.2', - state=kubernetes.KUBE_UPGRADE_STARTED, + state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES, ) # Upgrade the control plane @@ -627,6 +888,40 @@ class TestPostKubeUpgrades(TestHost): self.assertIn("must be unlocked and available", result.json['error_message']) + def test_kube_upgrade_control_plane_already_in_progress(self): + # Test upgrading kubernetes control plane with upgrade in progress + + # Create controller-0 + self._create_controller_0( + invprovision=constants.PROVISIONED, + administrative=constants.ADMIN_UNLOCKED, + operational=constants.OPERATIONAL_ENABLED, + availability=constants.AVAILABILITY_ONLINE) + + # Create the upgrade + dbutils.create_test_kube_upgrade( + from_version='v1.42.1', + to_version='v1.42.2', + state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES, + ) + + # Mark the kube host as already upgrading + values = {'status': kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE} + self.dbapi.kube_host_upgrade_update(1, values) + + # Upgrade the control plane + result = self.post_json( + '/ihosts/controller-0/kube_upgrade_control_plane', + {}, headers={'User-Agent': 'sysinv-test'}, + expect_errors=True) + + # Verify the failure + self.assertEqual(result.content_type, 'application/json') + self.assertEqual(http_client.BAD_REQUEST, result.status_int) + self.assertTrue(result.json['error_message']) + self.assertIn("control plane on this host is already being upgraded", + result.json['error_message']) + def test_kube_upgrade_kubelet_controller_0(self): # Test upgrading kubernetes kubelet on controller-0 @@ -657,6 +952,11 @@ class TestPostKubeUpgrades(TestHost): self.fake_conductor_api.kube_upgrade_kubelet.\ assert_called_with(mock.ANY, c0.uuid) + # Verify that the status was updated + result = self.get_json('/kube_host_upgrades/1') + self.assertEqual(result['status'], + kubernetes.KUBE_HOST_UPGRADING_KUBELET) + # Verify that the upgrade state was updated result = self.get_json('/kube_upgrade/%s' % kube_upgrade.uuid) self.assertEqual(result['state'], @@ -697,6 +997,11 @@ class TestPostKubeUpgrades(TestHost): self.fake_conductor_api.kube_upgrade_kubelet.\ assert_called_with(mock.ANY, c1.uuid) + # Verify that the status was updated + result = self.get_json('/kube_host_upgrades/2') + self.assertEqual(result['status'], + kubernetes.KUBE_HOST_UPGRADING_KUBELET) + # Verify that the upgrade state was updated result = self.get_json('/kube_upgrade/%s' % kube_upgrade.uuid) self.assertEqual(result['state'], @@ -745,9 +1050,11 @@ class TestPostKubeUpgrades(TestHost): # Verify the host was returned self.assertEqual(result.json['hostname'], 'worker-0') - # Verify that the target version was updated + # Verify that the target version and status was updated result = self.get_json('/kube_host_upgrades/3') self.assertEqual(result['target_version'], 'v1.42.2') + self.assertEqual(result['status'], + kubernetes.KUBE_HOST_UPGRADING_KUBELET) # Verify the kubelet was upgraded self.fake_conductor_api.kube_upgrade_kubelet.\ @@ -887,6 +1194,11 @@ class TestPostKubeUpgrades(TestHost): self.fake_conductor_api.kube_upgrade_kubelet.\ assert_called_with(mock.ANY, c0.uuid) + # Verify that the status was updated + result = self.get_json('/kube_host_upgrades/1') + self.assertEqual(result['status'], + kubernetes.KUBE_HOST_UPGRADING_KUBELET) + # Verify that the upgrade state was updated result = self.get_json('/kube_upgrade/%s' % kube_upgrade.uuid) self.assertEqual(result['state'], @@ -907,7 +1219,7 @@ class TestPostKubeUpgrades(TestHost): dbutils.create_test_kube_upgrade( from_version='v1.42.1', to_version='v1.42.2', - state=kubernetes.KUBE_UPGRADE_STARTED, + state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES, ) # Upgrade the kubelet @@ -1094,6 +1406,41 @@ class TestPostKubeUpgrades(TestHost): self.assertIn("The host must be locked and online", result.json['error_message']) + def test_kube_upgrade_kubelet_already_in_progress(self): + # Test upgrading kubernetes kubelet with upgrade in progress + + # Create controller-0 + self._create_controller_0( + invprovision=constants.PROVISIONED, + administrative=constants.ADMIN_LOCKED, + operational=constants.OPERATIONAL_DISABLED, + availability=constants.AVAILABILITY_ONLINE) + + # Create the upgrade + dbutils.create_test_kube_upgrade( + from_version='v1.42.1', + to_version='v1.42.2', + state=kubernetes.KUBE_UPGRADED_SECOND_MASTER, + ) + + # Mark the kube host as already upgrading + values = {'status': kubernetes.KUBE_HOST_UPGRADING_KUBELET} + self.dbapi.kube_host_upgrade_update(1, values) + + # Upgrade the kubelet + body = {} + result = self.post_json( + '/ihosts/controller-0/kube_upgrade_kubelet', + body, headers={'User-Agent': 'sysinv-test'}, + expect_errors=True) + + # Verify the failure + self.assertEqual(result.content_type, 'application/json') + self.assertEqual(http_client.BAD_REQUEST, result.status_int) + self.assertTrue(result.json['error_message']) + self.assertIn("The kubelet on this host is already being upgraded", + result.json['error_message']) + class TestDelete(TestHost): diff --git a/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py b/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py index eda6813dc7..f035e7b3f7 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_upgrade.py @@ -173,14 +173,11 @@ class TestPostKubeUpgrade(TestKubeUpgrade, dbbase.ControllerHostTestCase): result = self.post_json('/kube_upgrade', create_dict, headers={'User-Agent': 'sysinv-test'}) - # Verify that the images were downloaded - self.fake_conductor_api.kube_download_images.\ - assert_called_with(mock.ANY, 'v1.43.2') # Verify that the upgrade has the expected attributes self.assertEqual(result.json['from_version'], 'v1.43.1') self.assertEqual(result.json['to_version'], 'v1.43.2') self.assertEqual(result.json['state'], - kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES) + kubernetes.KUBE_UPGRADE_STARTED) # Verify that the target version for the host was updated kube_host_upgrade = self.dbapi.kube_host_upgrade_get_by_host( self.host.id) @@ -278,7 +275,7 @@ class TestPostKubeUpgrade(TestKubeUpgrade, dbbase.ControllerHostTestCase): self.assertEqual(result.json['from_version'], 'v1.43.1') self.assertEqual(result.json['to_version'], 'v1.43.2') self.assertEqual(result.json['state'], - kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES) + kubernetes.KUBE_UPGRADE_STARTED) def test_create_no_patches_required(self): # Test creation of upgrade when no applied patches are required @@ -298,7 +295,7 @@ class TestPostKubeUpgrade(TestKubeUpgrade, dbbase.ControllerHostTestCase): self.assertEqual(result.json['from_version'], 'v1.43.2') self.assertEqual(result.json['to_version'], 'v1.43.3') self.assertEqual(result.json['state'], - kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES) + kubernetes.KUBE_UPGRADE_STARTED) def test_create_applied_patch_missing(self): # Test creation of upgrade when applied patch is missing @@ -331,6 +328,98 @@ class TestPostKubeUpgrade(TestKubeUpgrade, dbbase.ControllerHostTestCase): class TestPatch(TestKubeUpgrade): + def test_update_state_download_images(self): + # Test updating the state of an upgrade to download images + + # Create the upgrade + kube_upgrade = dbutils.create_test_kube_upgrade( + from_version='v1.43.1', + to_version='v1.43.2', + state=kubernetes.KUBE_UPGRADE_STARTED) + uuid = kube_upgrade.uuid + + # Update state + new_state = kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES + response = self.patch_json('/kube_upgrade', + [{'path': '/state', + 'value': new_state, + 'op': 'replace'}], + headers={'User-Agent': 'sysinv-test'}) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + self.assertEqual(response.json['from_version'], 'v1.43.1') + self.assertEqual(response.json['to_version'], 'v1.43.2') + self.assertEqual(response.json['state'], new_state) + + # Verify that the images were downloaded + self.fake_conductor_api.kube_download_images.\ + assert_called_with(mock.ANY, 'v1.43.2') + + # Verify that the upgrade was updated with the new state + result = self.get_json('/kube_upgrade/%s' % uuid) + self.assertEqual(result['from_version'], 'v1.43.1') + self.assertEqual(result['to_version'], 'v1.43.2') + self.assertEqual(result['state'], new_state) + + def test_update_state_download_images_after_failure(self): + # Test updating the state of an upgrade to download images after a + # failure + + # Create the upgrade + kube_upgrade = dbutils.create_test_kube_upgrade( + from_version='v1.43.1', + to_version='v1.43.2', + state=kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED) + uuid = kube_upgrade.uuid + + # Update state + new_state = kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES + response = self.patch_json('/kube_upgrade', + [{'path': '/state', + 'value': new_state, + 'op': 'replace'}], + headers={'User-Agent': 'sysinv-test'}) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + self.assertEqual(response.json['from_version'], 'v1.43.1') + self.assertEqual(response.json['to_version'], 'v1.43.2') + self.assertEqual(response.json['state'], new_state) + + # Verify that the images were downloaded + self.fake_conductor_api.kube_download_images.\ + assert_called_with(mock.ANY, 'v1.43.2') + + # Verify that the upgrade was updated with the new state + result = self.get_json('/kube_upgrade/%s' % uuid) + self.assertEqual(result['from_version'], 'v1.43.1') + self.assertEqual(result['to_version'], 'v1.43.2') + self.assertEqual(result['state'], new_state) + + def test_update_state_download_images_invalid_state(self): + # Test updating the state of an upgrade to download images in an + # invalid state + + # Create the upgrade + dbutils.create_test_kube_upgrade( + from_version='v1.43.1', + to_version='v1.43.2', + state=kubernetes.KUBE_UPGRADING_KUBELETS) + + # Update state + new_state = kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES + result = self.patch_json('/kube_upgrade', + [{'path': '/state', + 'value': new_state, + 'op': 'replace'}], + headers={'User-Agent': 'sysinv-test'}, + expect_errors=True) + + # Verify the failure + self.assertEqual(result.content_type, 'application/json') + self.assertEqual(http_client.BAD_REQUEST, result.status_int) + self.assertIn("Kubernetes upgrade must be in", + result.json['error_message']) + def test_update_state_upgrade_networking(self): # Test updating the state of an upgrade to upgrade networking @@ -364,6 +453,65 @@ class TestPatch(TestKubeUpgrade): self.assertEqual(result['to_version'], 'v1.43.2') self.assertEqual(result['state'], new_state) + def test_update_state_upgrade_networking_after_failure(self): + # Test updating the state of an upgrade to upgrade networking after a + # failure + + # Create the upgrade + kube_upgrade = dbutils.create_test_kube_upgrade( + from_version='v1.43.1', + to_version='v1.43.2', + state=kubernetes.KUBE_UPGRADING_NETWORKING_FAILED) + uuid = kube_upgrade.uuid + + # Update state + new_state = kubernetes.KUBE_UPGRADING_NETWORKING + response = self.patch_json('/kube_upgrade', + [{'path': '/state', + 'value': new_state, + 'op': 'replace'}], + headers={'User-Agent': 'sysinv-test'}) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(response.status_code, http_client.OK) + self.assertEqual(response.json['from_version'], 'v1.43.1') + self.assertEqual(response.json['to_version'], 'v1.43.2') + self.assertEqual(response.json['state'], new_state) + + # Verify that networking was upgraded + self.fake_conductor_api.kube_upgrade_networking.\ + assert_called_with(mock.ANY, 'v1.43.2') + + # Verify that the upgrade was updated with the new state + result = self.get_json('/kube_upgrade/%s' % uuid) + self.assertEqual(result['from_version'], 'v1.43.1') + self.assertEqual(result['to_version'], 'v1.43.2') + self.assertEqual(result['state'], new_state) + + def test_update_state_upgrade_networking_invalid_state(self): + # Test updating the state of an upgrade to upgrade networking in an + # invalid state + + # Create the upgrade + dbutils.create_test_kube_upgrade( + from_version='v1.43.1', + to_version='v1.43.2', + state=kubernetes.KUBE_UPGRADING_KUBELETS) + + # Update state + new_state = kubernetes.KUBE_UPGRADING_NETWORKING + result = self.patch_json('/kube_upgrade', + [{'path': '/state', + 'value': new_state, + 'op': 'replace'}], + headers={'User-Agent': 'sysinv-test'}, + expect_errors=True) + + # Verify the failure + self.assertEqual(result.content_type, 'application/json') + self.assertEqual(http_client.BAD_REQUEST, result.status_int) + self.assertIn("Kubernetes upgrade must be in", + result.json['error_message']) + def test_update_state_complete(self): # Test updating the state of an upgrade to complete self.kube_get_version_states_result = {'v1.42.1': 'available', diff --git a/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_version.py b/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_version.py index 2347546a7e..bb890b0087 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_version.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/api/test_kube_version.py @@ -114,7 +114,7 @@ class TestListKubeVersions(TestKubeVersion): dbutils.create_test_kube_upgrade( from_version='v1.43.1', to_version='v1.43.2', - state=kubernetes.KUBE_UPGRADE_STARTED, + state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES, ) result = self.get_json('/kube_versions/v1.43.2') @@ -157,7 +157,7 @@ class TestListKubeVersions(TestKubeVersion): dbutils.create_test_kube_upgrade( from_version='v1.43.1', to_version='v1.43.2', - state=kubernetes.KUBE_UPGRADE_STARTED, + state=kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES, ) data = self.get_json('/kube_versions') diff --git a/sysinv/sysinv/sysinv/sysinv/tests/common/test_kubernetes.py b/sysinv/sysinv/sysinv/sysinv/tests/common/test_kubernetes.py index 1d12221850..9d763b22d2 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/common/test_kubernetes.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/common/test_kubernetes.py @@ -305,6 +305,99 @@ class TestKubeOperator(base.TestCase): ), } + self.cp_pods_missing_result = { + 'kube-apiserver-test-node-1': + kubernetes.client.V1PodList( + api_version="v1", + items=[ + kubernetes.client.V1Pod( + api_version="v1", + kind="Pod", + metadata=kubernetes.client.V1ObjectMeta( + name="kube-apiserver-test-node-1", + namespace="kube-system"), + spec=kubernetes.client.V1PodSpec( + containers=[ + kubernetes.client.V1Container( + name="kube-apiserver", + image="test-image-1:v1.42.1"), + ], + ), + ), + ], + ), + 'kube-controller-manager-test-node-1': + kubernetes.client.V1PodList( + api_version="v1", + items=[], + ), + 'kube-scheduler-test-node-1': + kubernetes.client.V1PodList( + api_version="v1", + items=[ + kubernetes.client.V1Pod( + api_version="v1", + kind="Pod", + metadata=kubernetes.client.V1ObjectMeta( + name="kube-scheduler-test-node-1", + namespace="kube-system"), + spec=kubernetes.client.V1PodSpec( + containers=[ + kubernetes.client.V1Container( + name="kube-scheduler", + image="test-image-3:v1.42.1"), + ], + ), + ), + ], + ), + 'kube-apiserver-test-node-2': + kubernetes.client.V1PodList( + api_version="v1", + items=[ + kubernetes.client.V1Pod( + api_version="v1", + kind="Pod", + metadata=kubernetes.client.V1ObjectMeta( + name="kube-apiserver-test-node-2", + namespace="kube-system"), + spec=kubernetes.client.V1PodSpec( + containers=[ + kubernetes.client.V1Container( + name="kube-apiserver", + image="test-image-1:v1.42.1"), + ], + ), + ), + ], + ), + 'kube-controller-manager-test-node-2': + kubernetes.client.V1PodList( + api_version="v1", + items=[ + kubernetes.client.V1Pod( + api_version="v1", + kind="Pod", + metadata=kubernetes.client.V1ObjectMeta( + name="kube-controller-manager-test-node-2", + namespace="kube-system"), + spec=kubernetes.client.V1PodSpec( + containers=[ + kubernetes.client.V1Container( + name="kube-controller-manager", + image="test-image-2:v1.42.1"), + ], + ), + ), + ], + ), + 'kube-scheduler-test-node-2': + kubernetes.client.V1PodList( + api_version="v1", + items=[], + ), + } + self.single_node_result = kubernetes.client.V1NodeList( api_version="v1", items=[ @@ -489,6 +582,20 @@ class TestKubeOperator(base.TestCase): result = self.kube_operator.kube_get_control_plane_versions() assert result == {'test-node-1': 'v1.42.0'} + def test_kube_get_control_plane_versions_missing_component(self): + + self.list_namespaced_pod_result = self.cp_pods_missing_result + self.list_node_result = self.multi_node_result + + self.cp_pods_missing_result['kube-apiserver-test-node-1'].\ + items[0].spec.containers[0].image = "test-image-1:v1.42.0" + self.cp_pods_missing_result['kube-controller-manager-test-node-2'].\ + items[0].spec.containers[0].image = "test-image-3:v1.42.3" + + result = self.kube_operator.kube_get_control_plane_versions() + assert result == {'test-node-1': 'v1.42.0', + 'test-node-2': 'v1.42.1'} + def test_kube_get_control_plane_versions_multi_node(self): self.list_namespaced_pod_result = self.cp_pods_result diff --git a/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py b/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py index dc5c589d01..dd4e44e9a4 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py @@ -426,7 +426,8 @@ class ManagerTestCase(base.DbTestCase): # Verify that the upgrade state was updated updated_upgrade = self.dbapi.kube_upgrade_get_one() - self.assertEqual(updated_upgrade.state, kubernetes.KUBE_UPGRADE_STARTED) + self.assertEqual(updated_upgrade.state, + kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES) def test_kube_download_images_ansible_fail(self): # Create an upgrade @@ -443,7 +444,67 @@ class ManagerTestCase(base.DbTestCase): # Verify that the upgrade state was updated updated_upgrade = self.dbapi.kube_upgrade_get_one() - self.assertEqual(updated_upgrade.state, kubernetes.KUBE_UPGRADE_FAILED) + self.assertEqual(updated_upgrade.state, + kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED) + + def test_kube_upgrade_init_actions(self): + # Create controller-0 + config_uuid = str(uuid.uuid4()) + self._create_test_ihost( + personality=constants.CONTROLLER, + hostname='controller-0', + uuid=str(uuid.uuid4()), + config_status=None, + config_applied=config_uuid, + config_target=config_uuid, + invprovision=constants.PROVISIONED, + administrative=constants.ADMIN_UNLOCKED, + operational=constants.OPERATIONAL_ENABLED, + availability=constants.AVAILABILITY_ONLINE, + ) + + # Test the handling of transitory upgrade states + expected_fail_results = [ + (kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES, + kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED), + (kubernetes.KUBE_UPGRADING_FIRST_MASTER, + kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED), + (kubernetes.KUBE_UPGRADING_NETWORKING, + kubernetes.KUBE_UPGRADING_NETWORKING_FAILED), + (kubernetes.KUBE_UPGRADING_SECOND_MASTER, + kubernetes.KUBE_UPGRADING_SECOND_MASTER_FAILED), + ] + + for current_state, fail_state in expected_fail_results: + utils.create_test_kube_upgrade( + from_version='v1.42.1', + to_version='v1.42.2', + state=current_state, + ) + self.service._kube_upgrade_init_actions() + updated_upgrade = self.dbapi.kube_upgrade_get_one() + self.assertEqual(updated_upgrade.state, fail_state) + self.dbapi.kube_upgrade_destroy(updated_upgrade.id) + + # Test the handling of transitory host upgrade states + expected_fail_results = [ + (kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE, + kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED), + (kubernetes.KUBE_HOST_UPGRADING_KUBELET, + kubernetes.KUBE_HOST_UPGRADING_KUBELET_FAILED), + ] + + utils.create_test_kube_upgrade( + from_version='v1.42.1', + to_version='v1.42.2', + state=kubernetes.KUBE_UPGRADING_KUBELETS, + ) + + for current_status, fail_status in expected_fail_results: + self.dbapi.kube_host_upgrade_update(1, {'status': current_status}) + self.service._kube_upgrade_init_actions() + updated_host_upgrade = self.dbapi.kube_host_upgrade_get(1) + self.assertEqual(updated_host_upgrade.status, fail_status) def test_kube_download_images_one_controller(self): # Create an upgrade @@ -476,7 +537,8 @@ class ManagerTestCase(base.DbTestCase): # Verify that the upgrade state was updated updated_upgrade = self.dbapi.kube_upgrade_get_one() - self.assertEqual(updated_upgrade.state, kubernetes.KUBE_UPGRADE_STARTED) + self.assertEqual(updated_upgrade.state, + kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES) def test_kube_download_images_one_controller_manifest_timeout(self): # Create an upgrade @@ -513,7 +575,7 @@ class ManagerTestCase(base.DbTestCase): # Verify that the upgrade state was updated updated_upgrade = self.dbapi.kube_upgrade_get_one() self.assertEqual(updated_upgrade.state, - kubernetes.KUBE_UPGRADE_FAILED) + kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED) def test_kube_download_images_two_controllers(self): # Create an upgrade @@ -562,7 +624,8 @@ class ManagerTestCase(base.DbTestCase): # Verify that the upgrade state was updated updated_upgrade = self.dbapi.kube_upgrade_get_one() - self.assertEqual(updated_upgrade.state, kubernetes.KUBE_UPGRADE_STARTED) + self.assertEqual(updated_upgrade.state, + kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES) def test_kube_upgrade_control_plane_first_master(self): # Create an upgrade @@ -645,11 +708,12 @@ class ManagerTestCase(base.DbTestCase): # Verify that the upgrade state was updated updated_upgrade = self.dbapi.kube_upgrade_get_one() self.assertEqual(updated_upgrade.state, - kubernetes.KUBE_UPGRADE_FAILED) + kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED) # Verify that the host upgrade status was set updated_host_upgrade = self.dbapi.kube_host_upgrade_get(1) - self.assertIsNotNone(updated_host_upgrade.status) + self.assertEqual(updated_host_upgrade.status, + kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED) def test_kube_upgrade_control_plane_first_master_upgrade_fail(self): # Create an upgrade @@ -686,7 +750,7 @@ class ManagerTestCase(base.DbTestCase): # Verify that the upgrade state was updated updated_upgrade = self.dbapi.kube_upgrade_get_one() self.assertEqual(updated_upgrade.state, - kubernetes.KUBE_UPGRADE_FAILED) + kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED) # Verify that the host upgrade status was cleared updated_host_upgrade = self.dbapi.kube_host_upgrade_get(1) @@ -870,7 +934,7 @@ class ManagerTestCase(base.DbTestCase): utils.create_test_kube_upgrade( from_version='v1.42.1', to_version='v1.42.2', - state=kubernetes.KUBE_UPGRADING_FIRST_MASTER, + state=kubernetes.KUBE_UPGRADING_KUBELETS, ) # Create controller-0 config_uuid = str(uuid.uuid4()) @@ -898,21 +962,22 @@ class ManagerTestCase(base.DbTestCase): # Upgrade the kubelet self.service.kube_upgrade_kubelet(self.context, c0.uuid) - # Verify that the upgrade state was updated + # Verify that the upgrade state was not updated updated_upgrade = self.dbapi.kube_upgrade_get_one() self.assertEqual(updated_upgrade.state, - kubernetes.KUBE_UPGRADE_FAILED) + kubernetes.KUBE_UPGRADING_KUBELETS) # Verify that the host upgrade status was set updated_host_upgrade = self.dbapi.kube_host_upgrade_get(1) - self.assertIsNotNone(updated_host_upgrade.status) + self.assertEqual(updated_host_upgrade.status, + kubernetes.KUBE_HOST_UPGRADING_KUBELET_FAILED) def test_kube_upgrade_kubelet_controller_upgrade_fail(self): # Create an upgrade utils.create_test_kube_upgrade( from_version='v1.42.1', to_version='v1.42.2', - state=kubernetes.KUBE_UPGRADING_FIRST_MASTER, + state=kubernetes.KUBE_UPGRADING_KUBELETS, ) # Create controller-0 config_uuid = str(uuid.uuid4()) @@ -939,10 +1004,10 @@ class ManagerTestCase(base.DbTestCase): # Upgrade the kubelet self.service.kube_upgrade_kubelet(self.context, c0.uuid) - # Verify that the upgrade state was updated + # Verify that the upgrade state was not updated updated_upgrade = self.dbapi.kube_upgrade_get_one() self.assertEqual(updated_upgrade.state, - kubernetes.KUBE_UPGRADE_FAILED) + kubernetes.KUBE_UPGRADING_KUBELETS) # Verify that the host upgrade status was cleared updated_host_upgrade = self.dbapi.kube_host_upgrade_get(1)