419 lines
16 KiB
Python
Executable File
419 lines
16 KiB
Python
Executable File
#
|
|
# Copyright (c) 2019 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
|
|
import pecan
|
|
from pecan import rest
|
|
import os
|
|
import six
|
|
import wsme
|
|
from wsme import types as wtypes
|
|
import wsmeext.pecan as wsme_pecan
|
|
|
|
from oslo_log import log
|
|
|
|
from sysinv._i18n import _
|
|
from sysinv.api.controllers.v1 import base
|
|
from sysinv.api.controllers.v1 import collection
|
|
from sysinv.api.controllers.v1 import link
|
|
from sysinv.api.controllers.v1 import patch_api
|
|
from sysinv.api.controllers.v1 import types
|
|
from sysinv.common import constants
|
|
from sysinv.common import dc_api
|
|
from sysinv.common import exception
|
|
from sysinv.common import kubernetes
|
|
from sysinv.common import utils as cutils
|
|
from sysinv import objects
|
|
|
|
LOG = log.getLogger(__name__)
|
|
|
|
|
|
class KubeUpgradePatchType(types.JsonPatchType):
|
|
|
|
@staticmethod
|
|
def mandatory_attrs():
|
|
return ['/state']
|
|
|
|
|
|
class KubeUpgrade(base.APIBase):
|
|
"""API representation of a Kubernetes Upgrade."""
|
|
|
|
id = int
|
|
"Unique ID for this entry"
|
|
|
|
uuid = types.uuid
|
|
"Unique UUID for this entry"
|
|
|
|
from_version = wtypes.text
|
|
"The from version for the kubernetes upgrade"
|
|
|
|
to_version = wtypes.text
|
|
"The to version for the kubernetes upgrade"
|
|
|
|
state = wtypes.text
|
|
"Kubernetes upgrade state"
|
|
|
|
links = [link.Link]
|
|
"A list containing a self link and associated kubernetes upgrade links"
|
|
|
|
def __init__(self, **kwargs):
|
|
self.fields = objects.kube_upgrade.fields.keys()
|
|
for k in self.fields:
|
|
if not hasattr(self, k):
|
|
continue
|
|
setattr(self, k, kwargs.get(k, wtypes.Unset))
|
|
|
|
@classmethod
|
|
def convert_with_links(cls, rpc_kube_upgrade, expand=True):
|
|
kube_upgrade = KubeUpgrade(**rpc_kube_upgrade.as_dict())
|
|
if not expand:
|
|
kube_upgrade.unset_fields_except(['uuid', 'from_version',
|
|
'to_version', 'state'])
|
|
|
|
kube_upgrade.links = [
|
|
link.Link.make_link('self', pecan.request.host_url,
|
|
'kube_upgrade', kube_upgrade.uuid),
|
|
link.Link.make_link('bookmark',
|
|
pecan.request.host_url,
|
|
'kube_upgrade', kube_upgrade.uuid,
|
|
bookmark=True)
|
|
]
|
|
return kube_upgrade
|
|
|
|
|
|
class KubeUpgradeCollection(collection.Collection):
|
|
"""API representation of a collection of kubernetes upgrades."""
|
|
|
|
kube_upgrades = [KubeUpgrade]
|
|
"A list containing kubernetes upgrade objects"
|
|
|
|
def __init__(self, **kwargs):
|
|
self._type = 'kube_upgrades'
|
|
|
|
@classmethod
|
|
def convert_with_links(cls, rpc_kube_upgrade, expand=True, **kwargs):
|
|
collection = KubeUpgradeCollection()
|
|
collection.kube_upgrades = [KubeUpgrade.convert_with_links(p, expand)
|
|
for p in rpc_kube_upgrade]
|
|
return collection
|
|
|
|
|
|
LOCK_NAME = 'KubeUpgradeController'
|
|
|
|
|
|
class KubeUpgradeController(rest.RestController):
|
|
"""REST controller for kubernetes upgrades."""
|
|
|
|
def __init__(self):
|
|
self._kube_operator = kubernetes.KubeOperator()
|
|
|
|
def _get_updates(self, patch):
|
|
"""Retrieve the updated attributes from the patch request."""
|
|
updates = {}
|
|
for p in patch:
|
|
attribute = p['path'] if p['path'][0] != '/' else p['path'][1:]
|
|
updates[attribute] = p['value']
|
|
return updates
|
|
|
|
@staticmethod
|
|
def _check_patch_requirements(region_name,
|
|
applied_patches=None,
|
|
available_patches=None):
|
|
"""Checks whether specified patches are applied or available"""
|
|
|
|
api_token = None
|
|
if applied_patches:
|
|
patches_applied = patch_api.patch_is_applied(
|
|
token=api_token,
|
|
timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS,
|
|
region_name=region_name,
|
|
patches=applied_patches
|
|
)
|
|
if not patches_applied:
|
|
raise wsme.exc.ClientSideError(_(
|
|
"The following patches must be applied before doing "
|
|
"the kubernetes upgrade: %s" % applied_patches))
|
|
|
|
if available_patches:
|
|
patches_available = patch_api.patch_is_available(
|
|
token=api_token,
|
|
timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS,
|
|
region_name=region_name,
|
|
patches=available_patches
|
|
)
|
|
if not patches_available:
|
|
raise wsme.exc.ClientSideError(_(
|
|
"The following patches must be available before doing "
|
|
"the kubernetes upgrade: %s" %
|
|
available_patches))
|
|
|
|
@staticmethod
|
|
def _check_installed_apps_compatibility(apps, kube_version):
|
|
"""Checks whether all installed applications are compatible
|
|
with the new k8s version"""
|
|
|
|
for app in apps:
|
|
if app.status != constants.APP_APPLY_SUCCESS:
|
|
continue
|
|
|
|
kube_min_version, kube_max_version = \
|
|
cutils.get_app_supported_kube_version(app.name, app.app_version)
|
|
|
|
if not kubernetes.is_kube_version_supported(
|
|
kube_version, kube_min_version, kube_max_version):
|
|
raise wsme.exc.ClientSideError(_(
|
|
"The installed Application %s (%s) is incompatible with the "
|
|
"new Kubernetes version %s." % (app.name, app.app_version, kube_version)))
|
|
|
|
@wsme_pecan.wsexpose(KubeUpgradeCollection)
|
|
def get_all(self):
|
|
"""Retrieve a list of kubernetes upgrades."""
|
|
|
|
kube_upgrades = pecan.request.dbapi.kube_upgrade_get_list()
|
|
return KubeUpgradeCollection.convert_with_links(kube_upgrades)
|
|
|
|
@wsme_pecan.wsexpose(KubeUpgrade, types.uuid)
|
|
def get_one(self, uuid):
|
|
"""Retrieve information about the given kubernetes upgrade."""
|
|
|
|
rpc_kube_upgrade = objects.kube_upgrade.get_by_uuid(
|
|
pecan.request.context, uuid)
|
|
return KubeUpgrade.convert_with_links(rpc_kube_upgrade)
|
|
|
|
@cutils.synchronized(LOCK_NAME)
|
|
@wsme_pecan.wsexpose(KubeUpgrade, wtypes.text, body=six.text_type)
|
|
def post(self, to_version, body):
|
|
"""Create a new Kubernetes Upgrade and start upgrade."""
|
|
|
|
force = body.get('force', False) is True
|
|
alarm_ignore_list = body.get('alarm_ignore_list')
|
|
|
|
# There must not be a platform upgrade in progress
|
|
try:
|
|
pecan.request.dbapi.software_upgrade_get_one()
|
|
except exception.NotFound:
|
|
pass
|
|
else:
|
|
raise wsme.exc.ClientSideError(_(
|
|
"A kubernetes upgrade cannot be done while a platform upgrade "
|
|
"is in progress"))
|
|
|
|
# There must not already be a kubernetes upgrade in progress
|
|
try:
|
|
pecan.request.dbapi.kube_upgrade_get_one()
|
|
except exception.NotFound:
|
|
pass
|
|
else:
|
|
raise wsme.exc.ClientSideError(_(
|
|
"A kubernetes upgrade is already in progress"))
|
|
|
|
# The target version must be available
|
|
try:
|
|
target_version_obj = objects.kube_version.get_by_version(
|
|
to_version)
|
|
except exception.KubeVersionNotFound:
|
|
raise wsme.exc.ClientSideError(_(
|
|
"Kubernetes version %s is not available" % to_version))
|
|
|
|
# The upgrade path must be supported
|
|
current_kube_version = \
|
|
self._kube_operator.kube_get_kubernetes_version()
|
|
if not target_version_obj.can_upgrade_from(current_kube_version):
|
|
raise wsme.exc.ClientSideError(_(
|
|
"The installed Kubernetes version %s cannot upgrade to "
|
|
"version %s" % (current_kube_version,
|
|
target_version_obj.version)))
|
|
|
|
# The current kubernetes version must be active
|
|
version_states = self._kube_operator.kube_get_version_states()
|
|
if version_states.get(current_kube_version) != \
|
|
kubernetes.KUBE_STATE_ACTIVE:
|
|
raise wsme.exc.ClientSideError(_(
|
|
"The installed Kubernetes version %s is not active on all "
|
|
"hosts" % current_kube_version))
|
|
|
|
# Verify patching requirements
|
|
system = pecan.request.dbapi.isystem_get_one()
|
|
self._check_patch_requirements(
|
|
system.region_name,
|
|
applied_patches=target_version_obj.applied_patches,
|
|
available_patches=target_version_obj.available_patches)
|
|
|
|
# Check that all installed applications support new k8s version
|
|
apps = pecan.request.dbapi.kube_app_get_all()
|
|
self._check_installed_apps_compatibility(apps, to_version)
|
|
|
|
# TODO: check that tiller/armada support new k8s version
|
|
|
|
# The system must be healthy
|
|
success, output = pecan.request.rpcapi.get_system_health(
|
|
pecan.request.context,
|
|
force=force,
|
|
kube_upgrade=True,
|
|
alarm_ignore_list=alarm_ignore_list)
|
|
if not success:
|
|
LOG.info("Health query failure during kubernetes upgrade start: %s"
|
|
% output)
|
|
if os.path.exists(constants.SYSINV_RUNNING_IN_LAB) and force:
|
|
LOG.info("Running in lab, ignoring health errors.")
|
|
else:
|
|
raise wsme.exc.ClientSideError(_(
|
|
"System is not in a valid state for kubernetes upgrade. "
|
|
"Run system health-query-kube-upgrade for more details."))
|
|
|
|
# Create upgrade record.
|
|
create_values = {'from_version': current_kube_version,
|
|
'to_version': to_version,
|
|
'state': kubernetes.KUBE_UPGRADE_STARTED}
|
|
new_upgrade = pecan.request.dbapi.kube_upgrade_create(create_values)
|
|
|
|
# Set the target version for each host to the current version
|
|
update_values = {'target_version': current_kube_version}
|
|
kube_host_upgrades = pecan.request.dbapi.kube_host_upgrade_get_list()
|
|
for kube_host_upgrade in kube_host_upgrades:
|
|
pecan.request.dbapi.kube_host_upgrade_update(kube_host_upgrade.id,
|
|
update_values)
|
|
|
|
LOG.info("Started kubernetes upgrade from version: %s to version: %s"
|
|
% (current_kube_version, to_version))
|
|
|
|
return KubeUpgrade.convert_with_links(new_upgrade)
|
|
|
|
@cutils.synchronized(LOCK_NAME)
|
|
@wsme.validate([KubeUpgradePatchType])
|
|
@wsme_pecan.wsexpose(KubeUpgrade, body=[KubeUpgradePatchType])
|
|
def patch(self, patch):
|
|
"""Updates attributes of a Kubernetes Upgrade."""
|
|
|
|
updates = self._get_updates(patch)
|
|
|
|
# Get the current upgrade
|
|
try:
|
|
kube_upgrade_obj = objects.kube_upgrade.get_one(
|
|
pecan.request.context)
|
|
except exception.NotFound:
|
|
raise wsme.exc.ClientSideError(_(
|
|
"A kubernetes upgrade is not in progress"))
|
|
|
|
if updates['state'] == kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES:
|
|
# Make sure upgrade is in the correct state to download images
|
|
if kube_upgrade_obj.state not in [
|
|
kubernetes.KUBE_UPGRADE_STARTED,
|
|
kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED]:
|
|
raise wsme.exc.ClientSideError(_(
|
|
"Kubernetes upgrade must be in %s or %s state to download "
|
|
"images" %
|
|
(kubernetes.KUBE_UPGRADE_STARTED,
|
|
kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED)))
|
|
|
|
# Verify patching requirements (since the api server is not
|
|
# upgraded yet, patches could have been removed)
|
|
system = pecan.request.dbapi.isystem_get_one()
|
|
target_version_obj = objects.kube_version.get_by_version(
|
|
kube_upgrade_obj.to_version)
|
|
self._check_patch_requirements(
|
|
system.region_name,
|
|
applied_patches=target_version_obj.applied_patches,
|
|
available_patches=target_version_obj.available_patches)
|
|
|
|
# Update the upgrade state
|
|
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_DOWNLOADING_IMAGES
|
|
kube_upgrade_obj.save()
|
|
|
|
# Tell the conductor to download the images for the new version
|
|
pecan.request.rpcapi.kube_download_images(
|
|
pecan.request.context, kube_upgrade_obj.to_version)
|
|
|
|
LOG.info("Downloading kubernetes images for version: %s" %
|
|
kube_upgrade_obj.to_version)
|
|
return KubeUpgrade.convert_with_links(kube_upgrade_obj)
|
|
|
|
elif updates['state'] == kubernetes.KUBE_UPGRADING_NETWORKING:
|
|
# Make sure upgrade is in the correct state to upgrade networking
|
|
if kube_upgrade_obj.state not in [
|
|
kubernetes.KUBE_UPGRADED_FIRST_MASTER,
|
|
kubernetes.KUBE_UPGRADING_NETWORKING_FAILED]:
|
|
raise wsme.exc.ClientSideError(_(
|
|
"Kubernetes upgrade must be in %s or %s state to upgrade "
|
|
"networking" %
|
|
(kubernetes.KUBE_UPGRADED_FIRST_MASTER,
|
|
kubernetes.KUBE_UPGRADING_NETWORKING_FAILED)))
|
|
|
|
# Update the upgrade state
|
|
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADING_NETWORKING
|
|
kube_upgrade_obj.save()
|
|
|
|
# Tell the conductor to upgrade networking
|
|
pecan.request.rpcapi.kube_upgrade_networking(
|
|
pecan.request.context, kube_upgrade_obj.to_version)
|
|
|
|
LOG.info("Upgrading kubernetes networking to version: %s" %
|
|
kube_upgrade_obj.to_version)
|
|
return KubeUpgrade.convert_with_links(kube_upgrade_obj)
|
|
|
|
elif updates['state'] == kubernetes.KUBE_UPGRADE_COMPLETE:
|
|
# Make sure upgrade is in the correct state to complete
|
|
if kube_upgrade_obj.state != \
|
|
kubernetes.KUBE_UPGRADING_KUBELETS:
|
|
raise wsme.exc.ClientSideError(_(
|
|
"Kubernetes upgrade must be in %s state to complete" %
|
|
kubernetes.KUBE_UPGRADING_KUBELETS))
|
|
|
|
# Make sure no hosts are in a transitory or failed state
|
|
kube_host_upgrades = \
|
|
pecan.request.dbapi.kube_host_upgrade_get_list()
|
|
for kube_host_upgrade in kube_host_upgrades:
|
|
if kube_host_upgrade.status is not None:
|
|
raise wsme.exc.ClientSideError(_(
|
|
"At least one host has not completed the kubernetes "
|
|
"upgrade"))
|
|
|
|
# Make sure the target version is active
|
|
version_states = self._kube_operator.kube_get_version_states()
|
|
if version_states.get(kube_upgrade_obj.to_version, None) != \
|
|
kubernetes.KUBE_STATE_ACTIVE:
|
|
raise wsme.exc.ClientSideError(_(
|
|
"Kubernetes to_version must be active to complete"))
|
|
|
|
# All is well, mark the upgrade as complete
|
|
kube_upgrade_obj.state = kubernetes.KUBE_UPGRADE_COMPLETE
|
|
kube_upgrade_obj.save()
|
|
|
|
LOG.info("Completed kubernetes upgrade to version: %s" %
|
|
kube_upgrade_obj.to_version)
|
|
|
|
# If applicable, notify dcmanager upgrade is complete
|
|
system = pecan.request.dbapi.isystem_get_one()
|
|
role = system.get('distributed_cloud_role')
|
|
if role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER:
|
|
dc_api.notify_dcmanager_kubernetes_upgrade_completed()
|
|
return KubeUpgrade.convert_with_links(kube_upgrade_obj)
|
|
|
|
else:
|
|
raise wsme.exc.ClientSideError(_(
|
|
"Invalid state %s supplied" % updates['state']))
|
|
|
|
@cutils.synchronized(LOCK_NAME)
|
|
@wsme_pecan.wsexpose(None)
|
|
def delete(self):
|
|
"""Delete Kubernetes Upgrade."""
|
|
|
|
# An upgrade must be in progress
|
|
try:
|
|
kube_upgrade_obj = pecan.request.dbapi.kube_upgrade_get_one()
|
|
except exception.NotFound:
|
|
raise wsme.exc.ClientSideError(_(
|
|
"A kubernetes upgrade is not in progress"))
|
|
|
|
# The upgrade must be complete
|
|
if kube_upgrade_obj.state != \
|
|
kubernetes.KUBE_UPGRADE_COMPLETE:
|
|
raise wsme.exc.ClientSideError(_(
|
|
"Kubernetes upgrade must be in %s state to delete" %
|
|
kubernetes.KUBE_UPGRADE_COMPLETE))
|
|
|
|
# Delete the upgrade
|
|
pecan.request.dbapi.kube_upgrade_destroy(kube_upgrade_obj.id)
|