Finish kubernetes networking upgrade support

The commit completes the RPC kube_upgrade_networking
in sysinv-conductor to run ansible playbook
upgrade-k8s-networking.yml to upgrade networking pods
and also updates the networking upgrade function called
as part of sysinv-conductor startup to provide a current
kubernetes version when running the upgrade playbook.

The second control plane upgrade can only be performed
after the networking upgrade is done, fix the semantic
check in sysinv api.

Change-Id: I8dcf5a2baedfaefb0a7ca037eb47bf7cacd686f8
Story: 2006781
Task: 37584
Depends-On: https://review.opendev.org/#/c/705310/
Signed-off-by: Angie Wang <angie.wang@windriver.com>
This commit is contained in:
Angie Wang 2020-01-29 21:18:18 -05:00
parent ca4aff3591
commit 7e2fda0102
5 changed files with 94 additions and 12 deletions

View File

@ -6586,8 +6586,7 @@ class HostController(rest.RestController):
# Verify the upgrade is in the correct state
if kube_upgrade_obj.state in [
kubernetes.KUBE_UPGRADE_DOWNLOADED_IMAGES,
kubernetes.KUBE_UPGRADED_NETWORKING,
kubernetes.KUBE_UPGRADED_FIRST_MASTER]:
kubernetes.KUBE_UPGRADED_NETWORKING]:
# We are upgrading a control plane
pass
elif kube_upgrade_obj.state in [

View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2013-2019 Wind River Systems, Inc.
# Copyright (c) 2013-2020 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -82,6 +82,22 @@ def get_kube_versions():
]
def get_kube_networking_upgrade_version(kube_upgrade):
"""Determine the version that kubernetes networking
should be upgraded to."""
if kube_upgrade.state in [
KUBE_UPGRADE_STARTED,
KUBE_UPGRADE_DOWNLOADING_IMAGES,
KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED,
KUBE_UPGRADE_DOWNLOADED_IMAGES,
KUBE_UPGRADING_FIRST_MASTER,
KUBE_UPGRADING_FIRST_MASTER_FAILED,
KUBE_UPGRADED_FIRST_MASTER]:
return kube_upgrade.from_version
else:
return kube_upgrade.to_version
class KubeOperator(object):
def __init__(self):

View File

@ -16,7 +16,7 @@
# License for the specific language governing permissions and limitations
# under the License.
#
# Copyright (c) 2013-2019 Wind River Systems, Inc.
# Copyright (c) 2013-2020 Wind River Systems, Inc.
#
"""Conduct all activity related system inventory.
@ -5280,12 +5280,27 @@ class ConductorManager(service.PeriodicService):
wait_fixed=(CONF.conductor.kube_upgrade_downgrade_retry_interval * 1000))
def _upgrade_downgrade_kube_networking(self):
try:
LOG.info(
"_upgrade_downgrade_kube_networking executing playbook: %s " %
constants.ANSIBLE_KUBE_NETWORKING_PLAYBOOK)
# Get the kubernetes version from the upgrade table
# if an upgrade exists
kube_upgrade = self.dbapi.kube_upgrade_get_one()
kube_version = \
kubernetes.get_kube_networking_upgrade_version(kube_upgrade)
except exception.NotFound:
# Not upgrading kubernetes, get the kubernetes version
# from the kubeadm config map
kube_version = self._kube.kube_get_kubernetes_version()
if not kube_version:
LOG.error("Unable to get the current kubernetes version.")
return False
try:
LOG.info("_upgrade_downgrade_kube_networking executing"
" playbook: %s for version %s" %
(constants.ANSIBLE_KUBE_NETWORKING_PLAYBOOK, kube_version))
proc = subprocess.Popen(
['ansible-playbook',
['ansible-playbook', '-e', 'kubernetes_version=%s' % kube_version,
constants.ANSIBLE_KUBE_NETWORKING_PLAYBOOK],
stdout=subprocess.PIPE)
out, _ = proc.communicate()
@ -11029,8 +11044,26 @@ class ConductorManager(service.PeriodicService):
def kube_upgrade_networking(self, context, kube_version):
"""Upgrade kubernetes networking for this kubernetes version"""
# TODO: Upgrade kubernetes networking.
LOG.info("Upgrade kubernetes networking here")
LOG.info("executing playbook: %s for version %s" %
(constants.ANSIBLE_KUBE_NETWORKING_PLAYBOOK, kube_version))
proc = subprocess.Popen(
['ansible-playbook', '-e', 'kubernetes_version=%s' % kube_version,
constants.ANSIBLE_KUBE_NETWORKING_PLAYBOOK],
stdout=subprocess.PIPE)
out, _ = proc.communicate()
LOG.info("ansible-playbook: %s." % out)
if proc.returncode:
LOG.warning("ansible-playbook returned an error: %s" %
proc.returncode)
# Update the upgrade state
kube_upgrade_obj = objects.kube_upgrade.get_one(context)
kube_upgrade_obj.state = \
kubernetes.KUBE_UPGRADING_NETWORKING_FAILED
kube_upgrade_obj.save()
return
# Indicate that networking upgrade is complete
kube_upgrade_obj = objects.kube_upgrade.get_one(context)

View File

@ -2,7 +2,7 @@
# -*- encoding: utf-8 -*-
#
#
# Copyright (c) 2013-2019 Wind River Systems, Inc.
# Copyright (c) 2013-2020 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -542,7 +542,7 @@ class TestPostKubeUpgrades(TestHost):
kube_upgrade = dbutils.create_test_kube_upgrade(
from_version='v1.42.1',
to_version='v1.42.2',
state=kubernetes.KUBE_UPGRADED_FIRST_MASTER,
state=kubernetes.KUBE_UPGRADED_NETWORKING,
)
# Mark the first kube host upgrade as OK

View File

@ -1162,6 +1162,40 @@ class ManagerTestCase(base.DbTestCase):
updated_host_upgrade = self.dbapi.kube_host_upgrade_get(1)
self.assertIsNotNone(updated_host_upgrade.status)
def test_kube_upgrade_networking(self):
# Create an upgrade
utils.create_test_kube_upgrade(
from_version='v1.42.1',
to_version='v1.42.2',
state=kubernetes.KUBE_UPGRADING_NETWORKING,
)
# Upgrade kubernetes networking
self.service.kube_upgrade_networking(self.context, 'v1.42.2')
# Verify that the upgrade state was updated
updated_upgrade = self.dbapi.kube_upgrade_get_one()
self.assertEqual(updated_upgrade.state,
kubernetes.KUBE_UPGRADED_NETWORKING)
def test_kube_upgrade_networking_ansible_fail(self):
# Create an upgrade
utils.create_test_kube_upgrade(
from_version='v1.42.1',
to_version='v1.42.2',
state=kubernetes.KUBE_UPGRADING_NETWORKING,
)
# Fake an ansible failure
self.fake_subprocess_popen.returncode = 1
# Upgrade kubernetes networking
self.service.kube_upgrade_networking(self.context, 'v1.42.2')
# Verify that the upgrade state was updated
updated_upgrade = self.dbapi.kube_upgrade_get_one()
self.assertEqual(updated_upgrade.state,
kubernetes.KUBE_UPGRADING_NETWORKING_FAILED)
def test_configure_out_of_date(self):
config_applied = self.service._config_set_reboot_required(uuid.uuid4())
config_target = self.service._config_set_reboot_required(uuid.uuid4())