starlingx
/
config
Code Issues Proposed changes

Kubernetes networking upgrade prep 

This commit adds code that calls ansible-playbook
upgrade-k8s-networking.yml when the conductor starts up.

Story: 2006590
Task: 36726
Change-Id: I59e62654bc2a9b2a0980f2e2b60cc93a74a3deb6
Signed-off-by: Kristine Bujold <kristine.bujold@windriver.com>
This commit is contained in:
Kristine Bujold 2019-10-03 16:01:46 -04:00
parent 8e51a4bb54
commit 210cbe029a
3 changed files with 57 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
sysinv/sysinv/centos/build_srpm.data
View File

@ -1,2 +1,2 @@
SRC_DIR="sysinv"
TIS_PATCH_VER=333
TIS_PATCH_VER=334

4
sysinv/sysinv/sysinv/sysinv/common/constants.py
View File

@ -1468,6 +1468,10 @@ ANSIBLE_BOOTSTRAP_FLAG = os.path.join(tsc.VOLATILE_PATH, ".ansible_bootstrap")
UNLOCK_READY_FLAG = os.path.join(tsc.PLATFORM_CONF_PATH, ".unlock_ready")
INVENTORY_WAIT_TIMEOUT_IN_SECS = 90
# Ansible kubernetes networking playbook
ANSIBLE_KUBE_NETWORKING_PLAYBOOK = \
'/usr/share/ansible/stx-ansible/playbooks/upgrade-k8s-networking.yml'
# Clock synchronization types
NTP = 'ntp'
PTP = 'ptp'

54
sysinv/sysinv/sysinv/sysinv/conductor/manager.py
View File

@ -227,8 +227,8 @@ class ConductorManager(service.PeriodicService):
self._handle_restore_in_progress()
# Upgrade/Downgrade tiller if required
greenthread.spawn(self._upgrade_downgrade_tiller())
# Upgrade/Downgrade kubernetes components
greenthread.spawn(self._upgrade_downgrade_kube_components())
LOG.info("sysinv-conductor start committed system=%s" %
system.as_dict())
@ -5124,12 +5124,42 @@ class ConductorManager(service.PeriodicService):
return
self.reapply_app(context, app_name)
def _upgrade_downgrade_kube_components(self):
self._upgrade_downgrade_tiller()
self._upgrade_downgrade_kube_networking()
def _upgrade_downgrade_tiller(self):
"""Check if tiller needs to be upgraded or downgraded"""
LOG.info("_upgrade_downgrade_tiller")
FIVE_MIN_IN_SECS = 300
in_progress_statuses = [constants.APP_APPLY_IN_PROGRESS,
constants.APP_UPLOAD_IN_PROGRESS,
constants.APP_REMOVE_IN_PROGRESS,
constants.APP_UPDATE_IN_PROGRESS,
constants.APP_RECOVER_IN_PROGRESS]
# Check if we are in the middle of an application apply. If so wait
# 5 minutes and retry.
while True:
try:
in_progress = False
for app in self.dbapi.kube_app_get_all():
if app.status in in_progress_statuses:
LOG.info("_upgrade_downgrade_tiller kubernetes application "
"'%s' in progress, status is '%s'" %
(app.name, app.status))
in_progress = True
break
if in_progress:
greenthread.sleep(FIVE_MIN_IN_SECS)
continue
except Exception as e:
LOG.error("{}. Failed to get kubernetes application list.".format(e))
break
# Upgrade or downgrade the tiller image
try:
running_image = self._kube.kube_get_image_by_selector(
image_versions.TILLER_SELECTOR_NAME,
@ -5194,6 +5224,26 @@ class ConductorManager(service.PeriodicService):
except Exception as e:
LOG.error("{}. Failed to upgrade/downgrade tiller.".format(e))
def _upgrade_downgrade_kube_networking(self):
try:
LOG.info(
"_upgrade_downgrade_kube_networking executing playbook: %s " %
constants.ANSIBLE_KUBE_NETWORKING_PLAYBOOK)
proc = subprocess.Popen(
['ansible-playbook',
constants.ANSIBLE_KUBE_NETWORKING_PLAYBOOK],
stdout=subprocess.PIPE)
out, _ = proc.communicate()
LOG.info("ansible-playbook: %s." % out)
if proc.returncode:
raise Exception("ansible-playbook returned an error: %s" % proc.returncode)
except Exception as e:
LOG.error("Failed to upgrade/downgrade kubernetes "
"networking images: {}".format(e))
def check_nodes_stable(self):
hosts = self.dbapi.ihost_get_list()
if (utils.is_host_simplex_controller(hosts[0]) and