Kubernetes Upgrade Orchestration

Provides the new CLI command:
  sw-manager kube-upgrade-strategy

VIM build stages are:
 - query-alarms
 - query-kube-upgrade
 - query-kube-versions
 - query-patches
 - query-patch-hosts

VIM apply stages are:
 - kube-upgrade-start
 - download images
 - first control plane
 - networking
 - second control plane
 - apply second kubernetes patch
  -- applies the patch
  -- host-install on each controller
  -- host-install on each storage
  -- host-install on each worker
 - kubelets (controllers)
 - kubelets (workers)
 - complete
 - cleanup

Functionality includes:
 - kube-upgrade API endpoint for orchestration.
 - new rpc messages for create kube strategy and intermediate actions.
 - kube-upgrade event handling, as well as alarm and event logs.
 - 'upgrade start' uses the latest sysinv health api to include
 the vim auto apply alarm in the ignore list for the health check.

New unit tests:
 - build strategy phase
 - simplex controller
 - duplex controller (no existing kube upgrade)

Story: 2008137
Task: 41436
Depends-On: https://review.opendev.org/c/starlingx/fault/+/767374
Depends-On: https://review.opendev.org/c/starlingx/stx-puppet/+/775824
Signed-off-by: albailey <Al.Bailey@windriver.com>
Change-Id: I36e1b3ff3550a9d656ba40754b47570acc82a525
changes/21/767421/18
albailey 2 years ago
parent af3bddc28d
commit 90b480c345
  1. 5
      nfv/nfv-client/nfv_client/openstack/sw_update.py
  2. 149
      nfv/nfv-client/nfv_client/shell.py
  3. 3
      nfv/nfv-client/nfv_client/sw_update/__init__.py
  4. 5
      nfv/nfv-client/nfv_client/sw_update/_sw_update.py
  5. 85
      nfv/nfv-client/scripts/sw-manager.completion
  6. 5
      nfv/nfv-common/nfv_common/alarm/objects/v1/_alarm_defs.py
  7. 13
      nfv/nfv-common/nfv_common/event_log/objects/v1/_event_log_defs.py
  8. 4
      nfv/nfv-common/nfv_common/strategy/_strategy_stage.py
  9. 8
      nfv/nfv-plugins/nfv_plugins/alarm_handlers/fm.py
  10. 24
      nfv/nfv-plugins/nfv_plugins/event_log_handlers/fm.py
  11. 700
      nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py
  12. 70
      nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_sw_mgmt_api.py
  13. 16
      nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/patching.py
  14. 208
      nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py
  15. 734
      nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py
  16. 37
      nfv/nfv-vim/nfv_vim/alarm/_sw_update.py
  17. 7
      nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/_controller.py
  18. 3
      nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/__init__.py
  19. 59
      nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_kube_upgrade.py
  20. 6
      nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_sw_update_defs.py
  21. 88
      nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_sw_update_strategy.py
  22. 6
      nfv/nfv-vim/nfv_vim/database/_database_sw_update.py
  23. 3
      nfv/nfv-vim/nfv_vim/directors/_directors_defs.py
  24. 157
      nfv/nfv-vim/nfv_vim/directors/_host_director.py
  25. 56
      nfv/nfv-vim/nfv_vim/directors/_sw_mgmt_director.py
  26. 168
      nfv/nfv-vim/nfv_vim/event_log/_sw_update.py
  27. 5
      nfv/nfv-vim/nfv_vim/events/_vim_api_events.py
  28. 4
      nfv/nfv-vim/nfv_vim/events/_vim_nfvi_events.py
  29. 21
      nfv/nfv-vim/nfv_vim/events/_vim_sw_update_api_events.py
  30. 13
      nfv/nfv-vim/nfv_vim/nfvi/__init__.py
  31. 107
      nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py
  32. 12
      nfv/nfv-vim/nfv_vim/nfvi/_nfvi_sw_mgmt_module.py
  33. 6
      nfv/nfv-vim/nfv_vim/nfvi/objects/v1/__init__.py
  34. 102
      nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py
  35. 3
      nfv/nfv-vim/nfv_vim/objects/__init__.py
  36. 189
      nfv/nfv-vim/nfv_vim/objects/_kube_upgrade.py
  37. 3
      nfv/nfv-vim/nfv_vim/objects/_sw_update.py
  38. 3
      nfv/nfv-vim/nfv_vim/rpc/__init__.py
  39. 3
      nfv/nfv-vim/nfv_vim/rpc/_rpc_defs.py
  40. 4
      nfv/nfv-vim/nfv_vim/rpc/_rpc_message.py
  41. 26
      nfv/nfv-vim/nfv_vim/rpc/_rpc_message_sw_update.py
  42. 14
      nfv/nfv-vim/nfv_vim/strategy/__init__.py
  43. 1019
      nfv/nfv-vim/nfv_vim/strategy/_strategy.py
  44. 6
      nfv/nfv-vim/nfv_vim/strategy/_strategy_defs.py
  45. 16
      nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py
  46. 743
      nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py
  47. 2
      nfv/pylint.rc

@ -1,5 +1,5 @@
#
# Copyright (c) 2016,2020 Wind River Systems, Inc.
# Copyright (c) 2016-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -231,6 +231,9 @@ def create_strategy(token_id, url, strategy_name, controller_apply_type,
elif 'fw-update' == strategy_name:
api_cmd_payload['controller-apply-type'] = controller_apply_type
api_cmd_payload['default-instance-action'] = default_instance_action
elif 'kube-upgrade' == strategy_name:
# required: 'to_version' passed to strategy as 'to-version'
api_cmd_payload['to-version'] = kwargs['to_version']
elif 'sw-upgrade' == strategy_name:
if 'start_upgrade' in kwargs and kwargs['start_upgrade']:
api_cmd_payload['start-upgrade'] = True

@ -1,5 +1,5 @@
#
# Copyright (c) 2016,2020 Wind River Systems, Inc.
# Copyright (c) 2016-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -11,6 +11,78 @@ import sys
from nfv_client import sw_update
def setup_kube_upgrade_parser(commands):
# Kubernetes Upgrade Commands
kube_upgrade_parser = commands.add_parser('kube-upgrade-strategy',
help='Kubernetes Upgrade Strategy')
kube_upgrade_parser.set_defaults(cmd_area='kube-upgrade-strategy')
kube_upgrade_cmds = kube_upgrade_parser.add_subparsers(
title='Kubernetes Upgrade Commands', metavar='')
kube_upgrade_cmds.required = True
kube_upgrade_create_strategy_cmd \
= kube_upgrade_cmds.add_parser('create', help='Create a strategy')
kube_upgrade_create_strategy_cmd.set_defaults(cmd='create')
kube_upgrade_create_strategy_cmd.add_argument('--controller-apply-type',
default=sw_update.APPLY_TYPE_SERIAL,
choices=[sw_update.APPLY_TYPE_SERIAL, sw_update.APPLY_TYPE_IGNORE],
help='defaults to serial')
kube_upgrade_create_strategy_cmd.add_argument('--storage-apply-type',
default=sw_update.APPLY_TYPE_SERIAL,
choices=[sw_update.APPLY_TYPE_SERIAL, sw_update.APPLY_TYPE_IGNORE],
help='defaults to serial')
kube_upgrade_create_strategy_cmd.add_argument('--worker-apply-type',
default=sw_update.APPLY_TYPE_SERIAL,
choices=[sw_update.APPLY_TYPE_SERIAL,
sw_update.APPLY_TYPE_PARALLEL,
sw_update.APPLY_TYPE_IGNORE],
help='defaults to serial')
kube_upgrade_create_strategy_cmd.add_argument(
'--max-parallel-worker-hosts', type=int, choices=range(2, 6),
help='maximum worker hosts to update in parallel')
kube_upgrade_create_strategy_cmd.add_argument('--instance-action',
default=sw_update.INSTANCE_ACTION_STOP_START,
choices=[sw_update.INSTANCE_ACTION_MIGRATE,
sw_update.INSTANCE_ACTION_STOP_START],
help='defaults to stop-start')
kube_upgrade_create_strategy_cmd.add_argument('--alarm-restrictions',
default=sw_update.ALARM_RESTRICTIONS_STRICT,
choices=[sw_update.ALARM_RESTRICTIONS_STRICT,
sw_update.ALARM_RESTRICTIONS_RELAXED],
help='defaults to strict')
kube_upgrade_create_strategy_cmd.add_argument(
'--to-version', required=True, help='The kubernetes version')
kube_upgrade_delete_strategy_cmd \
= kube_upgrade_cmds.add_parser('delete', help='Delete a strategy')
kube_upgrade_delete_strategy_cmd.set_defaults(cmd='delete')
kube_upgrade_delete_strategy_cmd.add_argument(
'--force', action='store_true', help=argparse.SUPPRESS)
kube_upgrade_apply_strategy_cmd \
= kube_upgrade_cmds.add_parser('apply', help='Apply a strategy')
kube_upgrade_apply_strategy_cmd.set_defaults(cmd='apply')
kube_upgrade_apply_strategy_cmd.add_argument(
'--stage-id', default=None, help='stage identifier to apply')
kube_upgrade_abort_strategy_cmd \
= kube_upgrade_cmds.add_parser('abort', help='Abort a strategy')
kube_upgrade_abort_strategy_cmd.set_defaults(cmd='abort')
kube_upgrade_abort_strategy_cmd.add_argument(
'--stage-id', help='stage identifier to abort')
kube_upgrade_show_strategy_cmd \
= kube_upgrade_cmds.add_parser('show', help='Show a strategy')
kube_upgrade_show_strategy_cmd.set_defaults(cmd='show')
kube_upgrade_show_strategy_cmd.add_argument(
'--details', action='store_true', help='show strategy details')
def process_main(argv=sys.argv[1:]): # pylint: disable=dangerous-default-value
"""
Client - Main
@ -222,6 +294,9 @@ def process_main(argv=sys.argv[1:]): # pylint: disable=dangerous-default-value
fw_update_show_strategy_cmd.add_argument(
'--details', action='store_true', help='show strategy details')
# Register kubernetes upgrade command parser
setup_kube_upgrade_parser(commands)
args = parser.parse_args(argv)
if args.debug:
@ -468,6 +543,78 @@ def process_main(argv=sys.argv[1:]): # pylint: disable=dangerous-default-value
raise ValueError("Unknown command, %s, "
"given for fw-update-strategy"
% args.cmd)
elif 'kube-upgrade-strategy' == args.cmd_area:
strategy_type = sw_update.STRATEGY_NAME_KUBE_UPGRADE
if 'create' == args.cmd:
sw_update.create_strategy(
args.os_auth_url,
args.os_project_name,
args.os_project_domain_name,
args.os_username,
args.os_password,
args.os_user_domain_name,
args.os_region_name,
args.os_interface,
strategy_type,
args.controller_apply_type,
args.storage_apply_type,
sw_update.APPLY_TYPE_IGNORE,
args.worker_apply_type,
args.max_parallel_worker_hosts,
args.instance_action,
args.alarm_restrictions,
to_version=args.to_version)
elif 'delete' == args.cmd:
sw_update.delete_strategy(args.os_auth_url,
args.os_project_name,
args.os_project_domain_name,
args.os_username,
args.os_password,
args.os_user_domain_name,
args.os_region_name,
args.os_interface,
strategy_type,
args.force)
elif 'apply' == args.cmd:
sw_update.apply_strategy(args.os_auth_url,
args.os_project_name,
args.os_project_domain_name,
args.os_username,
args.os_password,
args.os_user_domain_name,
args.os_region_name,
args.os_interface,
strategy_type,
args.stage_id)
elif 'abort' == args.cmd:
sw_update.abort_strategy(args.os_auth_url,
args.os_project_name,
args.os_project_domain_name,
args.os_username,
args.os_password,
args.os_user_domain_name,
args.os_region_name,
args.os_interface,
strategy_type,
args.stage_id)
elif 'show' == args.cmd:
sw_update.show_strategy(args.os_auth_url,
args.os_project_name,
args.os_project_domain_name,
args.os_username,
args.os_password,
args.os_user_domain_name,
args.os_region_name,
args.os_interface,
strategy_type,
args.details)
else:
raise ValueError("Unknown command, %s , given for %s"
% args.cmd, args.cmd_area)
else:
raise ValueError("Unknown command area, %s, given" % args.cmd_area)

@ -1,4 +1,4 @@
# Copyright (c) 2016, 2020 Wind River Systems, Inc.
# Copyright (c) 2016-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -15,5 +15,6 @@ from nfv_client.sw_update._sw_update import INSTANCE_ACTION_MIGRATE # noqa: F40
from nfv_client.sw_update._sw_update import INSTANCE_ACTION_STOP_START # noqa: F401
from nfv_client.sw_update._sw_update import show_strategy # noqa: F401
from nfv_client.sw_update._sw_update import STRATEGY_NAME_FW_UPDATE # noqa: F401
from nfv_client.sw_update._sw_update import STRATEGY_NAME_KUBE_UPGRADE # noqa: F401
from nfv_client.sw_update._sw_update import STRATEGY_NAME_SW_PATCH # noqa: F401
from nfv_client.sw_update._sw_update import STRATEGY_NAME_SW_UPGRADE # noqa: F401

@ -1,5 +1,5 @@
#
# Copyright (c) 2016, 2020 Wind River Systems, Inc.
# Copyright (c) 2016-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -9,6 +9,7 @@ from nfv_client.openstack import sw_update
STRATEGY_NAME_SW_PATCH = 'sw-patch'
STRATEGY_NAME_SW_UPGRADE = 'sw-upgrade'
STRATEGY_NAME_FW_UPDATE = 'fw-update'
STRATEGY_NAME_KUBE_UPGRADE = 'kube-upgrade'
APPLY_TYPE_SERIAL = 'serial'
APPLY_TYPE_PARALLEL = 'parallel'
@ -108,6 +109,8 @@ def _display_strategy(strategy, details=False):
print("Strategy Upgrade Strategy:")
elif strategy.name == STRATEGY_NAME_FW_UPDATE:
print("Strategy Firmware Update Strategy:")
elif strategy.name == STRATEGY_NAME_KUBE_UPGRADE:
print("Strategy Kubernetes Upgrade Strategy:")
else:
print("Strategy Unknown Strategy:")

@ -1,5 +1,5 @@
#
# Copyright (c) 2016-2017 Wind River Systems, Inc.
# Copyright (c) 2016-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -22,6 +22,7 @@ function _swmanager()
patch-strategy
upgrade-strategy
fw-update-strategy
kube-upgrade-strategy
"
if [ $COMP_CWORD -gt 1 ]; then
@ -250,6 +251,88 @@ function _swmanager()
esac
fi
# Provide actions for completion
COMPREPLY=($(compgen -W "${actions}" -- ${cur}))
return 0
;;
kube-upgrade-strategy)
local actions="
create
delete
apply
abort
show
"
if [ $COMP_CWORD -gt 2 ]; then
local action=${COMP_WORDS[2]}
#
# Complete the arguments for each action
#
case "$action" in
create)
local createopts="
--controller-apply-type
--storage-apply-type
--worker-apply-type
--max-parallel-worker-hosts
--instance-action
--alarm-restrictions
--to-version
"
local createopt=${prev}
case "$createopt" in
--controller-apply-type|--storage-apply-type)
COMPREPLY=($(compgen -W "serial" -- ${cur}))
return 0
;;
--worker-apply-type)
COMPREPLY=($(compgen -W "serial parallel ignore" -- ${cur}))
return 0
;;
--max-parallel-worker-hosts)
COMPREPLY=( $(compgen -- ${cur}))
return 0
;;
--instance-action)
COMPREPLY=($(compgen -W "migrate stop-start" -- ${cur}))
return 0
;;
--alarm-restrictions)
COMPREPLY=($(compgen -W "strict relaxed" -- ${cur}))
return 0
;;
--to-version)
COMPREPLY=( $(compgen -- ${cur}))
return 0
;;
*)
;;
esac
COMPREPLY=($(compgen -W "${createopts}" -- ${cur}))
return 0
;;
apply|abort)
if [ "${prev}" = "${action}" ]; then
COMPREPLY=($(compgen -W "--stage-id" -- ${cur}))
fi
return 0
;;
show)
if [ "${prev}" = "${action}" ]; then
COMPREPLY=($(compgen -W "--details" -- ${cur}))
fi
return 0
;;
delete)
# These subcommands have no options/arguments
COMPREPLY=( $(compgen -- ${cur}) )
return 0
;;
*)
;;
esac
fi
# Provide actions for completion
COMPREPLY=($(compgen -W "${actions}" -- ${cur}))
return 0

@ -1,5 +1,5 @@
#
# Copyright (c) 2015-2020 Wind River Systems, Inc.
# Copyright (c) 2015-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -44,6 +44,9 @@ class _AlarmType(Constants):
FW_UPDATE_AUTO_APPLY_INPROGRESS = Constant('fw-update-auto-apply-inprogress')
FW_UPDATE_AUTO_APPLY_ABORTING = Constant('fw-update-auto-apply-aborting')
FW_UPDATE_AUTO_APPLY_FAILED = Constant('fw-update-auto-apply-failed')
KUBE_UPGRADE_AUTO_APPLY_INPROGRESS = Constant('kube-upgrade-auto-apply-inprogress')
KUBE_UPGRADE_AUTO_APPLY_ABORTING = Constant('kube-upgrade-auto-apply-aborting')
KUBE_UPGRADE_AUTO_APPLY_FAILED = Constant('kube-upgrade-auto-apply-failed')
@six.add_metaclass(Singleton)

@ -1,5 +1,5 @@
#
# Copyright (c) 2015-2020 Wind River Systems, Inc.
# Copyright (c) 2015-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -171,6 +171,17 @@ class _EventId(Constants):
FW_UPDATE_AUTO_APPLY_ABORT_REJECTED = Constant('fw-update-auto-apply-abort-rejected')
FW_UPDATE_AUTO_APPLY_ABORT_FAILED = Constant('fw-update-auto-apply-abort-failed')
FW_UPDATE_AUTO_APPLY_ABORTED = Constant('fw-update-auto-apply-aborted')
KUBE_UPGRADE_AUTO_APPLY_START = Constant('kube-upgrade-auto-apply-started')
KUBE_UPGRADE_AUTO_APPLY_INPROGRESS = Constant('kube-upgrade-auto-apply-inprogress')
KUBE_UPGRADE_AUTO_APPLY_REJECTED = Constant('kube-upgrade-auto-apply-rejected')
KUBE_UPGRADE_AUTO_APPLY_CANCELLED = Constant('kube-upgrade-auto-apply-cancelled')
KUBE_UPGRADE_AUTO_APPLY_FAILED = Constant('kube-upgrade-auto-apply-failed')
KUBE_UPGRADE_AUTO_APPLY_COMPLETED = Constant('kube-upgrade-auto-apply-completed')
KUBE_UPGRADE_AUTO_APPLY_ABORT = Constant('kube-upgrade-auto-apply-abort')
KUBE_UPGRADE_AUTO_APPLY_ABORTING = Constant('kube-upgrade-auto-apply-aborting')
KUBE_UPGRADE_AUTO_APPLY_ABORT_REJECTED = Constant('kube-upgrade-auto-apply-abort-rejected')
KUBE_UPGRADE_AUTO_APPLY_ABORT_FAILED = Constant('kube-upgrade-auto-apply-abort-failed')
KUBE_UPGRADE_AUTO_APPLY_ABORTED = Constant('kube-upgrade-auto-apply-aborted')
@six.add_metaclass(Singleton)

@ -1,5 +1,5 @@
#
# Copyright (c) 2015-2016 Wind River Systems, Inc.
# Copyright (c) 2015-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -324,6 +324,8 @@ class StrategyStage(object):
self._step_timer_id = None
step.start_date_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
DLOG.debug("Stage (%s) Step (%s) (apply) called"
% (self.name, step.name))
step.result, step.result_reason = step.apply()
self._current_step = idx

@ -1,5 +1,5 @@
#
# Copyright (c) 2015-2020 Wind River Systems, Inc.
# Copyright (c) 2015-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -78,6 +78,12 @@ _fm_alarm_id_mapping = dict([
fm_constants.FM_ALARM_ID_FW_UPDATE_AUTO_APPLY_ABORTING),
(alarm_objects_v1.ALARM_TYPE.FW_UPDATE_AUTO_APPLY_FAILED,
fm_constants.FM_ALARM_ID_FW_UPDATE_AUTO_APPLY_FAILED),
(alarm_objects_v1.ALARM_TYPE.KUBE_UPGRADE_AUTO_APPLY_INPROGRESS,
fm_constants.FM_ALARM_ID_KUBE_UPGRADE_AUTO_APPLY_INPROGRESS),
(alarm_objects_v1.ALARM_TYPE.KUBE_UPGRADE_AUTO_APPLY_ABORTING,
fm_constants.FM_ALARM_ID_KUBE_UPGRADE_AUTO_APPLY_ABORTING),
(alarm_objects_v1.ALARM_TYPE.KUBE_UPGRADE_AUTO_APPLY_FAILED,
fm_constants.FM_ALARM_ID_KUBE_UPGRADE_AUTO_APPLY_FAILED),
])
_fm_alarm_type_mapping = dict([

@ -1,5 +1,5 @@
#
# Copyright (c) 2015-2016,2020 Wind River Systems, Inc.
# Copyright (c) 2015-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -334,6 +334,28 @@ _fm_event_id_mapping = dict([
fm_constants.FM_LOG_ID_FW_UPDATE_AUTO_APPLY_ABORT_FAILED),
(event_log_objects_v1.EVENT_ID.FW_UPDATE_AUTO_APPLY_ABORTED,
fm_constants.FM_LOG_ID_FW_UPDATE_AUTO_APPLY_ABORTED),
(event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_START,
fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_START),
(event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_INPROGRESS,
fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_INPROGRESS),
(event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_REJECTED,
fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_REJECTED),
(event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_CANCELLED,
fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_CANCELLED),
(event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_FAILED,
fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_FAILED),
(event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_COMPLETED,
fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_COMPLETED),
(event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORT,
fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_ABORT),
(event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORTING,
fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_ABORTING),
(event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORT_REJECTED,
fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_ABORT_REJECTED),
(event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORT_FAILED,
fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_ABORT_FAILED),
(event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORTED,
fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_ABORTED),
])
_fm_event_type_mapping = dict([

@ -1,5 +1,5 @@
#
# Copyright (c) 2015-2020 Wind River Systems, Inc.
# Copyright (c) 2015-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -839,6 +839,704 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
callback.send(response)
callback.close()
def _extract_kube_host_upgrade_list(self,
kube_host_upgrade_list,
host_list):
"""
Return a list of KubeHostUpgrade objects from sysinv api results.
"""
# Map the ID to the uuid from host_list
host_map = dict()
for host in host_list:
host_map[host['id']] = host['uuid']
result_list = []
for host_data in kube_host_upgrade_list:
host_uuid = host_map[host_data['host_id']]
result_list.append(
nfvi.objects.v1.KubeHostUpgrade(
host_data['host_id'],
host_uuid,
host_data['target_version'],
host_data['control_plane_version'],
host_data['kubelet_version'],
host_data['status'])
)
return result_list
def get_kube_host_upgrade_list(self, future, callback):
"""
Get information about the kube host upgrade list from the plugin
"""
response = dict()
response['completed'] = False
response['reason'] = ''
activity = "SysInv get-kube-host-upgrades"
try:
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._platform_token is None or \
self._platform_token.is_expired():
future.work(openstack.get_token, self._platform_directory)
future.result = (yield)
if not future.result.is_complete() or \
future.result.data is None:
DLOG.error("OpenStack get-token did not complete.")
return
self._platform_token = future.result.data
# Query the kube host upgrade list
future.work(sysinv.get_kube_host_upgrades, self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("Sysinv Get Kube Host Upgrades did not complete.")
return
kube_host_upgrade_list = future.result.data["kube_host_upgrades"]
# Also query the host list, kube_host_upgrades does not have uuid
future.work(sysinv.get_hosts, self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("Sysinv Get-Hosts did not complete.")
return
host_list = future.result.data["ihosts"]
results_obj = \
self._extract_kube_host_upgrade_list(kube_host_upgrade_list,
host_list)
response['result-data'] = results_obj
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
if self._platform_token is not None:
self._platform_token.set_expired()
else:
DLOG.exception("Caught exception kube host upgrade list err=%s"
% e)
except Exception as e:
DLOG.exception("Caught exception kube host upgrade list err=%s"
% e)
finally:
callback.send(response)
callback.close()
def _extract_kube_upgrade(self, kube_upgrade_data_list):
"""
Return a KubeUpgrade object from sysinv api results.
Returns None if there are no items in the list.
Returns first kube upgrade object, but the API should never return
more than one object.
"""
if 1 < len(kube_upgrade_data_list):
DLOG.critical("Too many kube upgrades returned, num=%i"
% len(kube_upgrade_data_list))
elif 0 == len(kube_upgrade_data_list):
DLOG.info("No kube upgrade exists, num=%i"
% len(kube_upgrade_data_list))
kube_upgrade_obj = None
for kube_upgrade_data in kube_upgrade_data_list:
kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade(
kube_upgrade_data['state'],
kube_upgrade_data['from_version'],
kube_upgrade_data['to_version'])
break
return kube_upgrade_obj
def get_kube_upgrade(self, future, callback):
"""
Get information about the kube upgrade from the plugin
"""
response = dict()
response['completed'] = False
response['reason'] = ''
action_type = 'get-kube-upgrade'
try:
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._platform_token is None or \
self._platform_token.is_expired():
future.work(openstack.get_token, self._platform_directory)
future.result = (yield)
if not future.result.is_complete() or \
future.result.data is None:
DLOG.error("OpenStack get-token did not complete.")
return
self._platform_token = future.result.data
future.work(sysinv.get_kube_upgrade, self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("SysInv get-kube-upgrade did not complete.")
return
kube_upgrade_data_list = future.result.data['kube_upgrades']
kube_upgrade_obj = \
self._extract_kube_upgrade(kube_upgrade_data_list)
response['result-data'] = kube_upgrade_obj
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
if self._platform_token is not None:
self._platform_token.set_expired()
else:
DLOG.exception("Caught API exception while trying %s. error=%s"
% (action_type, e))
except Exception as e:
DLOG.exception("Caught exception while trying %s. error=%s"
% (action_type, e))
finally:
callback.send(response)
callback.close()
def _extract_kube_version(self, kube_data):
"""
Return a KubeVersion from sysinv API results.
"""
# sysinv api returns a field called 'version' which is a reserved field
# in vim object data structure. It is stored as kube_version
return nfvi.objects.v1.KubeVersion(kube_data['version'],
kube_data['state'],
kube_data['target'],
kube_data['upgrade_from'],
kube_data['downgrade_to'],
kube_data['applied_patches'],
kube_data['available_patches'])
def get_kube_version_list(self, future, callback):
"""
Get information about the kube versions list from the plugin
"""
response = dict()
response['completed'] = False
response['reason'] = ''
action_type = 'get-kube-versions'
try:
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._platform_token is None or \
self._platform_token.is_expired():
future.work(openstack.get_token, self._platform_directory)
future.result = (yield)
if not future.result.is_complete() or \
future.result.data is None:
DLOG.error("OpenStack get-token did not complete.")
return
self._platform_token = future.result.data
# get_kube_versions only returns a limited amount of data about the
# kubernetes versions. Individual API calls get the patch info.
future.work(sysinv.get_kube_versions, self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("%s did not complete." % action_type)
return
# walk the list of versions and get the patch info
kube_versions_list = list()
limited_kube_version_list = future.result.data['kube_versions']
for kube_list_entry in limited_kube_version_list:
kube_ver = kube_list_entry['version']
future.work(sysinv.get_kube_version,
self._platform_token,
kube_ver)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("%s for version:%s did not complete."
% (action_type, kube_ver))
return
# returns a single object
kube_ver_data = future.result.data
kube_versions_list.append(
self._extract_kube_version(kube_ver_data))
response['result-data'] = kube_versions_list
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
if self._platform_token is not None:
self._platform_token.set_expired()
else:
DLOG.exception("Caught API exception while trying %s. error=%s"
% (action_type, e))
except Exception as e:
DLOG.exception("Caught exception while trying %s. error=%s"
% (action_type, e))
finally:
callback.send(response)
callback.close()
def kube_host_upgrade_control_plane(self,
future,
host_uuid,
host_name,
force,
callback):
"""
Start kube host upgrade 'control plane' for a particular controller
"""
response = dict()
response['completed'] = False
response['host_uuid'] = host_uuid
response['host_name'] = host_name
response['reason'] = ''
action_type = 'kube-host-upgrade-control-plane'
sysinv_method = sysinv.kube_host_upgrade_control_plane
try:
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._platform_token is None or \
self._platform_token.is_expired():
future.work(openstack.get_token, self._platform_directory)
future.result = (yield)
if not future.result.is_complete() or \
future.result.data is None:
DLOG.error("OpenStack get-token did not complete.")
return
self._platform_token = future.result.data
# invoke the actual kube_host_upgrade method
future.work(sysinv_method,
self._platform_token,
host_uuid,
force)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("%s did not complete." % action_type)
return
# result was a host object. Need to query to get kube upgrade obj
future.work(sysinv.get_kube_upgrade, self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("SysInv get-kube-upgrade did not complete.")
return
kube_upgrade_data_list = future.result.data['kube_upgrades']
kube_upgrade_obj = \
self._extract_kube_upgrade(kube_upgrade_data_list)
response['result-data'] = kube_upgrade_obj
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
if self._platform_token is not None:
self._platform_token.set_expired()
else:
DLOG.exception("Caught API exception while trying %s. error=%s"
% (action_type, e))
response['reason'] = e.http_response_reason
except Exception as e:
DLOG.exception("Caught exception while trying %s. error=%s"
% (action_type, e))
finally:
callback.send(response)
callback.close()
def kube_host_upgrade_kubelet(self,
future,
host_uuid,
host_name,
force,
callback):
"""
Start kube host upgrade 'kubelet' for a particular host
"""
response = dict()
response['completed'] = False
response['host_uuid'] = host_uuid
response['host_name'] = host_name
response['reason'] = ''
action_type = 'kube-host-upgrade-kubelet'
sysinv_method = sysinv.kube_host_upgrade_kubelet
try:
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._platform_token is None or \
self._platform_token.is_expired():
future.work(openstack.get_token, self._platform_directory)
future.result = (yield)
if not future.result.is_complete() or \
future.result.data is None:
DLOG.error("OpenStack get-token did not complete.")
return
self._platform_token = future.result.data
# invoke the actual kube_host_upgrade method
future.work(sysinv_method,
self._platform_token,
host_uuid,
force)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("%s did not complete." % action_type)
return
# result was a host object. Need to query to get kube upgrade obj
future.work(sysinv.get_kube_upgrade, self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("SysInv get-kube-upgrade did not complete.")
return
kube_upgrade_data_list = future.result.data['kube_upgrades']
kube_upgrade_obj = \
self._extract_kube_upgrade(kube_upgrade_data_list)
response['result-data'] = kube_upgrade_obj
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
if self._platform_token is not None:
self._platform_token.set_expired()
else:
DLOG.exception("Caught API exception while trying %s. error=%s"
% (action_type, e))
response['reason'] = e.http_response_reason
except Exception as e:
DLOG.exception("Caught exception while trying %s. error=%s"
% (action_type, e))
finally:
callback.send(response)
callback.close()
def kube_upgrade_cleanup(self, future, callback):
"""
kube upgrade cleanup
"""
response = dict()
response['completed'] = False
response['reason'] = ''
action_type = 'kube-upgrade-cleanup'
try:
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._platform_token is None or \
self._platform_token.is_expired():
future.work(openstack.get_token, self._platform_directory)
future.result = (yield)
if not future.result.is_complete() or \
future.result.data is None:
DLOG.error("OpenStack get-token did not complete.")
return
self._platform_token = future.result.data
future.work(sysinv.kube_upgrade_cleanup, self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("%s did not complete." % action_type)
return
# The result should be empty. no result data to report back
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
if self._platform_token is not None:
self._platform_token.set_expired()
else:
DLOG.exception("Caught API exception while trying %s. error=%s"
% (action_type, e))
response['reason'] = e.http_response_reason
except Exception as e:
DLOG.exception("Caught exception while trying %s. error=%s"
% (action_type, e))
finally:
callback.send(response)
callback.close()
def kube_upgrade_complete(self, future, callback):
"""
kube upgrade complete
"""
response = dict()
response['completed'] = False
response['reason'] = ''
action_type = 'kube-upgrade-complete'
try:
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._platform_token is None or \
self._platform_token.is_expired():
future.work(openstack.get_token, self._platform_directory)
future.result = (yield)
if not future.result.is_complete() or \
future.result.data is None:
DLOG.error("OpenStack get-token did not complete.")
return
self._platform_token = future.result.data
future.work(sysinv.kube_upgrade_complete, self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("%s did not complete." % action_type)
return
kube_upgrade_data = future.result.data
kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade(
kube_upgrade_data['state'],
kube_upgrade_data['from_version'],
kube_upgrade_data['to_version'])
response['result-data'] = kube_upgrade_obj
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
if self._platform_token is not None:
self._platform_token.set_expired()
else:
DLOG.exception("Caught API exception while trying %s. error=%s"
% (action_type, e))
response['reason'] = e.http_response_reason
except Exception as e:
DLOG.exception("Caught exception while trying %s. error=%s"
% (action_type, e))
finally:
callback.send(response)
callback.close()
def kube_upgrade_download_images(self, future, callback):
"""
Start kube upgrade download images
"""
response = dict()
response['completed'] = False
response['reason'] = ''
action_type = 'kube-upgrade-download-images'
try:
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._platform_token is None or \
self._platform_token.is_expired():
future.work(openstack.get_token, self._platform_directory)
future.result = (yield)
if not future.result.is_complete() or \
future.result.data is None:
DLOG.error("OpenStack get-token did not complete.")
return
self._platform_token = future.result.data
future.work(sysinv.kube_upgrade_download_images,
self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("%s did not complete." % action_type)
return
kube_upgrade_data = future.result.data
kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade(
kube_upgrade_data['state'],
kube_upgrade_data['from_version'],
kube_upgrade_data['to_version'])
response['result-data'] = kube_upgrade_obj
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
if self._platform_token is not None:
self._platform_token.set_expired()
else:
DLOG.exception("Caught API exception while trying %s. error=%s"
% (action_type, e))
except Exception as e:
DLOG.exception("Caught exception while trying %s. error=%s"
% (action_type, e))
finally:
callback.send(response)
callback.close()
def kube_upgrade_networking(self, future, callback):
"""
Start kube upgrade networking
"""
response = dict()
response['completed'] = False
response['reason'] = ''
action_type = 'kube-upgrade-networking'
try:
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._platform_token is None or \
self._platform_token.is_expired():
future.work(openstack.get_token, self._platform_directory)
future.result = (yield)
if not future.result.is_complete() or \
future.result.data is None:
DLOG.error("OpenStack get-token did not complete.")
return
self._platform_token = future.result.data
future.work(sysinv.kube_upgrade_networking, self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("%s did not complete." % action_type)
return
kube_upgrade_data = future.result.data
kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade(
kube_upgrade_data['state'],
kube_upgrade_data['from_version'],
kube_upgrade_data['to_version'])
response['result-data'] = kube_upgrade_obj
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
if self._platform_token is not None:
self._platform_token.set_expired()
else:
DLOG.exception("Caught API exception while trying %s. error=%s"
% (action_type, e))
except Exception as e:
DLOG.exception("Caught exception while trying %s. error=%s"
% (action_type, e))
finally:
callback.send(response)
callback.close()
def kube_upgrade_start(self, future, to_version, force, alarm_ignore_list,
callback):
"""
Start a kube upgrade
"""
response = dict()
response['completed'] = False
response['reason'] = ''
action_type = 'kube-upgrade-start'
try:
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._platform_token is None or \
self._platform_token.is_expired():
future.work(openstack.get_token, self._platform_directory)
future.result = (yield)
if not future.result.is_complete() or \
future.result.data is None:
DLOG.error("OpenStack get-token did not complete.")
return
self._platform_token = future.result.data
future.work(sysinv.kube_upgrade_start,
self._platform_token,
to_version,
force=force,
alarm_ignore_list=alarm_ignore_list)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("SysInv kube-upgrade-start did not complete.")
response['reason'] = "did not complete."
return
future.work(sysinv.get_kube_upgrade, self._platform_token)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("SysInv get-kube-upgrade did not complete.")
response['reason'] = "did not complete."
return
kube_upgrade_data_list = future.result.data['kube_upgrades']
kube_upgrade_obj = \
self._extract_kube_upgrade(kube_upgrade_data_list)
response['result-data'] = kube_upgrade_obj
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
if self._platform_token is not None:
self._platform_token.set_expired()
else:
DLOG.exception("Caught API exception while trying %s. error=%s"
% (action_type, e))
response['reason'] = e.http_response_reason
except Exception as e:
DLOG.exception("Caught exception while trying %s. error=%s"
% (action_type, e))
finally:
callback.send(response)
callback.close()
def get_upgrade(self, future, callback):
"""
Get information about the upgrade from the plugin

@ -1,5 +1,5 @@
#
# Copyright (c) 2016-2018 Wind River Systems, Inc.
# Copyright (c) 2016-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -216,6 +216,74 @@ class NFVISwMgmtAPI(nfvi.api.v1.NFVISwMgmtAPI):
callback.send(response)
callback.close()
def apply_patches(self, future, patch_names, callback):
"""
Apply a software patch that has already been uploaded
"""
response = dict()
response['completed'] = False
response['reason'] = ''
try:
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._token is None or self._token.is_expired():
future.work(openstack.get_token, self._directory)
future.result = (yield)
if not future.result.is_complete() or \
future.result.data is None:
return
self._token = future.result.data
for patch_name in patch_names:
future.work(patching.apply_patch, self._token, patch_name)
future.result = (yield)
if not future.result.is_complete():
return
# query the patches and return their state
future.work(patching.query_patches, self._token)
future.result = (yield)
if not future.result.is_complete():
return
sw_patches = list()
if future.result.data is not None:
sw_patch_data_list = future.result.data.get('pd', [])
for sw_patch_name in sw_patch_data_list.keys():
sw_patch_data = sw_patch_data_list[sw_patch_name]
sw_patch = nfvi.objects.v1.SwPatch(
sw_patch_name, sw_patch_data['sw_version'],
sw_patch_data['repostate'], sw_patch_data['patchstate'])
sw_patches.append(sw_patch)
response['result-data'] = sw_patches
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
if self._token is not None:
self._token.set_expired()
else:
DLOG.exception("Caught exception while trying to apply "
"software patches [%s], error=%s."
% (patch_names, e))
except Exception as e:
DLOG.exception("Caught exception while trying to apply "
"software patches [%s], error=%s."
% (patch_names, e))
finally:
callback.send(response)
callback.close()
def update_hosts(self, future, host_names, callback):
"""
Apply a software update to a list of hosts

@ -1,5 +1,5 @@
#
# Copyright (c) 2016-2018 Wind River Systems, Inc.
# Copyright (c) 2016-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -39,6 +39,20 @@ def query_hosts(token):
return response
def apply_patch(token, patch_name):
"""
Asks Patch Controller to apply a patch that is already uploaded
"""
url = token.get_service_url(PLATFORM_SERVICE.PATCHING, strip_version=True)
if url is None:
raise ValueError("OpenStack Patching URL is invalid")
api_cmd = url + "/v1/apply/%s" % str(patch_name)
response = rest_api_request(token, "POST", api_cmd)
return response
def host_install_async(token, host_name):
"""
Asks Patch Controller to perform a software upgrade on a host

@ -1,8 +1,9 @@
#
# Copyright (c) 2015-2020 Wind River Systems, Inc.
# Copyright (c) 2015-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import copy
import json
from nfv_common import debug
@ -97,6 +98,211 @@ def get_host_labels(token, host_uuid):
return response
def get_kube_host_upgrades(token):
"""
Asks System Inventory for information about the kube host upgrades
"""
url = token.get_service_url(PLATFORM_SERVICE.SYSINV)