From 90b480c3450823b8f2980904db6a30462377f636 Mon Sep 17 00:00:00 2001 From: albailey Date: Thu, 19 Nov 2020 16:17:36 -0600 Subject: [PATCH] Kubernetes Upgrade Orchestration Provides the new CLI command: sw-manager kube-upgrade-strategy VIM build stages are: - query-alarms - query-kube-upgrade - query-kube-versions - query-patches - query-patch-hosts VIM apply stages are: - kube-upgrade-start - download images - first control plane - networking - second control plane - apply second kubernetes patch -- applies the patch -- host-install on each controller -- host-install on each storage -- host-install on each worker - kubelets (controllers) - kubelets (workers) - complete - cleanup Functionality includes: - kube-upgrade API endpoint for orchestration. - new rpc messages for create kube strategy and intermediate actions. - kube-upgrade event handling, as well as alarm and event logs. - 'upgrade start' uses the latest sysinv health api to include the vim auto apply alarm in the ignore list for the health check. New unit tests: - build strategy phase - simplex controller - duplex controller (no existing kube upgrade) Story: 2008137 Task: 41436 Depends-On: https://review.opendev.org/c/starlingx/fault/+/767374 Depends-On: https://review.opendev.org/c/starlingx/stx-puppet/+/775824 Signed-off-by: albailey Change-Id: I36e1b3ff3550a9d656ba40754b47570acc82a525 --- .../nfv_client/openstack/sw_update.py | 5 +- nfv/nfv-client/nfv_client/shell.py | 149 ++- .../nfv_client/sw_update/__init__.py | 3 +- .../nfv_client/sw_update/_sw_update.py | 5 +- nfv/nfv-client/scripts/sw-manager.completion | 85 +- .../alarm/objects/v1/_alarm_defs.py | 5 +- .../event_log/objects/v1/_event_log_defs.py | 13 +- .../nfv_common/strategy/_strategy_stage.py | 4 +- .../nfv_plugins/alarm_handlers/fm.py | 8 +- .../nfv_plugins/event_log_handlers/fm.py | 24 +- .../nfvi_plugins/nfvi_infrastructure_api.py | 700 ++++++++++- .../nfvi_plugins/nfvi_sw_mgmt_api.py | 70 +- .../nfvi_plugins/openstack/patching.py | 16 +- .../nfvi_plugins/openstack/sysinv.py | 208 +++- .../tests/test_kube_upgrade_strategy.py | 734 ++++++++++++ nfv/nfv-vim/nfv_vim/alarm/_sw_update.py | 37 +- .../v1/orchestration/_controller.py | 7 +- .../v1/orchestration/sw_update/__init__.py | 3 +- .../orchestration/sw_update/_kube_upgrade.py | 59 + .../sw_update/_sw_update_defs.py | 6 +- .../sw_update/_sw_update_strategy.py | 88 +- .../nfv_vim/database/_database_sw_update.py | 6 +- .../nfv_vim/directors/_directors_defs.py | 3 +- .../nfv_vim/directors/_host_director.py | 157 ++- .../nfv_vim/directors/_sw_mgmt_director.py | 56 +- nfv/nfv-vim/nfv_vim/event_log/_sw_update.py | 168 ++- nfv/nfv-vim/nfv_vim/events/_vim_api_events.py | 5 +- .../nfv_vim/events/_vim_nfvi_events.py | 4 +- .../events/_vim_sw_update_api_events.py | 21 +- nfv/nfv-vim/nfv_vim/nfvi/__init__.py | 13 +- .../nfvi/_nfvi_infrastructure_module.py | 107 +- .../nfv_vim/nfvi/_nfvi_sw_mgmt_module.py | 12 +- .../nfv_vim/nfvi/objects/v1/__init__.py | 6 +- .../nfv_vim/nfvi/objects/v1/_kube_upgrade.py | 102 ++ nfv/nfv-vim/nfv_vim/objects/__init__.py | 3 +- nfv/nfv-vim/nfv_vim/objects/_kube_upgrade.py | 189 +++ nfv/nfv-vim/nfv_vim/objects/_sw_update.py | 3 +- nfv/nfv-vim/nfv_vim/rpc/__init__.py | 3 +- nfv/nfv-vim/nfv_vim/rpc/_rpc_defs.py | 3 +- nfv/nfv-vim/nfv_vim/rpc/_rpc_message.py | 4 +- .../nfv_vim/rpc/_rpc_message_sw_update.py | 26 +- nfv/nfv-vim/nfv_vim/strategy/__init__.py | 14 +- nfv/nfv-vim/nfv_vim/strategy/_strategy.py | 1019 ++++++++++++++++- .../nfv_vim/strategy/_strategy_defs.py | 6 +- .../nfv_vim/strategy/_strategy_stages.py | 16 +- .../nfv_vim/strategy/_strategy_steps.py | 743 +++++++++++- nfv/pylint.rc | 2 +- 47 files changed, 4872 insertions(+), 48 deletions(-) create mode 100755 nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py create mode 100755 nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_kube_upgrade.py create mode 100755 nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py create mode 100644 nfv/nfv-vim/nfv_vim/objects/_kube_upgrade.py diff --git a/nfv/nfv-client/nfv_client/openstack/sw_update.py b/nfv/nfv-client/nfv_client/openstack/sw_update.py index fa3f0d91..607ebda5 100755 --- a/nfv/nfv-client/nfv_client/openstack/sw_update.py +++ b/nfv/nfv-client/nfv_client/openstack/sw_update.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2016,2020 Wind River Systems, Inc. +# Copyright (c) 2016-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -231,6 +231,9 @@ def create_strategy(token_id, url, strategy_name, controller_apply_type, elif 'fw-update' == strategy_name: api_cmd_payload['controller-apply-type'] = controller_apply_type api_cmd_payload['default-instance-action'] = default_instance_action + elif 'kube-upgrade' == strategy_name: + # required: 'to_version' passed to strategy as 'to-version' + api_cmd_payload['to-version'] = kwargs['to_version'] elif 'sw-upgrade' == strategy_name: if 'start_upgrade' in kwargs and kwargs['start_upgrade']: api_cmd_payload['start-upgrade'] = True diff --git a/nfv/nfv-client/nfv_client/shell.py b/nfv/nfv-client/nfv_client/shell.py index 841069a3..96b690fa 100755 --- a/nfv/nfv-client/nfv_client/shell.py +++ b/nfv/nfv-client/nfv_client/shell.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2016,2020 Wind River Systems, Inc. +# Copyright (c) 2016-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -11,6 +11,78 @@ import sys from nfv_client import sw_update +def setup_kube_upgrade_parser(commands): + # Kubernetes Upgrade Commands + kube_upgrade_parser = commands.add_parser('kube-upgrade-strategy', + help='Kubernetes Upgrade Strategy') + kube_upgrade_parser.set_defaults(cmd_area='kube-upgrade-strategy') + + kube_upgrade_cmds = kube_upgrade_parser.add_subparsers( + title='Kubernetes Upgrade Commands', metavar='') + kube_upgrade_cmds.required = True + + kube_upgrade_create_strategy_cmd \ + = kube_upgrade_cmds.add_parser('create', help='Create a strategy') + kube_upgrade_create_strategy_cmd.set_defaults(cmd='create') + kube_upgrade_create_strategy_cmd.add_argument('--controller-apply-type', + default=sw_update.APPLY_TYPE_SERIAL, + choices=[sw_update.APPLY_TYPE_SERIAL, sw_update.APPLY_TYPE_IGNORE], + help='defaults to serial') + kube_upgrade_create_strategy_cmd.add_argument('--storage-apply-type', + default=sw_update.APPLY_TYPE_SERIAL, + choices=[sw_update.APPLY_TYPE_SERIAL, sw_update.APPLY_TYPE_IGNORE], + help='defaults to serial') + kube_upgrade_create_strategy_cmd.add_argument('--worker-apply-type', + default=sw_update.APPLY_TYPE_SERIAL, + choices=[sw_update.APPLY_TYPE_SERIAL, + sw_update.APPLY_TYPE_PARALLEL, + sw_update.APPLY_TYPE_IGNORE], + help='defaults to serial') + + kube_upgrade_create_strategy_cmd.add_argument( + '--max-parallel-worker-hosts', type=int, choices=range(2, 6), + help='maximum worker hosts to update in parallel') + + kube_upgrade_create_strategy_cmd.add_argument('--instance-action', + default=sw_update.INSTANCE_ACTION_STOP_START, + choices=[sw_update.INSTANCE_ACTION_MIGRATE, + sw_update.INSTANCE_ACTION_STOP_START], + help='defaults to stop-start') + + kube_upgrade_create_strategy_cmd.add_argument('--alarm-restrictions', + default=sw_update.ALARM_RESTRICTIONS_STRICT, + choices=[sw_update.ALARM_RESTRICTIONS_STRICT, + sw_update.ALARM_RESTRICTIONS_RELAXED], + help='defaults to strict') + + kube_upgrade_create_strategy_cmd.add_argument( + '--to-version', required=True, help='The kubernetes version') + + kube_upgrade_delete_strategy_cmd \ + = kube_upgrade_cmds.add_parser('delete', help='Delete a strategy') + kube_upgrade_delete_strategy_cmd.set_defaults(cmd='delete') + kube_upgrade_delete_strategy_cmd.add_argument( + '--force', action='store_true', help=argparse.SUPPRESS) + + kube_upgrade_apply_strategy_cmd \ + = kube_upgrade_cmds.add_parser('apply', help='Apply a strategy') + kube_upgrade_apply_strategy_cmd.set_defaults(cmd='apply') + kube_upgrade_apply_strategy_cmd.add_argument( + '--stage-id', default=None, help='stage identifier to apply') + + kube_upgrade_abort_strategy_cmd \ + = kube_upgrade_cmds.add_parser('abort', help='Abort a strategy') + kube_upgrade_abort_strategy_cmd.set_defaults(cmd='abort') + kube_upgrade_abort_strategy_cmd.add_argument( + '--stage-id', help='stage identifier to abort') + + kube_upgrade_show_strategy_cmd \ + = kube_upgrade_cmds.add_parser('show', help='Show a strategy') + kube_upgrade_show_strategy_cmd.set_defaults(cmd='show') + kube_upgrade_show_strategy_cmd.add_argument( + '--details', action='store_true', help='show strategy details') + + def process_main(argv=sys.argv[1:]): # pylint: disable=dangerous-default-value """ Client - Main @@ -222,6 +294,9 @@ def process_main(argv=sys.argv[1:]): # pylint: disable=dangerous-default-value fw_update_show_strategy_cmd.add_argument( '--details', action='store_true', help='show strategy details') + # Register kubernetes upgrade command parser + setup_kube_upgrade_parser(commands) + args = parser.parse_args(argv) if args.debug: @@ -468,6 +543,78 @@ def process_main(argv=sys.argv[1:]): # pylint: disable=dangerous-default-value raise ValueError("Unknown command, %s, " "given for fw-update-strategy" % args.cmd) + elif 'kube-upgrade-strategy' == args.cmd_area: + strategy_type = sw_update.STRATEGY_NAME_KUBE_UPGRADE + if 'create' == args.cmd: + sw_update.create_strategy( + args.os_auth_url, + args.os_project_name, + args.os_project_domain_name, + args.os_username, + args.os_password, + args.os_user_domain_name, + args.os_region_name, + args.os_interface, + strategy_type, + args.controller_apply_type, + args.storage_apply_type, + sw_update.APPLY_TYPE_IGNORE, + args.worker_apply_type, + args.max_parallel_worker_hosts, + args.instance_action, + args.alarm_restrictions, + to_version=args.to_version) + + elif 'delete' == args.cmd: + sw_update.delete_strategy(args.os_auth_url, + args.os_project_name, + args.os_project_domain_name, + args.os_username, + args.os_password, + args.os_user_domain_name, + args.os_region_name, + args.os_interface, + strategy_type, + args.force) + + elif 'apply' == args.cmd: + sw_update.apply_strategy(args.os_auth_url, + args.os_project_name, + args.os_project_domain_name, + args.os_username, + args.os_password, + args.os_user_domain_name, + args.os_region_name, + args.os_interface, + strategy_type, + args.stage_id) + + elif 'abort' == args.cmd: + sw_update.abort_strategy(args.os_auth_url, + args.os_project_name, + args.os_project_domain_name, + args.os_username, + args.os_password, + args.os_user_domain_name, + args.os_region_name, + args.os_interface, + strategy_type, + args.stage_id) + + elif 'show' == args.cmd: + sw_update.show_strategy(args.os_auth_url, + args.os_project_name, + args.os_project_domain_name, + args.os_username, + args.os_password, + args.os_user_domain_name, + args.os_region_name, + args.os_interface, + strategy_type, + args.details) + else: + raise ValueError("Unknown command, %s , given for %s" + % args.cmd, args.cmd_area) else: raise ValueError("Unknown command area, %s, given" % args.cmd_area) diff --git a/nfv/nfv-client/nfv_client/sw_update/__init__.py b/nfv/nfv-client/nfv_client/sw_update/__init__.py index 1006504c..1beaf1a5 100755 --- a/nfv/nfv-client/nfv_client/sw_update/__init__.py +++ b/nfv/nfv-client/nfv_client/sw_update/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016, 2020 Wind River Systems, Inc. +# Copyright (c) 2016-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -15,5 +15,6 @@ from nfv_client.sw_update._sw_update import INSTANCE_ACTION_MIGRATE # noqa: F40 from nfv_client.sw_update._sw_update import INSTANCE_ACTION_STOP_START # noqa: F401 from nfv_client.sw_update._sw_update import show_strategy # noqa: F401 from nfv_client.sw_update._sw_update import STRATEGY_NAME_FW_UPDATE # noqa: F401 +from nfv_client.sw_update._sw_update import STRATEGY_NAME_KUBE_UPGRADE # noqa: F401 from nfv_client.sw_update._sw_update import STRATEGY_NAME_SW_PATCH # noqa: F401 from nfv_client.sw_update._sw_update import STRATEGY_NAME_SW_UPGRADE # noqa: F401 diff --git a/nfv/nfv-client/nfv_client/sw_update/_sw_update.py b/nfv/nfv-client/nfv_client/sw_update/_sw_update.py index 459e958a..56a21e52 100755 --- a/nfv/nfv-client/nfv_client/sw_update/_sw_update.py +++ b/nfv/nfv-client/nfv_client/sw_update/_sw_update.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2016, 2020 Wind River Systems, Inc. +# Copyright (c) 2016-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -9,6 +9,7 @@ from nfv_client.openstack import sw_update STRATEGY_NAME_SW_PATCH = 'sw-patch' STRATEGY_NAME_SW_UPGRADE = 'sw-upgrade' STRATEGY_NAME_FW_UPDATE = 'fw-update' +STRATEGY_NAME_KUBE_UPGRADE = 'kube-upgrade' APPLY_TYPE_SERIAL = 'serial' APPLY_TYPE_PARALLEL = 'parallel' @@ -108,6 +109,8 @@ def _display_strategy(strategy, details=False): print("Strategy Upgrade Strategy:") elif strategy.name == STRATEGY_NAME_FW_UPDATE: print("Strategy Firmware Update Strategy:") + elif strategy.name == STRATEGY_NAME_KUBE_UPGRADE: + print("Strategy Kubernetes Upgrade Strategy:") else: print("Strategy Unknown Strategy:") diff --git a/nfv/nfv-client/scripts/sw-manager.completion b/nfv/nfv-client/scripts/sw-manager.completion index 8b72fc44..643450bf 100755 --- a/nfv/nfv-client/scripts/sw-manager.completion +++ b/nfv/nfv-client/scripts/sw-manager.completion @@ -1,5 +1,5 @@ # -# Copyright (c) 2016-2017 Wind River Systems, Inc. +# Copyright (c) 2016-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -22,6 +22,7 @@ function _swmanager() patch-strategy upgrade-strategy fw-update-strategy + kube-upgrade-strategy " if [ $COMP_CWORD -gt 1 ]; then @@ -250,6 +251,88 @@ function _swmanager() esac fi + # Provide actions for completion + COMPREPLY=($(compgen -W "${actions}" -- ${cur})) + return 0 + ;; + kube-upgrade-strategy) + local actions=" + create + delete + apply + abort + show + " + if [ $COMP_CWORD -gt 2 ]; then + local action=${COMP_WORDS[2]} + # + # Complete the arguments for each action + # + case "$action" in + create) + local createopts=" + --controller-apply-type + --storage-apply-type + --worker-apply-type + --max-parallel-worker-hosts + --instance-action + --alarm-restrictions + --to-version + " + local createopt=${prev} + case "$createopt" in + --controller-apply-type|--storage-apply-type) + COMPREPLY=($(compgen -W "serial" -- ${cur})) + return 0 + ;; + --worker-apply-type) + COMPREPLY=($(compgen -W "serial parallel ignore" -- ${cur})) + return 0 + ;; + --max-parallel-worker-hosts) + COMPREPLY=( $(compgen -- ${cur})) + return 0 + ;; + --instance-action) + COMPREPLY=($(compgen -W "migrate stop-start" -- ${cur})) + return 0 + ;; + --alarm-restrictions) + COMPREPLY=($(compgen -W "strict relaxed" -- ${cur})) + return 0 + ;; + --to-version) + COMPREPLY=( $(compgen -- ${cur})) + return 0 + ;; + *) + ;; + esac + COMPREPLY=($(compgen -W "${createopts}" -- ${cur})) + return 0 + ;; + apply|abort) + if [ "${prev}" = "${action}" ]; then + COMPREPLY=($(compgen -W "--stage-id" -- ${cur})) + fi + return 0 + ;; + show) + if [ "${prev}" = "${action}" ]; then + COMPREPLY=($(compgen -W "--details" -- ${cur})) + fi + return 0 + ;; + delete) + # These subcommands have no options/arguments + COMPREPLY=( $(compgen -- ${cur}) ) + return 0 + ;; + *) + ;; + esac + fi + # Provide actions for completion COMPREPLY=($(compgen -W "${actions}" -- ${cur})) return 0 diff --git a/nfv/nfv-common/nfv_common/alarm/objects/v1/_alarm_defs.py b/nfv/nfv-common/nfv_common/alarm/objects/v1/_alarm_defs.py index d2da9615..5523c54a 100755 --- a/nfv/nfv-common/nfv_common/alarm/objects/v1/_alarm_defs.py +++ b/nfv/nfv-common/nfv_common/alarm/objects/v1/_alarm_defs.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -44,6 +44,9 @@ class _AlarmType(Constants): FW_UPDATE_AUTO_APPLY_INPROGRESS = Constant('fw-update-auto-apply-inprogress') FW_UPDATE_AUTO_APPLY_ABORTING = Constant('fw-update-auto-apply-aborting') FW_UPDATE_AUTO_APPLY_FAILED = Constant('fw-update-auto-apply-failed') + KUBE_UPGRADE_AUTO_APPLY_INPROGRESS = Constant('kube-upgrade-auto-apply-inprogress') + KUBE_UPGRADE_AUTO_APPLY_ABORTING = Constant('kube-upgrade-auto-apply-aborting') + KUBE_UPGRADE_AUTO_APPLY_FAILED = Constant('kube-upgrade-auto-apply-failed') @six.add_metaclass(Singleton) diff --git a/nfv/nfv-common/nfv_common/event_log/objects/v1/_event_log_defs.py b/nfv/nfv-common/nfv_common/event_log/objects/v1/_event_log_defs.py index e40196df..b3f0c237 100755 --- a/nfv/nfv-common/nfv_common/event_log/objects/v1/_event_log_defs.py +++ b/nfv/nfv-common/nfv_common/event_log/objects/v1/_event_log_defs.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -171,6 +171,17 @@ class _EventId(Constants): FW_UPDATE_AUTO_APPLY_ABORT_REJECTED = Constant('fw-update-auto-apply-abort-rejected') FW_UPDATE_AUTO_APPLY_ABORT_FAILED = Constant('fw-update-auto-apply-abort-failed') FW_UPDATE_AUTO_APPLY_ABORTED = Constant('fw-update-auto-apply-aborted') + KUBE_UPGRADE_AUTO_APPLY_START = Constant('kube-upgrade-auto-apply-started') + KUBE_UPGRADE_AUTO_APPLY_INPROGRESS = Constant('kube-upgrade-auto-apply-inprogress') + KUBE_UPGRADE_AUTO_APPLY_REJECTED = Constant('kube-upgrade-auto-apply-rejected') + KUBE_UPGRADE_AUTO_APPLY_CANCELLED = Constant('kube-upgrade-auto-apply-cancelled') + KUBE_UPGRADE_AUTO_APPLY_FAILED = Constant('kube-upgrade-auto-apply-failed') + KUBE_UPGRADE_AUTO_APPLY_COMPLETED = Constant('kube-upgrade-auto-apply-completed') + KUBE_UPGRADE_AUTO_APPLY_ABORT = Constant('kube-upgrade-auto-apply-abort') + KUBE_UPGRADE_AUTO_APPLY_ABORTING = Constant('kube-upgrade-auto-apply-aborting') + KUBE_UPGRADE_AUTO_APPLY_ABORT_REJECTED = Constant('kube-upgrade-auto-apply-abort-rejected') + KUBE_UPGRADE_AUTO_APPLY_ABORT_FAILED = Constant('kube-upgrade-auto-apply-abort-failed') + KUBE_UPGRADE_AUTO_APPLY_ABORTED = Constant('kube-upgrade-auto-apply-aborted') @six.add_metaclass(Singleton) diff --git a/nfv/nfv-common/nfv_common/strategy/_strategy_stage.py b/nfv/nfv-common/nfv_common/strategy/_strategy_stage.py index c7e0cf27..8bb52770 100755 --- a/nfv/nfv-common/nfv_common/strategy/_strategy_stage.py +++ b/nfv/nfv-common/nfv_common/strategy/_strategy_stage.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2016 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -324,6 +324,8 @@ class StrategyStage(object): self._step_timer_id = None step.start_date_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + DLOG.debug("Stage (%s) Step (%s) (apply) called" + % (self.name, step.name)) step.result, step.result_reason = step.apply() self._current_step = idx diff --git a/nfv/nfv-plugins/nfv_plugins/alarm_handlers/fm.py b/nfv/nfv-plugins/nfv_plugins/alarm_handlers/fm.py index 3d84c7bb..7ee086ce 100755 --- a/nfv/nfv-plugins/nfv_plugins/alarm_handlers/fm.py +++ b/nfv/nfv-plugins/nfv_plugins/alarm_handlers/fm.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -78,6 +78,12 @@ _fm_alarm_id_mapping = dict([ fm_constants.FM_ALARM_ID_FW_UPDATE_AUTO_APPLY_ABORTING), (alarm_objects_v1.ALARM_TYPE.FW_UPDATE_AUTO_APPLY_FAILED, fm_constants.FM_ALARM_ID_FW_UPDATE_AUTO_APPLY_FAILED), + (alarm_objects_v1.ALARM_TYPE.KUBE_UPGRADE_AUTO_APPLY_INPROGRESS, + fm_constants.FM_ALARM_ID_KUBE_UPGRADE_AUTO_APPLY_INPROGRESS), + (alarm_objects_v1.ALARM_TYPE.KUBE_UPGRADE_AUTO_APPLY_ABORTING, + fm_constants.FM_ALARM_ID_KUBE_UPGRADE_AUTO_APPLY_ABORTING), + (alarm_objects_v1.ALARM_TYPE.KUBE_UPGRADE_AUTO_APPLY_FAILED, + fm_constants.FM_ALARM_ID_KUBE_UPGRADE_AUTO_APPLY_FAILED), ]) _fm_alarm_type_mapping = dict([ diff --git a/nfv/nfv-plugins/nfv_plugins/event_log_handlers/fm.py b/nfv/nfv-plugins/nfv_plugins/event_log_handlers/fm.py index 186f6d1c..56c6d4ef 100755 --- a/nfv/nfv-plugins/nfv_plugins/event_log_handlers/fm.py +++ b/nfv/nfv-plugins/nfv_plugins/event_log_handlers/fm.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2016,2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -334,6 +334,28 @@ _fm_event_id_mapping = dict([ fm_constants.FM_LOG_ID_FW_UPDATE_AUTO_APPLY_ABORT_FAILED), (event_log_objects_v1.EVENT_ID.FW_UPDATE_AUTO_APPLY_ABORTED, fm_constants.FM_LOG_ID_FW_UPDATE_AUTO_APPLY_ABORTED), + (event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_START, + fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_START), + (event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_INPROGRESS, + fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_INPROGRESS), + (event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_REJECTED, + fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_REJECTED), + (event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_CANCELLED, + fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_CANCELLED), + (event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_FAILED, + fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_FAILED), + (event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_COMPLETED, + fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_COMPLETED), + (event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORT, + fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_ABORT), + (event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORTING, + fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_ABORTING), + (event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORT_REJECTED, + fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_ABORT_REJECTED), + (event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORT_FAILED, + fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_ABORT_FAILED), + (event_log_objects_v1.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORTED, + fm_constants.FM_LOG_ID_KUBE_UPGRADE_AUTO_APPLY_ABORTED), ]) _fm_event_type_mapping = dict([ diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py index 012c1c73..70a6b5cc 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -839,6 +839,704 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI): callback.send(response) callback.close() + def _extract_kube_host_upgrade_list(self, + kube_host_upgrade_list, + host_list): + """ + Return a list of KubeHostUpgrade objects from sysinv api results. + """ + + # Map the ID to the uuid from host_list + host_map = dict() + for host in host_list: + host_map[host['id']] = host['uuid'] + result_list = [] + for host_data in kube_host_upgrade_list: + host_uuid = host_map[host_data['host_id']] + result_list.append( + nfvi.objects.v1.KubeHostUpgrade( + host_data['host_id'], + host_uuid, + host_data['target_version'], + host_data['control_plane_version'], + host_data['kubelet_version'], + host_data['status']) + ) + return result_list + + def get_kube_host_upgrade_list(self, future, callback): + """ + Get information about the kube host upgrade list from the plugin + """ + response = dict() + response['completed'] = False + response['reason'] = '' + + activity = "SysInv get-kube-host-upgrades" + + try: + future.set_timeouts(config.CONF.get('nfvi-timeouts', None)) + + if self._platform_token is None or \ + self._platform_token.is_expired(): + future.work(openstack.get_token, self._platform_directory) + future.result = (yield) + + if not future.result.is_complete() or \ + future.result.data is None: + DLOG.error("OpenStack get-token did not complete.") + return + + self._platform_token = future.result.data + + # Query the kube host upgrade list + future.work(sysinv.get_kube_host_upgrades, self._platform_token) + future.result = (yield) + if not future.result.is_complete(): + DLOG.error("Sysinv Get Kube Host Upgrades did not complete.") + return + kube_host_upgrade_list = future.result.data["kube_host_upgrades"] + + # Also query the host list, kube_host_upgrades does not have uuid + future.work(sysinv.get_hosts, self._platform_token) + future.result = (yield) + if not future.result.is_complete(): + DLOG.error("Sysinv Get-Hosts did not complete.") + return + host_list = future.result.data["ihosts"] + + results_obj = \ + self._extract_kube_host_upgrade_list(kube_host_upgrade_list, + host_list) + response['result-data'] = results_obj + response['completed'] = True + + except exceptions.OpenStackRestAPIException as e: + if httplib.UNAUTHORIZED == e.http_status_code: + response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED + if self._platform_token is not None: + self._platform_token.set_expired() + + else: + DLOG.exception("Caught exception kube host upgrade list err=%s" + % e) + except Exception as e: + DLOG.exception("Caught exception kube host upgrade list err=%s" + % e) + finally: + callback.send(response) + callback.close() + + def _extract_kube_upgrade(self, kube_upgrade_data_list): + """ + Return a KubeUpgrade object from sysinv api results. + + Returns None if there are no items in the list. + Returns first kube upgrade object, but the API should never return + more than one object. + """ + + if 1 < len(kube_upgrade_data_list): + DLOG.critical("Too many kube upgrades returned, num=%i" + % len(kube_upgrade_data_list)) + + elif 0 == len(kube_upgrade_data_list): + DLOG.info("No kube upgrade exists, num=%i" + % len(kube_upgrade_data_list)) + + kube_upgrade_obj = None + for kube_upgrade_data in kube_upgrade_data_list: + kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade( + kube_upgrade_data['state'], + kube_upgrade_data['from_version'], + kube_upgrade_data['to_version']) + break + return kube_upgrade_obj + + def get_kube_upgrade(self, future, callback): + """ + Get information about the kube upgrade from the plugin + """ + response = dict() + response['completed'] = False + response['reason'] = '' + action_type = 'get-kube-upgrade' + + try: + future.set_timeouts(config.CONF.get('nfvi-timeouts', None)) + + if self._platform_token is None or \ + self._platform_token.is_expired(): + future.work(openstack.get_token, self._platform_directory) + future.result = (yield) + + if not future.result.is_complete() or \ + future.result.data is None: + DLOG.error("OpenStack get-token did not complete.") + return + + self._platform_token = future.result.data + + future.work(sysinv.get_kube_upgrade, self._platform_token) + future.result = (yield) + + if not future.result.is_complete(): + DLOG.error("SysInv get-kube-upgrade did not complete.") + return + + kube_upgrade_data_list = future.result.data['kube_upgrades'] + kube_upgrade_obj = \ + self._extract_kube_upgrade(kube_upgrade_data_list) + response['result-data'] = kube_upgrade_obj + response['completed'] = True + + except exceptions.OpenStackRestAPIException as e: + if httplib.UNAUTHORIZED == e.http_status_code: + response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED + if self._platform_token is not None: + self._platform_token.set_expired() + + else: + DLOG.exception("Caught API exception while trying %s. error=%s" + % (action_type, e)) + + except Exception as e: + DLOG.exception("Caught exception while trying %s. error=%s" + % (action_type, e)) + + finally: + callback.send(response) + callback.close() + + def _extract_kube_version(self, kube_data): + """ + Return a KubeVersion from sysinv API results. + """ + # sysinv api returns a field called 'version' which is a reserved field + # in vim object data structure. It is stored as kube_version + return nfvi.objects.v1.KubeVersion(kube_data['version'], + kube_data['state'], + kube_data['target'], + kube_data['upgrade_from'], + kube_data['downgrade_to'], + kube_data['applied_patches'], + kube_data['available_patches']) + + def get_kube_version_list(self, future, callback): + """ + Get information about the kube versions list from the plugin + """ + response = dict() + response['completed'] = False + response['reason'] = '' + action_type = 'get-kube-versions' + + try: + future.set_timeouts(config.CONF.get('nfvi-timeouts', None)) + if self._platform_token is None or \ + self._platform_token.is_expired(): + future.work(openstack.get_token, self._platform_directory) + future.result = (yield) + if not future.result.is_complete() or \ + future.result.data is None: + DLOG.error("OpenStack get-token did not complete.") + return + self._platform_token = future.result.data + + # get_kube_versions only returns a limited amount of data about the + # kubernetes versions. Individual API calls get the patch info. + future.work(sysinv.get_kube_versions, self._platform_token) + future.result = (yield) + + if not future.result.is_complete(): + DLOG.error("%s did not complete." % action_type) + return + + # walk the list of versions and get the patch info + kube_versions_list = list() + limited_kube_version_list = future.result.data['kube_versions'] + for kube_list_entry in limited_kube_version_list: + kube_ver = kube_list_entry['version'] + future.work(sysinv.get_kube_version, + self._platform_token, + kube_ver) + future.result = (yield) + if not future.result.is_complete(): + DLOG.error("%s for version:%s did not complete." + % (action_type, kube_ver)) + return + # returns a single object + kube_ver_data = future.result.data + kube_versions_list.append( + self._extract_kube_version(kube_ver_data)) + + response['result-data'] = kube_versions_list + response['completed'] = True + + except exceptions.OpenStackRestAPIException as e: + if httplib.UNAUTHORIZED == e.http_status_code: + response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED + if self._platform_token is not None: + self._platform_token.set_expired() + else: + DLOG.exception("Caught API exception while trying %s. error=%s" + % (action_type, e)) + except Exception as e: + DLOG.exception("Caught exception while trying %s. error=%s" + % (action_type, e)) + finally: + callback.send(response) + callback.close() + + def kube_host_upgrade_control_plane(self, + future, + host_uuid, + host_name, + force, + callback): + """ + Start kube host upgrade 'control plane' for a particular controller + """ + response = dict() + response['completed'] = False + response['host_uuid'] = host_uuid + response['host_name'] = host_name + response['reason'] = '' + action_type = 'kube-host-upgrade-control-plane' + sysinv_method = sysinv.kube_host_upgrade_control_plane + + try: + future.set_timeouts(config.CONF.get('nfvi-timeouts', None)) + + if self._platform_token is None or \ + self._platform_token.is_expired(): + future.work(openstack.get_token, self._platform_directory) + future.result = (yield) + + if not future.result.is_complete() or \ + future.result.data is None: + DLOG.error("OpenStack get-token did not complete.") + return + + self._platform_token = future.result.data + + # invoke the actual kube_host_upgrade method + future.work(sysinv_method, + self._platform_token, + host_uuid, + force) + future.result = (yield) + + if not future.result.is_complete(): + DLOG.error("%s did not complete." % action_type) + return + # result was a host object. Need to query to get kube upgrade obj + future.work(sysinv.get_kube_upgrade, self._platform_token) + future.result = (yield) + + if not future.result.is_complete(): + DLOG.error("SysInv get-kube-upgrade did not complete.") + return + + kube_upgrade_data_list = future.result.data['kube_upgrades'] + kube_upgrade_obj = \ + self._extract_kube_upgrade(kube_upgrade_data_list) + response['result-data'] = kube_upgrade_obj + response['completed'] = True + + except exceptions.OpenStackRestAPIException as e: + if httplib.UNAUTHORIZED == e.http_status_code: + response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED + if self._platform_token is not None: + self._platform_token.set_expired() + + else: + DLOG.exception("Caught API exception while trying %s. error=%s" + % (action_type, e)) + response['reason'] = e.http_response_reason + + except Exception as e: + DLOG.exception("Caught exception while trying %s. error=%s" + % (action_type, e)) + + finally: + callback.send(response) + callback.close() + + def kube_host_upgrade_kubelet(self, + future, + host_uuid, + host_name, + force, + callback): + """ + Start kube host upgrade 'kubelet' for a particular host + """ + response = dict() + response['completed'] = False + response['host_uuid'] = host_uuid + response['host_name'] = host_name + response['reason'] = '' + action_type = 'kube-host-upgrade-kubelet' + sysinv_method = sysinv.kube_host_upgrade_kubelet + + try: + future.set_timeouts(config.CONF.get('nfvi-timeouts', None)) + + if self._platform_token is None or \ + self._platform_token.is_expired(): + future.work(openstack.get_token, self._platform_directory) + future.result = (yield) + + if not future.result.is_complete() or \ + future.result.data is None: + DLOG.error("OpenStack get-token did not complete.") + return + + self._platform_token = future.result.data + + # invoke the actual kube_host_upgrade method + future.work(sysinv_method, + self._platform_token, + host_uuid, + force) + future.result = (yield) + + if not future.result.is_complete(): + DLOG.error("%s did not complete." % action_type) + return + # result was a host object. Need to query to get kube upgrade obj + future.work(sysinv.get_kube_upgrade, self._platform_token) + future.result = (yield) + + if not future.result.is_complete(): + DLOG.error("SysInv get-kube-upgrade did not complete.") + return + + kube_upgrade_data_list = future.result.data['kube_upgrades'] + kube_upgrade_obj = \ + self._extract_kube_upgrade(kube_upgrade_data_list) + response['result-data'] = kube_upgrade_obj + response['completed'] = True + + except exceptions.OpenStackRestAPIException as e: + if httplib.UNAUTHORIZED == e.http_status_code: + response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED + if self._platform_token is not None: + self._platform_token.set_expired() + + else: + DLOG.exception("Caught API exception while trying %s. error=%s" + % (action_type, e)) + response['reason'] = e.http_response_reason + + except Exception as e: + DLOG.exception("Caught exception while trying %s. error=%s" + % (action_type, e)) + + finally: + callback.send(response) + callback.close() + + def kube_upgrade_cleanup(self, future, callback): + """ + kube upgrade cleanup + """ + response = dict() + response['completed'] = False + response['reason'] = '' + action_type = 'kube-upgrade-cleanup' + + try: + future.set_timeouts(config.CONF.get('nfvi-timeouts', None)) + + if self._platform_token is None or \ + self._platform_token.is_expired(): + future.work(openstack.get_token, self._platform_directory) + future.result = (yield) + + if not future.result.is_complete() or \ + future.result.data is None: + DLOG.error("OpenStack get-token did not complete.") + return + + self._platform_token = future.result.data + + future.work(sysinv.kube_upgrade_cleanup, self._platform_token) + future.result = (yield) + + if not future.result.is_complete(): + DLOG.error("%s did not complete." % action_type) + return + + # The result should be empty. no result data to report back + response['completed'] = True + + except exceptions.OpenStackRestAPIException as e: + if httplib.UNAUTHORIZED == e.http_status_code: + response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED + if self._platform_token is not None: + self._platform_token.set_expired() + + else: + DLOG.exception("Caught API exception while trying %s. error=%s" + % (action_type, e)) + response['reason'] = e.http_response_reason + + except Exception as e: + DLOG.exception("Caught exception while trying %s. error=%s" + % (action_type, e)) + + finally: + callback.send(response) + callback.close() + + def kube_upgrade_complete(self, future, callback): + """ + kube upgrade complete + """ + response = dict() + response['completed'] = False + response['reason'] = '' + action_type = 'kube-upgrade-complete' + + try: + future.set_timeouts(config.CONF.get('nfvi-timeouts', None)) + + if self._platform_token is None or \ + self._platform_token.is_expired(): + future.work(openstack.get_token, self._platform_directory) + future.result = (yield) + + if not future.result.is_complete() or \ + future.result.data is None: + DLOG.error("OpenStack get-token did not complete.") + return + + self._platform_token = future.result.data + + future.work(sysinv.kube_upgrade_complete, self._platform_token) + future.result = (yield) + + if not future.result.is_complete(): + DLOG.error("%s did not complete." % action_type) + return + + kube_upgrade_data = future.result.data + kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade( + kube_upgrade_data['state'], + kube_upgrade_data['from_version'], + kube_upgrade_data['to_version']) + + response['result-data'] = kube_upgrade_obj + response['completed'] = True + + except exceptions.OpenStackRestAPIException as e: + if httplib.UNAUTHORIZED == e.http_status_code: + response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED + if self._platform_token is not None: + self._platform_token.set_expired() + + else: + DLOG.exception("Caught API exception while trying %s. error=%s" + % (action_type, e)) + response['reason'] = e.http_response_reason + + except Exception as e: + DLOG.exception("Caught exception while trying %s. error=%s" + % (action_type, e)) + + finally: + callback.send(response) + callback.close() + + def kube_upgrade_download_images(self, future, callback): + """ + Start kube upgrade download images + """ + response = dict() + response['completed'] = False + response['reason'] = '' + action_type = 'kube-upgrade-download-images' + + try: + future.set_timeouts(config.CONF.get('nfvi-timeouts', None)) + + if self._platform_token is None or \ + self._platform_token.is_expired(): + future.work(openstack.get_token, self._platform_directory) + future.result = (yield) + + if not future.result.is_complete() or \ + future.result.data is None: + DLOG.error("OpenStack get-token did not complete.") + return + + self._platform_token = future.result.data + + future.work(sysinv.kube_upgrade_download_images, + self._platform_token) + future.result = (yield) + + if not future.result.is_complete(): + DLOG.error("%s did not complete." % action_type) + return + + kube_upgrade_data = future.result.data + kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade( + kube_upgrade_data['state'], + kube_upgrade_data['from_version'], + kube_upgrade_data['to_version']) + + response['result-data'] = kube_upgrade_obj + response['completed'] = True + + except exceptions.OpenStackRestAPIException as e: + if httplib.UNAUTHORIZED == e.http_status_code: + response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED + if self._platform_token is not None: + self._platform_token.set_expired() + + else: + DLOG.exception("Caught API exception while trying %s. error=%s" + % (action_type, e)) + + except Exception as e: + DLOG.exception("Caught exception while trying %s. error=%s" + % (action_type, e)) + + finally: + callback.send(response) + callback.close() + + def kube_upgrade_networking(self, future, callback): + """ + Start kube upgrade networking + """ + response = dict() + response['completed'] = False + response['reason'] = '' + action_type = 'kube-upgrade-networking' + + try: + future.set_timeouts(config.CONF.get('nfvi-timeouts', None)) + + if self._platform_token is None or \ + self._platform_token.is_expired(): + future.work(openstack.get_token, self._platform_directory) + future.result = (yield) + + if not future.result.is_complete() or \ + future.result.data is None: + DLOG.error("OpenStack get-token did not complete.") + return + + self._platform_token = future.result.data + + future.work(sysinv.kube_upgrade_networking, self._platform_token) + future.result = (yield) + + if not future.result.is_complete(): + DLOG.error("%s did not complete." % action_type) + return + + kube_upgrade_data = future.result.data + kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade( + kube_upgrade_data['state'], + kube_upgrade_data['from_version'], + kube_upgrade_data['to_version']) + + response['result-data'] = kube_upgrade_obj + response['completed'] = True + + except exceptions.OpenStackRestAPIException as e: + if httplib.UNAUTHORIZED == e.http_status_code: + response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED + if self._platform_token is not None: + self._platform_token.set_expired() + + else: + DLOG.exception("Caught API exception while trying %s. error=%s" + % (action_type, e)) + + except Exception as e: + DLOG.exception("Caught exception while trying %s. error=%s" + % (action_type, e)) + + finally: + callback.send(response) + callback.close() + + def kube_upgrade_start(self, future, to_version, force, alarm_ignore_list, + callback): + """ + Start a kube upgrade + """ + response = dict() + response['completed'] = False + response['reason'] = '' + action_type = 'kube-upgrade-start' + + try: + future.set_timeouts(config.CONF.get('nfvi-timeouts', None)) + + if self._platform_token is None or \ + self._platform_token.is_expired(): + future.work(openstack.get_token, self._platform_directory) + future.result = (yield) + + if not future.result.is_complete() or \ + future.result.data is None: + DLOG.error("OpenStack get-token did not complete.") + return + + self._platform_token = future.result.data + + future.work(sysinv.kube_upgrade_start, + self._platform_token, + to_version, + force=force, + alarm_ignore_list=alarm_ignore_list) + future.result = (yield) + + if not future.result.is_complete(): + DLOG.error("SysInv kube-upgrade-start did not complete.") + response['reason'] = "did not complete." + return + + future.work(sysinv.get_kube_upgrade, self._platform_token) + future.result = (yield) + + if not future.result.is_complete(): + DLOG.error("SysInv get-kube-upgrade did not complete.") + response['reason'] = "did not complete." + return + + kube_upgrade_data_list = future.result.data['kube_upgrades'] + kube_upgrade_obj = \ + self._extract_kube_upgrade(kube_upgrade_data_list) + response['result-data'] = kube_upgrade_obj + response['completed'] = True + + except exceptions.OpenStackRestAPIException as e: + if httplib.UNAUTHORIZED == e.http_status_code: + response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED + if self._platform_token is not None: + self._platform_token.set_expired() + + else: + DLOG.exception("Caught API exception while trying %s. error=%s" + % (action_type, e)) + response['reason'] = e.http_response_reason + + except Exception as e: + DLOG.exception("Caught exception while trying %s. error=%s" + % (action_type, e)) + + finally: + callback.send(response) + callback.close() + def get_upgrade(self, future, callback): """ Get information about the upgrade from the plugin diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_sw_mgmt_api.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_sw_mgmt_api.py index e836c646..8f6a7cc3 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_sw_mgmt_api.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_sw_mgmt_api.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2016-2018 Wind River Systems, Inc. +# Copyright (c) 2016-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -216,6 +216,74 @@ class NFVISwMgmtAPI(nfvi.api.v1.NFVISwMgmtAPI): callback.send(response) callback.close() + def apply_patches(self, future, patch_names, callback): + """ + Apply a software patch that has already been uploaded + """ + response = dict() + response['completed'] = False + response['reason'] = '' + + try: + future.set_timeouts(config.CONF.get('nfvi-timeouts', None)) + + if self._token is None or self._token.is_expired(): + future.work(openstack.get_token, self._directory) + future.result = (yield) + + if not future.result.is_complete() or \ + future.result.data is None: + return + + self._token = future.result.data + + for patch_name in patch_names: + future.work(patching.apply_patch, self._token, patch_name) + future.result = (yield) + + if not future.result.is_complete(): + return + + # query the patches and return their state + future.work(patching.query_patches, self._token) + future.result = (yield) + + if not future.result.is_complete(): + return + + sw_patches = list() + + if future.result.data is not None: + sw_patch_data_list = future.result.data.get('pd', []) + for sw_patch_name in sw_patch_data_list.keys(): + sw_patch_data = sw_patch_data_list[sw_patch_name] + sw_patch = nfvi.objects.v1.SwPatch( + sw_patch_name, sw_patch_data['sw_version'], + sw_patch_data['repostate'], sw_patch_data['patchstate']) + sw_patches.append(sw_patch) + + response['result-data'] = sw_patches + response['completed'] = True + + except exceptions.OpenStackRestAPIException as e: + if httplib.UNAUTHORIZED == e.http_status_code: + response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED + if self._token is not None: + self._token.set_expired() + + else: + DLOG.exception("Caught exception while trying to apply " + "software patches [%s], error=%s." + % (patch_names, e)) + + except Exception as e: + DLOG.exception("Caught exception while trying to apply " + "software patches [%s], error=%s." + % (patch_names, e)) + finally: + callback.send(response) + callback.close() + def update_hosts(self, future, host_names, callback): """ Apply a software update to a list of hosts diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/patching.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/patching.py index 41eb6a83..299025f9 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/patching.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/patching.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2016-2018 Wind River Systems, Inc. +# Copyright (c) 2016-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -39,6 +39,20 @@ def query_hosts(token): return response +def apply_patch(token, patch_name): + """ + Asks Patch Controller to apply a patch that is already uploaded + """ + url = token.get_service_url(PLATFORM_SERVICE.PATCHING, strip_version=True) + if url is None: + raise ValueError("OpenStack Patching URL is invalid") + + api_cmd = url + "/v1/apply/%s" % str(patch_name) + + response = rest_api_request(token, "POST", api_cmd) + return response + + def host_install_async(token, host_name): """ Asks Patch Controller to perform a software upgrade on a host diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py index 3ce47472..63793696 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py @@ -1,8 +1,9 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # +import copy import json from nfv_common import debug @@ -97,6 +98,211 @@ def get_host_labels(token, host_uuid): return response +def get_kube_host_upgrades(token): + """ + Asks System Inventory for information about the kube host upgrades + """ + url = token.get_service_url(PLATFORM_SERVICE.SYSINV) + if url is None: + raise ValueError("OpenStack SysInv URL is invalid") + + api_cmd = url + "/kube_host_upgrades" + + response = rest_api_request(token, "GET", api_cmd, + timeout_in_secs=REST_API_REQUEST_TIMEOUT) + return response + + +def get_kube_upgrade(token): + """ + Asks System Inventory for information about the kube upgrade + """ + url = token.get_service_url(PLATFORM_SERVICE.SYSINV) + if url is None: + raise ValueError("OpenStack SysInv URL is invalid") + + api_cmd = url + "/kube_upgrade" + + response = rest_api_request(token, "GET", api_cmd, + timeout_in_secs=REST_API_REQUEST_TIMEOUT) + return response + + +def get_kube_version(token, kube_version): + """ + Asks System Inventory for information a kube version + """ + url = token.get_service_url(PLATFORM_SERVICE.SYSINV) + if url is None: + raise ValueError("OpenStack SysInv URL is invalid") + + api_cmd = url + "/kube_versions/" + kube_version + + response = rest_api_request(token, "GET", api_cmd, + timeout_in_secs=REST_API_REQUEST_TIMEOUT) + return response + + +def get_kube_versions(token): + """ + Asks System Inventory for information about the kube versions + """ + url = token.get_service_url(PLATFORM_SERVICE.SYSINV) + if url is None: + raise ValueError("OpenStack SysInv URL is invalid") + + api_cmd = url + "/kube_versions" + + response = rest_api_request(token, "GET", api_cmd, + timeout_in_secs=REST_API_REQUEST_TIMEOUT) + return response + + +def kube_upgrade_start(token, to_version, force=False, alarm_ignore_list=None): + """ + Ask System Inventory to start a kube upgrade + """ + url = token.get_service_url(PLATFORM_SERVICE.SYSINV) + if url is None: + raise ValueError("OpenStack SysInv URL is invalid") + + api_cmd = url + "/kube_upgrade" + + api_cmd_headers = dict() + api_cmd_headers['Content-Type'] = "application/json" + api_cmd_headers['User-Agent'] = "vim/1.0" + + api_cmd_payload = dict() + api_cmd_payload['to_version'] = to_version + api_cmd_payload['force'] = force + if alarm_ignore_list is not None: + api_cmd_payload['alarm_ignore_list'] = copy.copy(alarm_ignore_list) + + response = rest_api_request(token, + "POST", + api_cmd, + api_cmd_headers, + json.dumps(api_cmd_payload), + timeout_in_secs=REST_API_REQUEST_TIMEOUT) + return response + + +def _patch_kube_upgrade_state(token, new_value): + url = token.get_service_url(PLATFORM_SERVICE.SYSINV) + if url is None: + raise ValueError("OpenStack SysInv URL is invalid") + + api_cmd = url + "/kube_upgrade" + + api_cmd_headers = dict() + api_cmd_headers['Content-Type'] = "application/json" + api_cmd_headers['User-Agent'] = "vim/1.0" + + api_cmd_payload = list() + host_data = dict() + host_data['path'] = "/state" + host_data['value'] = new_value + host_data['op'] = "replace" + api_cmd_payload.append(host_data) + + return rest_api_request(token, + "PATCH", + api_cmd, + api_cmd_headers, + json.dumps(api_cmd_payload), + timeout_in_secs=REST_API_REQUEST_TIMEOUT) + + +def kube_upgrade_cleanup(token): + """ + Ask System Inventory to delete the kube upgrade + """ + url = token.get_service_url(PLATFORM_SERVICE.SYSINV) + if url is None: + raise ValueError("OpenStack SysInv URL is invalid") + + api_cmd = url + "/kube_upgrade" + + api_cmd_headers = dict() + api_cmd_headers['Content-Type'] = "application/json" + api_cmd_headers['User-Agent'] = "vim/1.0" + + response = rest_api_request(token, "DELETE", api_cmd, api_cmd_headers, + timeout_in_secs=REST_API_REQUEST_TIMEOUT) + return response + + +def kube_upgrade_complete(token): + """ + Ask System Inventory to kube upgrade complete + """ + return _patch_kube_upgrade_state(token, "upgrade-complete") + + +def kube_upgrade_download_images(token): + """ + Ask System Inventory to kube upgrade download images + """ + return _patch_kube_upgrade_state(token, "downloading-images") + + +def kube_upgrade_networking(token): + """ + Ask System Inventory to kube upgrade networking + """ + return _patch_kube_upgrade_state(token, "upgrading-networking") + + +def _kube_host_upgrade(token, host_uuid, target_operation, force): + """ + Invoke a POST for a host kube-upgrade operation + + target_operation one of: kube_upgrade_control_plane, kube_upgrade_kubelet + force is a 'string' + """ + + url = token.get_service_url(PLATFORM_SERVICE.SYSINV) + if url is None: + raise ValueError("OpenStack SysInv URL is invalid") + + api_cmd = url + "/ihosts/%s/%s" % (host_uuid, target_operation) + + api_cmd_headers = dict() + api_cmd_headers['Content-Type'] = "application/json" + api_cmd_headers['User-Agent'] = "vim/1.0" + + api_cmd_payload = dict() + api_cmd_payload['force'] = force + + response = rest_api_request(token, + "POST", + api_cmd, + api_cmd_headers, + json.dumps(api_cmd_payload), + timeout_in_secs=REST_API_REQUEST_TIMEOUT) + return response + + +def kube_host_upgrade_control_plane(token, host_uuid, force="true"): + """ + Ask System Inventory to kube HOST upgrade control plane + """ + return _kube_host_upgrade(token, + host_uuid, + "kube_upgrade_control_plane", + force) + + +def kube_host_upgrade_kubelet(token, host_uuid, force="true"): + """ + Ask System Inventory to kube HOST upgrade kubelet + """ + return _kube_host_upgrade(token, + host_uuid, + "kube_upgrade_kubelet", + force) + + def get_upgrade(token): """ Asks System Inventory for information about the upgrade diff --git a/nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py b/nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py new file mode 100755 index 00000000..2c749a10 --- /dev/null +++ b/nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py @@ -0,0 +1,734 @@ +# +# Copyright (c) 2020-2021 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +import mock +import uuid + +from nfv_common import strategy as common_strategy +from nfv_vim import nfvi + +from nfv_vim.nfvi.objects.v1 import HostSwPatch +from nfv_vim.nfvi.objects.v1 import KUBE_UPGRADE_STATE +from nfv_vim.nfvi.objects.v1 import KubeVersion +from nfv_vim.nfvi.objects.v1 import SwPatch +from nfv_vim.objects import KubeUpgrade +from nfv_vim.objects import SW_UPDATE_ALARM_RESTRICTION +from nfv_vim.objects import SW_UPDATE_APPLY_TYPE +from nfv_vim.strategy._strategy import KubeUpgradeStrategy + +from . import sw_update_testcase # noqa: H304 + + +FROM_KUBE_VERSION = '1.2.3' +TO_KUBE_VERSION = '1.2.4' + +FAKE_LOAD = '12.01' + +KUBE_PATCH_1 = 'KUBE.1' # the control plane patch +KUBE_PATCH_2 = 'KUBE.2' # the kubelet patch + + +@mock.patch('nfv_vim.event_log._instance._event_issue', + sw_update_testcase.fake_event_issue) +@mock.patch('nfv_vim.objects._sw_update.SwUpdate.save', + sw_update_testcase.fake_save) +@mock.patch('nfv_vim.objects._sw_update.timers.timers_create_timer', + sw_update_testcase.fake_timer) +@mock.patch('nfv_vim.nfvi.nfvi_compute_plugin_disabled', + sw_update_testcase.fake_nfvi_compute_plugin_disabled) +class TestBuildStrategy(sw_update_testcase.SwUpdateStrategyTestCase): + + def _create_kube_upgrade_strategy(self, + sw_update_obj, + storage_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL, + worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL, + max_parallel_worker_hosts=10, + alarm_restrictions=SW_UPDATE_ALARM_RESTRICTION.STRICT, + to_version=TO_KUBE_VERSION, + single_controller=False, + nfvi_kube_upgrade=None): + """ + Create a kube upgrade strategy + """ + strategy = KubeUpgradeStrategy( + uuid=str(uuid.uuid4()), + storage_apply_type=storage_apply_type, + worker_apply_type=worker_apply_type, + max_parallel_worker_hosts=max_parallel_worker_hosts, + alarm_restrictions=alarm_restrictions, + ignore_alarms=[], + to_version=to_version, + single_controller=single_controller + ) + strategy.sw_update_obj = sw_update_obj # this is a weakref + strategy.nfvi_kube_upgrade = nfvi_kube_upgrade + return strategy + + @mock.patch('nfv_common.strategy._strategy.Strategy._build') + def test_kube_upgrade_strategy_build_steps(self, fake_build): + """ + Verify build phase steps and stages for kube_upgrade strategy creation. + """ + # setup a minimal host environment + self.create_host('controller-0') + + # construct the strategy. the update_obj MUST be declared here and not + # in the create method, because it is a weakref and will be cleaned up + # when it goes out of scope. + update_obj = KubeUpgrade() + strategy = self._create_kube_upgrade_strategy(update_obj, + single_controller=True) + # The 'build' constructs a strategy that includes multiple queries + # the results of those queries are not used until build_complete + # mock away '_build', which invokes the build steps and their api calls + fake_build.return_value = None + strategy.build() + + # verify the build phase and steps + build_phase = strategy.build_phase.as_dict() + query_steps = [ + {'name': 'query-alarms'}, + {'name': 'query-kube-versions'}, + {'name': 'query-kube-upgrade'}, + {'name': 'query-kube-host-upgrade'}, + {'name': 'query-sw-patches'}, + {'name': 'query-sw-patch-hosts'}, + ] + expected_results = { + 'total_stages': 1, + 'stages': [ + {'name': 'kube-upgrade-query', + 'total_steps': len(query_steps), + 'steps': query_steps, + }, + ], + } + sw_update_testcase.validate_phase(build_phase, expected_results) + + +class SimplexKubeUpgradeMixin(object): + FAKE_PATCH_HOSTS_LIST = [ + HostSwPatch('controller-0', 'controller', FAKE_LOAD, + True, False, 'idle', False, False), + ] + FAKE_KUBE_HOST_UPGRADES_LIST = [] + + def setUp(self): + super(SimplexKubeUpgradeMixin, self).setUp() + + def is_simplex(self): + return True + + def is_duplex(self): + return False + + def _kube_upgrade_kubelet_controller_stage(self, host): + """duplex needs to swact/lock/unlock whereas simplex does not""" + steps = [ + {'name': 'kube-host-upgrade-kubelet', + 'entity_names': [host], + 'entity_type': 'hosts', }, + {'name': 'system-stabilize', }, + ] + + return { + 'name': 'kube-upgrade-kubelets-controllers', + 'total_steps': len(steps), + 'steps': steps, + } + + +class DuplexKubeUpgradeMixin(object): + FAKE_PATCH_HOSTS_LIST = [ + HostSwPatch('controller-0', 'controller', FAKE_LOAD, + True, False, 'idle', False, False), + HostSwPatch('controller-1', 'controller', FAKE_LOAD, + True, False, 'idle', False, False), + ] + FAKE_KUBE_HOST_UPGRADES_LIST = [] + + def setUp(self): + super(DuplexKubeUpgradeMixin, self).setUp() + + def is_simplex(self): + return False + + def is_duplex(self): + return True + + def _kube_upgrade_kubelet_controller_stage(self, host): + """duplex needs to swact/lock/unlock whereas simplex does not""" + steps = [ + {'name': 'swact-hosts', + 'entity_names': [host], + 'entity_type': 'hosts', }, + {'name': 'lock-hosts', + 'entity_names': [host], + 'entity_type': 'hosts', }, + {'name': 'kube-host-upgrade-kubelet', + 'entity_names': [host], + 'entity_type': 'hosts', }, + {'name': 'system-stabilize', }, + {'name': 'unlock-hosts', + 'entity_names': [host], + 'entity_type': 'hosts', }, + {'name': 'wait-alarms-clear', }, + ] + return { + 'name': 'kube-upgrade-kubelets-controllers', + 'total_steps': len(steps), + 'steps': steps, + } + + +class KubePatchMixin(object): + """This Mixin represents the patches for a kube upgrade in proper state""" + + FAKE_KUBE_VERSIONS_LIST = [ + KubeVersion( + FROM_KUBE_VERSION, # kube_version + 'active', # state + True, # target + [], # upgrade_from + [], # downgrade_to + [], # applied_patches + [] # available_patches + ), + KubeVersion( + TO_KUBE_VERSION, # kube_version + 'available', # state + False, # target + [FROM_KUBE_VERSION], # upgrade_from + [], # downgrade_to + [KUBE_PATCH_1], # applied_patches + [KUBE_PATCH_2] # available_patches + ) + ] + + FAKE_PATCHES_LIST = [ + SwPatch(KUBE_PATCH_1, FAKE_LOAD, 'Applied', 'Applied'), + SwPatch(KUBE_PATCH_2, FAKE_LOAD, 'Available', 'Available'), + ] + + def setUp(self): + super(KubePatchMixin, self).setUp() + + def _kube_upgrade_patch_stage(self, + controller_list, + storage_list=None, + worker_list=None): + """hosts are patched in the following order + controller-0 in simplex, + controller-1, controller-0 for duplex + storage hosts after controllers + workers after storage hosts + """ + if storage_list is None: + storage_list = [] + if worker_list is None: + worker_list = [] + patch_steps = [ + {'name': 'apply-patches', + 'entity_type': 'patches', + 'entity_names': ['KUBE.2']} + ] + for host_name in controller_list: + patch_steps.append({'name': 'sw-patch-hosts', + 'entity_type': 'hosts', + 'entity_names': [host_name]}) + # storage and workers may be processed in bulk in the future + for host_name in storage_list: + patch_steps.append({'name': 'sw-patch-hosts', + 'entity_type': 'hosts', + 'entity_names': [host_name]}) + for host_name in worker_list: + patch_steps.append({'name': 'sw-patch-hosts', + 'entity_type': 'hosts', + 'entity_names': [host_name]}) + return { + 'name': 'kube-upgrade-patch', + 'total_steps': len(patch_steps), + 'steps': patch_steps + } + + +class ApplyStageMixin(object): + """This Mixin will not work unless combined with other mixins. + PatchMixin - to provide the setup patches and kube versions + HostMixin - to provide the patch hosts and kube host upgrade states + """ + + def setUp(self): + super(ApplyStageMixin, self).setUp() + + def _create_kube_upgrade_obj(self, + state, + from_version=FROM_KUBE_VERSION, + to_version=TO_KUBE_VERSION): + """ + Create a kube upgrade db object + """ + return nfvi.objects.v1.KubeUpgrade(state=state, + from_version=from_version, + to_version=to_version) + + def _create_built_kube_upgrade_strategy(self, + sw_update_obj, + to_version=TO_KUBE_VERSION, + single_controller=False, + kube_upgrade=None, + alarms_list=None, + patch_list=None, + patch_hosts_list=None, + kube_versions_list=None, + kube_hosts_list=None): + """ + Create a kube upgrade strategy + populate the API query results from the build steps + """ + storage_apply_type = SW_UPDATE_APPLY_TYPE.IGNORE + worker_apply_type = SW_UPDATE_APPLY_TYPE.IGNORE + max_parallel_worker_hosts = 10 + alarm_restrictions = SW_UPDATE_ALARM_RESTRICTION.STRICT + + strategy = KubeUpgradeStrategy( + uuid=str(uuid.uuid4()), + storage_apply_type=storage_apply_type, + worker_apply_type=worker_apply_type, + max_parallel_worker_hosts=max_parallel_worker_hosts, + alarm_restrictions=alarm_restrictions, + ignore_alarms=[], + to_version=to_version, + single_controller=single_controller + ) + strategy.sw_update_obj = sw_update_obj # warning: this is a weakref + strategy.nfvi_kube_upgrade = kube_upgrade + + # If any of the input lists are None, replace with defaults + # this is done to prevent passing a list as a default + if patch_list is None: + patch_list = self.FAKE_PATCHES_LIST + strategy.nfvi_sw_patches = patch_list + + if patch_hosts_list is None: + patch_hosts_list = self.FAKE_PATCH_HOSTS_LIST + strategy.nfvi_sw_patch_hosts = patch_hosts_list + + if kube_versions_list is None: + kube_versions_list = self.FAKE_KUBE_VERSIONS_LIST + strategy.nfvi_kube_versions_list = kube_versions_list + + if kube_hosts_list is None: + kube_hosts_list = self.FAKE_KUBE_HOST_UPGRADES_LIST + strategy.nfvi_kube_host_upgrade_list = kube_hosts_list + + return strategy + + def _kube_upgrade_start_stage(self): + return { + 'name': 'kube-upgrade-start', + 'total_steps': 1, + 'steps': [ + {'name': 'kube-upgrade-start', + 'success_state': 'upgrade-started'}, + ], + } + + def _kube_upgrade_download_images_stage(self): + return { + 'name': 'kube-upgrade-download-images', + 'total_steps': 1, + 'steps': [ + {'name': 'kube-upgrade-download-images', + 'success_state': 'downloaded-images', + 'fail_state': 'downloading-images-failed'}, + ], + } + + def _kube_upgrade_first_control_plane_stage(self): + return { + 'name': 'kube-upgrade-first-control-plane', + 'total_steps': 1, + 'steps': [ + {'name': 'kube-host-upgrade-control-plane', + 'success_state': 'upgraded-first-master', + 'fail_state': 'upgrading-first-master-failed'}, + ], + } + + def _kube_upgrade_networking_stage(self): + return { + 'name': 'kube-upgrade-networking', + 'total_steps': 1, + 'steps': [ + {'name': 'kube-upgrade-networking', + 'success_state': 'upgraded-networking', + 'fail_state': 'upgrading-networking-failed'}, + ], + } + + def _kube_upgrade_second_control_plane_stage(self): + """This stage only executes on a duplex system""" + return { + 'name': 'kube-upgrade-second-control-plane', + 'total_steps': 1, + 'steps': [ + {'name': 'kube-host-upgrade-control-plane', + 'success_state': 'upgraded-second-master', + 'fail_state': 'upgrading-second-master-failed'}, + ], + } + + def _kube_upgrade_complete_stage(self): + return { + 'name': 'kube-upgrade-complete', + 'total_steps': 1, + 'steps': [ + {'name': 'kube-upgrade-complete', + 'success_state': 'upgrade-complete'}, + ], + } + + def _kube_upgrade_cleanup_stage(self): + return { + 'name': 'kube-upgrade-cleanup', + 'total_steps': 1, + 'steps': [ + {'name': 'kube-upgrade-cleanup'}, + ], + } + + def _kube_upgrade_kubelet_worker_stage(self, host): + steps = [ + {'name': 'lock-hosts', + 'entity_names': [host], }, + {'name': 'kube-host-upgrade-kubelet', + 'entity_names': [host], + 'entity_type': 'hosts', }, + {'name': 'system-stabilize', }, + {'name': 'unlock-hosts', + 'entity_names': [host], + 'entity_type': 'hosts', }, + {'name': 'wait-alarms-clear', }, + ] + return { + 'name': 'kube-upgrade-kubelets-workers', + 'total_steps': len(steps), + 'steps': steps, + } + + def _kube_upgrade_kubelet_stages(self, controller_list, worker_list=None): + """This section will change as more host types are supported""" + if worker_list is None: + worker_list = [] + kubelet_stages = [] + for host_name in controller_list: + kubelet_stages.append( + self._kube_upgrade_kubelet_controller_stage(host_name)) + for host_name in worker_list: + kubelet_stages.append( + self._kube_upgrade_kubelet_worker_stage(host_name)) + return kubelet_stages + + def validate_apply_phase(self, single_controller, kube_upgrade, stages): + # sw_update_obj is a weak ref. it must be defined here + update_obj = KubeUpgrade() + + # create a strategy for a system with no existing kube_upgrade + strategy = self._create_built_kube_upgrade_strategy( + update_obj, + single_controller=single_controller, + kube_upgrade=kube_upgrade) + + strategy.build_complete(common_strategy.STRATEGY_RESULT.SUCCESS, "") + + self.assertFalse(strategy.is_build_failed()) + self.assertEqual(strategy.build_phase.result_reason, "") + + apply_phase = strategy.apply_phase.as_dict() + expected_results = { + 'total_stages': len(stages), + 'stages': stages + } + sw_update_testcase.validate_strategy_persists(strategy) + sw_update_testcase.validate_phase(apply_phase, expected_results) + + def build_stage_list(self, + controller_list, + worker_list, + add_start=True, add_download=True, + add_first_plane=True, add_networking=True, + add_second_plane=True, add_patches=True, + add_kubelets=True, + add_complete=True, add_cleanup=True): + """The order of the host_list determines the patch and kubelets""" + stages = [] + if add_start: + stages.append(self._kube_upgrade_start_stage()) + if add_download: + stages.append(self._kube_upgrade_download_images_stage()) + if add_first_plane: + stages.append(self._kube_upgrade_first_control_plane_stage()) + if add_networking: + stages.append(self._kube_upgrade_networking_stage()) + if add_second_plane: + stages.append(self._kube_upgrade_second_control_plane_stage()) + if add_patches: + stages.append(self._kube_upgrade_patch_stage(controller_list, + worker_list)) + if add_kubelets: + stages.extend(self._kube_upgrade_kubelet_stages(controller_list, + worker_list)) + if add_complete: + stages.append(self._kube_upgrade_complete_stage()) + if add_cleanup: + stages.append(self._kube_upgrade_cleanup_stage()) + return stages + + def test_no_existing_upgrade(self): + """ + Test the kube_upgrade strategy creation for the hosts when there is + no existing kube upgrade exists. + A duplex env will have more steps than a simplex environment + """ + kube_upgrade = None + # default stage list includes all , however second plane is duplex only + stages = self.build_stage_list(self.controller_list, + self.worker_list, + add_second_plane=self.is_duplex()) + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + + def test_resume_after_upgrade_started(self): + """ + Test the kube_upgrade strategy creation when the upgrade was created + already (upgrade-started) + The 'start stage should be skipped and the upgrade resumes at the + 'downloading images' stage + """ + kube_upgrade = self._create_kube_upgrade_obj( + KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTED) + # explicity bypass the start stage + stages = self.build_stage_list(self.controller_list, + self.worker_list, + add_start=False, + add_second_plane=self.is_duplex()) + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + + def test_resume_after_upgrade_complete(self): + """ + Test the kube_upgrade strategy creation when the upgrade had previously + stopped after upgrade-completed. + It is expected to resume at the cleanup stage + """ + kube_upgrade = self._create_kube_upgrade_obj( + KUBE_UPGRADE_STATE.KUBE_UPGRADE_COMPLETE) + # not using build_stage_list utility since the list of stages is small + stages = [ + self._kube_upgrade_cleanup_stage(), + ] + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + + +@mock.patch('nfv_vim.event_log._instance._event_issue', + sw_update_testcase.fake_event_issue) +@mock.patch('nfv_vim.objects._sw_update.SwUpdate.save', + sw_update_testcase.fake_save) +@mock.patch('nfv_vim.objects._sw_update.timers.timers_create_timer', + sw_update_testcase.fake_timer) +@mock.patch('nfv_vim.nfvi.nfvi_compute_plugin_disabled', + sw_update_testcase.fake_nfvi_compute_plugin_disabled) +class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, + KubePatchMixin, + ApplyStageMixin, + SimplexKubeUpgradeMixin): + def setUp(self): + super(TestSimplexApplyStrategy, self).setUp() + self.create_host('controller-0') + self.controller_list = ['controller-0', ] + self.worker_list = [] + + def test_resume_after_download_images_failed(self): + """ + Test the kube_upgrade strategy creation when the upgrade had previously + stopped with 'downloading-images-failed' + It is expected to resume at the 'downloading images' stage + """ + kube_upgrade = self._create_kube_upgrade_obj( + KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED) + stages = [ + self._kube_upgrade_download_images_stage(), + self._kube_upgrade_first_control_plane_stage(), + self._kube_upgrade_networking_stage(), + self._kube_upgrade_patch_stage(['controller-0']), + self._kube_upgrade_kubelet_controller_stage('controller-0'), + self._kube_upgrade_complete_stage(), + self._kube_upgrade_cleanup_stage(), + ] + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + + def test_resume_after_download_images_succeeded(self): + """ + Test the kube_upgrade strategy creation when the upgrade had previously + stopped with 'downloaded-images' + It is expected to resume at the 'first control plane' stage. + """ + kube_upgrade = self._create_kube_upgrade_obj( + KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADED_IMAGES) + stages = [ + self._kube_upgrade_first_control_plane_stage(), + self._kube_upgrade_networking_stage(), + self._kube_upgrade_patch_stage(['controller-0']), + self._kube_upgrade_kubelet_controller_stage('controller-0'), + self._kube_upgrade_complete_stage(), + self._kube_upgrade_cleanup_stage(), + ] + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + + def test_resume_after_first_control_plane_failed(self): + """ + Test the kube_upgrade strategy creation when there is only a simplex + and the upgrade had previously failed during the first control plane. + It is expected to resume and retry the 'first control plane' stage + """ + kube_upgrade = self._create_kube_upgrade_obj( + KUBE_UPGRADE_STATE.KUBE_UPGRADING_FIRST_MASTER_FAILED) + stages = [ + self._kube_upgrade_first_control_plane_stage(), + self._kube_upgrade_networking_stage(), + self._kube_upgrade_patch_stage(['controller-0']), + self._kube_upgrade_kubelet_controller_stage('controller-0'), + self._kube_upgrade_complete_stage(), + self._kube_upgrade_cleanup_stage(), + ] + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + + def test_resume_after_first_control_plane_succeeded(self): + """ + Test the kube_upgrade strategy creation when there is only a simplex + and the upgrade had previously stopped after the first control plane. + It is expected to resume at the networking stage + """ + kube_upgrade = self._create_kube_upgrade_obj( + KUBE_UPGRADE_STATE.KUBE_UPGRADED_FIRST_MASTER) + stages = [ + self._kube_upgrade_networking_stage(), + self._kube_upgrade_patch_stage(['controller-0']), + self._kube_upgrade_kubelet_controller_stage('controller-0'), + self._kube_upgrade_complete_stage(), + self._kube_upgrade_cleanup_stage(), + ] + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + + def test_resume_after_networking_failed(self): + """ + Test the kube_upgrade strategy creation when there is only a simplex + and the upgrade had previously failed during networking. + It is expected to retry and resume at the networking stage + """ + kube_upgrade = self._create_kube_upgrade_obj( + KUBE_UPGRADE_STATE.KUBE_UPGRADING_NETWORKING_FAILED) + stages = [ + self._kube_upgrade_networking_stage(), + self._kube_upgrade_patch_stage(['controller-0']), + self._kube_upgrade_kubelet_controller_stage('controller-0'), + self._kube_upgrade_complete_stage(), + self._kube_upgrade_cleanup_stage(), + ] + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + + def test_resume_after_networking_succeeded(self): + """ + Test the kube_upgrade strategy creation when there is only a simplex + and the upgrade had previously stopped after successful networking. + It is expected to resume at the patch stage + """ + kube_upgrade = self._create_kube_upgrade_obj( + KUBE_UPGRADE_STATE.KUBE_UPGRADED_NETWORKING) + stages = [ + self._kube_upgrade_patch_stage(['controller-0']), + self._kube_upgrade_kubelet_controller_stage('controller-0'), + self._kube_upgrade_complete_stage(), + self._kube_upgrade_cleanup_stage(), + ] + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + + def test_resume_after_invalid_second_master_state(self): + """ + Test the kube_upgrade strategy creation when there is only a simplex + and the upgrade had previously stopped after a second control plane + state is encountered. + There should never be a second control plane state in a simplex, so + the stages should skip over it to the patch stage. + """ + kube_upgrade = self._create_kube_upgrade_obj( + KUBE_UPGRADE_STATE.KUBE_UPGRADED_SECOND_MASTER) + stages = [ + self._kube_upgrade_patch_stage(['controller-0']), + self._kube_upgrade_kubelet_controller_stage('controller-0'), + self._kube_upgrade_complete_stage(), + self._kube_upgrade_cleanup_stage(), + ] + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + + def test_resume_after_invalid_second_master_fail_state(self): + """ + Test the kube_upgrade strategy creation when there is only a simplex + and the upgrade had previously stopped after a second control plane + failure state is encountered. + There should never be a second control plane state in a simplex, so + the stages should skip over it to the patch stage. + """ + kube_upgrade = self._create_kube_upgrade_obj( + KUBE_UPGRADE_STATE.KUBE_UPGRADING_SECOND_MASTER_FAILED) + stages = [ + self._kube_upgrade_patch_stage(['controller-0']), + self._kube_upgrade_kubelet_controller_stage('controller-0'), + self._kube_upgrade_complete_stage(), + self._kube_upgrade_cleanup_stage(), + ] + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + + +@mock.patch('nfv_vim.event_log._instance._event_issue', + sw_update_testcase.fake_event_issue) +@mock.patch('nfv_vim.objects._sw_update.SwUpdate.save', + sw_update_testcase.fake_save) +@mock.patch('nfv_vim.objects._sw_update.timers.timers_create_timer', + sw_update_testcase.fake_timer) +@mock.patch('nfv_vim.nfvi.nfvi_compute_plugin_disabled', + sw_update_testcase.fake_nfvi_compute_plugin_disabled) +class TestDuplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, + KubePatchMixin, + ApplyStageMixin, + DuplexKubeUpgradeMixin): + def setUp(self): + super(TestDuplexApplyStrategy, self).setUp() + self.create_host('controller-0') + self.create_host('controller-1') + # the order in which hosts should be patched + self.controller_list = ['controller-1', 'controller-0'] + self.worker_list = [] + + +@mock.patch('nfv_vim.event_log._instance._event_issue', + sw_update_testcase.fake_event_issue) +@mock.patch('nfv_vim.objects._sw_update.SwUpdate.save', + sw_update_testcase.fake_save) +@mock.patch('nfv_vim.objects._sw_update.timers.timers_create_timer', + sw_update_testcase.fake_timer) +@mock.patch('nfv_vim.nfvi.nfvi_compute_plugin_disabled', + sw_update_testcase.fake_nfvi_compute_plugin_disabled) +class TestDuplexPlusApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, + KubePatchMixin, + ApplyStageMixin, + DuplexKubeUpgradeMixin): + def setUp(self): + super(TestDuplexPlusApplyStrategy, self).setUp() + self.create_host('controller-0') + self.create_host('controller-1') + self.create_host('compute-0') # creates a worker + # the order in which hosts should be patched + self.controller_list = ['controller-1', 'controller-0'] + self.worker_list = ['compute-0'] diff --git a/nfv/nfv-vim/nfv_vim/alarm/_sw_update.py b/nfv/nfv-vim/nfv_vim/alarm/_sw_update.py index bdd7110b..367d9cd6 100755 --- a/nfv/nfv-vim/nfv_vim/alarm/_sw_update.py +++ b/nfv/nfv-vim/nfv_vim/alarm/_sw_update.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2016-2020 Wind River Systems, Inc. +# Copyright (c) 2016-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -114,6 +114,41 @@ _alarm_templates = { "problem persists contact next level of support"), 'exclude_alarm_context': [alarm.ALARM_CONTEXT.TENANT], }, + + alarm.ALARM_TYPE.KUBE_UPGRADE_AUTO_APPLY_INPROGRESS: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'event_type': alarm.ALARM_EVENT_TYPE.EQUIPMENT_ALARM, + 'severity': alarm.ALARM_SEVERITY.MAJOR, + 'probable_cause': alarm.ALARM_PROBABLE_CAUSE.UNKNOWN, + 'reason_text': "Kubernetes upgrade auto-apply inprogress", + 'repair_action': ("Wait for kubernetes upgrade auto-apply to complete; " + "if problem persists contact next level of support"), + 'exclude_alarm_context': [alarm.ALARM_CONTEXT.TENANT], + }, + alarm.ALARM_TYPE.KUBE_UPGRADE_AUTO_APPLY_ABORTING: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'event_type': alarm.ALARM_EVENT_TYPE.EQUIPMENT_ALARM, + 'severity': alarm.ALARM_SEVERITY.MAJOR, + 'probable_cause': alarm.ALARM_PROBABLE_CAUSE.UNKNOWN, + 'reason_text': "Kubernetes upgrade auto-apply aborting", + 'repair_action': ("Wait for kubernetes upgrade auto-apply abort to " + "complete; if problem persists contact next " + "level of support"), + 'exclude_alarm_context': [alarm.ALARM_CONTEXT.TENANT], + }, + alarm.ALARM_TYPE.KUBE_UPGRADE_AUTO_APPLY_FAILED: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'event_type': alarm.ALARM_EVENT_TYPE.EQUIPMENT_ALARM, + 'severity': alarm.ALARM_SEVERITY.CRITICAL, + 'probable_cause': alarm.ALARM_PROBABLE_CAUSE.UNKNOWN, + 'reason_text': "Kubernetes upgrade auto-apply failed", + 'repair_action': ("Attempt to apply kubernetes upgrade manually; if " + "problem persists contact next level of support"), + 'exclude_alarm_context': [alarm.ALARM_CONTEXT.TENANT], + }, } diff --git a/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/_controller.py b/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/_controller.py index 18329d11..bc261f5e 100755 --- a/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/_controller.py +++ b/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/_controller.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -11,6 +11,7 @@ import wsmeext.pecan as wsme_pecan from nfv_vim.api._link import Link from nfv_vim.api.controllers.v1.orchestration.sw_update import FwUpdateAPI +from nfv_vim.api.controllers.v1.orchestration.sw_update import KubeUpgradeAPI from nfv_vim.api.controllers.v1.orchestration.sw_update import SwPatchAPI from nfv_vim.api.controllers.v1.orchestration.sw_update import SwUpgradeAPI @@ -32,6 +33,8 @@ class OrchestrationDescription(wsme_types.Base): Link.make_link('self', url, 'orchestration'), Link.make_link('sw-patch', url, 'orchestration/sw-patch', ''), Link.make_link('sw-upgrade', url, 'orchestration/sw-upgrade', ''), + Link.make_link('kube-upgrade', + url, 'orchestration/kube-upgrade', ''), Link.make_link('fw-update', url, 'orchestration/fw-update', '')] return description @@ -48,6 +51,8 @@ class OrchestrationAPI(rest.RestController): return SwUpgradeAPI(), remainder elif 'fw-update' == key: return FwUpdateAPI(), remainder + elif 'kube-upgrade' == key: + return KubeUpgradeAPI(), remainder else: pecan.abort(httplib.NOT_FOUND) diff --git a/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/__init__.py b/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/__init__.py index c4d000b5..cc7154df 100755 --- a/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/__init__.py +++ b/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/__init__.py @@ -1,8 +1,9 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # from nfv_vim.api.controllers.v1.orchestration.sw_update._fw_update import FwUpdateAPI # noqa: F401 +from nfv_vim.api.controllers.v1.orchestration.sw_update._kube_upgrade import KubeUpgradeAPI # noqa: F401 from nfv_vim.api.controllers.v1.orchestration.sw_update._sw_patch import SwPatchAPI # noqa: F401 from nfv_vim.api.controllers.v1.orchestration.sw_update._sw_upgrade import SwUpgradeAPI # noqa: F401 diff --git a/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_kube_upgrade.py b/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_kube_upgrade.py new file mode 100755 index 00000000..1cff118e --- /dev/null +++ b/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_kube_upgrade.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2020-2021 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +import pecan +from pecan import rest +from six.moves import http_client as httplib +from wsme import types as wsme_types +import wsmeext.pecan as wsme_pecan + +from nfv_common import debug +from nfv_vim.api._link import Link +from nfv_vim.api.controllers.v1.orchestration.sw_update._sw_update_strategy \ + import KubeUpgradeStrategyAPI + +DLOG = debug.debug_get_logger('nfv_vim.api.kube_upgrade') + + +class KubeUpgradeDescription(wsme_types.Base): + """ + Kubernetes Update Description + """ + id = wsme_types.text + links = wsme_types.wsattr([Link], name='links') + + @classmethod + def convert(cls): + url = pecan.request.host_url + + description = KubeUpgradeDescription() + description.id = "kube-upgrade" + description.links = [ + Link.make_link('self', + url, + 'orchestration/kube-upgrade'), + Link.make_link('strategy', + url, + 'orchestration/kube-upgrade/strategy')] + return description + + +class KubeUpgradeAPI(rest.RestController): + """ + KubeUpgradeRest API + """ + @pecan.expose() + def _lookup(self, key, *remainder): + if 'strategy' == key: + return KubeUpgradeStrategyAPI(), remainder + else: + pecan.abort(httplib.NOT_FOUND) + + @wsme_pecan.wsexpose(KubeUpgradeDescription) + def get(self): + # NOTE: The reason why convert() is being called for every + # request is because we need to get the host url from + # the request object to make the links. + return KubeUpgradeDescription.convert() diff --git a/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_sw_update_defs.py b/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_sw_update_defs.py index 9359c73a..71c2ed5c 100755 --- a/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_sw_update_defs.py +++ b/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_sw_update_defs.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -19,6 +19,7 @@ class SwUpdateNames(Constants): SW_PATCH = Constant('sw-patch') SW_UPGRADE = Constant('sw-upgrade') FW_UPDATE = Constant('fw-update') + KUBE_UPGRADE = Constant('kube-upgrade') @six.add_metaclass(Singleton) @@ -72,7 +73,8 @@ SW_UPDATE_ALARM_RESTRICTION_TYPES = SwUpdateAlarmRestrictionTypes() SwUpdateNames = wsme_types.Enum(str, SW_UPDATE_NAME.SW_PATCH, SW_UPDATE_NAME.SW_UPGRADE, - SW_UPDATE_NAME.FW_UPDATE) + SW_UPDATE_NAME.FW_UPDATE, + SW_UPDATE_NAME.KUBE_UPGRADE) SwUpdateApplyTypes = wsme_types.Enum(str, SW_UPDATE_APPLY_TYPE.SERIAL, SW_UPDATE_APPLY_TYPE.PARALLEL, diff --git a/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_sw_update_strategy.py b/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_sw_update_strategy.py index e3e85d9a..a79c7856 100755 --- a/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_sw_update_strategy.py +++ b/nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_sw_update_strategy.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -33,6 +33,7 @@ MIN_PARALLEL_HOSTS = 2 MAX_PARALLEL_PATCH_HOSTS = 100 MAX_PARALLEL_UPGRADE_HOSTS = 10 MAX_PARALLEL_FW_UPDATE_HOSTS = 5 +MAX_PARALLEL_KUBE_UPGRADE_HOSTS = 10 def _get_sw_update_type_from_path(path): @@ -43,6 +44,8 @@ def _get_sw_update_type_from_path(path): return SW_UPDATE_NAME.SW_UPGRADE elif 'fw-update' in split_path: return SW_UPDATE_NAME.FW_UPDATE + elif 'kube-upgrade' in split_path: + return SW_UPDATE_NAME.KUBE_UPGRADE else: DLOG.error("Unknown sw_update_type in path: %s" % path) return 'unknown' @@ -201,6 +204,30 @@ class FwUpdateStrategyCreateData(wsme_types.Base): name='alarm-restrictions') +class KubeUpgradeStrategyCreateData(wsme_types.Base): + """ + Kubernetes Upgrade Strategy - Create Data + """ + to_version = wsme_types.wsattr(six.text_type, + mandatory=True, + name='to-version') + storage_apply_type = wsme_types.wsattr(SwUpdateApplyTypes, + mandatory=True, + name='storage-apply-type') + worker_apply_type = wsme_types.wsattr(SwUpdateApplyTypes, + mandatory=True, + name='worker-apply-type') + max_parallel_worker_hosts = wsme_types.wsattr( + int, + mandatory=False, + name='max-parallel-worker-hosts') + alarm_restrictions = wsme_types.wsattr( + SwUpdateAlarmRestrictionTypes, + mandatory=False, + default=SW_UPDATE_ALARM_RESTRICTION_TYPES.STRICT, + name='alarm-restrictions') + + class SwUpdateStrategyActionData(wsme_types.Base): """ Software Update Strategy - Action Data @@ -633,3 +660,62 @@ class FwUpdateStrategyAPI(SwUpdateStrategyAPI): DLOG.error("Unexpected result received, result=%s." % response.result) return pecan.abort(httplib.INTERNAL_SERVER_ERROR) + + +class KubeUpgradeStrategyAPI(SwUpdateStrategyAPI): + """ + Kubernetes Upgrade Strategy Rest API + """ + @wsme_pecan.wsexpose(SwUpdateStrategyQueryData, + body=KubeUpgradeStrategyCreateData, + status_code=httplib.OK) + def post(self, request_data): + rpc_request = rpc.APIRequestCreateKubeUpgradeStrategy() + rpc_request.sw_update_type = _get_sw_update_type_from_path( + pecan.request.path) + rpc_request.to_version = request_data.to_version + rpc_request.controller_apply_type = SW_UPDATE_APPLY_TYPE.SERIAL + rpc_request.storage_apply_type = request_data.storage_apply_type + rpc_request.worker_apply_type = request_data.worker_apply_type + if wsme_types.Unset != request_data.max_parallel_worker_hosts: + if request_data.max_parallel_worker_hosts < MIN_PARALLEL_HOSTS: + return pecan.abort( + httplib.BAD_REQUEST, + "Invalid value for max-parallel-worker-hosts:(%s) < (%s)" + % (request_data.max_parallel_worker_hosts, + MIN_PARALLEL_HOSTS)) + if (request_data.max_parallel_worker_hosts > + MAX_PARALLEL_KUBE_UPGRADE_HOSTS): + return pecan.abort( + httplib.BAD_REQUEST, + "Invalid value for max-parallel-worker-hosts:(%s) > (%s)" + % (request_data.max_parallel_worker_hosts, + MAX_PARALLEL_KUBE_UPGRADE_HOSTS)) + rpc_request.max_parallel_worker_hosts = \ + request_data.max_parallel_worker_hosts + rpc_request.default_instance_action = SW_UPDATE_INSTANCE_ACTION.MIGRATE + rpc_request.alarm_restrictions = request_data.alarm_restrictions + vim_connection = pecan.request.vim.open_connection() + vim_connection.send(rpc_request.serialize()) + msg = vim_connection.receive(timeout_in_secs=30) + if msg is None: + DLOG.error("No response received.") + return pecan.abort(httplib.INTERNAL_SERVER_ERROR) + + response = rpc.RPCMessage.deserialize(msg) + if (rpc.RPC_MSG_TYPE.CREATE_SW_UPDATE_STRATEGY_RESPONSE != + response.type): + DLOG.error("Unexpected message type received, msg_type=%s." + % response.type) + return pecan.abort(httplib.INTERNAL_SERVER_ERROR) + + if rpc.RPC_MSG_RESULT.SUCCESS == response.result: + strategy = json.loads(response.strategy) + query_data = SwUpdateStrategyQueryData() + query_data.convert_strategy(strategy) + return query_data + elif rpc.RPC_MSG_RESULT.CONFLICT == response.result: + return pecan.abort(httplib.CONFLICT) + + DLOG.error("Unexpected result received, result=%s." % response.result) + return pecan.abort(httplib.INTERNAL_SERVER_ERROR) diff --git a/nfv/nfv-vim/nfv_vim/database/_database_sw_update.py b/nfv/nfv-vim/nfv_vim/database/_database_sw_update.py index d107517f..7c91fd2e 100755 --- a/nfv/nfv-vim/nfv_vim/database/_database_sw_update.py +++ b/nfv/nfv-vim/nfv_vim/database/_database_sw_update.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2016,2020 Wind River Systems, Inc. +# Copyright (c) 2016-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -69,4 +69,8 @@ def database_sw_update_get_list(): elif objects.SW_UPDATE_TYPE.FW_UPDATE == sw_update.sw_update_type: fw_update_obj = objects.FwUpdate(sw_update.uuid, strategy_data) sw_update_objs.append(fw_update_obj) + elif objects.SW_UPDATE_TYPE.KUBE_UPGRADE == sw_update.sw_update_type: + kube_upgrade_obj = objects.KubeUpgrade(sw_update.uuid, + strategy_data) + sw_update_objs.append(kube_upgrade_obj) return sw_update_objs diff --git a/nfv/nfv-vim/nfv_vim/directors/_directors_defs.py b/nfv/nfv-vim/nfv_vim/directors/_directors_defs.py index b100a033..134aa1c6 100755 --- a/nfv/nfv-vim/nfv_vim/directors/_directors_defs.py +++ b/nfv/nfv-vim/nfv_vim/directors/_directors_defs.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -27,6 +27,7 @@ class OperationTypes(Constants): SWACT_HOSTS = Constant('swact-hosts') FW_UPDATE_HOSTS = Constant('fw-update-hosts') FW_UPDATE_ABORT_HOSTS = Constant('fw-update-abort-hosts') + KUBE_UPGRADE_HOSTS = Constant('kube-upgrade-hosts') START_INSTANCES = Constant('start-instances') START_INSTANCES_SERIAL = Constant('start-instances-serial') STOP_INSTANCES = Constant('stop-instances') diff --git a/nfv/nfv-vim/nfv_vim/directors/_host_director.py b/nfv/nfv-vim/nfv_vim/directors/_host_director.py index 4e646bce..155be2da 100755 --- a/nfv/nfv-vim/nfv_vim/directors/_host_director.py +++ b/nfv/nfv-vim/nfv_vim/directors/_host_director.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -803,6 +803,161 @@ class HostDirector(object): return host_operation + @coroutine + def _nfvi_kube_host_upgrade_control_plane_callback(self): + """ + NFVI Kube Host Upgrade Control Plane Callback + """ + from nfv_vim import directors + + response = (yield) + DLOG.verbose("NFVI Kube Host Upgrade Control Plane response=%s." + % response) + if not response['completed']: + DLOG.info("Kube Host Upgrade Control Plane failed. Host:%s, reason=%s." + % (response['host_name'], response['reason'])) + + host_table = tables.tables_get_host_table() + host = host_table.get(response['host_name'], None) + if host is None: + DLOG.verbose("Host %s does not exist." % response['host_name']) + return + + if self._host_operation is None: + DLOG.verbose("No host %s operation inprogress." % host.name) + return + + if OPERATION_TYPE.KUBE_UPGRADE_HOSTS != self._host_operation.operation_type: + DLOG.verbose("Unexpected host %s operation %s, ignoring." + % (host.name, self._host_operation.operation_type)) + return + + sw_mgmt_director = directors.get_sw_mgmt_director() + sw_mgmt_director.kube_host_upgrade_control_plane_failed(host) + + def _nfvi_kube_host_upgrade_control_plane(self, + host_uuid, + host_name, + force): + """ + NFVI Kube Host Upgrade Control Plane + """ + nfvi.nfvi_kube_host_upgrade_control_plane( + host_uuid, + host_name, + force, + self._nfvi_kube_host_upgrade_control_plane_callback()) + + def kube_upgrade_hosts_control_plane(self, host_names, force): + """ + Kube Upgrade Hosts Control Plane for multiple hosts + """ + DLOG.info("Kube Host Upgrade control plane for hosts: %s" % host_names) + + host_operation = \ + Operation(OPERATION_TYPE.KUBE_UPGRADE_HOSTS) + + if self._host_operation is not None: + DLOG.debug("Canceling previous host operation %s, before " + "continuing with host operation %s." + % (self._host_operation.operation_type, + host_operation.operation_type)) + self._host_operation = None + + host_table = tables.tables_get_host_table() + for host_name in host_names: + host = host_table.get(host_name, None) + if host is None: + reason = "Unknown host %s given." % host_name + DLOG.info(reason) + host_operation.set_failed(reason) + return host_operation + + host_operation.add_host(host.name, OPERATION_STATE.INPROGRESS) + self._nfvi_kube_host_upgrade_control_plane(host.uuid, + host.name, + force) + + if host_operation.is_inprogress(): + self._host_operation = host_operation + + return host_operation + + @coroutine + def _nfvi_kube_host_upgrade_kubelet_callback(self): + """ + NFVI Kube Host Upgrade Kubelet Callback (for a single host) + """ + from nfv_vim import directors + + response = (yield) + DLOG.verbose("NFVI Kube Host Upgrade Kubelet response=%s." + % response) + if not response['completed']: + DLOG.info("Kube Host Upgrade Kubelet failed. Host:%s, reason=%s." + % (response['host_name'], response['reason'])) + + host_table = tables.tables_get_host_table() + host = host_table.get(response['host_name'], None) + if host is None: + DLOG.verbose("Host %s does not exist." % response['host_name']) + return + + if self._host_operation is None: + DLOG.verbose("No host %s operation inprogress." % host.name) + return + + if OPERATION_TYPE.KUBE_UPGRADE_HOSTS != self._host_operation.operation_type: + DLOG.verbose("Unexpected host %s operation %s, ignoring." + % (host.name, self._host_operation.operation_type)) + return + + sw_mgmt_director = directors.get_sw_mgmt_director() + sw_mgmt_director.kube_host_upgrade_kubelet_failed(host) + + def _nfvi_kube_host_upgrade_kubelet(self, host_uuid, host_name, force): + """ + NFVI Kube Host Upgrade Kubelet + """ + nfvi.nfvi_kube_host_upgrade_kubelet( + host_uuid, + host_name, + force, + self._nfvi_kube_host_upgrade_kubelet_callback()) + + def kube_upgrade_hosts_kubelet(self, host_names, force): + """ + Kube Upgrade Hosts Kubelet for multiple hosts + """ + DLOG.info("Kube Host Upgrade kubelet for hosts: %s" % host_names) + + host_operation = \ + Operation(OPERATION_TYPE.KUBE_UPGRADE_HOSTS) + + if self._host_operation is not None: + DLOG.debug("Canceling previous host operation %s, before " + "continuing with host operation %s." + % (self._host_operation.operation_type, + host_operation.operation_type)) + self._host_operation = None + + host_table = tables.tables_get_host_table() + for host_name in host_names: + host = host_table.get(host_name, None) + if host is None: + reason = "Unknown host %s given." % host_name + DLOG.info(reason) + host_operation.set_failed(reason) + return host_operation + + host_operation.add_host(host.name, OPERATION_STATE.INPROGRESS) + self._nfvi_kube_host_upgrade_kubelet(host.uuid, host.name, force) + + if host_operation.is_inprogress(): + self._host_operation = host_operation + + return host_operation + def disable_host_services(self, host_names, service): """ Disable a host service on a list of hosts diff --git a/nfv/nfv-vim/nfv_vim/directors/_sw_mgmt_director.py b/nfv/nfv-vim/nfv_vim/directors/_sw_mgmt_director.py index 56ecde98..428d074d 100755 --- a/nfv/nfv-vim/nfv_vim/directors/_sw_mgmt_director.py +++ b/nfv/nfv-vim/nfv_vim/directors/_sw_mgmt_director.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -134,6 +134,42 @@ class SwMgmtDirector(object): self._sw_update.strategy) return strategy_uuid, '' + def create_kube_upgrade_strategy(self, + storage_apply_type, + worker_apply_type, + max_parallel_worker_hosts, + alarm_restrictions, + to_version, + callback): + """ + Create Kubernetes Upgrade Strategy + """ + strategy_uuid = str(uuid.uuid4()) + + if self._sw_update is not None: + # Do not schedule the callback - if creation failed because a + # strategy already exists, the callback will attempt to operate + # on the old strategy, which is not what we want. + reason = "strategy already exists" + return None, reason + + self._sw_update = objects.KubeUpgrade() + success, reason = self._sw_update.strategy_build( + strategy_uuid, + storage_apply_type, + worker_apply_type, + max_parallel_worker_hosts, + alarm_restrictions, + self._ignore_alarms, + to_version, + self._single_controller) + + schedule.schedule_function_call(callback, + success, + reason, + self._sw_update.strategy) + return strategy_uuid, '' + def apply_sw_update_strategy(self, strategy_uuid, stage_id, callback): """ Apply Software Update Strategy @@ -245,6 +281,24 @@ class SwMgmtDirector(object): self._sw_update.handle_event( strategy.STRATEGY_EVENT.HOST_FW_UPDATE_FAILED, host) + def kube_host_upgrade_control_plane_failed(self, host): + """ + Called when a kube host upgrade for control plane fails + """ + if self._sw_update is not None: + self._sw_update.handle_event( + strategy.STRATEGY_EVENT.KUBE_HOST_UPGRADE_CONTROL_PLANE_FAILED, + host) + + def kube_host_upgrade_kubelet_failed(self, host): + """ + Called when a kube host upgrade for kubelet fails + """ + if self._sw_update is not None: + self._sw_update.handle_event( + strategy.STRATEGY_EVENT.KUBE_HOST_UPGRADE_KUBELET_FAILED, + host) + def host_audit(self, host): """ Called when a host audit is to be performed diff --git a/nfv/nfv-vim/nfv_vim/event_log/_sw_update.py b/nfv/nfv-vim/nfv_vim/event_log/_sw_update.py index 1b1d284f..8e06e198 100755 --- a/nfv/nfv-vim/nfv_vim/event_log/_sw_update.py +++ b/nfv/nfv-vim/nfv_vim/event_log/_sw_update.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2016 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -342,6 +342,172 @@ _event_templates = { } } }, + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_START: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'event_type': event_log.EVENT_TYPE.ACTION_EVENT, + 'importance': event_log.EVENT_IMPORTANCE.HIGH, + 'reason_text': "Kubernetes upgrade auto-apply start", + 'exclude_event_context': [], + 'event_context_data': { + event_log.EVENT_CONTEXT.ADMIN: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'reason_text': "Kubernetes upgrade auto-apply start", + } + } + }, + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_INPROGRESS: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'event_type': event_log.EVENT_TYPE.ACTION_EVENT, + 'importance': event_log.EVENT_IMPORTANCE.HIGH, + 'reason_text': "Kubernetes upgrade auto-apply inprogress", + 'exclude_event_context': [], + 'event_context_data': { + event_log.EVENT_CONTEXT.ADMIN: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'reason_text': "Kubernetes upgrade auto-apply inprogress", + } + } + }, + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_REJECTED: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'event_type': event_log.EVENT_TYPE.ACTION_EVENT, + 'importance': event_log.EVENT_IMPORTANCE.HIGH, + 'reason_text': "Kubernetes upgrade auto-apply rejected", + 'exclude_event_context': [], + 'event_context_data': { + event_log.EVENT_CONTEXT.ADMIN: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'reason_text': "Kubernetes upgrade auto-apply rejected%(reason)s", + } + } + }, + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_CANCELLED: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'event_type': event_log.EVENT_TYPE.ACTION_EVENT, + 'importance': event_log.EVENT_IMPORTANCE.HIGH, + 'reason_text': "Kubernetes upgrade auto-apply cancelled", + 'exclude_event_context': [], + 'event_context_data': { + event_log.EVENT_CONTEXT.ADMIN: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'reason_text': "Kubernetes upgrade auto-apply cancelled%(reason)s", + } + } + }, + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_FAILED: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'event_type': event_log.EVENT_TYPE.ACTION_EVENT, + 'importance': event_log.EVENT_IMPORTANCE.HIGH, + 'reason_text': "Kubernetes upgrade auto-apply failed", + 'exclude_event_context': [], + 'event_context_data': { + event_log.EVENT_CONTEXT.ADMIN: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'reason_text': "Kubernetes upgrade auto-apply failed%(reason)s", + } + } + }, + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_COMPLETED: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'event_type': event_log.EVENT_TYPE.ACTION_EVENT, + 'importance': event_log.EVENT_IMPORTANCE.HIGH, + 'reason_text': "Kubernetes upgrade auto-apply completed", + 'exclude_event_context': [], + 'event_context_data': { + event_log.EVENT_CONTEXT.ADMIN: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'reason_text': "Kubernetes upgrade auto-apply completed", + } + } + }, + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORT: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'event_type': event_log.EVENT_TYPE.ACTION_EVENT, + 'importance': event_log.EVENT_IMPORTANCE.HIGH, + 'reason_text': "Kubernetes upgrade auto-apply abort", + 'exclude_event_context': [], + 'event_context_data': { + event_log.EVENT_CONTEXT.ADMIN: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'reason_text': "Kubernetes upgrade auto-apply abort", + } + } + }, + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORTING: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'event_type': event_log.EVENT_TYPE.ACTION_EVENT, + 'importance': event_log.EVENT_IMPORTANCE.HIGH, + 'reason_text': "Kubernetes upgrade auto-apply aborting", + 'exclude_event_context': [], + 'event_context_data': { + event_log.EVENT_CONTEXT.ADMIN: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'reason_text': "Kubernetes upgrade auto-apply aborting", + } + } + }, + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORT_REJECTED: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'event_type': event_log.EVENT_TYPE.ACTION_EVENT, + 'importance': event_log.EVENT_IMPORTANCE.HIGH, + 'reason_text': "Kubernetes upgrade auto-apply abort rejected", + 'exclude_event_context': [], + 'event_context_data': { + event_log.EVENT_CONTEXT.ADMIN: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'reason_text': "Kubernetes upgrade auto-apply abort " + "rejected%(reason)s", + } + } + }, + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORT_FAILED: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'event_type': event_log.EVENT_TYPE.ACTION_EVENT, + 'importance': event_log.EVENT_IMPORTANCE.HIGH, + 'reason_text': "Kubernetes upgrade auto-apply abort failed", + 'exclude_event_context': [], + 'event_context_data': { + event_log.EVENT_CONTEXT.ADMIN: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'reason_text': "Kubernetes upgrade auto-apply abort failed%(reason)s", + } + } + }, + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORTED: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'event_type': event_log.EVENT_TYPE.ACTION_EVENT, + 'importance': event_log.EVENT_IMPORTANCE.HIGH, + 'reason_text': "Kubernetes upgrade auto-apply aborted", + 'exclude_event_context': [], + 'event_context_data': { + event_log.EVENT_CONTEXT.ADMIN: { + 'entity_type': "orchestration", + 'entity': "orchestration=kube-upgrade", + 'reason_text': "Kubernetes upgrade auto-apply aborted", + } + } + }, } diff --git a/nfv/nfv-vim/nfv_vim/events/_vim_api_events.py b/nfv/nfv-vim/nfv_vim/events/_vim_api_events.py index 0bef8d8b..b86c3f6f 100755 --- a/nfv/nfv-vim/nfv_vim/events/_vim_api_events.py +++ b/nfv/nfv-vim/nfv_vim/events/_vim_api_events.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2016 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -187,6 +187,9 @@ def _vim_api_message_handler(connection, msg): elif rpc.RPC_MSG_TYPE.CREATE_SW_UPDATE_STRATEGY_REQUEST == msg.type: vim_sw_update_api_create_strategy(connection, msg) + elif rpc.RPC_MSG_TYPE.CREATE_KUBE_UPGRADE_STRATEGY_REQUEST == msg.type: + vim_sw_update_api_create_strategy(connection, msg) + elif rpc.RPC_MSG_TYPE.CREATE_SW_UPGRADE_STRATEGY_REQUEST == msg.type: vim_sw_update_api_create_strategy(connection, msg) diff --git a/nfv/nfv-vim/nfv_vim/events/_vim_nfvi_events.py b/nfv/nfv-vim/nfv_vim/events/_vim_nfvi_events.py index 2b094dcb..365e2d6a 100755 --- a/nfv/nfv-vim/nfv_vim/events/_vim_nfvi_events.py +++ b/nfv/nfv-vim/nfv_vim/events/_vim_nfvi_events.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -124,6 +124,8 @@ def _nfvi_sw_update_get_callback(): sw_update_type = 'sw-upgrade' elif sw_update.sw_update_type == objects.SW_UPDATE_TYPE.FW_UPDATE: sw_update_type = 'fw-update' + elif sw_update.sw_update_type == objects.SW_UPDATE_TYPE.KUBE_UPGRADE: + sw_update_type = 'kube-upgrade' if sw_update.strategy.is_applying() or sw_update.strategy.is_aborting(): in_progress = True diff --git a/nfv/nfv-vim/nfv_vim/events/_vim_sw_update_api_events.py b/nfv/nfv-vim/nfv_vim/events/_vim_sw_update_api_events.py index a3bc1f43..aa860611 100755 --- a/nfv/nfv-vim/nfv_vim/events/_vim_sw_update_api_events.py +++ b/nfv/nfv-vim/nfv_vim/events/_vim_sw_update_api_events.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -113,6 +113,15 @@ def vim_sw_update_api_create_strategy(connection, msg): default_instance_action, alarm_restrictions, _vim_sw_update_api_create_strategy_callback) + elif 'kube-upgrade' == msg.sw_update_type: + to_version = msg.to_version + uuid, reason = sw_mgmt_director.create_kube_upgrade_strategy( + storage_apply_type, + worker_apply_type, + max_parallel_worker_hosts, + alarm_restrictions, + to_version, + _vim_sw_update_api_create_strategy_callback) else: DLOG.error("Invalid message name: %s" % msg.sw_update_type) response = rpc.APIResponseCreateSwUpdateStrategy() @@ -164,13 +173,15 @@ def vim_sw_update_api_apply_strategy(connection, msg): """ Handle Sw-Update Apply Strategy API request """ - DLOG.info("Apply sw-update strategy.") + DLOG.info("Apply sw-update strategy: (%s) called." % msg.sw_update_type) if 'sw-patch' == msg.sw_update_type: sw_update_type = objects.SW_UPDATE_TYPE.SW_PATCH elif 'sw-upgrade' == msg.sw_update_type: sw_update_type = objects.SW_UPDATE_TYPE.SW_UPGRADE elif 'fw-update' == msg.sw_update_type: sw_update_type = objects.SW_UPDATE_TYPE.FW_UPDATE + elif 'kube-upgrade' == msg.sw_update_type: + sw_update_type = objects.SW_UPDATE_TYPE.KUBE_UPGRADE else: DLOG.error("Invalid message name: %s" % msg.sw_update_type) sw_update_type = 'unknown' @@ -226,6 +237,8 @@ def vim_sw_update_api_abort_strategy(connection, msg): sw_update_type = objects.SW_UPDATE_TYPE.SW_UPGRADE elif 'fw-update' == msg.sw_update_type: sw_update_type = objects.SW_UPDATE_TYPE.FW_UPDATE + elif 'kube-upgrade' == msg.sw_update_type: + sw_update_type = objects.SW_UPDATE_TYPE.KUBE_UPGRADE else: DLOG.error("Invalid message name: %s" % msg.sw_update_type) sw_update_type = 'unknown' @@ -278,6 +291,8 @@ def vim_sw_update_api_delete_strategy(connection, msg): sw_update_type = objects.SW_UPDATE_TYPE.SW_UPGRADE elif 'fw-update' == msg.sw_update_type: sw_update_type = objects.SW_UPDATE_TYPE.FW_UPDATE + elif 'kube-upgrade' == msg.sw_update_type: + sw_update_type = objects.SW_UPDATE_TYPE.KUBE_UPGRADE else: DLOG.error("Invalid message name: %s" % msg.sw_update_type) sw_update_type = 'unknown' @@ -309,6 +324,8 @@ def vim_sw_update_api_get_strategy(connection, msg): sw_update_type = objects.SW_UPDATE_TYPE.SW_UPGRADE elif 'fw-update' == msg.sw_update_type: sw_update_type = objects.SW_UPDATE_TYPE.FW_UPDATE + elif 'kube-upgrade' == msg.sw_update_type: + sw_update_type = objects.SW_UPDATE_TYPE.KUBE_UPGRADE else: DLOG.error("Invalid message name: %s" % msg.sw_update_type) sw_update_type = 'unknown' diff --git a/nfv/nfv-vim/nfv_vim/nfvi/__init__.py b/nfv/nfv-vim/nfv_vim/nfvi/__init__.py index 8ef4093f..a499b322 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/__init__.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -100,6 +100,9 @@ from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_get_host # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_get_host_device # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_get_host_devices # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_get_hosts # noqa: F401 +from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_get_kube_host_upgrade_list # noqa: F401 +from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_get_kube_upgrade # noqa: F401 +from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_get_kube_version_list # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_get_logs # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_get_system_info # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_get_system_state # noqa: F401 @@ -107,6 +110,13 @@ from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_get_terminating_pods from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_get_upgrade # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_host_device_image_update # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_host_device_image_update_abort # noqa: F401 +from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_upgrade_control_plane # noqa: F401 +from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_upgrade_kubelet # noqa: F401 +from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_upgrade_cleanup # noqa: F401 +from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_upgrade_complete # noqa: F401 +from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_upgrade_download_images # noqa: F401 +from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_upgrade_networking # noqa: F401 +from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_upgrade_start # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_lock_host # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_notify_host_failed # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_notify_host_services_delete_failed # noqa: F401 @@ -160,6 +170,7 @@ from nfv_vim.nfvi._nfvi_network_module import nfvi_remove_router_from_agent # n from nfv_vim.nfvi._nfvi_network_module import nfvi_update_network # noqa: F401 from nfv_vim.nfvi._nfvi_network_module import nfvi_update_subnet # noqa: F401 +from nfv_vim.nfvi._nfvi_sw_mgmt_module import nfvi_sw_mgmt_apply_updates # noqa: F401 from nfv_vim.nfvi._nfvi_sw_mgmt_module import nfvi_sw_mgmt_query_hosts # noqa: F401 from nfv_vim.nfvi._nfvi_sw_mgmt_module import nfvi_sw_mgmt_query_updates # noqa: F401 from nfv_vim.nfvi._nfvi_sw_mgmt_module import nfvi_sw_mgmt_update_host # noqa: F401 diff --git a/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py b/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py index 429b1e7f..cc697922 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -100,6 +100,111 @@ def nfvi_host_device_image_update_abort(host_uuid, host_name, callback): return cmd_id +def nfvi_kube_host_upgrade_control_plane(host_uuid, host_name, force, callback): + """ + Kube Host Upgrade Control Plane + """ + cmd_id = _infrastructure_plugin.invoke_plugin( + 'kube_host_upgrade_control_plane', + host_uuid, + host_name, + force, + callback=callback) + return cmd_id + + +def nfvi_kube_host_upgrade_kubelet(host_uuid, host_name, force, callback): + """ + Kube Host Upgrade Kubelet + """ + cmd_id = _infrastructure_plugin.invoke_plugin( + 'kube_host_upgrade_kubelet', + host_uuid, + host_name, + force, + callback=callback) + return cmd_id + + +def nfvi_kube_upgrade_cleanup(callback): + """ + Kube Upgrade Cleanup + """ + cmd_id = _infrastructure_plugin.invoke_plugin( + 'kube_upgrade_cleanup', + callback=callback) + return cmd_id + + +def nfvi_kube_upgrade_complete(callback): + """ + Kube Upgrade Complete + """ + cmd_id = _infrastructure_plugin.invoke_plugin( + 'kube_upgrade_complete', + callback=callback) + return cmd_id + + +def nfvi_kube_upgrade_download_images(callback): + """ + Kube Upgrade Download Images + """ + cmd_id = _infrastructure_plugin.invoke_plugin( + 'kube_upgrade_download_images', + callback=callback) + return cmd_id + + +def nfvi_kube_upgrade_networking(callback): + """ + Kube Upgrade Networking + """ + cmd_id = _infrastructure_plugin.invoke_plugin('kube_upgrade_networking', + callback=callback) + return cmd_id + + +def nfvi_kube_upgrade_start(to_version, force, alarm_ignore_list, callback): + """ + Kube Upgrade Start + """ + cmd_id = _infrastructure_plugin.invoke_plugin( + 'kube_upgrade_start', + to_version=to_version, + force=force, + alarm_ignore_list=alarm_ignore_list, + callback=callback) + return cmd_id + + +def nfvi_get_kube_host_upgrade_list(callback): + """ + Get kube host upgrade list + """ + cmd_id = _infrastructure_plugin.invoke_plugin('get_kube_host_upgrade_list', + callback=callback) + return cmd_id + + +def nfvi_get_kube_upgrade(callback): + """ + Get kube upgrade + """ + cmd_id = _infrastructure_plugin.invoke_plugin('get_kube_upgrade', + callback=callback) + return cmd_id + + +def nfvi_get_kube_version_list(callback): + """ + Get kube version list + """ + cmd_id = _infrastructure_plugin.invoke_plugin('get_kube_version_list', + callback=callback) + return cmd_id + + def nfvi_get_upgrade(callback): """ Get upgrade diff --git a/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_sw_mgmt_module.py b/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_sw_mgmt_module.py index 00b5a9cb..88795127 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_sw_mgmt_module.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_sw_mgmt_module.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2016 Wind River Systems, Inc. +# Copyright (c) 2016-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -37,6 +37,16 @@ def nfvi_sw_mgmt_update_host(host_name, callback): return cmd_id +def nfvi_sw_mgmt_apply_updates(patch_names, callback): + """ + Apply Software Patches + """ + cmd_id = _sw_mgmt_plugin.invoke_plugin('apply_patches', + patch_names, + callback=callback) + return cmd_id + + def nfvi_sw_mgmt_update_hosts(host_names, callback): """ Apply Software Patch to a list of hosts diff --git a/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/__init__.py b/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/__init__.py index 16dca43a..9410c77c 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/__init__.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -48,6 +48,10 @@ from nfv_vim.nfvi.objects.v1._instance_group import InstanceGroup # noqa: F401 from nfv_vim.nfvi.objects.v1._instance_type import INSTANCE_TYPE_EXTENSION # noqa: F401 from nfv_vim.nfvi.objects.v1._instance_type import InstanceType # noqa: F401 from nfv_vim.nfvi.objects.v1._instance_type import InstanceTypeAttributes # noqa: F401 +from nfv_vim.nfvi.objects.v1._kube_upgrade import KUBE_UPGRADE_STATE # noqa: F401 +from nfv_vim.nfvi.objects.v1._kube_upgrade import KubeHostUpgrade # noqa: F401 +from nfv_vim.nfvi.objects.v1._kube_upgrade import KubeUpgrade # noqa: F401 +from nfv_vim.nfvi.objects.v1._kube_upgrade import KubeVersion # noqa: F401 from nfv_vim.nfvi.objects.v1._network import Network # noqa: F401 from nfv_vim.nfvi.objects.v1._network import NETWORK_ADMIN_STATE # noqa: F401 from nfv_vim.nfvi.objects.v1._network import NETWORK_AVAIL_STATUS # noqa: F401 diff --git a/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py b/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py new file mode 100755 index 00000000..b7d03763 --- /dev/null +++ b/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py @@ -0,0 +1,102 @@ +# +# Copyright (c) 2016-2021 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +import six + +from nfv_common.helpers import Constant +from nfv_common.helpers import Constants +from nfv_common.helpers import Singleton + +from nfv_vim.nfvi.objects.v1._object import ObjectData + + +@six.add_metaclass(Singleton) +class KubeUpgradeState(Constants): + """ + Kube Upgrade State Constants + These values are copied from sysinv/common/kubernetes.py + """ + + KUBE_UPGRADE_STARTED = Constant('upgrade-started') + KUBE_UPGRADE_DOWNLOADING_IMAGES = Constant('downloading-images') + KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED = Constant('downloading-images-failed') + KUBE_UPGRADE_DOWNLOADED_IMAGES = Constant('downloaded-images') + KUBE_UPGRADING_FIRST_MASTER = Constant('upgrading-first-master') + KUBE_UPGRADING_FIRST_MASTER_FAILED = Constant('upgrading-first-master-failed') + KUBE_UPGRADED_FIRST_MASTER = Constant('upgraded-first-master') + KUBE_UPGRADING_NETWORKING = Constant('upgrading-networking') + KUBE_UPGRADING_NETWORKING_FAILED = Constant('upgrading-networking-failed') + KUBE_UPGRADED_NETWORKING = Constant('upgraded-networking') + KUBE_UPGRADING_SECOND_MASTER = Constant('upgrading-second-master') + KUBE_UPGRADING_SECOND_MASTER_FAILED = Constant('upgrading-second-master-failed') + KUBE_UPGRADED_SECOND_MASTER = Constant('upgraded-second-master') + KUBE_UPGRADING_KUBELETS = Constant('upgrading-kubelets') + KUBE_UPGRADE_COMPLETE = Constant('upgrade-complete') + + +# Kube Upgrade Constant Instantiation +KUBE_UPGRADE_STATE = KubeUpgradeState() + + +class KubeHostUpgrade(ObjectData): + """ + NFVI Kube Host Upgrade Object + """ + def __init__(self, + host_id, + host_uuid, + target_version, + control_plane_version, + kubelet_version, + status): + super(KubeHostUpgrade, self).__init__('1.0.0') + self.update( + dict(host_id=host_id, + host_uuid=host_uuid, + target_version=target_version, + control_plane_version=control_plane_version, + kubelet_version=kubelet_version, + status=status + ) + ) + + +class KubeUpgrade(ObjectData): + """ + NFVI Kube Upgrade Object + """ + def __init__(self, state, from_version, to_version): + super(KubeUpgrade, self).__init__('1.0.0') + self.update( + dict(state=state, + from_version=from_version, + to_version=to_version + ) + ) + + +class KubeVersion(ObjectData): + """ + NFVI Kube Version Object + """ + def __init__(self, + kube_version, + state, + target, + upgrade_from, + downgrade_to, + applied_patches, + available_patches): + super(KubeVersion, self).__init__('1.0.0') + self.update( + dict(kube_version=kube_version, + state=state, + target=target, + upgrade_from=upgrade_from, + downgrade_to=downgrade_to, + applied_patches=applied_patches, + available_patches=available_patches + ) + ) diff --git a/nfv/nfv-vim/nfv_vim/objects/__init__.py b/nfv/nfv-vim/nfv_vim/objects/__init__.py index edd69f72..2a42fd60 100755 --- a/nfv/nfv-vim/nfv_vim/objects/__init__.py +++ b/nfv/nfv-vim/nfv_vim/objects/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -26,6 +26,7 @@ from nfv_vim.objects._instance_group import InstanceGroup # noqa: F401 from nfv_vim.objects._instance_type import INSTANCE_TYPE_EXTENSION # noqa: F401 from nfv_vim.objects._instance_type import InstanceType # noqa: F401 from nfv_vim.objects._instance_type import InstanceTypeAttributes # noqa: F401 +from nfv_vim.objects._kube_upgrade import KubeUpgrade # noqa: F401 from nfv_vim.objects._network import Network # noqa: F401 from nfv_vim.objects._network import NetworkProviderData # noqa: F401 from nfv_vim.objects._service_host import ServiceHost # noqa: F401 diff --git a/nfv/nfv-vim/nfv_vim/objects/_kube_upgrade.py b/nfv/nfv-vim/nfv_vim/objects/_kube_upgrade.py new file mode 100644 index 00000000..014b17cb --- /dev/null +++ b/nfv/nfv-vim/nfv_vim/objects/_kube_upgrade.py @@ -0,0 +1,189 @@ +# +# Copyright (c) 2020-2021 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +from nfv_common import debug + +from nfv_common.helpers import coroutine + +from nfv_vim import alarm +from nfv_vim import event_log +from nfv_vim import nfvi + +from nfv_vim.objects._sw_update import SW_UPDATE_ALARM_TYPES +from nfv_vim.objects._sw_update import SW_UPDATE_EVENT_IDS +from nfv_vim.objects._sw_update import SW_UPDATE_TYPE +from nfv_vim.objects._sw_update import SwUpdate + +DLOG = debug.debug_get_logger('nfv_vim.objects.kube_upgrade') + + +class KubeUpgrade(SwUpdate): + """ + Kubernetes Upgrade Object + """ + def __init__(self, sw_update_uuid=None, strategy_data=None): + super(KubeUpgrade, self).__init__( + sw_update_type=SW_UPDATE_TYPE.KUBE_UPGRADE, + sw_update_uuid=sw_update_uuid, + strategy_data=strategy_data) + + self._kube_upgrade_hosts = list() + + def strategy_build(self, + strategy_uuid, + storage_apply_type, + worker_apply_type, + max_parallel_worker_hosts, + alarm_restrictions, + ignore_alarms, + to_version, + single_controller): + """ + Create a kubernetes upgrade strategy + """ + from nfv_vim import strategy + + if self._strategy: + reason = "strategy already exists" + return False, reason + + self._strategy = \ + strategy.KubeUpgradeStrategy(strategy_uuid, + storage_apply_type, + worker_apply_type, + max_parallel_worker_hosts, + alarm_restrictions, + ignore_alarms, + to_version, + single_controller) + self._strategy.sw_update_obj = self + self._strategy.build() + self._persist() + return True, '' + + def strategy_build_complete(self, success, reason): + """ + Creation of a kubernetes upgrade strategy complete + """ + DLOG.info("Kubernetes upgrade strategy build complete.") + pass + + @staticmethod + def alarm_type(alarm_type): + """ + Returns ALARM_TYPE corresponding to SW_UPDATE_ALARM_TYPES + """ + ALARM_TYPE_MAPPING = { + SW_UPDATE_ALARM_TYPES.APPLY_INPROGRESS: + alarm.ALARM_TYPE.KUBE_UPGRADE_AUTO_APPLY_INPROGRESS, + SW_UPDATE_ALARM_TYPES.APPLY_ABORTING: + alarm.ALARM_TYPE.KUBE_UPGRADE_AUTO_APPLY_ABORTING, + SW_UPDATE_ALARM_TYPES.APPLY_FAILED: + alarm.ALARM_TYPE.KUBE_UPGRADE_AUTO_APPLY_FAILED, + } + return ALARM_TYPE_MAPPING[alarm_type] + + @staticmethod + def event_id(event_id): + """ + Returns EVENT_ID corresponding to SW_UPDATE_EVENT_IDS + """ + EVENT_ID_MAPPING = { + SW_UPDATE_EVENT_IDS.APPLY_START: + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_START, + SW_UPDATE_EVENT_IDS.APPLY_INPROGRESS: + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_INPROGRESS, + SW_UPDATE_EVENT_IDS.APPLY_REJECTED: + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_REJECTED, + SW_UPDATE_EVENT_IDS.APPLY_CANCELLED: + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_CANCELLED, + SW_UPDATE_EVENT_IDS.APPLY_FAILED: + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_FAILED, + SW_UPDATE_EVENT_IDS.APPLY_COMPLETED: + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_COMPLETED, + SW_UPDATE_EVENT_IDS.APPLY_ABORT: + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORT, + SW_UPDATE_EVENT_IDS.APPLY_ABORTING: + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORTING, + SW_UPDATE_EVENT_IDS.APPLY_ABORT_REJECTED: + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORT_REJECTED, + SW_UPDATE_EVENT_IDS.APPLY_ABORT_FAILED: + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORT_FAILED, + SW_UPDATE_EVENT_IDS.APPLY_ABORTED: + event_log.EVENT_ID.KUBE_UPGRADE_AUTO_APPLY_ABORTED, + } + return EVENT_ID_MAPPING[event_id] + + def nfvi_update(self): + """ + NFVI Update + """ + if self._strategy is None: + if self._alarms: + alarm.clear_sw_update_alarm(self._alarms) + return False + + if self.strategy.is_applying(): + if not self._alarms: + self._alarms = alarm.raise_sw_update_alarm( + self.alarm_type(SW_UPDATE_ALARM_TYPES.APPLY_INPROGRESS)) + event_log.sw_update_issue_log( + self.event_id(SW_UPDATE_EVENT_IDS.APPLY_INPROGRESS)) + + elif (self.strategy.is_apply_failed() or + self.strategy.is_apply_timed_out()): + for kube_upgrade_host in self._kube_upgrade_hosts: + if not self._alarms: + self._alarms = alarm.raise_sw_update_alarm( + self.alarm_type(SW_UPDATE_ALARM_TYPES.APPLY_FAILED)) + event_log.sw_update_issue_log( + self.event_id(SW_UPDATE_EVENT_IDS.APPLY_FAILED)) + break + + else: + if self._alarms: + alarm.clear_sw_update_alarm(self._alarms) + return False + + elif self.strategy.is_aborting(): + if not self._alarms: + self._alarms = alarm.raise_sw_update_alarm( + self.alarm_type(SW_UPDATE_ALARM_TYPES.APPLY_ABORTING)) + event_log.sw_update_issue_log( + self.event_id(SW_UPDATE_EVENT_IDS.APPLY_ABORTING)) + + else: + if self._alarms: + alarm.clear_sw_update_alarm(self._alarms) + return False + + return True + + @coroutine + def nfvi_audit(self): + """ + Audit NFVI layer + """ + while True: + timer_id = (yield) + + DLOG.debug("Audit alarms, timer_id=%s." % timer_id) + self.nfvi_alarms_clear() + nfvi.nfvi_get_alarms(self.nfvi_alarms_callback(timer_id)) + if not nfvi.nfvi_fault_mgmt_plugin_disabled(): + nfvi.nfvi_get_openstack_alarms( + self.nfvi_alarms_callback(timer_id)) + self._nfvi_audit_inprogress = True + while self._nfvi_audit_inprogress: + timer_id = (yield) + + if not self.nfvi_update(): + DLOG.info("Audit no longer needed.") + break + + DLOG.verbose("Audit kube upgrade still running, timer_id=%s." % + timer_id) + + self._nfvi_timer_id = None diff --git a/nfv/nfv-vim/nfv_vim/objects/_sw_update.py b/nfv/nfv-vim/nfv_vim/objects/_sw_update.py index 1eee2083..2e7c49fa 100755 --- a/nfv/nfv-vim/nfv_vim/objects/_sw_update.py +++ b/nfv/nfv-vim/nfv_vim/objects/_sw_update.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2016-2020 Wind River Systems, Inc. +# Copyright (c) 2016-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -30,6 +30,7 @@ class SwUpdateTypes(Constants): SW_PATCH = Constant('sw-patch') SW_UPGRADE = Constant('sw-upgrade') FW_UPDATE = Constant('fw-update') + KUBE_UPGRADE = Constant('kube-upgrade') @six.add_metaclass(Singleton) diff --git a/nfv/nfv-vim/nfv_vim/rpc/__init__.py b/nfv/nfv-vim/nfv_vim/rpc/__init__.py index 016215e1..5be05bb7 100755 --- a/nfv/nfv-vim/nfv_vim/rpc/__init__.py +++ b/nfv/nfv-vim/nfv_vim/rpc/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2016 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -73,6 +73,7 @@ from nfv_vim.rpc._rpc_message_network import APIResponseUpdateNetwork # noqa: F from nfv_vim.rpc._rpc_message_sw_update import APIRequestAbortSwUpdateStrategy # noqa: F401 from nfv_vim.rpc._rpc_message_sw_update import APIRequestApplySwUpdateStrategy # noqa: F401 +from nfv_vim.rpc._rpc_message_sw_update import APIRequestCreateKubeUpgradeStrategy # noqa: F401 from nfv_vim.rpc._rpc_message_sw_update import APIRequestCreateSwUpdateStrategy # noqa: F401 from nfv_vim.rpc._rpc_message_sw_update import APIRequestCreateSwUpgradeStrategy # noqa: F401 from nfv_vim.rpc._rpc_message_sw_update import APIRequestDeleteSwUpdateStrategy # noqa: F401 diff --git a/nfv/nfv-vim/nfv_vim/rpc/_rpc_defs.py b/nfv/nfv-vim/nfv_vim/rpc/_rpc_defs.py index f09c3247..8670f2f9 100755 --- a/nfv/nfv-vim/nfv_vim/rpc/_rpc_defs.py +++ b/nfv/nfv-vim/nfv_vim/rpc/_rpc_defs.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2016 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -96,6 +96,7 @@ class _RPCMessageType(Constants): # Software Update Definitions CREATE_SW_UPDATE_STRATEGY_REQUEST = Constant('create-sw-update-strategy-request') + CREATE_KUBE_UPGRADE_STRATEGY_REQUEST = Constant('create-kube-upgrade-strategy-request') CREATE_SW_UPGRADE_STRATEGY_REQUEST = Constant('create-sw-upgrade-strategy-request') CREATE_SW_UPDATE_STRATEGY_RESPONSE = Constant('create-sw-update-strategy-response') APPLY_SW_UPDATE_STRATEGY_REQUEST = Constant('apply-sw-update-strategy-request') diff --git a/nfv/nfv-vim/nfv_vim/rpc/_rpc_message.py b/nfv/nfv-vim/nfv_vim/rpc/_rpc_message.py index 88353de6..f012afd4 100755 --- a/nfv/nfv-vim/nfv_vim/rpc/_rpc_message.py +++ b/nfv/nfv-vim/nfv_vim/rpc/_rpc_message.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2016 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -143,6 +143,7 @@ class RPCMessageFactory(object): from nfv_vim.rpc._rpc_message_sw_update import APIRequestAbortSwUpdateStrategy from nfv_vim.rpc._rpc_message_sw_update import APIRequestApplySwUpdateStrategy + from nfv_vim.rpc._rpc_message_sw_update import APIRequestCreateKubeUpgradeStrategy from nfv_vim.rpc._rpc_message_sw_update import APIRequestCreateSwUpdateStrategy from nfv_vim.rpc._rpc_message_sw_update import APIRequestCreateSwUpgradeStrategy from nfv_vim.rpc._rpc_message_sw_update import APIRequestDeleteSwUpdateStrategy @@ -224,6 +225,7 @@ class RPCMessageFactory(object): # Software Update Mapping RPC_MSG_TYPE.CREATE_SW_UPDATE_STRATEGY_REQUEST: APIRequestCreateSwUpdateStrategy, + RPC_MSG_TYPE.CREATE_KUBE_UPGRADE_STRATEGY_REQUEST: APIRequestCreateKubeUpgradeStrategy, RPC_MSG_TYPE.CREATE_SW_UPGRADE_STRATEGY_REQUEST: APIRequestCreateSwUpgradeStrategy, RPC_MSG_TYPE.CREATE_SW_UPDATE_STRATEGY_RESPONSE: APIResponseCreateSwUpdateStrategy, RPC_MSG_TYPE.APPLY_SW_UPDATE_STRATEGY_REQUEST: APIRequestApplySwUpdateStrategy, diff --git a/nfv/nfv-vim/nfv_vim/rpc/_rpc_message_sw_update.py b/nfv/nfv-vim/nfv_vim/rpc/_rpc_message_sw_update.py index d054eaf9..065adba2 100755 --- a/nfv/nfv-vim/nfv_vim/rpc/_rpc_message_sw_update.py +++ b/nfv/nfv-vim/nfv_vim/rpc/_rpc_message_sw_update.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2016 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -84,6 +84,30 @@ class APIRequestCreateSwUpgradeStrategy(APIRequestCreateSwUpdateStrategy): return "create-sw-upgrade-strategy request: %s" % self.deserialize_payload +class APIRequestCreateKubeUpgradeStrategy(APIRequestCreateSwUpdateStrategy): + """ + RPC API Request Message - Create Kube Upgrade Strategy + """ + to_version = None + + def __init__(self, msg_version=RPC_MSG_VERSION.VERSION_1_0, + msg_type=RPC_MSG_TYPE.CREATE_KUBE_UPGRADE_STRATEGY_REQUEST, + msg_result=RPC_MSG_RESULT.SUCCESS): + super(APIRequestCreateKubeUpgradeStrategy, self).__init__( + msg_version, msg_type, msg_result) + + def serialize_payload(self, msg): + super(APIRequestCreateKubeUpgradeStrategy, self).serialize_payload(msg) + msg['to_version'] = self.to_version + + def deserialize_payload(self, msg): + super(APIRequestCreateKubeUpgradeStrategy, self).deserialize_payload(msg) + self.to_version = msg.get('to_version', None) + + def __str__(self): + return "create-kube-upgrade-strategy request: %s" % self.deserialize_payload + + class APIResponseCreateSwUpdateStrategy(RPCMessage): """ RPC API Response Message - Create Software Update Strategy diff --git a/nfv/nfv-vim/nfv_vim/strategy/__init__.py b/nfv/nfv-vim/nfv_vim/strategy/__init__.py index c439e7ab..ef4f0e96 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/__init__.py +++ b/nfv/nfv-vim/nfv_vim/strategy/__init__.py @@ -1,22 +1,34 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # from nfv_common.strategy import * # noqa: F401,F403 from nfv_vim.strategy._strategy import FwUpdateStrategy # noqa: F401 +from nfv_vim.strategy._strategy import KubeUpgradeStrategy # noqa: F401 from nfv_vim.strategy._strategy import strategy_rebuild_from_dict # noqa: F401 from nfv_vim.strategy._strategy import SwPatchStrategy # noqa: F401 from nfv_vim.strategy._strategy import SwUpgradeStrategy # noqa: F401 from nfv_vim.strategy._strategy_defs import STRATEGY_EVENT # noqa: F401 from nfv_vim.strategy._strategy_stages import STRATEGY_STAGE_NAME # noqa: F401 +from nfv_vim.strategy._strategy_steps import ApplySwPatchesStep # noqa: F401 from nfv_vim.strategy._strategy_steps import DisableHostServicesStep # noqa: F401 from nfv_vim.strategy._strategy_steps import FwUpdateAbortHostsStep # noqa: F401 from nfv_vim.strategy._strategy_steps import FwUpdateHostsStep # noqa: F401 +from nfv_vim.strategy._strategy_steps import KubeHostUpgradeControlPlaneStep # noqa: F401 +from nfv_vim.strategy._strategy_steps import KubeHostUpgradeKubeletStep # noqa: F401 +from nfv_vim.strategy._strategy_steps import KubeUpgradeCleanupStep # noqa: F401 +from nfv_vim.strategy._strategy_steps import KubeUpgradeCompleteStep # noqa: F401 +from nfv_vim.strategy._strategy_steps import KubeUpgradeDownloadImagesStep # noqa: F401 +from nfv_vim.strategy._strategy_steps import KubeUpgradeNetworkingStep # noqa: F401 +from nfv_vim.strategy._strategy_steps import KubeUpgradeStartStep # noqa: F401 from nfv_vim.strategy._strategy_steps import LockHostsStep # noqa: F401 from nfv_vim.strategy._strategy_steps import MigrateInstancesStep # noqa: F401 from nfv_vim.strategy._strategy_steps import QueryAlarmsStep # noqa: F401 from nfv_vim.strategy._strategy_steps import QueryFwUpdateHostStep # noqa: F401 +from nfv_vim.strategy._strategy_steps import QueryKubeHostUpgradeStep # noqa: F401 +from nfv_vim.strategy._strategy_steps import QueryKubeUpgradeStep # noqa: F401 +from nfv_vim.strategy._strategy_steps import QueryKubeVersionsStep # noqa: F401 from nfv_vim.strategy._strategy_steps import QuerySwPatchesStep # noqa: F401 from nfv_vim.strategy._strategy_steps import QuerySwPatchHostsStep # noqa: F401 from nfv_vim.strategy._strategy_steps import QueryUpgradeStep # noqa: F401 diff --git a/nfv/nfv-vim/nfv_vim/strategy/_strategy.py b/nfv/nfv-vim/nfv_vim/strategy/_strategy.py index e1a25b17..69fe3df3 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/_strategy.py +++ b/nfv/nfv-vim/nfv_vim/strategy/_strategy.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -22,7 +22,6 @@ from nfv_vim.objects import INSTANCE_GROUP_POLICY from nfv_vim.objects import SW_UPDATE_APPLY_TYPE from nfv_vim.objects import SW_UPDATE_INSTANCE_ACTION - DLOG = debug.debug_get_logger('nfv_vim.strategy') @@ -34,6 +33,7 @@ class StrategyNames(Constants): SW_PATCH = Constant('sw-patch') SW_UPGRADE = Constant('sw-upgrade') FW_UPDATE = Constant('fw-update') + KUBE_UPGRADE = Constant('kube-upgrade') # Constant Instantiation @@ -46,6 +46,10 @@ MTCE_DELAY = 15 # a no-reboot patch can stabilize in 30 seconds NO_REBOOT_DELAY = 30 +# constants used by the patching API for state and repo state +PATCH_REPO_STATE_APPLIED = 'Applied' +PATCH_STATE_APPLIED = 'Applied' + ################################################################### # @@ -556,7 +560,8 @@ class SwPatchStrategy(SwUpdateStrategy): stage = strategy.StrategyStage( strategy.STRATEGY_STAGE_NAME.SW_PATCH_QUERY) - stage.add_step(strategy.QueryAlarmsStep(ignore_alarms=self._ignore_alarms)) + stage.add_step( + strategy.QueryAlarmsStep(ignore_alarms=self._ignore_alarms)) stage.add_step(strategy.QuerySwPatchesStep()) stage.add_step(strategy.QuerySwPatchHostsStep()) self.build_phase.add_stage(stage) @@ -2079,6 +2084,1012 @@ class FwUpdateStrategy(SwUpdateStrategy): return data +# Query steps require fields on the strategy to populate and store results +# each query step should have a mixin to simplify using it for a strategy + +class QueryMixinBase(object): + """ + QueryMixinBase stubs the query mixin classes. + + The methods do not call super, and stop the method invocation chain. + """ + + def initialize_mixin(self): + pass + + def mixin_from_dict(self, data): + pass + + def mixin_as_dict(self, data): + pass + + +class QuerySwPatchesMixin(QueryMixinBase): + """This mixin is used through the QuerySwPatchesStep class""" + + def initialize_mixin(self): + super(QuerySwPatchesMixin, self).initialize_mixin() + self._nfvi_sw_patches = list() + + @property + def nfvi_sw_patches(self): + """ + Returns the software patches from the NFVI layer + """ + return self._nfvi_sw_patches + + @nfvi_sw_patches.setter + def nfvi_sw_patches(self, nfvi_sw_patches): + """ + Save the software patches from the NFVI Layer + """ + self._nfvi_sw_patches = nfvi_sw_patches + + def mixin_from_dict(self, data): + """ + Extracts this mixin data from a dictionary + """ + super(QuerySwPatchesMixin, self).mixin_from_dict(data) + + from nfv_vim import nfvi + + mixin_data = list() + for sw_patch_data in data['nfvi_sw_patches_data']: + sw_patch = nfvi.objects.v1.SwPatch( + sw_patch_data['name'], + sw_patch_data['sw_version'], + sw_patch_data['repo_state'], + sw_patch_data['patch_state']) + mixin_data.append(sw_patch) + self._nfvi_sw_patches = mixin_data + + def mixin_as_dict(self, data): + """ + Updates the dictionary with this mixin data + """ + super(QuerySwPatchesMixin, self).mixin_as_dict(data) + mixin_data = list() + for sw_patch in self._nfvi_sw_patches: + mixin_data.append(sw_patch.as_dict()) + data['nfvi_sw_patches_data'] = mixin_data + + +class QuerySwPatchHostsMixin(QueryMixinBase): + """This mixin is used through the QuerySwPatchHostsStep class""" + + def initialize_mixin(self): + super(QuerySwPatchHostsMixin, self).initialize_mixin() + self._nfvi_sw_patch_hosts = list() + + @property + def nfvi_sw_patch_hosts(self): + """ + Returns the software patch hosts from the NFVI layer + """ + return self._nfvi_sw_patch_hosts + + @nfvi_sw_patch_hosts.setter + def nfvi_sw_patch_hosts(self, nfvi_sw_patch_hosts): + """ + Save the software patch hosts from the NFVI Layer + """ + self._nfvi_sw_patch_hosts = nfvi_sw_patch_hosts + + def mixin_from_dict(self, data): + """ + Extracts this mixin data from a dictionary + """ + super(QuerySwPatchHostsMixin, self).mixin_from_dict(data) + + from nfv_vim import nfvi + + mixin_data = list() + for host_data in data['nfvi_sw_patch_hosts_data']: + host = nfvi.objects.v1.HostSwPatch( + host_data['name'], host_data['personality'], + host_data['sw_version'], host_data['requires_reboot'], + host_data['patch_current'], host_data['state'], + host_data['patch_failed'], host_data['interim_state']) + mixin_data.append(host) + self._nfvi_sw_patch_hosts = mixin_data + + def mixin_as_dict(self, data): + """ + Updates the dictionary with this mixin data + """ + super(QuerySwPatchHostsMixin, self).mixin_as_dict(data) + mixin_data = list() + for host in self._nfvi_sw_patch_hosts: + mixin_data.append(host.as_dict()) + data['nfvi_sw_patch_hosts_data'] = mixin_data + + +class QueryKubeUpgradesMixin(QueryMixinBase): + """This mixin is used through the QueryKubeUpgradesStep class""" + + def initialize_mixin(self): + super(QueryKubeUpgradesMixin, self).initialize_mixin() + self._nfvi_kube_upgrade = None + + @property + def nfvi_kube_upgrade(self): + """ + Returns the kube upgrade from the NFVI layer + """ + return self._nfvi_kube_upgrade + + @nfvi_kube_upgrade.setter + def nfvi_kube_upgrade(self, nfvi_kube_upgrade): + """ + Save the kube upgrade from the NFVI Layer + """ + self._nfvi_kube_upgrade = nfvi_kube_upgrade + + def mixin_from_dict(self, data): + """ + Extracts this mixin data from a dictionary + """ + super(QueryKubeUpgradesMixin, self).mixin_from_dict(data) + + from nfv_vim import nfvi + + mixin_data = data['nfvi_kube_upgrade_data'] + if mixin_data: + self._nfvi_kube_upgrade = nfvi.objects.v1.KubeUpgrade( + mixin_data['state'], + mixin_data['from_version'], + mixin_data['to_version']) + else: + self._nfvi_kube_upgrade = None + + def mixin_as_dict(self, data): + """ + Updates the dictionary with this mixin data + """ + super(QueryKubeUpgradesMixin, self).mixin_as_dict(data) + mixin_data = None + if self._nfvi_kube_upgrade: + mixin_data = self._nfvi_kube_upgrade.as_dict() + data['nfvi_kube_upgrade_data'] = mixin_data + + +class QueryKubeHostUpgradesMixin(QueryMixinBase): + """This mixin is used through the QueryKubeHostUpgradesStep class""" + + def initialize_mixin(self): + super(QueryKubeHostUpgradesMixin, self).initialize_mixin() + self._nfvi_kube_host_upgrade_list = list() + + @property + def nfvi_kube_host_upgrade_list(self): + """ + Returns the kube host upgrade list from the NFVI layer + """ + return self._nfvi_kube_host_upgrade_list + + @nfvi_kube_host_upgrade_list.setter + def nfvi_kube_host_upgrade_list(self, nfvi_kube_host_upgrade_list): + """ + Save the kube host upgrade list from the NFVI Layer + """ + self._nfvi_kube_host_upgrade_list = nfvi_kube_host_upgrade_list + + def mixin_from_dict(self, data): + """ + Extracts this mixin data from a dictionary + """ + super(QueryKubeHostUpgradesMixin, self).mixin_from_dict(data) + + from nfv_vim import nfvi + + mixin_data = list() + for kube_host_upgrade_data in data['nfvi_kube_host_upgrade_list_data']: + kube_host_upgrade = nfvi.objects.v1.KubeHostUpgrade( + kube_host_upgrade_data['host_id'], + kube_host_upgrade_data['host_uuid'], + kube_host_upgrade_data['target_version'], + kube_host_upgrade_data['control_plane_version'], + kube_host_upgrade_data['kubelet_version'], + kube_host_upgrade_data['status']) + mixin_data.append(kube_host_upgrade) + self._nfvi_kube_host_upgrade_list = mixin_data + + def mixin_as_dict(self, data): + """ + Updates the dictionary with this mixin data + """ + super(QueryKubeHostUpgradesMixin, self).mixin_as_dict(data) + mixin_data = list() + for kube_host_upgrade in self._nfvi_kube_host_upgrade_list: + mixin_data.append(kube_host_upgrade.as_dict()) + data['nfvi_kube_host_upgrade_list_data'] = mixin_data + + +class QueryKubeVersionsMixin(QueryMixinBase): + """This mixin is used through the QueryKubeVersionsStep class""" + + def initialize_mixin(self): + super(QueryKubeVersionsMixin, self).initialize_mixin() + self._nfvi_kube_versions_list = list() + + @property + def nfvi_kube_versions_list(self): + """ + Returns the kube versions list from the NFVI layer + """ + return self._nfvi_kube_versions_list + + @nfvi_kube_versions_list.setter + def nfvi_kube_versions_list(self, nfvi_kube_versions_list): + """ + Save the kube versions list from the NFVI Layer + """ + self._nfvi_kube_versions_list = nfvi_kube_versions_list + + def mixin_from_dict(self, data): + """ + Extracts this mixin data from a dictionary + """ + super(QueryKubeVersionsMixin, self).mixin_from_dict(data) + + from nfv_vim import nfvi + + mixin_data = list() + for data_item in data['nfvi_kube_versions_list_data']: + mixin_object = nfvi.objects.v1.KubeVersion( + data_item['kube_version'], + data_item['state'], + data_item['target'], + data_item['upgrade_from'], + data_item['downgrade_to'], + data_item['applied_patches'], + data_item['available_patches']) + mixin_data.append(mixin_object) + self._nfvi_kube_versions_list = mixin_data + + def mixin_as_dict(self, data): + """ + Updates the dictionary with this mixin data + """ + super(QueryKubeVersionsMixin, self).mixin_as_dict(data) + mixin_data = list() + for mixin_obj in self._nfvi_kube_versions_list: + mixin_data.append(mixin_obj.as_dict()) + data['nfvi_kube_versions_list_data'] = mixin_data + + +################################################################### +# +# The Kubernetes Upgrade Strategy +# +################################################################### +class KubeUpgradeStrategy(SwUpdateStrategy, + QueryKubeUpgradesMixin, + QueryKubeHostUpgradesMixin, + QueryKubeVersionsMixin, + QuerySwPatchesMixin, + QuerySwPatchHostsMixin): + """ + Kubernetes Upgrade - Strategy + """ + def __init__(self, + uuid, + storage_apply_type, + worker_apply_type, + max_parallel_worker_hosts, + alarm_restrictions, + ignore_alarms, + to_version, + single_controller): + super(KubeUpgradeStrategy, self).__init__( + uuid, + STRATEGY_NAME.KUBE_UPGRADE, + SW_UPDATE_APPLY_TYPE.SERIAL, + storage_apply_type, + SW_UPDATE_APPLY_TYPE.IGNORE, + worker_apply_type, + max_parallel_worker_hosts, + SW_UPDATE_INSTANCE_ACTION.MIGRATE, + alarm_restrictions, + ignore_alarms) + + # The following alarms will NOT prevent a kube upgrade operation + # Note: if an alarm is critical (ex: memory), it will still block the + # kube upgrade due to the host being degraded. + IGNORE_ALARMS = [ + '100.103', # Memory threshold exceeded + '200.001', # Locked Host + '280.001', # Subcloud resource off-line + '280.002', # Subcloud resource out-of-sync + '700.004', # VM stopped + '750.006', # Configuration change requires reapply of cert-manager + '900.007', # Kube Upgrade in progress + '900.401', # kube-upgrade-auto-apply-inprogress + ] + # self._ignore_alarms is declared in parent class + self._ignore_alarms += IGNORE_ALARMS + + # to_version and single_controller MUST be serialized + self._to_version = to_version + self._single_controller = single_controller + + # initialize the variables required by the mixins + self.initialize_mixin() + + @property + def to_version(self): + """ + Returns the read only kube upgrade 'to_version' for this strategy + """ + return self._to_version + + @property + def is_simplex(self): + """ + Returns the read only 'single_controller' attribute which indicates + if this is a simplex system (only one controller) + """ + return self._single_controller + + @property + def is_duplex(self): + """ + Indicate if this is a duplex based on the read only single_controller + attribute (if its not a single controller, its a duplex) + """ + return not self._single_controller + + def build(self): + """ + Build the strategy + """ + from nfv_vim import strategy + + # Initial stage is a query of existing kube upgrade + stage = strategy.StrategyStage( + strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_QUERY) + stage.add_step(strategy.QueryAlarmsStep( + ignore_alarms=self._ignore_alarms)) + # these query steps are paired with mixins that process their results + stage.add_step(strategy.QueryKubeVersionsStep()) + stage.add_step(strategy.QueryKubeUpgradeStep()) + stage.add_step(strategy.QueryKubeHostUpgradeStep()) + stage.add_step(strategy.QuerySwPatchesStep()) + stage.add_step(strategy.QuerySwPatchHostsStep()) + + self.build_phase.add_stage(stage) + super(KubeUpgradeStrategy, self).build() + + def _kubelet_map(self): + """Map the host kubelet versions by the host uuid. + Leave the kubelet version empty, if the status is not None, + since that means the kubelet may not be running the version + indicated. ie: upgrading-kubelet-failed + """ + kubelet_map = dict() + for host in self.nfvi_kube_host_upgrade_list: + # if host status is anything but None, it means the kubelet may + # not yet be fully upgraded to the version indicated. + if host.status is None: + kubelet_map[host.host_uuid] = host.kubelet_version + return kubelet_map + + def _add_controller_kubelet_stages(self, controllers, reboot): + """ + Add controller kube upgrade strategy stages to upgrade kubelets + For most controller configurations the steps are: + swact away/ lock / upgrade kubelet / unlock + For AIO-SX there is only + upgrade kubelet + (the controller must not be locked) + todo(abailey): Similar logic for VMs is needed as with reboot patches + This method must return a tuple: Boolean,String + """ + # declare a utility method for adding the kubelet stage for controller + def create_kubelet_stage(host): + """Utility method to declare a upgrade kubelet controller stage""" + from nfv_vim import strategy + + # force=True to allow recovery from previously attempts to + # upgrade kubelet that failed. + force = True + host_list = [host] + stage = strategy.StrategyStage( + strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_KUBELETS_CONTROLLERS) + if self.is_duplex: + stage.add_step(strategy.SwactHostsStep(host_list)) + stage.add_step(strategy.LockHostsStep(host_list)) + stage.add_step(strategy.KubeHostUpgradeKubeletStep(host, force)) + stage.add_step( + strategy.SystemStabilizeStep(timeout_in_secs=MTCE_DELAY)) + if self.is_duplex: + stage.add_step(strategy.UnlockHostsStep(host_list)) + # todo(abailey): add support related to vms restart + stage.add_step(strategy.WaitAlarmsClearStep( + timeout_in_secs=30 * 60, + ignore_alarms=self._ignore_alarms)) + return stage + + # determine which controller is controller-0 and controller-1 + controller_0_host = None + controller_1_host = None + for host in controllers: + if HOST_NAME.CONTROLLER_0 == host.name: + controller_0_host = host + elif HOST_NAME.CONTROLLER_1 == host.name: + controller_1_host = host + + kubelet_map = self._kubelet_map() + # always add controller-1 stage before controller-0 + if controller_1_host is not None: + if kubelet_map.get(controller_1_host.uuid) == self._to_version: + DLOG.info("Controller-1 kubelet already up to date") + else: + self.apply_phase.add_stage( + create_kubelet_stage(controller_1_host)) + if controller_0_host is not None: + if kubelet_map.get(controller_0_host.uuid) == self._to_version: + DLOG.info("Controller-0 kubelet already up to date") + else: + self.apply_phase.add_stage( + create_kubelet_stage(controller_0_host)) + return True, '' + + def _add_worker_kubelet_stages(self, hosts, reboot): + """ + Add worker kube upgrade strategy stages to upgrade kubelets + For most workers the steps are: + lock / upgrade kubelet / unlock + todo(abailey): Similar logic for VMs is needed as with reboot patches + This method must return a tuple: Boolean,String + """ + def create_kubelet_worker_stage(host): + """Utility method to declare a upgrade kubelet worker stage""" + from nfv_vim import strategy + + host_list = [host] + stage = strategy.StrategyStage( + strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_KUBELETS_WORKERS) + stage.add_step(strategy.LockHostsStep(host_list)) + # force flag allows re-attempt if a previous kubelet upgrade failed + stage.add_step(strategy.KubeHostUpgradeKubeletStep(host, + force=True)) + stage.add_step( + strategy.SystemStabilizeStep(timeout_in_secs=MTCE_DELAY)) + stage.add_step(strategy.UnlockHostsStep(host_list)) + # todo(abailey): add support related to vms restart + stage.add_step(strategy.WaitAlarmsClearStep( + timeout_in_secs=30 * 60, + ignore_alarms=self._ignore_alarms)) + return stage + + kubelet_map = self._kubelet_map() + for host in hosts: + if kubelet_map.get(host.uuid) == self._to_version: + DLOG.info("%s kubelet already up to date" % host.name) + else: + self.apply_phase.add_stage(create_kubelet_worker_stage(host)) + return True, '' + + def _add_kube_upgrade_start_stage(self): + """ + Add upgrade start strategy stage + This stage only occurs when no kube upgrade has been initiated. + """ + from nfv_vim import strategy + stage = strategy.StrategyStage( + strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_START) + stage.add_step(strategy.KubeUpgradeStartStep(self._to_version, + force=True)) + self.apply_phase.add_stage(stage) + # Add the stage that comes after the kube upgrade start stage + self._add_kube_upgrade_download_images_stage() + + def _add_kube_upgrade_download_images_stage(self): + """ + Add downloading images stage + This stage only occurs when kube upgrade has been started. + It then proceeds to the next stage + """ + from nfv_vim import strategy + + stage = strategy.StrategyStage( + strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_DOWNLOAD_IMAGES) + stage.add_step(strategy.KubeUpgradeDownloadImagesStep()) + self.apply_phase.add_stage(stage) + # Next stage after download images is upgrade control plane for first + self._add_kube_upgrade_first_control_plane_stage() + + def _add_kube_upgrade_first_control_plane_stage(self): + """ + Add first controller control plane kube upgrade stage + This stage only occurs after images are downloaded. + It then proceeds to the next stage + """ + from nfv_vim import nfvi + from nfv_vim import strategy + + stage = strategy.StrategyStage( + strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_FIRST_CONTROL_PLANE) + first_host = self.get_first_host() + # force argument is ignored by control plane API + force = True + stage.add_step(strategy.KubeHostUpgradeControlPlaneStep( + first_host, + force, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADED_FIRST_MASTER, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADING_FIRST_MASTER_FAILED) + ) + self.apply_phase.add_stage(stage) + # Next stage after first control plane is networking + self._add_kube_upgrade_networking_stage() + + def _add_kube_upgrade_networking_stage(self): + """ + Add kube upgrade networking stage. + This stage only occurs after the first control plane is upgraded. + It then proceeds to the next stage + """ + from nfv_vim import strategy + + stage = strategy.StrategyStage( + strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_NETWORKING) + stage.add_step(strategy.KubeUpgradeNetworkingStep()) + self.apply_phase.add_stage(stage) + # Next stage after networking is second control plane (if duplex) + self._add_kube_upgrade_second_control_plane_stage() + + def _add_kube_upgrade_second_control_plane_stage(self): + """ + Add second control plane kube upgrade stage + This stage only occurs after networking and if this is a duplex. + It then proceeds to the next stage + """ + from nfv_vim import nfvi + from nfv_vim import strategy + + second_host = self.get_second_host() + if second_host is not None: + # force argument is ignored by control plane API + force = True + stage = strategy.StrategyStage( + strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_SECOND_CONTROL_PLANE) + stage.add_step(strategy.KubeHostUpgradeControlPlaneStep( + second_host, + force, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADED_SECOND_MASTER, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADING_SECOND_MASTER_FAILED) + ) + self.apply_phase.add_stage(stage) + # Next stage after second control plane is to apply kube patch + self._add_kube_upgrade_patch_stage() + + def _check_host_patch_current(self, host, new_patches): + # If any new patches have been applied, assume the host + # if not patch current. If a patch was controller or worker only + # then this assumption may not be true. + if new_patches: + return False + for host_entry in self._nfvi_sw_patch_hosts: + if host_entry['name'] == host.name: + return host_entry['patch_current'] + # Did not find a matching entry in the sw patch hosts list. + # Since we cannot determine if it is patch current, return False + return False + + def _add_kube_upgrade_patch_stage(self): + """ + Add patch steps for the kubelet patch + If required 'applied' patches have not already been applied, fail this + stage. This stage is meant to apply the patches tagged as 'available' + for the kube upgrade. The patches are then installed on the hosts. + """ + from nfv_vim import strategy + from nfv_vim import tables + + applied_patches = None + available_patches = None + for kube_version_object in self.nfvi_kube_versions_list: + if kube_version_object['kube_version'] == self._to_version: + applied_patches = kube_version_object['applied_patches'] + available_patches = kube_version_object['available_patches'] + break + + # todo(abailey): handle 'committed' state + + # This section validates the 'applied_patches' for a kube upgrade. + # Note: validation fails on the first required patch in wrong state + # it does not indicate all pre-requisite patches that are invalid. + if applied_patches: + for kube_patch in applied_patches: + matching_patch = None + for patch in self.nfvi_sw_patches: + if patch['name'] == kube_patch: + matching_patch = patch + break + # - Fail if the required patch is missing + # - Fail if the required patch is not applied + # - Fail if the required patch is not installed on all hosts + if matching_patch is None: + self.report_build_failure("Missing a required patch: [%s]" + % kube_patch) + return + elif matching_patch['repo_state'] != PATCH_REPO_STATE_APPLIED: + self.report_build_failure( + "Required pre-applied patch: [%s] is not applied." + % kube_patch) + return + elif matching_patch['patch_state'] != PATCH_STATE_APPLIED: + self.report_build_failure( + "Required patch: [%s] is not installed on all hosts." + % kube_patch) + return + else: + DLOG.debug("Verified patch: [%s] is applied and installed" + % kube_patch) + + # This section validates the 'available_patches' for a kube upgrade. + # It also sets up the apply and install steps. + # 'available_patches' are the patches that need to be applied and + # installed on all hosts during kube upgrade orchestration after the + # control plane has been setup. + patches_to_apply = [] + patches_need_host_install = False + if available_patches: + for kube_patch in available_patches: + matching_patch = None + for patch in self.nfvi_sw_patches: + if patch['name'] == kube_patch: + matching_patch = patch + break + # - Fail if the required patch is missing + # - Apply the patch if it is not yet applied + # - Install the patch on any hosts where it is not installed. + if matching_patch is None: + self.report_build_failure("Missing a required patch: [%s]" + % kube_patch) + return + # if there is an applied_patch that is not applied, fail + elif matching_patch['repo_state'] != PATCH_REPO_STATE_APPLIED: + DLOG.debug("Preparing to apply available patch %s" + % kube_patch) + patches_to_apply.append(kube_patch) + # we apply the patch, so it must be installed on the hosts + patches_need_host_install = True + elif matching_patch['patch_state'] != PATCH_STATE_APPLIED: + # One of the patches is not fully installed on all hosts + patches_need_host_install = True + else: + DLOG.debug("Skipping available patch %s already applied" + % kube_patch) + + if patches_to_apply or patches_need_host_install: + # Combine the patch apply with the host patch install in one stage + stage_populated = False + stage = strategy.StrategyStage( + strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_PATCH) + + # First step are the patches if they must be applied + if patches_to_apply: + stage.add_step(strategy.ApplySwPatchesStep(patches_to_apply)) + stage_populated = True + + controller_0_host = None + controller_1_host = None + worker_hosts = [] + storage_hosts = [] + host_table = tables.tables_get_host_table() + for host in host_table.values(): + # filter the host out if we do not need to patch it + if not self._check_host_patch_current(host, patches_to_apply): + if HOST_NAME.CONTROLLER_0 == host.name: + controller_0_host = host + elif HOST_NAME.CONTROLLER_1 == host.name: + controller_1_host = host + elif HOST_PERSONALITY.WORKER in host.personality: + worker_hosts.append(host) + elif HOST_PERSONALITY.STORAGE in host.personality: + storage_hosts.append(host) + else: + DLOG.error("Logic Error. Unprocessed host: %s of %s" + % (host.name, host.personality)) + + # Process controller-1 first, if it needs to be patched + if controller_1_host: + # add controller-1 as a list to this step + stage.add_step(strategy.SwPatchHostsStep([controller_1_host])) + stage_populated = True + # Process controller-0 if it needs to be patched + if controller_0_host: + # add controller-0 as a list to this step + stage.add_step(strategy.SwPatchHostsStep([controller_0_host])) + stage_populated = True + # We can patch storage hosts next. kubernetes does not run on + # storage hosts, but its rpms are still installed there + if storage_hosts: + stage.add_step(strategy.SwPatchHostsStep(storage_hosts)) + stage_populated = True + # do worker hosts last. AIO were done during controller phase + if worker_hosts: + stage.add_step(strategy.SwPatchHostsStep(worker_hosts)) + stage_populated = True + + # this stage_populated check should not be necessary, once all + # hosts are able to be patched. + if not stage_populated: + DLOG.error("Logic Error. Upgrade Patch stage is empty.") + + self.apply_phase.add_stage(stage) + else: + DLOG.info("No 'available_patches' need to be applied or installed") + + # next stage after this are kubelets, which are updated for all hosts + self._add_kube_upgrade_kubelets_stage() + + def _add_kube_upgrade_kubelets_stage(self): + from nfv_vim import tables + + host_table = tables.tables_get_host_table() + + controller_hosts = list() + worker_hosts = list() + + # sort the hosts by their type (controller, storage, worker) + # there are no kubelets running on storage nodes + for host in host_table.values(): + if HOST_PERSONALITY.CONTROLLER in host.personality: + controller_hosts.append(host) + elif HOST_PERSONALITY.WORKER in host.personality: + worker_hosts.append(host) + else: + DLOG.info("No kubelet stage required for host %s" % host.name) + + # kubelet order is: controllers, storage (N/A), then workers + HOST_STAGES = [ + (self._add_controller_kubelet_stages, controller_hosts, True), + (self._add_worker_kubelet_stages, worker_hosts, True) + ] + for add_kubelet_stages_function, host_list, reboot in HOST_STAGES: + if host_list: + success, reason = add_kubelet_stages_function(host_list, + reboot) + if not success: + self.report_build_failure(reason) + return + # stage after kubelets is kube upgrade complete stage + self._add_kube_upgrade_complete_stage() + + def _add_kube_upgrade_complete_stage(self): + """ + Add kube upgrade complete strategy stage + This stage occurs after all kubelets are upgraded + """ + from nfv_vim import strategy + stage = strategy.StrategyStage( + strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_COMPLETE) + stage.add_step(strategy.KubeUpgradeCompleteStep()) + self.apply_phase.add_stage(stage) + # stage after kube upgrade complete stage, cleans up the kube upgrade + self._add_kube_upgrade_cleanup_stage() + + def _add_kube_upgrade_cleanup_stage(self): + """ + kube upgrade cleanup stage deletes the kube upgrade. + This stage occurs after all kube upgrade is completed + """ + from nfv_vim import strategy + stage = strategy.StrategyStage( + strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_CLEANUP) + stage.add_step(strategy.KubeUpgradeCleanupStep()) + self.apply_phase.add_stage(stage) + + def report_build_failure(self, reason): + DLOG.warn("Strategy Build Failed: %s" % reason) + self._state = strategy.STRATEGY_STATE.BUILD_FAILED + self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED + self.build_phase.result_reason = reason + self.sw_update_obj.strategy_build_complete( + False, + self.build_phase.result_reason) + self.save() + + def get_first_host(self): + """ + This corresponds to the first host that should be updated. + In simplex env, first host: controller-0. In duplex env: controller-1 + """ + from nfv_vim import tables + + controller_0_host = None + controller_1_host = None + host_table = tables.tables_get_host_table() + for host in host_table.get_by_personality(HOST_PERSONALITY.CONTROLLER): + if HOST_NAME.CONTROLLER_0 == host.name: + controller_0_host = host + if HOST_NAME.CONTROLLER_1 == host.name: + controller_1_host = host + if controller_1_host is None: + # simplex + return controller_0_host + else: + # duplex + return controller_1_host + + def get_second_host(self): + """ + This corresponds to the second host that should be updated. + In simplex env, second host: None. In duplex env: controller-0 + """ + from nfv_vim import tables + controller_0_host = None + controller_1_host = None + host_table = tables.tables_get_host_table() + for host in host_table.get_by_personality(HOST_PERSONALITY.CONTROLLER): + if HOST_NAME.CONTROLLER_0 == host.name: + controller_0_host = host + if HOST_NAME.CONTROLLER_1 == host.name: + controller_1_host = host + if controller_1_host is None: + # simplex + return None + else: + # duplex + return controller_0_host + + def build_complete(self, result, result_reason): + """ + Strategy Build Complete + """ + from nfv_vim import nfvi + from nfv_vim import strategy + + # Note: there are no resume states for actions that are still running + # ie: KUBE_UPGRADE_DOWNLOADING_IMAGES + RESUME_STATE = { + # after upgrade-started -> download images + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTED: + self._add_kube_upgrade_download_images_stage, + + # if downloading images failed, resume at downloading images + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED: + self._add_kube_upgrade_download_images_stage, + + # After downloaing images -> upgrade first control plane + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADED_IMAGES: + self._add_kube_upgrade_first_control_plane_stage, + + # if upgrading first control plane failed, resume there + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADING_FIRST_MASTER_FAILED: + self._add_kube_upgrade_first_control_plane_stage, + + # After first control plane, upgrade networking + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADED_FIRST_MASTER: + self._add_kube_upgrade_networking_stage, + + # if networking state failed, resyne at networking state + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADING_NETWORKING_FAILED: + self._add_kube_upgrade_networking_stage, + + # After networking , upgrade second control plane + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADED_NETWORKING: + self._add_kube_upgrade_second_control_plane_stage, + + # if upgrading second control plane failed, resume there + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADING_SECOND_MASTER_FAILED: + self._add_kube_upgrade_second_control_plane_stage, + + # After second control plane , proceed with patching + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADED_SECOND_MASTER: + self._add_kube_upgrade_patch_stage, + + # kubelets are next kube upgrade phase after second patch applied + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADING_KUBELETS: + self._add_kube_upgrade_kubelets_stage, + + # kubelets applied and upgrade is completed, delete the upgrade + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_COMPLETE: + self._add_kube_upgrade_cleanup_stage, + } + + result, result_reason = \ + super(KubeUpgradeStrategy, self).build_complete(result, + result_reason) + + DLOG.verbose("Build Complete Callback, result=%s, reason=%s." + % (result, result_reason)) + + if result in [strategy.STRATEGY_RESULT.SUCCESS, + strategy.STRATEGY_RESULT.DEGRADED]: + + matching_version_upgraded = False + for kube_version_object in self._nfvi_kube_versions_list: + if kube_version_object['kube_version'] == self._to_version: + # found a matching version. check if already upgraded + matching_version_upgraded = (kube_version_object['target'] + and kube_version_object['state'] == 'active') + break + else: + # the for loop above did not find a matching kube version + DLOG.warn("Invalid to_version(%s) for the kube upgrade" + % self._to_version) + self.report_build_failure("Invalid to_version value: '%s'" + % self._to_version) + return + + if self._nfvi_alarms: + # Fail create strategy if unignored alarms present + # add the alarm ids to the result reason. + # eliminate duplicates using a set, and sort the list + alarm_id_set = set() + for alarm_data in self._nfvi_alarms: + alarm_id_set.add(alarm_data['alarm_id']) + alarm_id_list = ", ".join(sorted(alarm_id_set)) + + DLOG.warn("Cannot upgrade kube: Active alarms present [ %s ]" + % alarm_id_list) + self.report_build_failure("active alarms present [ %s ]" + % alarm_id_list) + return + + if self.nfvi_kube_upgrade is None: + # We only reject creating a new kube upgrade for an already + # upgraded version if no kube_upgrade exists + if matching_version_upgraded: + self.report_build_failure( + "Kubernetes is already upgraded to: %s" + % self._to_version) + return + # Do NOT start a kube upgrade if none exists AND the + # to_version is already active + # Start upgrade which adds all stages + self._add_kube_upgrade_start_stage() + else: + # Determine which stage to resume at + current_state = self.nfvi_kube_upgrade.state + resume_from_stage = RESUME_STATE.get(current_state) + if resume_from_stage is None: + self.report_build_failure( + "Unable to resume kube upgrade from state: %s" + % current_state) + return + else: + # Invoke the method that resumes the build from the stage + resume_from_stage() + + else: + # build did not succeed. set failed. + self.report_build_failure(result_reason) + return + + # successful build + self.sw_update_obj.strategy_build_complete(True, '') + self.save() + + def from_dict(self, data, build_phase=None, apply_phase=None, + abort_phase=None): + """ + Initializes a kube upgrade strategy object using the given dictionary + """ + super(KubeUpgradeStrategy, self).from_dict(data, + build_phase, + apply_phase, + abort_phase) + self._to_version = data['to_version'] + self._single_controller = data['single_controller'] + self.mixin_from_dict(data) + return self + + def as_dict(self): + """ + Represent the kube upgrade strategy as a dictionary + """ + data = super(KubeUpgradeStrategy, self).as_dict() + data['to_version'] = self._to_version + data['single_controller'] = self._single_controller + self.mixin_as_dict(data) + return data + + def strategy_rebuild_from_dict(data): """ Returns the strategy object initialized using the given dictionary @@ -2098,6 +3109,8 @@ def strategy_rebuild_from_dict(data): strategy_obj = object.__new__(SwUpgradeStrategy) elif STRATEGY_NAME.FW_UPDATE == data['name']: strategy_obj = object.__new__(FwUpdateStrategy) + elif STRATEGY_NAME.KUBE_UPGRADE == data['name']: + strategy_obj = object.__new__(KubeUpgradeStrategy) else: strategy_obj = object.__new__(strategy.StrategyStage) diff --git a/nfv/nfv-vim/nfv_vim/strategy/_strategy_defs.py b/nfv/nfv-vim/nfv_vim/strategy/_strategy_defs.py index d865c9d3..ad258b1d 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/_strategy_defs.py +++ b/nfv/nfv-vim/nfv_vim/strategy/_strategy_defs.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -28,6 +28,10 @@ class EventNames(object): DISABLE_HOST_SERVICES_FAILED = Constant('disable-host-services-failed') ENABLE_HOST_SERVICES_FAILED = Constant('enable-host-services-failed') MIGRATE_INSTANCES_FAILED = Constant('migrate-instances-failed') + KUBE_HOST_UPGRADE_CONTROL_PLANE_FAILED = \ + Constant('kube-host-upgrade-control-plane-failed') + KUBE_HOST_UPGRADE_KUBELET_FAILED = \ + Constant('kube-host-upgrade-kubelet-failed') # Constants diff --git a/nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py b/nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py index 50227ad5..3be26d38 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py +++ b/nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -34,6 +34,20 @@ class StrategyStageNames(Constants): FW_UPDATE_HOSTS_QUERY = Constant('fw-update-hosts-query') FW_UPDATE_HOST_QUERY = Constant('fw-update-host-query') FW_UPDATE_WORKER_HOSTS = Constant('fw-update-worker-hosts') + KUBE_UPGRADE_QUERY = Constant('kube-upgrade-query') + KUBE_UPGRADE_START = Constant('kube-upgrade-start') + KUBE_UPGRADE_DOWNLOAD_IMAGES = Constant('kube-upgrade-download-images') + KUBE_UPGRADE_FIRST_CONTROL_PLANE = \ + Constant('kube-upgrade-first-control-plane') + KUBE_UPGRADE_NETWORKING = Constant('kube-upgrade-networking') + KUBE_UPGRADE_SECOND_CONTROL_PLANE = \ + Constant('kube-upgrade-second-control-plane') + KUBE_UPGRADE_PATCH = Constant('kube-upgrade-patch') + KUBE_UPGRADE_KUBELETS_CONTROLLERS = \ + Constant('kube-upgrade-kubelets-controllers') + KUBE_UPGRADE_KUBELETS_WORKERS = Constant('kube-upgrade-kubelets-workers') + KUBE_UPGRADE_COMPLETE = Constant('kube-upgrade-complete') + KUBE_UPGRADE_CLEANUP = Constant('kube-upgrade-cleanup') # Constant Instantiation diff --git a/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py b/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py index 388426f8..b01a4f28 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py +++ b/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2020 Wind River Systems, Inc. +# Copyright (c) 2015-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -50,6 +50,18 @@ class StrategyStepNames(Constants): QUERY_UPGRADE = Constant('query-upgrade') DISABLE_HOST_SERVICES = Constant('disable-host-services') ENABLE_HOST_SERVICES = Constant('enable-host-services') + APPLY_PATCHES = Constant('apply-patches') + QUERY_KUBE_HOST_UPGRADE = Constant('query-kube-host-upgrade') + QUERY_KUBE_UPGRADE = Constant('query-kube-upgrade') + QUERY_KUBE_VERSIONS = Constant('query-kube-versions') + KUBE_UPGRADE_START = Constant('kube-upgrade-start') + KUBE_UPGRADE_CLEANUP = Constant('kube-upgrade-cleanup') + KUBE_UPGRADE_COMPLETE = Constant('kube-upgrade-complete') + KUBE_UPGRADE_DOWNLOAD_IMAGES = Constant('kube-upgrade-download-images') + KUBE_UPGRADE_NETWORKING = Constant('kube-upgrade-networking') + KUBE_HOST_UPGRADE_CONTROL_PLANE = \ + Constant('kube-host-upgrade-control-plane') + KUBE_HOST_UPGRADE_KUBELET = Constant('kube-host-upgrade-kubelet') # Constant Instantiation @@ -1498,6 +1510,7 @@ class QueryAlarmsStep(strategy.StrategyStep): "strictness" % (nfvi_alarm.alarm_id, nfvi_alarm.alarm_uuid)) elif nfvi_alarm.alarm_id not in self._ignore_alarms: + DLOG.warn("Alarm: %s" % nfvi_alarm.alarm_id) nfvi_alarms.append(nfvi_alarm) else: DLOG.warn("Ignoring alarm %s - uuid %s" % @@ -2545,11 +2558,737 @@ class EnableHostServicesStep(strategy.StrategyStep): return data +class AbstractStrategyStep(strategy.StrategyStep): + + def __init__(self, step_name, timeout_in_secs): + super(AbstractStrategyStep, self).__init__( + step_name, + timeout_in_secs=timeout_in_secs) + + def from_dict(self, data): + """ + Returns the step object initialized using the given dictionary + """ + super(AbstractStrategyStep, self).from_dict(data) + return self + + def as_dict(self): + """ + Represent the step as a dictionary + """ + data = super(AbstractStrategyStep, self).as_dict() + # Next 3 lines are required for all strategy steps and may be + # overridden by subclass in some cases + data['entity_type'] = '' + data['entity_names'] = list() + data['entity_uuids'] = list() + return data + + +class ApplySwPatchesStep(AbstractStrategyStep): + """ + Apply Patches using patch API + """ + def __init__(self, patches_to_apply): + super(ApplySwPatchesStep, self).__init__( + STRATEGY_STEP_NAME.APPLY_PATCHES, + timeout_in_secs=600) + self._patches_to_apply = patches_to_apply + + @coroutine + def _api_callback(self): + """ + Callback for the API method invoked in apply + """ + response = (yield) + DLOG.debug("%s callback response=%s." % (self._name, response)) + + if response['completed']: + if self.strategy is not None: + self.strategy.nfvi_sw_patches = response['result-data'] + + result = strategy.STRATEGY_STEP_RESULT.SUCCESS + self.stage.step_complete(result, "") + else: + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete(result, response['reason']) + + def apply(self): + """ + Apply patches + """ + from nfv_vim import nfvi + + nfvi.nfvi_sw_mgmt_apply_updates(self._patches_to_apply, + self._api_callback()) + return strategy.STRATEGY_STEP_RESULT.WAIT, "" + + def from_dict(self, data): + """ + Returns the step object initialized using the given dictionary + """ + super(ApplySwPatchesStep, self).from_dict(data) + # only the names are serialized + self._patches_to_apply = data['entity_names'] + return self + + def as_dict(self): + """ + Represent the step as a dictionary + """ + data = super(ApplySwPatchesStep, self).as_dict() + data['entity_type'] = 'patches' + data['entity_names'] = self._patches_to_apply + # there are no entity_uuids + return data + + +class QueryKubeUpgradeStep(AbstractStrategyStep): + """ + Query Kube Upgrade + """ + def __init__(self): + super(QueryKubeUpgradeStep, self).__init__( + STRATEGY_STEP_NAME.QUERY_KUBE_UPGRADE, timeout_in_secs=60) + + @coroutine + def _get_kube_upgrade_callback(self): + """ + Get Kube Upgrade Callback + """ + response = (yield) + DLOG.debug("%s callback response=%s." % (self._name, response)) + + if response['completed']: + if self.strategy is not None: + self.strategy.nfvi_kube_upgrade = response['result-data'] + + result = strategy.STRATEGY_STEP_RESULT.SUCCESS + self.stage.step_complete(result, "") + else: + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete(result, response['reason']) + + def apply(self): + """ + Query Kube Upgrade + """ + from nfv_vim import nfvi + + DLOG.info("Step (%s) apply." % self._name) + nfvi.nfvi_get_kube_upgrade(self._get_kube_upgrade_callback()) + return strategy.STRATEGY_STEP_RESULT.WAIT, "" + + +class QueryKubeVersionsStep(AbstractStrategyStep): + """ + Query Kube Versions + This step should be used with its matching QueryKubeVersionsMixin + """ + def __init__(self): + super(QueryKubeVersionsStep, self).__init__( + STRATEGY_STEP_NAME.QUERY_KUBE_VERSIONS, timeout_in_secs=60) + + @coroutine + def _query_callback(self): + """ + Get Kube Versions List Callback + """ + response = (yield) + DLOG.debug("%s callback response=%s." % (self._name, response)) + + if response['completed']: + if self.strategy is not None: + self.strategy.nfvi_kube_versions_list = response['result-data'] + + result = strategy.STRATEGY_STEP_RESULT.SUCCESS + self.stage.step_complete(result, "") + else: + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete(result, response['reason']) + + def apply(self): + """ + Query Kube Versions List + """ + from nfv_vim import nfvi + + nfvi.nfvi_get_kube_version_list(self._query_callback()) + return strategy.STRATEGY_STEP_RESULT.WAIT, "" + + +class QueryKubeHostUpgradeStep(AbstractStrategyStep): + """ + Query Kube Host Upgrade list + """ + def __init__(self): + super(QueryKubeHostUpgradeStep, self).__init__( + STRATEGY_STEP_NAME.QUERY_KUBE_HOST_UPGRADE, timeout_in_secs=60) + + @coroutine + def _get_kube_host_upgrade_list_callback(self): + """ + Get Kube Host Upgrade List Callback + """ + response = (yield) + DLOG.debug("%s callback response=%s." % (self._name, response)) + + if response['completed']: + if self.strategy is not None: + self.strategy.nfvi_kube_host_upgrade_list = \ + response['result-data'] + + result = strategy.STRATEGY_STEP_RESULT.SUCCESS + self.stage.step_complete(result, "") + else: + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete(result, response['reason']) + + def apply(self): + """ + Query Kube Host Upgrade List + """ + from nfv_vim import nfvi + nfvi.nfvi_get_kube_host_upgrade_list( + self._get_kube_host_upgrade_list_callback()) + return strategy.STRATEGY_STEP_RESULT.WAIT, "" + + +class AbstractKubeUpgradeStep(AbstractStrategyStep): + + def __init__(self, + step_name, + success_state, + fail_state, + timeout_in_secs=600): + super(AbstractKubeUpgradeStep, self).__init__(step_name, + timeout_in_secs) + # These two attributes are not persisted + self._wait_time = 0 + self._query_inprogress = False + # success and fail state validators are persisted + self._success_state = success_state + self._fail_state = fail_state + + @coroutine + def _get_kube_upgrade_callback(self): + """Get Upgrade Callback""" + response = (yield) + DLOG.debug("(%s) callback response=%s." % (self._name, response)) + + self._query_inprogress = False + if response['completed']: + if self.strategy is None: + # there is no longer a strategy. abort. + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete(result, 'strategy no longer exists') + + kube_upgrade_obj = response['result-data'] + # replace the object in the strategy with the most recent object + self.strategy.nfvi_kube_upgrade = kube_upgrade_obj + + # break out of the loop if fail or success states match + if kube_upgrade_obj.state == self._success_state: + DLOG.debug("(%s) successfully reached (%s)." + % (self._name, self._success_state)) + result = strategy.STRATEGY_STEP_RESULT.SUCCESS + self.stage.step_complete(result, "") + elif (self._fail_state is not None + and kube_upgrade_obj.state == self._fail_state): + DLOG.warn("(%s) encountered failure state(%s)." + % (self._name, self._fail_state)) + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete( + result, + '(%s) failed:(%s)' % (self._name, self._fail_state) + ) + else: + # Keep waiting for upgrade to reach success or fail state + # timeout will occur if it is never reached. + DLOG.debug("(%s) in state (%s) waiting for (%s) or (%s)." + % (self._name, + kube_upgrade_obj.state, + self._success_state, + self._fail_state)) + pass + else: + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete(result, response['reason']) + + def handle_event(self, event, event_data=None): + """Handle Host events""" + + from nfv_vim import nfvi + + DLOG.debug("Step (%s) handle event (%s)." % (self._name, event)) + if event == STRATEGY_EVENT.HOST_AUDIT: + if 0 == self._wait_time: + self._wait_time = timers.get_monotonic_timestamp_in_ms() + + now_ms = timers.get_monotonic_timestamp_in_ms() + secs_expired = (now_ms - self._wait_time) / 1000 + # Wait at least 60 seconds before checking upgrade for first time + if 60 <= secs_expired and not self._query_inprogress: + self._query_inprogress = True + nfvi.nfvi_get_kube_upgrade(self._get_kube_upgrade_callback()) + return True + return False + + def from_dict(self, data): + """ + Returns the step object initialized using the given dictionary + """ + super(AbstractKubeUpgradeStep, self).from_dict(data) + # these two attributes are not persisted + self._wait_time = 0 + self._query_inprogress = False + # validation states are persisted + self._success_state = data['success_state'] + self._fail_state = data['fail_state'] + return self + + def as_dict(self): + """ + Represent the kube upgrade step as a dictionary + """ + data = super(AbstractKubeUpgradeStep, self).as_dict() + data['success_state'] = self._success_state + data['fail_state'] = self._fail_state + return data + + +class KubeUpgradeStartStep(AbstractKubeUpgradeStep): + """Kube Upgrade Start - Strategy Step""" + + def __init__(self, to_version, force=False): + + from nfv_vim import nfvi + + super(KubeUpgradeStartStep, self).__init__( + STRATEGY_STEP_NAME.KUBE_UPGRADE_START, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTED, + None) # there is no failure state if upgrade-start fails + # next 2 attributes must be persisted through from_dict/as_dict + self._to_version = to_version + self._force = force + + def from_dict(self, data): + """ + Returns the step object initialized using the given dictionary + """ + super(KubeUpgradeStartStep, self).from_dict(data) + self._to_version = data['to_version'] + self._force = data['force'] + return self + + def as_dict(self): + """ + Represent the kube upgrade step as a dictionary + """ + data = super(KubeUpgradeStartStep, self).as_dict() + data['to_version'] = self._to_version + data['force'] = self._force + return data + + @coroutine + def _response_callback(self): + """Kube Upgrade Start - Callback""" + + response = (yield) + DLOG.debug("%s callback response=%s." % (self._name, response)) + + if response['completed']: + if self.strategy is not None: + self.strategy.nfvi_kube_upgrade = response['result-data'] + # We do not set 'success' here, let the handle_event do this + else: + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete(result, response['reason']) + + def apply(self): + """Kube Upgrade Start""" + + from nfv_vim import nfvi + + alarm_ignore_list = ["900.401", ] # ignore the auto apply alarm + nfvi.nfvi_kube_upgrade_start(self._to_version, + self._force, + alarm_ignore_list, + self._response_callback()) + return strategy.STRATEGY_STEP_RESULT.WAIT, "" + + +class KubeUpgradeCleanupStep(AbstractKubeUpgradeStep): + """Kube Upgrade Cleanup - Strategy Step""" + + def __init__(self): + super(KubeUpgradeCleanupStep, self).__init__( + STRATEGY_STEP_NAME.KUBE_UPGRADE_CLEANUP, + None, # there is no success state for this cleanup activity + None) # there is no failure state for this cleanup activity + + @coroutine + def _response_callback(self): + """Kube Upgrade Cleanup - Callback""" + + response = (yield) + DLOG.debug("%s callback response=%s." % (self._name, response)) + + if response['completed']: + if self.strategy is not None: + # cleanup deletes the kube upgrade, clear it from the strategy + self.strategy.nfvi_kube_upgrade = None + result = strategy.STRATEGY_STEP_RESULT.SUCCESS + self.stage.step_complete(result, "") + else: + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete(result, response['reason']) + + def apply(self): + """Kube Upgrade Cleanup""" + + from nfv_vim import nfvi + + nfvi.nfvi_kube_upgrade_cleanup(self._response_callback()) + return strategy.STRATEGY_STEP_RESULT.WAIT, "" + + +class KubeUpgradeCompleteStep(AbstractKubeUpgradeStep): + """Kube Upgrade Complete - Strategy Step""" + + def __init__(self): + from nfv_vim import nfvi + super(KubeUpgradeCompleteStep, self).__init__( + STRATEGY_STEP_NAME.KUBE_UPGRADE_COMPLETE, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_COMPLETE, + None) # there is no failure state for upgrade-complete + + @coroutine + def _response_callback(self): + """Kube Upgrade Complete - Callback""" + + response = (yield) + DLOG.debug("%s callback response=%s." % (self._name, response)) + + if response['completed']: + if self.strategy is not None: + self.strategy.nfvi_kube_upgrade = response['result-data'] + else: + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete(result, response['reason']) + + def apply(self): + """Kube Upgrade Complete """ + + from nfv_vim import nfvi + + nfvi.nfvi_kube_upgrade_complete(self._response_callback()) + return strategy.STRATEGY_STEP_RESULT.WAIT, "" + + +class KubeUpgradeDownloadImagesStep(AbstractKubeUpgradeStep): + """Kube Upgrade Download Images - Strategy Step""" + + def __init__(self): + from nfv_vim import nfvi + super(KubeUpgradeDownloadImagesStep, self).__init__( + STRATEGY_STEP_NAME.KUBE_UPGRADE_DOWNLOAD_IMAGES, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADED_IMAGES, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED) + + @coroutine + def _response_callback(self): + """Kube Upgrade Download Images - Callback""" + + response = (yield) + DLOG.debug("%s callback response=%s." % (self._name, response)) + + if response['completed']: + if self.strategy is not None: + self.strategy.nfvi_kube_upgrade = response['result-data'] + else: + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete(result, response['reason']) + + def apply(self): + """Kube Upgrade Download Images """ + + from nfv_vim import nfvi + + nfvi.nfvi_kube_upgrade_download_images(self._response_callback()) + return strategy.STRATEGY_STEP_RESULT.WAIT, "" + + +class KubeUpgradeNetworkingStep(AbstractKubeUpgradeStep): + """Kube Upgrade Networking - Strategy Step""" + + def __init__(self): + from nfv_vim import nfvi + super(KubeUpgradeNetworkingStep, self).__init__( + STRATEGY_STEP_NAME.KUBE_UPGRADE_NETWORKING, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADED_NETWORKING, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADING_NETWORKING_FAILED) + + @coroutine + def _response_callback(self): + """Kube Upgrade Networking - Callback""" + + response = (yield) + DLOG.debug("%s callback response=%s." % (self._name, response)) + + if response['completed']: + if self.strategy is not None: + self.strategy.nfvi_kube_upgrade = response['result-data'] + else: + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete(result, response['reason']) + + def apply(self): + """Kube Upgrade Networking""" + + from nfv_vim import nfvi + + nfvi.nfvi_kube_upgrade_networking(self._response_callback()) + return strategy.STRATEGY_STEP_RESULT.WAIT, "" + + +class AbstractKubeHostUpgradeStep(AbstractKubeUpgradeStep): + """Kube Upgrade Host - Abtsract Strategy Step + + This operation issues a host command, which updates the kube upgrade object + """ + def __init__(self, + host, + force, + step_name, + success_state, + fail_state, + timeout_in_secs=600): + super(AbstractKubeHostUpgradeStep, self).__init__( + step_name, + success_state, + fail_state, + timeout_in_secs=timeout_in_secs) + self._force = force + # This class accepts only a single host + # but serializes as a list of hosts (list size of one) + self._hosts = list() + self._host_names = list() + self._host_uuids = list() + self._hosts.append(host) + self._host_names.append(host.name) + self._host_uuids.append(host.uuid) + + def from_dict(self, data): + """ + Returns the step object initialized using the given dictionary + """ + super(AbstractKubeHostUpgradeStep, self).from_dict(data) + self._force = data['force'] + self._hosts = list() + self._host_uuids = list() + self._host_names = data['entity_names'] + host_table = tables.tables_get_host_table() + for host_name in self._host_names: + host = host_table.get(host_name, None) + if host is not None: + self._hosts.append(host) + self._host_uuids.append(host.uuid) + return self + + def as_dict(self): + """ + Represent the step as a dictionary + """ + data = super(AbstractKubeHostUpgradeStep, self).as_dict() + data['force'] = self._force + data['entity_type'] = 'hosts' + data['entity_names'] = self._host_names + data['entity_uuids'] = self._host_uuids + return data + + +class KubeHostUpgradeControlPlaneStep(AbstractKubeHostUpgradeStep): + """Kube Host Upgrade Control Plane - Strategy Step + + This operation issues a host command, which updates the kube upgrade object + """ + + def __init__(self, host, force, target_state, target_failure_state): + super(KubeHostUpgradeControlPlaneStep, self).__init__( + host, + force, + STRATEGY_STEP_NAME.KUBE_HOST_UPGRADE_CONTROL_PLANE, + target_state, + target_failure_state, + timeout_in_secs=600) + + def handle_event(self, event, event_data=None): + """ + Handle Host events - does not query kube host upgrade list but + instead queries kube host upgrade directly. + """ + DLOG.debug("Step (%s) handle event (%s)." % (self._name, event)) + + if event == STRATEGY_EVENT.KUBE_HOST_UPGRADE_CONTROL_PLANE_FAILED: + host = event_data + if host is not None and host.name in self._host_names: + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete( + result, + "kube host upgrade control plane (%s) failed" % host.name) + return True + # return handle_event of parent class + return super(KubeHostUpgradeControlPlaneStep, self).handle_event( + event, event_data=event_data) + + def apply(self): + """Kube Host Upgrade Control Plane""" + + from nfv_vim import directors + + DLOG.info("Step (%s) apply to hostnames (%s)." + % (self._name, self._host_names)) + host_director = directors.get_host_director() + operation = \ + host_director.kube_upgrade_hosts_control_plane(self._host_names, + self._force) + + if operation.is_inprogress(): + return strategy.STRATEGY_STEP_RESULT.WAIT, "" + elif operation.is_failed(): + return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason + + return strategy.STRATEGY_STEP_RESULT.SUCCESS, "" + + +class KubeHostUpgradeKubeletStep(AbstractKubeHostUpgradeStep): + """Kube Host Upgrade Kubelet - Strategy Step + + This operation issues a host command, which indirectly updates the kube + upgrade object, however additional calls to other hosts do not change it. + This step should only be invoked on a locked host. + """ + + def __init__(self, host, force): + super(KubeHostUpgradeKubeletStep, self).__init__( + host, + force, + STRATEGY_STEP_NAME.KUBE_HOST_UPGRADE_KUBELET, + None, # there is no kube upgrade success state for kubelets + None, # there is no kube upgrade failure state for kubelets + timeout_in_secs=900) # kubelet takes longer than control plane + + @coroutine + def _get_kube_host_upgrade_list_callback(self): + """Get Kube Host Upgrade List Callback""" + + response = (yield) + DLOG.debug("(%s) callback response=%s." % (self._name, response)) + + self._query_inprogress = False + if response['completed']: + self.strategy.nfvi_kube_host_upgrade_list = response['result-data'] + + host_count = 0 + match_count = 0 + for host_uuid in self._host_uuids: + for k_host in self.strategy.nfvi_kube_host_upgrade_list: + if k_host.host_uuid == host_uuid: + if k_host.kubelet_version == self.strategy.to_version: + match_count += 1 + host_count += 1 + # break out of inner loop, since uuids match + break + if host_count == len(self._host_uuids): + # this is a pointless break + break + if match_count == len(self._host_uuids): + result = strategy.STRATEGY_STEP_RESULT.SUCCESS + self.stage.step_complete(result, "") + else: + # keep waiting for kubelet state to change + pass + else: + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete(result, response['reason']) + + def handle_event(self, event, event_data=None): + """ + Handle Host events - queries kube host upgrade list + Override to bypass checking for kube upgrade state. + """ + from nfv_vim import nfvi + + DLOG.debug("Step (%s) handle event (%s)." % (self._name, event)) + + if event == STRATEGY_EVENT.KUBE_HOST_UPGRADE_KUBELET_FAILED: + host = event_data + if host is not None and host.name in self._host_names: + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete( + result, + "kube host upgrade kubelet (%s) failed" % host.name) + return True + elif event == STRATEGY_EVENT.HOST_AUDIT: + if 0 == self._wait_time: + self._wait_time = timers.get_monotonic_timestamp_in_ms() + + now_ms = timers.get_monotonic_timestamp_in_ms() + secs_expired = (now_ms - self._wait_time) / 1000 + # Wait at least 60 seconds before checking upgrade for first time + if 60 <= secs_expired and not self._query_inprogress: + self._query_inprogress = True + nfvi.nfvi_get_kube_host_upgrade_list( + self._get_kube_host_upgrade_list_callback()) + return True + return False + + def apply(self): + """Kube Upgrade Kubelet""" + + from nfv_vim import directors + + DLOG.info("Step (%s) apply to hostnames (%s)." + % (self._name, self._host_names)) + host_director = directors.get_host_director() + operation = \ + host_director.kube_upgrade_hosts_kubelet(self._host_names, + self._force) + + if operation.is_inprogress(): + return strategy.STRATEGY_STEP_RESULT.WAIT, "" + elif operation.is_failed(): + return strategy.STRATEGY_STEP_RESULT.FAILED, operation.reason + + return strategy.STRATEGY_STEP_RESULT.SUCCESS, "" + + def strategy_step_rebuild_from_dict(data): """ Returns the strategy step object initialized using the given dictionary """ - if STRATEGY_STEP_NAME.SYSTEM_STABILIZE == data['name']: + rebuild_map = { + STRATEGY_STEP_NAME.APPLY_PATCHES: ApplySwPatchesStep, + STRATEGY_STEP_NAME.KUBE_HOST_UPGRADE_CONTROL_PLANE: + KubeHostUpgradeControlPlaneStep, + STRATEGY_STEP_NAME.KUBE_HOST_UPGRADE_KUBELET: + KubeHostUpgradeKubeletStep, + STRATEGY_STEP_NAME.KUBE_UPGRADE_CLEANUP: KubeUpgradeCleanupStep, + STRATEGY_STEP_NAME.KUBE_UPGRADE_COMPLETE: KubeUpgradeCompleteStep, + STRATEGY_STEP_NAME.KUBE_UPGRADE_DOWNLOAD_IMAGES: + KubeUpgradeDownloadImagesStep, + STRATEGY_STEP_NAME.KUBE_UPGRADE_NETWORKING: KubeUpgradeNetworkingStep, + STRATEGY_STEP_NAME.KUBE_UPGRADE_START: KubeUpgradeStartStep, + STRATEGY_STEP_NAME.QUERY_KUBE_HOST_UPGRADE: QueryKubeHostUpgradeStep, + STRATEGY_STEP_NAME.QUERY_KUBE_UPGRADE: QueryKubeUpgradeStep, + STRATEGY_STEP_NAME.QUERY_KUBE_VERSIONS: QueryKubeVersionsStep, + } + obj_type = rebuild_map.get(data['name']) + if obj_type is not None: + step_obj = object.__new__(obj_type) + + elif STRATEGY_STEP_NAME.SYSTEM_STABILIZE == data['name']: step_obj = object.__new__(SystemStabilizeStep) elif STRATEGY_STEP_NAME.UNLOCK_HOSTS == data['name']: diff --git a/nfv/pylint.rc b/nfv/pylint.rc index c53c6897..96e0acea 100755 --- a/nfv/pylint.rc +++ b/nfv/pylint.rc @@ -63,7 +63,7 @@ output-format=text files-output=no # Tells whether to display a full report or only the messages -reports=no +reports=yes # Python expression which should return a note less than 10 (10 is the highest # note). You have access to the variables errors warning, statement which