diff --git a/distributedcloud/dcmanager/api/controllers/v1/sw_update_strategy.py b/distributedcloud/dcmanager/api/controllers/v1/sw_update_strategy.py index 493e503d9..11dc4cc4d 100644 --- a/distributedcloud/dcmanager/api/controllers/v1/sw_update_strategy.py +++ b/distributedcloud/dcmanager/api/controllers/v1/sw_update_strategy.py @@ -15,10 +15,11 @@ # under the License. # +import os + from oslo_config import cfg from oslo_log import log as logging from oslo_messaging import RemoteError - import pecan from pecan import expose from pecan import request @@ -160,6 +161,11 @@ class SwUpdateStrategyController(object): consts.SUBCLOUD_APPLY_TYPE_SERIAL]: pecan.abort(400, _('subcloud-apply-type invalid')) + patch_file = payload.get('patch') + if patch_file and not os.path.isfile(patch_file): + message = f"Patch file {patch_file} is missing." + pecan.abort(400, _(message)) + max_parallel_subclouds_str = payload.get('max-parallel-subclouds') if max_parallel_subclouds_str is not None: max_parallel_subclouds = None diff --git a/distributedcloud/dcmanager/common/consts.py b/distributedcloud/dcmanager/common/consts.py index 63ed5740b..9ea11900d 100644 --- a/distributedcloud/dcmanager/common/consts.py +++ b/distributedcloud/dcmanager/common/consts.py @@ -379,6 +379,7 @@ EXTRA_ARGS_FORCE = 'force' # extra_args for patching EXTRA_ARGS_UPLOAD_ONLY = 'upload-only' +EXTRA_ARGS_PATCH = 'patch' # http request/response arguments for prestage PRESTAGE_SOFTWARE_VERSION = 'prestage-software-version' diff --git a/distributedcloud/dcmanager/orchestrator/states/patch/updating_patches.py b/distributedcloud/dcmanager/orchestrator/states/patch/updating_patches.py index 0b8e4d288..5b68fed5e 100644 --- a/distributedcloud/dcmanager/orchestrator/states/patch/updating_patches.py +++ b/distributedcloud/dcmanager/orchestrator/states/patch/updating_patches.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2023 Wind River Systems, Inc. +# Copyright (c) 2023-2024 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -33,100 +33,129 @@ class UpdatingPatchesState(BaseState): def set_job_data(self, job_data): """Store an orch_thread job data object""" self.region_one_patches = job_data.region_one_patches - self.region_one_applied_patch_ids = job_data.\ - region_one_applied_patch_ids + self.region_one_applied_patch_ids = job_data.region_one_applied_patch_ids self.extra_args = job_data.extra_args + def upload_patch(self, patch_file, strategy_step): + """Upload a patch file to the subcloud""" + + if not os.path.isfile(patch_file): + message = f"Patch file {patch_file} is missing" + self.error_log(strategy_step, message) + raise Exception(message) + + self.get_patching_client(self.region_name).upload([patch_file]) + if self.stopped(): + self.info_log(strategy_step, "Exiting because task is stopped") + raise StrategyStoppedException() + def perform_state_action(self, strategy_step): """Update patches in this subcloud""" self.info_log(strategy_step, "Updating patches") upload_only = self.extra_args.get(consts.EXTRA_ARGS_UPLOAD_ONLY) + patch_file = self.extra_args.get(consts.EXTRA_ARGS_PATCH) # Retrieve all subcloud patches try: - subcloud_patches = self.get_patching_client(self.region_name).\ - query() + subcloud_patches = self.get_patching_client(self.region_name).query() except Exception: - message = ("Cannot retrieve subcloud patches. Please see logs for" - " details.") + message = ("Cannot retrieve subcloud patches. Please see logs for " + "details.") self.exception_log(strategy_step, message) raise Exception(message) - patches_to_upload = [] - patches_to_apply = [] - patches_to_remove = [] - subcloud_patch_ids = subcloud_patches.keys() - # RegionOne applied patches not present on the subcloud needs to - # be uploaded and applied to the subcloud - for patch_id in self.region_one_applied_patch_ids: - if patch_id not in subcloud_patch_ids: - self.info_log(strategy_step, "Patch %s missing from subloud" % - patch_id) - patches_to_upload.append(patch_id) - patches_to_apply.append(patch_id) + # If a patch file is provided, upload and apply without checking RegionOne + # patches + if patch_file: + self.info_log( + strategy_step, + f"Patch {patch_file} will be uploaded and applied to subcloud" + ) + patch = os.path.basename(patch_file) + patch_id = os.path.splitext(patch)[0] + # raise Exception(subcloud_patch_ids) + if patch_id in subcloud_patch_ids: + message = f"Patch {patch_id} is already present in subcloud." + self.info_log(strategy_step, message) + else: + self.upload_patch(patch_file, strategy_step) - # Check that all applied patches in subcloud match RegionOne - if not upload_only: - for patch_id in subcloud_patch_ids: - repostate = subcloud_patches[patch_id]["repostate"] - if repostate == patching_v1.PATCH_STATE_APPLIED: - if patch_id not in self.region_one_applied_patch_ids: - self.info_log(strategy_step, - "Patch %s will be removed from subcloud" % - patch_id) - patches_to_remove.append(patch_id) - elif repostate == patching_v1.PATCH_STATE_COMMITTED: - if patch_id not in self.region_one_applied_patch_ids: - message = ("Patch %s is committed in subcloud but " - "not applied in SystemController" % patch_id) + if upload_only: + self.info_log( + strategy_step, + f"{consts.EXTRA_ARGS_UPLOAD_ONLY} option enabled, skipping " + f"execution. Forward to state: {consts.STRATEGY_STATE_COMPLETE}", + ) + return consts.STRATEGY_STATE_COMPLETE + + self.get_patching_client(self.region_name).apply([patch_id]) + else: + patches_to_upload = [] + patches_to_apply = [] + patches_to_remove = [] + + # RegionOne applied patches not present on the subcloud needs to + # be uploaded and applied to the subcloud + for patch_id in self.region_one_applied_patch_ids: + if patch_id not in subcloud_patch_ids: + self.info_log(strategy_step, "Patch %s missing from subloud " % + patch_id) + patches_to_upload.append(patch_id) + patches_to_apply.append(patch_id) + + # Check that all applied patches in subcloud match RegionOne + if not upload_only: + for patch_id in subcloud_patch_ids: + repostate = subcloud_patches[patch_id]["repostate"] + if repostate == patching_v1.PATCH_STATE_APPLIED: + if patch_id not in self.region_one_applied_patch_ids: + self.info_log(strategy_step, + "Patch %s will be removed from subcloud " % + patch_id) + patches_to_remove.append(patch_id) + elif repostate == patching_v1.PATCH_STATE_COMMITTED: + if patch_id not in self.region_one_applied_patch_ids: + message = ("Patch %s is committed in subcloud but " + "not applied in SystemController" % patch_id) + self.warn_log(strategy_step, message) + raise Exception(message) + elif repostate == patching_v1.PATCH_STATE_AVAILABLE: + if patch_id in self.region_one_applied_patch_ids: + patches_to_apply.append(patch_id) + + else: + # This patch is in an invalid state + message = ("Patch %s in subcloud is in an unexpected state: " + "%s" % (patch_id, repostate)) self.warn_log(strategy_step, message) raise Exception(message) - elif repostate == patching_v1.PATCH_STATE_AVAILABLE: - if patch_id in self.region_one_applied_patch_ids: - patches_to_apply.append(patch_id) - else: - # This patch is in an invalid state - message = ("Patch %s in subcloud is in an unexpected state:" - " %s" % (patch_id, repostate)) - self.warn_log(strategy_step, message) - raise Exception(message) + if patches_to_upload: + self.info_log(strategy_step, "Uploading patches %s to subcloud" % + patches_to_upload) + for patch in patches_to_upload: + patch_sw_version = self.region_one_patches[patch]["sw_version"] + patch_file = "%s/%s/%s.patch" % (consts.PATCH_VAULT_DIR, + patch_sw_version, patch) + self.upload_patch(patch_file, strategy_step) - if patches_to_upload: - self.info_log(strategy_step, "Uploading patches %s to subcloud" % - patches_to_upload) - for patch in patches_to_upload: - patch_sw_version = self.region_one_patches[patch]["sw_version"] - patch_file = "%s/%s/%s.patch" % (consts.PATCH_VAULT_DIR, - patch_sw_version, patch) - if not os.path.isfile(patch_file): - message = "Patch file %s is missing" % patch_file - self.error_log(strategy_step, message) - raise Exception(message) + if upload_only: + self.info_log(strategy_step, "%s option enabled, skipping forward" + " to state:(%s)" % (consts.EXTRA_ARGS_UPLOAD_ONLY, + consts.STRATEGY_STATE_COMPLETE)) + return consts.STRATEGY_STATE_COMPLETE - self.get_patching_client(self.region_name).upload([patch_file]) - if self.stopped(): - self.info_log(strategy_step, - "Exiting because task is stopped") - raise StrategyStoppedException() + if patches_to_remove: + self.info_log(strategy_step, "Removing patches %s from subcloud" % + patches_to_remove) + self.get_patching_client(self.region_name).remove(patches_to_remove) - if upload_only: - self.info_log(strategy_step, "%s option enabled, skipping forward" - " to state:(%s)" % (consts.EXTRA_ARGS_UPLOAD_ONLY, - consts.STRATEGY_STATE_COMPLETE)) - return consts.STRATEGY_STATE_COMPLETE - - if patches_to_remove: - self.info_log(strategy_step, "Removing patches %s from subcloud" % - patches_to_remove) - self.get_patching_client(self.region_name).remove(patches_to_remove) - - if patches_to_apply: - self.info_log(strategy_step, "Applying patches %s to subcloud" % - patches_to_apply) - self.get_patching_client(self.region_name).apply(patches_to_apply) + if patches_to_apply: + self.info_log(strategy_step, "Applying patches %s to subcloud" % + patches_to_apply) + self.get_patching_client(self.region_name).apply(patches_to_apply) # Now that we have applied/removed/uploaded patches, we need to give # the patch controller on this subcloud time to determine whether diff --git a/distributedcloud/dcmanager/orchestrator/sw_update_manager.py b/distributedcloud/dcmanager/orchestrator/sw_update_manager.py index a5270fd8d..70d574d24 100644 --- a/distributedcloud/dcmanager/orchestrator/sw_update_manager.py +++ b/distributedcloud/dcmanager/orchestrator/sw_update_manager.py @@ -122,12 +122,30 @@ class SwUpdateManager(manager.Manager): def _validate_subcloud_status_sync(self, strategy_type, subcloud_status, force, - availability_status): + subcloud, patch_file): """Check the appropriate subcloud_status fields for the strategy_type Returns: True if out of sync. """ + availability_status = subcloud.availability_status if strategy_type == consts.SW_UPDATE_TYPE_PATCH: + if patch_file: + # If a patch file is specified, we need to check the software version + # of the subcloud and the system controller. If the software versions + # are the same, we cannot apply the patch. + LOG.warning( + f"Patch file: {patch_file} specified for " + f"subcloud {subcloud.name}" + ) + if subcloud.software_version == SW_VERSION: + raise exceptions.BadRequest( + resource="strategy", + msg=( + f"Subcloud {subcloud.name} has the same software " + "version than the system controller. The --patch " + "option only works with n-1 subclouds." + ), + ) return (subcloud_status.endpoint_type == dccommon_consts.ENDPOINT_TYPE_PATCHING and subcloud_status.sync_status == @@ -319,6 +337,7 @@ class SwUpdateManager(manager.Manager): else: force = False + patch_file = payload.get('patch') installed_loads = [] software_version = None if payload.get(consts.PRESTAGE_REQUEST_RELEASE): @@ -401,6 +420,7 @@ class SwUpdateManager(manager.Manager): raise exceptions.BadRequest( resource='strategy', msg='Subcloud %s does not require patching' % cloud_name) + elif strategy_type == consts.SW_UPDATE_TYPE_PRESTAGE: # Do initial validation for subcloud try: @@ -449,7 +469,10 @@ class SwUpdateManager(manager.Manager): elif strategy_type == consts.SW_UPDATE_TYPE_PATCH: upload_only_str = payload.get(consts.EXTRA_ARGS_UPLOAD_ONLY) upload_only_bool = True if upload_only_str == 'true' else False - extra_args = {consts.EXTRA_ARGS_UPLOAD_ONLY: upload_only_bool} + extra_args = { + consts.EXTRA_ARGS_UPLOAD_ONLY: upload_only_bool, + consts.EXTRA_ARGS_PATCH: payload.get(consts.EXTRA_ARGS_PATCH) + } # Don't create a strategy if any of the subclouds is online and the # relevant sync status is unknown. Offline subcloud is skipped unless @@ -628,7 +651,8 @@ class SwUpdateManager(manager.Manager): if self._validate_subcloud_status_sync(strategy_type, status, force, - subcloud.availability_status): + subcloud, + patch_file): LOG.debug("Creating strategy_step for endpoint_type: %s, " "sync_status: %s, subcloud: %s, id: %s", status.endpoint_type, status.sync_status, diff --git a/distributedcloud/dcmanager/tests/unit/orchestrator/states/patch/test_updating_patches.py b/distributedcloud/dcmanager/tests/unit/orchestrator/states/patch/test_updating_patches.py index 23eb4e266..b84ad6016 100644 --- a/distributedcloud/dcmanager/tests/unit/orchestrator/states/patch/test_updating_patches.py +++ b/distributedcloud/dcmanager/tests/unit/orchestrator/states/patch/test_updating_patches.py @@ -10,6 +10,7 @@ import mock from dcmanager.common import consts from dcmanager.orchestrator.orch_thread import OrchThread +from dcmanager.orchestrator.states.base import BaseState from dcmanager.tests.unit.common import fake_strategy from dcmanager.tests.unit.orchestrator.states.fakes import FakeLoad from dcmanager.tests.unit.orchestrator.states.patch.test_base import \ @@ -79,6 +80,29 @@ SUBCLOUD_PATCHES_BAD_STATE = {"DC.1": {"sw_version": "20.12", "repostate": "Applied", "patchstate": "Partial-Apply"}} +SUBCLOUD_USM_PATCHES = { + "usm": { + "sw_version": "stx8", + "repostate": "Available", + "patchstate": "Available", + }, + "DC.3": { + "sw_version": "20.12", + "repostate": "Available", + "patchstate": "Partial-Remove", + }, + "DC.5": { + "sw_version": "20.12", + "repostate": "Unknown", + "patchstate": "Unknown" + }, + "DC.6": { + "sw_version": "20.12", + "repostate": "Applied", + "patchstate": "Partial-Apply", + }, +} + @mock.patch("dcmanager.orchestrator.states.patch.updating_patches." "DEFAULT_MAX_QUERIES", 3) @@ -120,9 +144,12 @@ class TestUpdatingPatchesStage(TestPatchState): self.fake_load = FakeLoad(1, software_version="20.12", state=consts.ACTIVE_LOAD_STATE) - def _create_fake_strategy(self, upload_only=False): + def _create_fake_strategy(self, upload_only=False, patch_file=None): # setup extra_args used by PatchJobData - extra_args = {consts.EXTRA_ARGS_UPLOAD_ONLY: upload_only} + extra_args = { + consts.EXTRA_ARGS_UPLOAD_ONLY: upload_only, + consts.EXTRA_ARGS_PATCH: patch_file + } return fake_strategy.create_fake_strategy(self.ctx, self.DEFAULT_STRATEGY_TYPE, extra_args=extra_args) @@ -183,6 +210,120 @@ class TestUpdatingPatchesStage(TestPatchState): self.assert_step_details(self.strategy_step.subcloud_id, "") + @mock.patch.object(os_path, "isfile") + def test_update_subcloud_patches_patch_file_success(self, mock_os_path_isfile): + """Test update_patches where the API call succeeds patch parameter.""" + + mock_os_path_isfile.return_value = True + + self.patching_client.query.side_effect = [ + REGION_ONE_PATCHES, + SUBCLOUD_PATCHES_SUCCESS, + ] + + self._create_fake_strategy(patch_file="usm.patch") + + # invoke the pre apply setup to create the PatchJobData object + self.worker.pre_apply_setup() + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + self.patching_client.upload.assert_called_with(["usm.patch"]) + + call_args, _ = self.patching_client.apply.call_args_list[0] + self.assertItemsEqual(["usm"], call_args[0]) + + # On success, the state should transition to the next state + self.assert_step_updated(self.strategy_step.subcloud_id, self.success_state) + + self.assert_step_details(self.strategy_step.subcloud_id, "") + + def test_update_subcloud_patches_patch_file_no_upload(self): + """Test update_patches where the API call patch parameter is not uploaded.""" + + self.patching_client.query.side_effect = [ + REGION_ONE_PATCHES, + SUBCLOUD_USM_PATCHES, + ] + + self._create_fake_strategy(patch_file="usm.patch") + + # invoke the pre apply setup to create the PatchJobData object + self.worker.pre_apply_setup() + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + self.patching_client.upload.assert_not_called() + + call_args, _ = self.patching_client.apply.call_args_list[0] + self.assertItemsEqual(["usm"], call_args[0]) + + # On success, the state should transition to the next state + self.assert_step_updated(self.strategy_step.subcloud_id, self.success_state) + + self.assert_step_details(self.strategy_step.subcloud_id, "") + + @mock.patch.object(os_path, "isfile") + def test_update_subcloud_patches_patch_file_upload_only_success( + self, mock_os_path_isfile + ): + """Test update_patches where the API call succeeds with patch/upload only.""" + + mock_os_path_isfile.return_value = True + + self.patching_client.query.side_effect = [ + REGION_ONE_PATCHES, + SUBCLOUD_PATCHES_SUCCESS, + ] + + self._create_fake_strategy(upload_only=True, patch_file="usm.patch") + + # invoke the pre apply setup to create the PatchJobData object + self.worker.pre_apply_setup() + + # invoke the strategy state operation on the orch thread + self.worker.perform_state_action(self.strategy_step) + + self.patching_client.upload.assert_called_with(["usm.patch"]) + + self.patching_client.remove.assert_not_called() + self.patching_client.apply.assert_not_called() + + self.assert_step_details(self.strategy_step.subcloud_id, "") + + # On success, the state should transition to the complete state + self.assert_step_updated( + self.strategy_step.subcloud_id, consts.STRATEGY_STATE_COMPLETE + ) + + @mock.patch.object(BaseState, "stopped") + @mock.patch.object(os_path, "isfile") + def test_updating_subcloud_patches_fails_when_stopped( + self, mock_os_path_isfile, mock_base_stopped + ): + """Test finish strategy fails when stopped""" + mock_os_path_isfile.return_value = True + + self.patching_client.query.side_effect = [ + REGION_ONE_PATCHES, + SUBCLOUD_PATCHES_SUCCESS, + ] + + self._create_fake_strategy(upload_only=True, patch_file="usm.patch") + + # invoke the pre apply setup to create the PatchJobData object + self.worker.pre_apply_setup() + + mock_base_stopped.return_value = True + + self.worker.perform_state_action(self.strategy_step) + + self.assert_step_updated( + self.strategy_step.subcloud_id, consts.STRATEGY_STATE_FAILED + ) + @mock.patch.object(os_path, "isfile") def test_update_subcloud_patches_bad_committed(self, mock_os_path_isfile): """Test update_patches where the API call fails.