Add patch file support to patch orchestration

This commit adds a new parameter (patch) to the patch orchestration,
allowing the upload and apply of a specific patch file to a subcloud.
This change is essencial for enabling the new USM feature on subclouds
running older version.

Test Plan:

PASS: Fail if perform patch orchestation using --patch parameter
      with the subcloud and systemcontroller with the same version.
PASS: Perform patch orchestration using --patch parameter
- The patch should be uploaded, applied and installed to the subcloud
PASS: Perform patch orchestration using --patch and --upload-only
- The patch should be uploaded to the subcloud

Obs.:
1. Tests were performed without the patch being applied to the
systemcontroller
2. Tests were performed with subcloud in-sync and out-of-sync

Story: 2010676
Task: 50012

Change-Id: I7eb2940c708668b17ff93977b5622c3cff4cb3da
Signed-off-by: Hugo Brito <hugo.brito@windriver.com>

distributedcloud/dcmanager/api/controllers/v1/sw_update_strategy.py

@@ -15,10 +15,11 @@
 #    under the License.
 #
 
+import os
+
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_messaging import RemoteError
-
 import pecan
 from pecan import expose
 from pecan import request
@@ -160,6 +161,11 @@ class SwUpdateStrategyController(object):
                         consts.SUBCLOUD_APPLY_TYPE_SERIAL]:
                     pecan.abort(400, _('subcloud-apply-type invalid'))
 
+            patch_file = payload.get('patch')
+            if patch_file and not os.path.isfile(patch_file):
+                message = f"Patch file {patch_file} is missing."
+                pecan.abort(400, _(message))
+
             max_parallel_subclouds_str = payload.get('max-parallel-subclouds')
             if max_parallel_subclouds_str is not None:
                 max_parallel_subclouds = None

distributedcloud/dcmanager/common/consts.py

@@ -379,6 +379,7 @@ EXTRA_ARGS_FORCE = 'force'
 
 # extra_args for patching
 EXTRA_ARGS_UPLOAD_ONLY = 'upload-only'
+EXTRA_ARGS_PATCH = 'patch'
 
 # http request/response arguments for prestage
 PRESTAGE_SOFTWARE_VERSION = 'prestage-software-version'

distributedcloud/dcmanager/orchestrator/states/patch/updating_patches.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2023 Wind River Systems, Inc.
+# Copyright (c) 2023-2024 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -33,100 +33,129 @@ class UpdatingPatchesState(BaseState):
     def set_job_data(self, job_data):
         """Store an orch_thread job data object"""
         self.region_one_patches = job_data.region_one_patches
-        self.region_one_applied_patch_ids = job_data.\
+        self.region_one_applied_patch_ids = job_data.region_one_applied_patch_ids
-            region_one_applied_patch_ids
         self.extra_args = job_data.extra_args
 
+    def upload_patch(self, patch_file, strategy_step):
+        """Upload a patch file to the subcloud"""
+
+        if not os.path.isfile(patch_file):
+            message = f"Patch file {patch_file} is missing"
+            self.error_log(strategy_step, message)
+            raise Exception(message)
+
+        self.get_patching_client(self.region_name).upload([patch_file])
+        if self.stopped():
+            self.info_log(strategy_step, "Exiting because task is stopped")
+            raise StrategyStoppedException()
+
     def perform_state_action(self, strategy_step):
         """Update patches in this subcloud"""
         self.info_log(strategy_step, "Updating patches")
         upload_only = self.extra_args.get(consts.EXTRA_ARGS_UPLOAD_ONLY)
+        patch_file = self.extra_args.get(consts.EXTRA_ARGS_PATCH)
 
         # Retrieve all subcloud patches
         try:
-            subcloud_patches = self.get_patching_client(self.region_name).\
+            subcloud_patches = self.get_patching_client(self.region_name).query()
-                query()
         except Exception:
-            message = ("Cannot retrieve subcloud patches. Please see logs for"
+            message = ("Cannot retrieve subcloud patches. Please see logs for "
-                       " details.")
+                       "details.")
             self.exception_log(strategy_step, message)
             raise Exception(message)
 
-        patches_to_upload = []
-        patches_to_apply = []
-        patches_to_remove = []
-
         subcloud_patch_ids = subcloud_patches.keys()
 
-        # RegionOne applied patches not present on the subcloud needs to
+        # If a patch file is provided, upload and apply without checking RegionOne
-        # be uploaded and applied to the subcloud
+        # patches
-        for patch_id in self.region_one_applied_patch_ids:
+        if patch_file:
-            if patch_id not in subcloud_patch_ids:
+            self.info_log(
-                self.info_log(strategy_step, "Patch %s missing from subloud" %
+                strategy_step,
-                              patch_id)
+                f"Patch {patch_file} will be uploaded and applied to subcloud"
-                patches_to_upload.append(patch_id)
+            )
-                patches_to_apply.append(patch_id)
+            patch = os.path.basename(patch_file)
+            patch_id = os.path.splitext(patch)[0]
+            # raise Exception(subcloud_patch_ids)
+            if patch_id in subcloud_patch_ids:
+                message = f"Patch {patch_id} is already present in subcloud."
+                self.info_log(strategy_step, message)
+            else:
+                self.upload_patch(patch_file, strategy_step)
 
-        # Check that all applied patches in subcloud match RegionOne
+            if upload_only:
-        if not upload_only:
+                self.info_log(
-            for patch_id in subcloud_patch_ids:
+                    strategy_step,
-                repostate = subcloud_patches[patch_id]["repostate"]
+                    f"{consts.EXTRA_ARGS_UPLOAD_ONLY} option enabled, skipping "
-                if repostate == patching_v1.PATCH_STATE_APPLIED:
+                    f"execution. Forward to state: {consts.STRATEGY_STATE_COMPLETE}",
-                    if patch_id not in self.region_one_applied_patch_ids:
+                )
-                        self.info_log(strategy_step,
+                return consts.STRATEGY_STATE_COMPLETE
-                                      "Patch %s will be removed from subcloud" %
+
-                                      patch_id)
+            self.get_patching_client(self.region_name).apply([patch_id])
-                        patches_to_remove.append(patch_id)
+        else:
-                elif repostate == patching_v1.PATCH_STATE_COMMITTED:
+            patches_to_upload = []
-                    if patch_id not in self.region_one_applied_patch_ids:
+            patches_to_apply = []
-                        message = ("Patch %s is committed in subcloud but "
+            patches_to_remove = []
-                                   "not applied in SystemController" % patch_id)
+
+            # RegionOne applied patches not present on the subcloud needs to
+            # be uploaded and applied to the subcloud
+            for patch_id in self.region_one_applied_patch_ids:
+                if patch_id not in subcloud_patch_ids:
+                    self.info_log(strategy_step, "Patch %s missing from subloud " %
+                                  patch_id)
+                    patches_to_upload.append(patch_id)
+                    patches_to_apply.append(patch_id)
+
+            # Check that all applied patches in subcloud match RegionOne
+            if not upload_only:
+                for patch_id in subcloud_patch_ids:
+                    repostate = subcloud_patches[patch_id]["repostate"]
+                    if repostate == patching_v1.PATCH_STATE_APPLIED:
+                        if patch_id not in self.region_one_applied_patch_ids:
+                            self.info_log(strategy_step,
+                                          "Patch %s will be removed from subcloud " %
+                                          patch_id)
+                            patches_to_remove.append(patch_id)
+                    elif repostate == patching_v1.PATCH_STATE_COMMITTED:
+                        if patch_id not in self.region_one_applied_patch_ids:
+                            message = ("Patch %s is committed in subcloud but "
+                                       "not applied in SystemController" % patch_id)
+                            self.warn_log(strategy_step, message)
+                            raise Exception(message)
+                    elif repostate == patching_v1.PATCH_STATE_AVAILABLE:
+                        if patch_id in self.region_one_applied_patch_ids:
+                            patches_to_apply.append(patch_id)
+
+                    else:
+                        # This patch is in an invalid state
+                        message = ("Patch %s in subcloud is in an unexpected state: "
+                                   "%s" % (patch_id, repostate))
                         self.warn_log(strategy_step, message)
                         raise Exception(message)
-                elif repostate == patching_v1.PATCH_STATE_AVAILABLE:
-                    if patch_id in self.region_one_applied_patch_ids:
-                        patches_to_apply.append(patch_id)
 
-                else:
+            if patches_to_upload:
-                    # This patch is in an invalid state
+                self.info_log(strategy_step, "Uploading patches %s to subcloud" %
-                    message = ("Patch %s in subcloud is in an unexpected state:"
+                              patches_to_upload)
-                               " %s" % (patch_id, repostate))
+                for patch in patches_to_upload:
-                    self.warn_log(strategy_step, message)
+                    patch_sw_version = self.region_one_patches[patch]["sw_version"]
-                    raise Exception(message)
+                    patch_file = "%s/%s/%s.patch" % (consts.PATCH_VAULT_DIR,
+                                                     patch_sw_version, patch)
+                    self.upload_patch(patch_file, strategy_step)
 
-        if patches_to_upload:
+            if upload_only:
-            self.info_log(strategy_step, "Uploading patches %s to subcloud" %
+                self.info_log(strategy_step, "%s option enabled, skipping forward"
-                          patches_to_upload)
+                              " to state:(%s)" % (consts.EXTRA_ARGS_UPLOAD_ONLY,
-            for patch in patches_to_upload:
+                                                  consts.STRATEGY_STATE_COMPLETE))
-                patch_sw_version = self.region_one_patches[patch]["sw_version"]
+                return consts.STRATEGY_STATE_COMPLETE
-                patch_file = "%s/%s/%s.patch" % (consts.PATCH_VAULT_DIR,
-                                                 patch_sw_version, patch)
-                if not os.path.isfile(patch_file):
-                    message = "Patch file %s is missing" % patch_file
-                    self.error_log(strategy_step, message)
-                    raise Exception(message)
 
-                self.get_patching_client(self.region_name).upload([patch_file])
+            if patches_to_remove:
-                if self.stopped():
+                self.info_log(strategy_step, "Removing patches %s from subcloud" %
-                    self.info_log(strategy_step,
+                              patches_to_remove)
-                                  "Exiting because task is stopped")
+                self.get_patching_client(self.region_name).remove(patches_to_remove)
-                    raise StrategyStoppedException()
 
-        if upload_only:
+            if patches_to_apply:
-            self.info_log(strategy_step, "%s option enabled, skipping forward"
+                self.info_log(strategy_step, "Applying patches %s to subcloud" %
-                          " to state:(%s)" % (consts.EXTRA_ARGS_UPLOAD_ONLY,
+                              patches_to_apply)
-                                              consts.STRATEGY_STATE_COMPLETE))
+                self.get_patching_client(self.region_name).apply(patches_to_apply)
-            return consts.STRATEGY_STATE_COMPLETE
-
-        if patches_to_remove:
-            self.info_log(strategy_step, "Removing patches %s from subcloud" %
-                          patches_to_remove)
-            self.get_patching_client(self.region_name).remove(patches_to_remove)
-
-        if patches_to_apply:
-            self.info_log(strategy_step, "Applying patches %s to subcloud" %
-                          patches_to_apply)
-            self.get_patching_client(self.region_name).apply(patches_to_apply)
 
         # Now that we have applied/removed/uploaded patches, we need to give
         # the patch controller on this subcloud time to determine whether

distributedcloud/dcmanager/orchestrator/sw_update_manager.py

@@ -122,12 +122,30 @@ class SwUpdateManager(manager.Manager):
 
     def _validate_subcloud_status_sync(self, strategy_type,
                                        subcloud_status, force,
-                                       availability_status):
+                                       subcloud, patch_file):
         """Check the appropriate subcloud_status fields for the strategy_type
 
            Returns: True if out of sync.
         """
+        availability_status = subcloud.availability_status
         if strategy_type == consts.SW_UPDATE_TYPE_PATCH:
+            if patch_file:
+                # If a patch file is specified, we need to check the software version
+                # of the subcloud and the system controller. If the software versions
+                # are the same, we cannot apply the patch.
+                LOG.warning(
+                    f"Patch file: {patch_file} specified for "
+                    f"subcloud {subcloud.name}"
+                )
+                if subcloud.software_version == SW_VERSION:
+                    raise exceptions.BadRequest(
+                        resource="strategy",
+                        msg=(
+                            f"Subcloud {subcloud.name} has the same software "
+                            "version than the system controller. The --patch "
+                            "option only works with n-1 subclouds."
+                        ),
+                    )
             return (subcloud_status.endpoint_type ==
                     dccommon_consts.ENDPOINT_TYPE_PATCHING and
                     subcloud_status.sync_status ==
@@ -319,6 +337,7 @@ class SwUpdateManager(manager.Manager):
             else:
                 force = False
 
+        patch_file = payload.get('patch')
         installed_loads = []
         software_version = None
         if payload.get(consts.PRESTAGE_REQUEST_RELEASE):
@@ -401,6 +420,7 @@ class SwUpdateManager(manager.Manager):
                     raise exceptions.BadRequest(
                         resource='strategy',
                         msg='Subcloud %s does not require patching' % cloud_name)
+
             elif strategy_type == consts.SW_UPDATE_TYPE_PRESTAGE:
                 # Do initial validation for subcloud
                 try:
@@ -449,7 +469,10 @@ class SwUpdateManager(manager.Manager):
         elif strategy_type == consts.SW_UPDATE_TYPE_PATCH:
             upload_only_str = payload.get(consts.EXTRA_ARGS_UPLOAD_ONLY)
             upload_only_bool = True if upload_only_str == 'true' else False
-            extra_args = {consts.EXTRA_ARGS_UPLOAD_ONLY: upload_only_bool}
+            extra_args = {
+                consts.EXTRA_ARGS_UPLOAD_ONLY: upload_only_bool,
+                consts.EXTRA_ARGS_PATCH: payload.get(consts.EXTRA_ARGS_PATCH)
+            }
 
         # Don't create a strategy if any of the subclouds is online and the
         # relevant sync status is unknown. Offline subcloud is skipped unless
@@ -628,7 +651,8 @@ class SwUpdateManager(manager.Manager):
                 if self._validate_subcloud_status_sync(strategy_type,
                                                        status,
                                                        force,
-                                                       subcloud.availability_status):
+                                                       subcloud,
+                                                       patch_file):
                     LOG.debug("Creating strategy_step for endpoint_type: %s, "
                               "sync_status: %s, subcloud: %s, id: %s",
                               status.endpoint_type, status.sync_status,

distributedcloud/dcmanager/tests/unit/orchestrator/states/patch/test_updating_patches.py

@@ -10,6 +10,7 @@ import mock
 
 from dcmanager.common import consts
 from dcmanager.orchestrator.orch_thread import OrchThread
+from dcmanager.orchestrator.states.base import BaseState
 from dcmanager.tests.unit.common import fake_strategy
 from dcmanager.tests.unit.orchestrator.states.fakes import FakeLoad
 from dcmanager.tests.unit.orchestrator.states.patch.test_base import \
@@ -79,6 +80,29 @@ SUBCLOUD_PATCHES_BAD_STATE = {"DC.1": {"sw_version": "20.12",
                                        "repostate": "Applied",
                                        "patchstate": "Partial-Apply"}}
 
+SUBCLOUD_USM_PATCHES = {
+    "usm": {
+        "sw_version": "stx8",
+        "repostate": "Available",
+        "patchstate": "Available",
+    },
+    "DC.3": {
+        "sw_version": "20.12",
+        "repostate": "Available",
+        "patchstate": "Partial-Remove",
+    },
+    "DC.5": {
+        "sw_version": "20.12",
+        "repostate": "Unknown",
+        "patchstate": "Unknown"
+    },
+    "DC.6": {
+        "sw_version": "20.12",
+        "repostate": "Applied",
+        "patchstate": "Partial-Apply",
+    },
+}
+
 
 @mock.patch("dcmanager.orchestrator.states.patch.updating_patches."
             "DEFAULT_MAX_QUERIES", 3)
@@ -120,9 +144,12 @@ class TestUpdatingPatchesStage(TestPatchState):
         self.fake_load = FakeLoad(1, software_version="20.12",
                                   state=consts.ACTIVE_LOAD_STATE)
 
-    def _create_fake_strategy(self, upload_only=False):
+    def _create_fake_strategy(self, upload_only=False, patch_file=None):
         # setup extra_args used by PatchJobData
-        extra_args = {consts.EXTRA_ARGS_UPLOAD_ONLY: upload_only}
+        extra_args = {
+            consts.EXTRA_ARGS_UPLOAD_ONLY: upload_only,
+            consts.EXTRA_ARGS_PATCH: patch_file
+        }
         return fake_strategy.create_fake_strategy(self.ctx,
                                                   self.DEFAULT_STRATEGY_TYPE,
                                                   extra_args=extra_args)
@@ -183,6 +210,120 @@ class TestUpdatingPatchesStage(TestPatchState):
 
         self.assert_step_details(self.strategy_step.subcloud_id, "")
 
+    @mock.patch.object(os_path, "isfile")
+    def test_update_subcloud_patches_patch_file_success(self, mock_os_path_isfile):
+        """Test update_patches where the API call succeeds patch parameter."""
+
+        mock_os_path_isfile.return_value = True
+
+        self.patching_client.query.side_effect = [
+            REGION_ONE_PATCHES,
+            SUBCLOUD_PATCHES_SUCCESS,
+        ]
+
+        self._create_fake_strategy(patch_file="usm.patch")
+
+        # invoke the pre apply setup to create the PatchJobData object
+        self.worker.pre_apply_setup()
+
+        # invoke the strategy state operation on the orch thread
+        self.worker.perform_state_action(self.strategy_step)
+
+        self.patching_client.upload.assert_called_with(["usm.patch"])
+
+        call_args, _ = self.patching_client.apply.call_args_list[0]
+        self.assertItemsEqual(["usm"], call_args[0])
+
+        # On success, the state should transition to the next state
+        self.assert_step_updated(self.strategy_step.subcloud_id, self.success_state)
+
+        self.assert_step_details(self.strategy_step.subcloud_id, "")
+
+    def test_update_subcloud_patches_patch_file_no_upload(self):
+        """Test update_patches where the API call patch parameter is not uploaded."""
+
+        self.patching_client.query.side_effect = [
+            REGION_ONE_PATCHES,
+            SUBCLOUD_USM_PATCHES,
+        ]
+
+        self._create_fake_strategy(patch_file="usm.patch")
+
+        # invoke the pre apply setup to create the PatchJobData object
+        self.worker.pre_apply_setup()
+
+        # invoke the strategy state operation on the orch thread
+        self.worker.perform_state_action(self.strategy_step)
+
+        self.patching_client.upload.assert_not_called()
+
+        call_args, _ = self.patching_client.apply.call_args_list[0]
+        self.assertItemsEqual(["usm"], call_args[0])
+
+        # On success, the state should transition to the next state
+        self.assert_step_updated(self.strategy_step.subcloud_id, self.success_state)
+
+        self.assert_step_details(self.strategy_step.subcloud_id, "")
+
+    @mock.patch.object(os_path, "isfile")
+    def test_update_subcloud_patches_patch_file_upload_only_success(
+            self, mock_os_path_isfile
+    ):
+        """Test update_patches where the API call succeeds with patch/upload only."""
+
+        mock_os_path_isfile.return_value = True
+
+        self.patching_client.query.side_effect = [
+            REGION_ONE_PATCHES,
+            SUBCLOUD_PATCHES_SUCCESS,
+        ]
+
+        self._create_fake_strategy(upload_only=True, patch_file="usm.patch")
+
+        # invoke the pre apply setup to create the PatchJobData object
+        self.worker.pre_apply_setup()
+
+        # invoke the strategy state operation on the orch thread
+        self.worker.perform_state_action(self.strategy_step)
+
+        self.patching_client.upload.assert_called_with(["usm.patch"])
+
+        self.patching_client.remove.assert_not_called()
+        self.patching_client.apply.assert_not_called()
+
+        self.assert_step_details(self.strategy_step.subcloud_id, "")
+
+        # On success, the state should transition to the complete state
+        self.assert_step_updated(
+            self.strategy_step.subcloud_id, consts.STRATEGY_STATE_COMPLETE
+        )
+
+    @mock.patch.object(BaseState, "stopped")
+    @mock.patch.object(os_path, "isfile")
+    def test_updating_subcloud_patches_fails_when_stopped(
+        self, mock_os_path_isfile, mock_base_stopped
+    ):
+        """Test finish strategy fails when stopped"""
+        mock_os_path_isfile.return_value = True
+
+        self.patching_client.query.side_effect = [
+            REGION_ONE_PATCHES,
+            SUBCLOUD_PATCHES_SUCCESS,
+        ]
+
+        self._create_fake_strategy(upload_only=True, patch_file="usm.patch")
+
+        # invoke the pre apply setup to create the PatchJobData object
+        self.worker.pre_apply_setup()
+
+        mock_base_stopped.return_value = True
+
+        self.worker.perform_state_action(self.strategy_step)
+
+        self.assert_step_updated(
+            self.strategy_step.subcloud_id, consts.STRATEGY_STATE_FAILED
+        )
+
     @mock.patch.object(os_path, "isfile")
     def test_update_subcloud_patches_bad_committed(self, mock_os_path_isfile):
         """Test update_patches where the API call fails.