Adapt the prestage to support subcloud rehoming

When a subcloud is rehomed from one DC to a new DC, It has to
be able to reach the release level of the new DC.

The prestage for software deploy prepares the release level
that the subcloud should have deployed based on the release
level of the new DC.

This change affects the following scenarios:
- Subcloud with the same patch level as the system
  controller.
- Subcloud with a patch level lower than that of the
  system controller.
- System controller with a pre-patched load and
  subcloud with the same patch level applied.
- Subcloud with release N-1 and USM support.

Test Plan:
Setup 1: - Bring up a DC#1 with 24.09 and at least 1 subcloud.
	 - Bring up a DC#2 with 24.09.

PASS: - Rehome subcloud from DC#1 to DC#2.
      - Verify that the rehome is completed.
      - On DC#2 prestage the subcloud for software deploy.
      - Verify that the prestage is skipped because subcloud
        has the same release as the system controller.

PASS: - Upload and deploy patch 24.09.1 on DC#2.
      - Prestage the subcloud for software deploy.
      - Verify that the 24.09.1 patch is prestaged on the subcloud.

PASS: - Rehome subcloud from DC#2 to DC#1.
      - Verify that the rehome is completed.
      - On DC#1 prestage the subcloud for software deploy.
      - Verify that the prestage is skipped because the subcloud
        has a higher patch level than the system controller.

Setup 2: - Bring up a DC#1 with 22.12p5 (without USM feature)
           and at least 1 subcloud with the same release.
         - Bring up a DC#2 with 24.09.

PASS: - Rehome subcloud from DC#1 to DC#2.
      - Verify that the rehome is completed.
      - On DC#2 prestage the subcloud for software deploy.
      - Verify that the prestage is rejected because the USM
        service is not available on the subcloud.

PASS: - Upload and apply USM patch on the subcloud.
      - Verify that the USM service is now available.
      - On DC#2 prestage the subcloud for software deploy.
      - Verify that the 24.09 release is prestaged on the subcloud.

PASS: - Bring up a DC#1 with 24.09 and at least 1 subcloud.
      - Bring up a DC#2 with 24.09 pre-patched load.
      - Rehome subcloud from DC#1 to DC#2.
      - On DC#2 prestage the subcloud for software deploy.
      - Verify that the 24.09 release is prestaged on the subcloud.

Story: 2010676
Task: 51037

Change-Id: I7c2a985972df57b18b9ddcff4ac9d6ffd0d10e8e
Signed-off-by: Cristian Mondo <cristian.mondo@windriver.com>

playbookconfig/src/playbooks/roles/prestage/prepare-env/tasks/prepare-sw-packages-prestage-env.yml

@@ -65,6 +65,7 @@
     force_ostree_dir_sync: "{{ 'true' if (prestage_source == 'remote' and
                                 host_software_version is version('22.12', '=') and
                                 not host_ostree_dir_exist) else 'false' }}"
+
 - debug:
     msg: |
       Prestaging type: {{ prestage_type }}

playbookconfig/src/playbooks/roles/prestage/sync-software-metadata/files/ostree-metadata-sync.sh

@@ -38,6 +38,9 @@ SW_VERSION=${SW_VERSION:-}
 DEBUG=${DEBUG:-}
 DRY_RUN=${DRY_RUN:-}
 
+USM_SOFTWARE_DIR=/opt/software
+USM_METADATA_DIR=${USM_SOFTWARE_DIR}/metadata
+
 help() {
 cat<<EOF
 ostree metadata synchronization utilities.
@@ -164,6 +167,7 @@ initialize_env() {
     export MAJOR_SW_VERSION
 
     export OSTREE_REPO="/var/www/pages/feed/rel-${MAJOR_SW_VERSION}/ostree_repo"
+    export OSTREE_SYSROOT_REPO="/sysroot/ostree/repo"
     export OSTREE_REMOTE=starlingx
     export OSTREE_BRANCH=starlingx
     export OSTREE_LOCAL_REF="${OSTREE_REMOTE}"
@@ -174,6 +178,7 @@ initialize_env() {
     log_debug_l "SW_VERSION: ${SW_VERSION}"\
         "MAJOR_SW_VERSION: ${MAJOR_SW_VERSION}"\
         "OSTREE_REPO: ${OSTREE_REPO}"\
+        "OSTREE_SYSROOT_REPO: ${OSTREE_SYSROOT_REPO}"\
         "OSTREE_LOCAL_REF: ${OSTREE_LOCAL_REF}"\
         "OSTREE_REMOTE_REF: ${OSTREE_REMOTE_REF}"\
         "METADATA_DIR: ${METADATA_DIR}"\
@@ -255,35 +260,46 @@ find_metadata_file_for_attrib_val() {
 }
 
 get_simple_xml_attrib_from_metadata() {
-    # Retrieve the value of given attribute.
+    # Get the value of given XML attribute.
-    # WARNING: This function performs very basic parsing:
+    # We embed Python to parse the XML within script.
-    #          It only works if the opening and closing
+    # Params:
-    #          <attrib> </attrib> are on the same line.
+    #      $1 = the metadata file path
+    #      $2 = the xml attribute to find. can be a path that
+    #           specifies the element. For example:
+    #               /root/item/subitem1
+    # Returns the value of the element if it exists.
     local meta_file=$1
     local attrib=$2
     local val
-    val=$(sed -n 's|<'"${attrib}"'>\(.*\)</'"${attrib}"'>|\1|p' "${meta_file}")
+
-    val=$(trim "${val}")
+val=$(python - <<END
+import xml.etree.ElementTree as et
+import sys
+
+try:
+    tree = et.parse('$meta_file')
+    root = tree.getroot()
+    tag = root.find(".//$attrib")
+
+    print(tag.text)
+except:
+    sys.exit()
+END
+)
     log_debug "metadata GET ${attrib}: ${val}"
     echo "${val}"
 }
 
 get_commit_hashes_from_metadata() {
-    # Retrieves the commit hashes of given metadata file.
+    # Retrieves the commit hash of given metadata file
-    # The base commit and empty commits are ignored.
+    # Currently the metadata file supports only a single commit.
-    # Only package commits are taken into account.
+    # We use the commit1 path to find the hash.
-    #
-    # Using a nameref to update the passed-in array,
-    # see https://mywiki.wooledge.org/BashProgramming?highlight=%28nameref%29#Functions
     local -n from_metadata_commit_hashes=$1
     local meta_file=$2
     local commit
-    while IFS= read -r commit; do
+    local commit_path="contents/ostree/commit1/commit"
-        commit=$(trim "${commit}")
+
-        if [ ! -z "$commit" ]; then
+    from_metadata_commit_hashes=$(get_simple_xml_attrib_from_metadata "${meta_file}" "${commit_path}")
-            from_metadata_commit_hashes+=( "${commit}" )
-        fi
-    done < <(sed '/<base>/,/<\/base>/d' "${meta_file}" | sed --quiet 's|<commit[0-9]*>\(.*\)</commit[0-9]*>|\1|p')
 }
 
 get_usm_state_from_path() {
@@ -318,12 +334,14 @@ get_usm_state_from_path() {
 }
 
 ostree_commit_exists() {
-    # Does given commit exist in ostree?  i.e. has it been pulled into our repo yet?
+    # To ensure that the patch is entirely deployed, it must be verified in both repositories.
-    # Note: this only works for locally defined ostree repositories.
+    # This ensures that when checking whether the commit of a given patch exists on the system,
-    # i.e. it can't get status from a remote server
+    # it considers both repos (ostree repo and sysroot)
     local commit_hash=$1
     local ref=${2:-${OSTREE_LOCAL_REF}}
-    ostree --repo="${OSTREE_REPO}" log "${ref}" | grep '^commit ' | grep --quiet "${commit_hash}"
+    ostree --repo="${OSTREE_REPO}" log "${ref}" | grep '^commit ' | grep --quiet "${commit_hash}" && \
+    ostree --repo="${OSTREE_SYSROOT_REPO}" log "${ref}" | grep '^commit ' | grep --quiet "${commit_hash}"
+    return $?
 }
 
 translate_central_metadata_path() {
@@ -333,33 +351,24 @@ translate_central_metadata_path() {
 }
 
 get_metadata_files_unique_to_central() {
-    # TODO ISSUE:
+    cat "${METADATA_SYNC_DIR}"/ostree-metadata-commits.central | awk -F ':' '{print $1;}'
-    # This gets flagged for removal which it shouldn't - it's just a stage change:
-    #
-    # [sysadmin@controller-0 ~(keystone_admin)]$ diff -s /opt/software/tmp/metadata-sync/ostree-metadata-commits.*
-    # 1d0
-    # < /opt/software/metadata/deployed/starlingx-24.09.1-metadata.xml:db313865837f9512b024a2356bd76106140ebcea783f8183e5fcc8d5cd28783b
-    # 2a2
-    # > /opt/software/metadata/available/starlingx-24.09.1-metadata.xml:db313865837f9512b024a2356bd76106140ebcea783f8183e5fcc8d5cd28783b
-
-    diff "${METADATA_SYNC_DIR}"/ostree-metadata-commits.{central,subcloud} | awk '/^</ {print $2;}' | awk -F ':' '{print $1;}'
 }
 
-get_metadata_files_unique_to_subcloud() {
+sync_ostree_repo() {
-    diff "${METADATA_SYNC_DIR}"/ostree-metadata-commits.{central,subcloud} | awk '/^>/ {print $2;}' | awk -F ':' '{print $1;}'
+    # Synchronizes the remote ostree repository (system controller) to the local subcloud.
-}
+    local tmp_ostree_sync_file="/tmp/sync-ostree-commits.log"
-
+    run_cmd ostree --repo="${OSTREE_REPO}" pull --mirror --depth=-1 "${OSTREE_REMOTE_REF}" > "${tmp_ostree_sync_file}" 2>&1
-pull_ostree_commit_to_subcloud() {
+    rc=$?
-    # Pulls given commit into subcloud feed repo
+    # To avoid showing all ostree pull progress, which generates a very large output
-    #
+    # in Ansible, we show only the report line in case of success.
-    local commit_hash=$1
+    # In case of error only the last 10 lines.
-    if ostree_commit_exists "${commit_hash}"; then
+    if [ "$rc" != "0" ]; then
-        log_info "ostree commit ${commit_hash}: already exists in ${OSTREE_LOCAL_REF}"
+        tail -10 "${tmp_ostree_sync_file}"
-    else
+        rm -f "${tmp_ostree_sync_file}"
-        log_info "Pulling ostree commit from system controller: ${commit_hash}"
+        check_rc_die $rc "Unable to synchronize ostree repository."
-        run_cmd ostree --repo="${OSTREE_REPO}" pull --mirror "${OSTREE_REMOTE_REF}" "${OSTREE_BRANCH}@${commit_hash}"
-        check_rc_die $? "ostree pull failed"
     fi
+    tail -1 "${tmp_ostree_sync_file}"
+    rm -f "${tmp_ostree_sync_file}"
 }
 
 configure_ostree_repo_for_central_pull() {
@@ -436,7 +445,6 @@ find_all_ostree_commits_for_release() {
             get_commit_hashes_from_metadata commit_hashes "${metadata_file}"
             if [ "${#commit_hashes[@]}" -gt 0 ]; then
                 # We only need to supply the first commit here.
-                # See how the sync_subcloud_metadata algorithm works - it only uses the first commit
                 echo "${metadata_file}:${commit_hashes[0]}"
             fi
         fi
@@ -456,19 +464,15 @@ sync_metadata_on_subcloud() {
     #         - this will include the 'ostree-commit-id' and 'committed' ATTRIBUTES from systemController
     #         * this has already been done by ansible
     #
+    #     We are deleting all files of the release before synchronization to ensure that the
+    #     metadata will be exactly the same as the SC. It is not necessary to update the commits since
+    #     the subcloud repository is a mirror of the SC repository.
+    #
     #     IF RELEASE does NOT EXIST on subcloud
     #         IF 'ostree-commit-id' == NULL
-    #             Create it with STATE = unavailable
-    #         ELSE
     #             Create it with STATE = available
-    #     ELSE   // RELEASE exists on subcloud
+    #         ELSE
-    #         IF subcloud STATE == deployed
+    #             Create it with STATE = deployed
-    #             Leave it as deployed
-    #         ELSE IF subcloud STATE == available or unavailable
-    #             IF ‘ostree-commit-id’ == NULL
-    #                 Set STATE = unavailable
-    #             ELSE
-    #                 Set STATE = available
     #
     # For each RELEASE on SUBCLOUD but NOT synchronized from systemController
     #     REMOVE RELEASE FROM SUBCLOUD
@@ -506,50 +510,16 @@ sync_metadata_on_subcloud() {
     if [ -z "${subcloud_metadata_file}" ]; then
         # Not found: RELEASE does NOT EXIST on subcloud
         if ostree_commit_exists "${commit_hash}"; then
-            # Create it with STATE = available
+            # Create it with STATE = deployed
             log_debug_l "sync_metadata_on_subcloud: commit exists in local ${OSTREE_LOCAL_REF}"\
                 "ref: ${commit_hash}"
-            new_state="available"
+            new_state="deployed"
         else
-            # Create it with STATE = unavailable
+            # Create it with STATE = available
-            new_state="unavailable"
+            new_state="available"
         fi
         log_info "${log_hdr} does not exist on subcloud, setting to ${new_state}"
         run_cmd cp "${central_metadata_file}" "${METADATA_DIR}/${new_state}"
-    else
-        # RELEASE exists on subcloud
-        local subcloud_state
-        subcloud_state=$(get_usm_state_from_path "${subcloud_metadata_file}")
-        case "${subcloud_state}" in
-            'deployed')
-                # Leave it as deployed
-                log_info "${log_hdr} is in sync (subcloud state: deployed)"
-                ;;
-            'available'|'unavailable')
-                # Not found: RELEASE does NOT EXIST on subcloud
-                if ostree_commit_exists "${commit_hash}"; then
-                    # Set STATE = available
-                    log_debug_l "sync_metadata_on_subcloud: commit exists in local ${OSTREE_LOCAL_REF}"\
-                        "ref: ${commit_hash}"
-                    new_state=available
-                else
-                    # Set STATE = unavailable
-                    new_state=unavailable
-                fi
-                log_info "${log_hdr} exists on subcloud, setting subcloud state: ${new_state}"
-                run_cmd rm "${subcloud_metadata_file}"
-                run_cmd cp "${central_metadata_file}" "${METADATA_DIR}/${new_state}"
-                ;;
-            'committed')
-                log_info "${log_hdr} subcloud state is ${subcloud_state} - ignoring"
-                ;;
-            'deploying'|'removing')
-                log_info "${log_hdr} subcloud state is ${subcloud_state} - transitional, ignoring"
-                ;;
-            *)
-                log_error "${log_hdr} subcloud state is unexpected: ${subcloud_state} - ignoring"
-                ;;
-        esac
     fi
 }
 
@@ -560,30 +530,37 @@ sync_subcloud_metadata() {
     #
     # When this is invoked, we have the following in place (via ansible):
     #
-    #   - "${METADATA_SYNC_DIR}"/ostree-metadata-commits.{central,subcloud}
+    #   - "${METADATA_SYNC_DIR}"/ostree-metadata-commits.central
-    #       - these files summarizing the metadata files / ostree commits matching our given release
     #   - "${METADATA_SYNC_DIR}/metadata
     #       - is a direct copy of the system controller /opt/software/medatada directory
-    #       - we use this to calculate the new subcloud state of the release
+    #       - we use this to ensure that the metadata files exist in the subcloud
-    #         and to pull the missing ostree commits to the subcloud
     #
     # Synchronization is done on a per-major-release basis.
     # For given major release:
-    # 1) Get a list of all update metadata files needing to be synchronized
+    # 1) Syncronize ostree repo
-    #    (this is done by comparing (using diff) the central and subcloud file in
+    # 2) Remove the metadata from the specified release to ensure that when synchronized,
-    #    "${METADATA_SYNC_DIR}"/ostree-metadata-commits.{central,subcloud}).
+    #    it is in the right directory, based on the state.
-    # 2) Ensure any ostree commit(s) for the update are pulled from central
+    # 3) Get a list of all update metadata files needing to be synchronized
-    #    controller if necessary.
+    # 4) Synchronize the update metadata file into the proper state-based location
-    # 3) Synchronize the update metadata file into the proper state-based location
     #    on the subcloud
     #
     local metadata_file commit_hash central_metadata_file
+    local sw_version=${1:-$SW_VERSION}
 
+    # Configure and sync ostree feed repo
+    # This will be able to ostree at the same level between System Controller and subcloud.
     configure_ostree_repo_for_central_pull
+    sync_ostree_repo
 
     local commit_hashes=()
     local commit_hash
-    # 1) Get list of metadata files requiring sync
+
+    # Remove current files for specified release
+    log_info "Removing directories for release ${sw_version}"
+    rm -Rf ${USM_SOFTWARE_DIR}/rel-${sw_version}.*
+    find ${USM_METADATA_DIR} -type f -name "*${sw_version}*" | xargs rm -f
+
+    # Get list of metadata files requiring sync
     for metadata_file in $(get_metadata_files_unique_to_central); do
         log_info "sync_subcloud_metadata: processing ${metadata_file} from central (sync)"
         central_metadata_file=$(translate_central_metadata_path "${metadata_file}")
@@ -595,44 +572,9 @@ sync_subcloud_metadata() {
             "commit_hashes: ${commit_hashes[*]}"
 
         if [ "${#commit_hashes[@]}" -gt 0 ]; then
-            for commit_hash in "${commit_hashes[@]}"; do
+            # Synchronize the metadata file
-                # 2) Pull from central controller if necessary
-
-                # TODO(kmacleod): check if previous_commit exists from metadata, fail
-
-                pull_ostree_commit_to_subcloud "${commit_hash}"
-            done
-            # 3) Synchronize the metadata file
             sync_metadata_on_subcloud "${metadata_file}" "${central_metadata_file}" commit_hashes
-        fi
+            commit_hashes=()
-    done
-    for metadata_file in $(get_metadata_files_unique_to_subcloud); do
-        log_info "sync_subcloud_metadata: processing ${metadata_file} from subcloud (check remove)"
-        commit_hashes=()
-        get_commit_hashes_from_metadata commit_hashes "${central_metadata_file}"
-        log_debug_l "sync_subcloud_metadata from subcloud (check remove): "\
-            "metadata_file: ${metadata_file}"\
-            "commit_hashes: ${commit_hashes[*]}"
-        local removed=
-        if [ "${#commit_hashes[@]}" -gt 0 ]; then
-            for commit_hash in "${commit_hashes[@]}"; do
-                if ! ostree_commit_exists "${commit_hash}"; then
-                    log_info "sync_subcloud_metadata from subcloud: commit '${commit_hash}' does not exist, removing '${metadata_file}'"
-                    removed=1
-                fi
-            done
-            if [ -n "${removed}" ]; then
-                rm "${metadata_file}"
-            fi
-        fi
-        if [ -n "${removed}" ]; then
-            log_info_l "sync_subcloud_metadata from subcloud, removed file for non-existing commit(s): "\
-                "metadata_file: ${metadata_file}"\
-                "commit_hashes: ${commit_hashes[*]}"
-        else
-            log_info_l "sync_subcloud_metadata from subcloud, commit is in use, not removing: "\
-                "metadata_file: ${metadata_file}"\
-                "commit_hashes: ${commit_hashes[*]}"
         fi
     done
 }

playbookconfig/src/playbooks/roles/prestage/sync-software-metadata/tasks/main.yml

@@ -150,33 +150,52 @@
     subcloud_releases: "{{ subcloud_software_list.stdout_lines | \
     map('regex_replace', '.*?-([0-9\\.]+).*', '\\1') | list }}"
 
-- name: Verify patch levels between system controller and subcloud
+# The subcloud must have at least one release for the given software version to be able
-  debug:
+# to compare with the system controller version. Otherwise, it is assumed that the
-    msg: >
+# release does not exist in the subcloud and the prestage is executed.
-      {% if system_controller_releases | length > subcloud_releases | length %}
+- block:
-        The system controller has a patch level higher than the subcloud
+    # It is necessary to compare the current patch level between the SC and the subcloud
-      {% elif system_controller_releases | length == subcloud_releases | length %}
+    # to determine if prestage should be executed. In this way all scenarios are covered,
+    # including pre-patched scenario.
+  - name: Extract current patch number from system controller release list
+    set_fact:
+      system_controller_current_patch: "{{ system_controller_releases[-1] | \
+      regex_replace('.*\\.', '') | int }}"
+
+  - name: Extract current patch number from subcloud release list
+    set_fact:
+      subcloud_current_patch: "{{ subcloud_releases[-1] | regex_replace('.*\\.', '') | int  }}"
+
+  # We need to compare the patch level, since we must take into account even the
+  # pre-patched scenario.
+  - debug:
+      msg: |
+        ostree revision from:                 {{ ostree_repo_release_feed }}:
+        system controller current patch:      {{ system_controller_releases[-1] }}
+        subcloud current patch:               {{ subcloud_releases[-1] }}
+        system controller current commit:     {{ ostree_commit_system_controller.stdout }}
+        subcloud current commit:              {{ ostree_commit_subcloud.stdout }}
+
+        {% if system_controller_current_patch == subcloud_current_patch %}
         Skipping for software deploy prestage as the subcloud has the same patch level
         than the system controller.
-      {% else %}
+        {% elif subcloud_current_patch > system_controller_current_patch %}
         Skipping for software deploy prestage as the subcloud has a higher patch level
         than the system controller.
-      {% endif %}
+        {% endif %}
 
-- name: "Subcloud does not require software prestage. Exiting..."
+  - name: "Subcloud does not require software prestage. Exiting..."
-  meta: end_play
+    meta: end_play
-  when: subcloud_releases | length >= system_controller_releases | length
+    when: subcloud_current_patch >= system_controller_current_patch
+  when: subcloud_releases | length > 0
 
 - block:
+  - debug:
+      msg: The system controller has a patch level higher than the subcloud
+
   - name: Gather system controller metadata commits
-    #
+    # Get the commits from the metadata, in the same way that software list does,
-    # Use the existing ostree_metadata_commits_central file if:
+    # avoiding ambiguous queries.
-    # 1) It exists, and
-    # 2) There are no new commits. Compare last line of the ostree_metadata_commits_central
-    #    file against current ostree repo commit.
-    # Otherwise, we generate a new ostree_metadata_commits_central file using the
-    # get-commits target to our script.
-    #
     # Parallel operations (for orchestration):
     # We use flock here because there may be many prestaging operations running
     # in parallel on system controller. flock behaviour:
@@ -184,15 +203,10 @@
     #   - the timeout is long just to ensure we never deadlock for any reason
     shell: |
       exec 3>/tmp/ostree_metadata_commits_central.lock
-      flock --exclusive --timeout 180 3 || echo "ERROR: flock failed: $?"
+      flock --exclusive --timeout 180 3 || \
-      if [ ! -f "{{ ostree_metadata_commits_central }}" ] \
+      { echo "ERROR: $? - flock failed while trying to get the commits."; exit 1; }
-        || ! diff -q <(ostree --repo="{{ ostree_repo_release_feed }}" rev-parse "{{ ostree_rev }}") \
+      {{ role_path }}/files/ostree-metadata-sync.sh --sw-version "{{ software_version }}" \
-        <(tail --lines=1 "{{ ostree_metadata_commits_central }}" | awk -F':' '{ print $2; }') > /dev/null 2>&1 ; then
+      --output "{{ ostree_metadata_commits_central }}" get-commits
-        {{ role_path }}/files/ostree-metadata-sync.sh \
-        --sw-version "{{ software_version }}" --output "{{ ostree_metadata_commits_central }}" get-commits
-      else
-        cat "{{ ostree_metadata_commits_central }}"
-      fi
       exec 3>&-  # release the lock
     register: system_controller_software_metadata_commits
     delegate_to: localhost
@@ -212,27 +226,6 @@
     debug:
       var: subcloud_software_metadata_commits
 
-  - debug:
-      msg:
-        - "ostree revision from {{ ostree_repo_release_feed }}:"
-        - "system controller: {{ ostree_commit_system_controller.stdout }}"
-        - "subcloud:          {{ ostree_commit_subcloud.stdout }}"
-        - "Software list:"
-        - "system controller:"
-        - "{{ system_controller_software_list.stdout }}"
-        - "{{ system_controller_software_metadata_commits.stdout }}"
-        - "subcloud:"
-        - "{{ subcloud_software_list.stdout }}"
-        - "{{ subcloud_software_metadata_commits.stdout }}"
-
-  - name: Gather system controller deployed release list
-    shell: |
-      while IFS= read -r line; do
-        echo "$line" | awk -F'|' '{split($2,p,"-"); print p[2]}'
-      done <<< "{{ system_controller_software_list.stdout }}"
-    register: minor_releases_list
-    delegate_to: localhost
-
   # It's necessary to temporarily change the owner to sysadmin so that
   # the system controller can push the files to the subcloud, since the files
   # are in the folders only the root can access but the synchronize only
@@ -245,21 +238,6 @@
       recurse: yes
     become: yes
 
-  - name: Copy system controller deployed release directory to subcloud
-    synchronize:
-      mode: "push"
-      src: "{{ usm_software_dir }}/rel-{{ item }}"
-      dest: "{{ usm_software_dir }}"
-    with_items: "{{ minor_releases_list.stdout_lines }}"
-    failed_when: false
-
-  - name: Copy system controller {{ usm_software_dir }}/software-scripts to subcloud
-    synchronize:
-      mode: "push"
-      src: "{{ usm_software_dir }}/software-scripts"
-      dest: "{{ usm_software_dir }}"
-    failed_when: false
-
   - name: Copy system controller {{ usm_metadata_dir }} to subcloud {{ tmp_metadata_sync_dir }}
     copy:
       src: "{{ usm_metadata_dir }}"
@@ -278,7 +256,7 @@
   # on the system controller as it does not exist on N-1 release e.g. 22.12
   - name: Synchronizing system controller ostree commits on subcloud
     script: "{{ role_path }}/files/ostree-metadata-sync.sh --sw-version {{ software_version }}
-            sync-subcloud 2>&1 | tee /tmp/sync-ostree-commits.log"
+            sync-subcloud"
     register: sync_software_commits
     become: true
 
@@ -290,6 +268,21 @@
         - "stderr: {{ sync_software_commits.stderr }}"
         - "stdout: {{ sync_software_commits.stdout }}"
 
+  - name: Copy system controller deployed release directory to subcloud
+    synchronize:
+      mode: "push"
+      src: "{{ usm_software_dir }}/rel-{{ item }}"
+      dest: "{{ usm_software_dir }}"
+    with_items: "{{ system_controller_releases }}"
+    failed_when: false
+
+  - name: Copy system controller {{ usm_software_dir }}/software-scripts to subcloud
+    synchronize:
+      mode: "push"
+      src: "{{ usm_software_dir }}/software-scripts"
+      dest: "{{ usm_software_dir }}"
+    failed_when: false
+
   always:
     - name: Restore the ownership of {{ usm_software_dir }}
       file: