Merge "Adapt the prestage to support subcloud rehoming"

playbookconfig/src/playbooks/roles/prestage/prepare-env/tasks/prepare-sw-packages-prestage-env.yml

@@ -65,6 +65,7 @@
     force_ostree_dir_sync: "{{ 'true' if (prestage_source == 'remote' and
                                 host_software_version is version('22.12', '=') and
                                 not host_ostree_dir_exist) else 'false' }}"
+
 - debug:
     msg: |
       Prestaging type: {{ prestage_type }}

playbookconfig/src/playbooks/roles/prestage/sync-software-metadata/files/ostree-metadata-sync.sh

@@ -38,6 +38,9 @@ SW_VERSION=${SW_VERSION:-}
 DEBUG=${DEBUG:-}
 DRY_RUN=${DRY_RUN:-}
 
+USM_SOFTWARE_DIR=/opt/software
+USM_METADATA_DIR=${USM_SOFTWARE_DIR}/metadata
+
 help() {
 cat<<EOF
 ostree metadata synchronization utilities.
@@ -164,6 +167,7 @@ initialize_env() {
     export MAJOR_SW_VERSION
 
     export OSTREE_REPO="/var/www/pages/feed/rel-${MAJOR_SW_VERSION}/ostree_repo"
+    export OSTREE_SYSROOT_REPO="/sysroot/ostree/repo"
     export OSTREE_REMOTE=starlingx
     export OSTREE_BRANCH=starlingx
     export OSTREE_LOCAL_REF="${OSTREE_REMOTE}"
@@ -174,6 +178,7 @@ initialize_env() {
     log_debug_l "SW_VERSION: ${SW_VERSION}"\
         "MAJOR_SW_VERSION: ${MAJOR_SW_VERSION}"\
         "OSTREE_REPO: ${OSTREE_REPO}"\
+        "OSTREE_SYSROOT_REPO: ${OSTREE_SYSROOT_REPO}"\
         "OSTREE_LOCAL_REF: ${OSTREE_LOCAL_REF}"\
         "OSTREE_REMOTE_REF: ${OSTREE_REMOTE_REF}"\
         "METADATA_DIR: ${METADATA_DIR}"\
@@ -255,35 +260,46 @@ find_metadata_file_for_attrib_val() {
 }
 
 get_simple_xml_attrib_from_metadata() {
-    # Retrieve the value of given attribute.
+    # Get the value of given XML attribute.
-    # WARNING: This function performs very basic parsing:
+    # We embed Python to parse the XML within script.
-    #          It only works if the opening and closing
+    # Params:
-    #          <attrib> </attrib> are on the same line.
+    #      $1 = the metadata file path
+    #      $2 = the xml attribute to find. can be a path that
+    #           specifies the element. For example:
+    #               /root/item/subitem1
+    # Returns the value of the element if it exists.
     local meta_file=$1
     local attrib=$2
     local val
-    val=$(sed -n 's|<'"${attrib}"'>\(.*\)</'"${attrib}"'>|\1|p' "${meta_file}")
+
-    val=$(trim "${val}")
+val=$(python - <<END
+import xml.etree.ElementTree as et
+import sys
+
+try:
+    tree = et.parse('$meta_file')
+    root = tree.getroot()
+    tag = root.find(".//$attrib")
+
+    print(tag.text)
+except:
+    sys.exit()
+END
+)
     log_debug "metadata GET ${attrib}: ${val}"
     echo "${val}"
 }
 
 get_commit_hashes_from_metadata() {
-    # Retrieves the commit hashes of given metadata file.
+    # Retrieves the commit hash of given metadata file
-    # The base commit and empty commits are ignored.
+    # Currently the metadata file supports only a single commit.
-    # Only package commits are taken into account.
+    # We use the commit1 path to find the hash.
-    #
-    # Using a nameref to update the passed-in array,
-    # see https://mywiki.wooledge.org/BashProgramming?highlight=%28nameref%29#Functions
     local -n from_metadata_commit_hashes=$1
     local meta_file=$2
     local commit
-    while IFS= read -r commit; do
+    local commit_path="contents/ostree/commit1/commit"
-        commit=$(trim "${commit}")
+
-        if [ ! -z "$commit" ]; then
+    from_metadata_commit_hashes=$(get_simple_xml_attrib_from_metadata "${meta_file}" "${commit_path}")
-            from_metadata_commit_hashes+=( "${commit}" )
-        fi
-    done < <(sed '/<base>/,/<\/base>/d' "${meta_file}" | sed --quiet 's|<commit[0-9]*>\(.*\)</commit[0-9]*>|\1|p')
 }
 
 get_usm_state_from_path() {
@@ -318,12 +334,14 @@ get_usm_state_from_path() {
 }
 
 ostree_commit_exists() {
-    # Does given commit exist in ostree?  i.e. has it been pulled into our repo yet?
+    # To ensure that the patch is entirely deployed, it must be verified in both repositories.
-    # Note: this only works for locally defined ostree repositories.
+    # This ensures that when checking whether the commit of a given patch exists on the system,
-    # i.e. it can't get status from a remote server
+    # it considers both repos (ostree repo and sysroot)
     local commit_hash=$1
     local ref=${2:-${OSTREE_LOCAL_REF}}
-    ostree --repo="${OSTREE_REPO}" log "${ref}" | grep '^commit ' | grep --quiet "${commit_hash}"
+    ostree --repo="${OSTREE_REPO}" log "${ref}" | grep '^commit ' | grep --quiet "${commit_hash}" && \
+    ostree --repo="${OSTREE_SYSROOT_REPO}" log "${ref}" | grep '^commit ' | grep --quiet "${commit_hash}"
+    return $?
 }
 
 translate_central_metadata_path() {
@@ -333,33 +351,24 @@ translate_central_metadata_path() {
 }
 
 get_metadata_files_unique_to_central() {
-    # TODO ISSUE:
+    cat "${METADATA_SYNC_DIR}"/ostree-metadata-commits.central | awk -F ':' '{print $1;}'
-    # This gets flagged for removal which it shouldn't - it's just a stage change:
-    #
-    # [sysadmin@controller-0 ~(keystone_admin)]$ diff -s /opt/software/tmp/metadata-sync/ostree-metadata-commits.*
-    # 1d0
-    # < /opt/software/metadata/deployed/starlingx-24.09.1-metadata.xml:db313865837f9512b024a2356bd76106140ebcea783f8183e5fcc8d5cd28783b
-    # 2a2
-    # > /opt/software/metadata/available/starlingx-24.09.1-metadata.xml:db313865837f9512b024a2356bd76106140ebcea783f8183e5fcc8d5cd28783b
-
-    diff "${METADATA_SYNC_DIR}"/ostree-metadata-commits.{central,subcloud} | awk '/^</ {print $2;}' | awk -F ':' '{print $1;}'
 }
 
-get_metadata_files_unique_to_subcloud() {
+sync_ostree_repo() {
-    diff "${METADATA_SYNC_DIR}"/ostree-metadata-commits.{central,subcloud} | awk '/^>/ {print $2;}' | awk -F ':' '{print $1;}'
+    # Synchronizes the remote ostree repository (system controller) to the local subcloud.
-}
+    local tmp_ostree_sync_file="/tmp/sync-ostree-commits.log"
-
+    run_cmd ostree --repo="${OSTREE_REPO}" pull --mirror --depth=-1 "${OSTREE_REMOTE_REF}" > "${tmp_ostree_sync_file}" 2>&1
-pull_ostree_commit_to_subcloud() {
+    rc=$?
-    # Pulls given commit into subcloud feed repo
+    # To avoid showing all ostree pull progress, which generates a very large output
-    #
+    # in Ansible, we show only the report line in case of success.
-    local commit_hash=$1
+    # In case of error only the last 10 lines.
-    if ostree_commit_exists "${commit_hash}"; then
+    if [ "$rc" != "0" ]; then
-        log_info "ostree commit ${commit_hash}: already exists in ${OSTREE_LOCAL_REF}"
+        tail -10 "${tmp_ostree_sync_file}"
-    else
+        rm -f "${tmp_ostree_sync_file}"
-        log_info "Pulling ostree commit from system controller: ${commit_hash}"
+        check_rc_die $rc "Unable to synchronize ostree repository."
-        run_cmd ostree --repo="${OSTREE_REPO}" pull --mirror "${OSTREE_REMOTE_REF}" "${OSTREE_BRANCH}@${commit_hash}"
-        check_rc_die $? "ostree pull failed"
     fi
+    tail -1 "${tmp_ostree_sync_file}"
+    rm -f "${tmp_ostree_sync_file}"
 }
 
 configure_ostree_repo_for_central_pull() {
@@ -436,7 +445,6 @@ find_all_ostree_commits_for_release() {
             get_commit_hashes_from_metadata commit_hashes "${metadata_file}"
             if [ "${#commit_hashes[@]}" -gt 0 ]; then
                 # We only need to supply the first commit here.
-                # See how the sync_subcloud_metadata algorithm works - it only uses the first commit
                 echo "${metadata_file}:${commit_hashes[0]}"
             fi
         fi
@@ -456,19 +464,15 @@ sync_metadata_on_subcloud() {
     #         - this will include the 'ostree-commit-id' and 'committed' ATTRIBUTES from systemController
     #         * this has already been done by ansible
     #
+    #     We are deleting all files of the release before synchronization to ensure that the
+    #     metadata will be exactly the same as the SC. It is not necessary to update the commits since
+    #     the subcloud repository is a mirror of the SC repository.
+    #
     #     IF RELEASE does NOT EXIST on subcloud
     #         IF 'ostree-commit-id' == NULL
-    #             Create it with STATE = unavailable
-    #         ELSE
     #             Create it with STATE = available
-    #     ELSE   // RELEASE exists on subcloud
-    #         IF subcloud STATE == deployed
-    #             Leave it as deployed
-    #         ELSE IF subcloud STATE == available or unavailable
-    #             IF ‘ostree-commit-id’ == NULL
-    #                 Set STATE = unavailable
     #         ELSE
-    #                 Set STATE = available
+    #             Create it with STATE = deployed
     #
     # For each RELEASE on SUBCLOUD but NOT synchronized from systemController
     #     REMOVE RELEASE FROM SUBCLOUD
@@ -506,50 +510,16 @@ sync_metadata_on_subcloud() {
     if [ -z "${subcloud_metadata_file}" ]; then
         # Not found: RELEASE does NOT EXIST on subcloud
         if ostree_commit_exists "${commit_hash}"; then
-            # Create it with STATE = available
+            # Create it with STATE = deployed
             log_debug_l "sync_metadata_on_subcloud: commit exists in local ${OSTREE_LOCAL_REF}"\
                 "ref: ${commit_hash}"
-            new_state="available"
+            new_state="deployed"
         else
-            # Create it with STATE = unavailable
+            # Create it with STATE = available
-            new_state="unavailable"
+            new_state="available"
         fi
         log_info "${log_hdr} does not exist on subcloud, setting to ${new_state}"
         run_cmd cp "${central_metadata_file}" "${METADATA_DIR}/${new_state}"
-    else
-        # RELEASE exists on subcloud
-        local subcloud_state
-        subcloud_state=$(get_usm_state_from_path "${subcloud_metadata_file}")
-        case "${subcloud_state}" in
-            'deployed')
-                # Leave it as deployed
-                log_info "${log_hdr} is in sync (subcloud state: deployed)"
-                ;;
-            'available'|'unavailable')
-                # Not found: RELEASE does NOT EXIST on subcloud
-                if ostree_commit_exists "${commit_hash}"; then
-                    # Set STATE = available
-                    log_debug_l "sync_metadata_on_subcloud: commit exists in local ${OSTREE_LOCAL_REF}"\
-                        "ref: ${commit_hash}"
-                    new_state=available
-                else
-                    # Set STATE = unavailable
-                    new_state=unavailable
-                fi
-                log_info "${log_hdr} exists on subcloud, setting subcloud state: ${new_state}"
-                run_cmd rm "${subcloud_metadata_file}"
-                run_cmd cp "${central_metadata_file}" "${METADATA_DIR}/${new_state}"
-                ;;
-            'committed')
-                log_info "${log_hdr} subcloud state is ${subcloud_state} - ignoring"
-                ;;
-            'deploying'|'removing')
-                log_info "${log_hdr} subcloud state is ${subcloud_state} - transitional, ignoring"
-                ;;
-            *)
-                log_error "${log_hdr} subcloud state is unexpected: ${subcloud_state} - ignoring"
-                ;;
-        esac
     fi
 }
 
@@ -560,30 +530,37 @@ sync_subcloud_metadata() {
     #
     # When this is invoked, we have the following in place (via ansible):
     #
-    #   - "${METADATA_SYNC_DIR}"/ostree-metadata-commits.{central,subcloud}
+    #   - "${METADATA_SYNC_DIR}"/ostree-metadata-commits.central
-    #       - these files summarizing the metadata files / ostree commits matching our given release
     #   - "${METADATA_SYNC_DIR}/metadata
     #       - is a direct copy of the system controller /opt/software/medatada directory
-    #       - we use this to calculate the new subcloud state of the release
+    #       - we use this to ensure that the metadata files exist in the subcloud
-    #         and to pull the missing ostree commits to the subcloud
     #
     # Synchronization is done on a per-major-release basis.
     # For given major release:
-    # 1) Get a list of all update metadata files needing to be synchronized
+    # 1) Syncronize ostree repo
-    #    (this is done by comparing (using diff) the central and subcloud file in
+    # 2) Remove the metadata from the specified release to ensure that when synchronized,
-    #    "${METADATA_SYNC_DIR}"/ostree-metadata-commits.{central,subcloud}).
+    #    it is in the right directory, based on the state.
-    # 2) Ensure any ostree commit(s) for the update are pulled from central
+    # 3) Get a list of all update metadata files needing to be synchronized
-    #    controller if necessary.
+    # 4) Synchronize the update metadata file into the proper state-based location
-    # 3) Synchronize the update metadata file into the proper state-based location
     #    on the subcloud
     #
     local metadata_file commit_hash central_metadata_file
+    local sw_version=${1:-$SW_VERSION}
 
+    # Configure and sync ostree feed repo
+    # This will be able to ostree at the same level between System Controller and subcloud.
     configure_ostree_repo_for_central_pull
+    sync_ostree_repo
 
     local commit_hashes=()
     local commit_hash
-    # 1) Get list of metadata files requiring sync
+
+    # Remove current files for specified release
+    log_info "Removing directories for release ${sw_version}"
+    rm -Rf ${USM_SOFTWARE_DIR}/rel-${sw_version}.*
+    find ${USM_METADATA_DIR} -type f -name "*${sw_version}*" | xargs rm -f
+
+    # Get list of metadata files requiring sync
     for metadata_file in $(get_metadata_files_unique_to_central); do
         log_info "sync_subcloud_metadata: processing ${metadata_file} from central (sync)"
         central_metadata_file=$(translate_central_metadata_path "${metadata_file}")
@@ -595,44 +572,9 @@ sync_subcloud_metadata() {
             "commit_hashes: ${commit_hashes[*]}"
 
         if [ "${#commit_hashes[@]}" -gt 0 ]; then
-            for commit_hash in "${commit_hashes[@]}"; do
+            # Synchronize the metadata file
-                # 2) Pull from central controller if necessary
-
-                # TODO(kmacleod): check if previous_commit exists from metadata, fail
-
-                pull_ostree_commit_to_subcloud "${commit_hash}"
-            done
-            # 3) Synchronize the metadata file
             sync_metadata_on_subcloud "${metadata_file}" "${central_metadata_file}" commit_hashes
-        fi
-    done
-    for metadata_file in $(get_metadata_files_unique_to_subcloud); do
-        log_info "sync_subcloud_metadata: processing ${metadata_file} from subcloud (check remove)"
             commit_hashes=()
-        get_commit_hashes_from_metadata commit_hashes "${central_metadata_file}"
-        log_debug_l "sync_subcloud_metadata from subcloud (check remove): "\
-            "metadata_file: ${metadata_file}"\
-            "commit_hashes: ${commit_hashes[*]}"
-        local removed=
-        if [ "${#commit_hashes[@]}" -gt 0 ]; then
-            for commit_hash in "${commit_hashes[@]}"; do
-                if ! ostree_commit_exists "${commit_hash}"; then
-                    log_info "sync_subcloud_metadata from subcloud: commit '${commit_hash}' does not exist, removing '${metadata_file}'"
-                    removed=1
-                fi
-            done
-            if [ -n "${removed}" ]; then
-                rm "${metadata_file}"
-            fi
-        fi
-        if [ -n "${removed}" ]; then
-            log_info_l "sync_subcloud_metadata from subcloud, removed file for non-existing commit(s): "\
-                "metadata_file: ${metadata_file}"\
-                "commit_hashes: ${commit_hashes[*]}"
-        else
-            log_info_l "sync_subcloud_metadata from subcloud, commit is in use, not removing: "\
-                "metadata_file: ${metadata_file}"\
-                "commit_hashes: ${commit_hashes[*]}"
         fi
     done
 }

playbookconfig/src/playbooks/roles/prestage/sync-software-metadata/tasks/main.yml

@@ -150,33 +150,52 @@
     subcloud_releases: "{{ subcloud_software_list.stdout_lines | \
     map('regex_replace', '.*?-([0-9\\.]+).*', '\\1') | list }}"
 
-- name: Verify patch levels between system controller and subcloud
+# The subcloud must have at least one release for the given software version to be able
-  debug:
+# to compare with the system controller version. Otherwise, it is assumed that the
-    msg: >
+# release does not exist in the subcloud and the prestage is executed.
-      {% if system_controller_releases | length > subcloud_releases | length %}
+- block:
-        The system controller has a patch level higher than the subcloud
+    # It is necessary to compare the current patch level between the SC and the subcloud
-      {% elif system_controller_releases | length == subcloud_releases | length %}
+    # to determine if prestage should be executed. In this way all scenarios are covered,
+    # including pre-patched scenario.
+  - name: Extract current patch number from system controller release list
+    set_fact:
+      system_controller_current_patch: "{{ system_controller_releases[-1] | \
+      regex_replace('.*\\.', '') | int }}"
+
+  - name: Extract current patch number from subcloud release list
+    set_fact:
+      subcloud_current_patch: "{{ subcloud_releases[-1] | regex_replace('.*\\.', '') | int  }}"
+
+  # We need to compare the patch level, since we must take into account even the
+  # pre-patched scenario.
+  - debug:
+      msg: |
+        ostree revision from:                 {{ ostree_repo_release_feed }}:
+        system controller current patch:      {{ system_controller_releases[-1] }}
+        subcloud current patch:               {{ subcloud_releases[-1] }}
+        system controller current commit:     {{ ostree_commit_system_controller.stdout }}
+        subcloud current commit:              {{ ostree_commit_subcloud.stdout }}
+
+        {% if system_controller_current_patch == subcloud_current_patch %}
         Skipping for software deploy prestage as the subcloud has the same patch level
         than the system controller.
-      {% else %}
+        {% elif subcloud_current_patch > system_controller_current_patch %}
         Skipping for software deploy prestage as the subcloud has a higher patch level
         than the system controller.
         {% endif %}
 
-- name: "Subcloud does not require software prestage. Exiting..."
+  - name: "Subcloud does not require software prestage. Exiting..."
     meta: end_play
-  when: subcloud_releases | length >= system_controller_releases | length
+    when: subcloud_current_patch >= system_controller_current_patch
+  when: subcloud_releases | length > 0
 
 - block:
+  - debug:
+      msg: The system controller has a patch level higher than the subcloud
+
   - name: Gather system controller metadata commits
-    #
+    # Get the commits from the metadata, in the same way that software list does,
-    # Use the existing ostree_metadata_commits_central file if:
+    # avoiding ambiguous queries.
-    # 1) It exists, and
-    # 2) There are no new commits. Compare last line of the ostree_metadata_commits_central
-    #    file against current ostree repo commit.
-    # Otherwise, we generate a new ostree_metadata_commits_central file using the
-    # get-commits target to our script.
-    #
     # Parallel operations (for orchestration):
     # We use flock here because there may be many prestaging operations running
     # in parallel on system controller. flock behaviour:
@@ -184,15 +203,10 @@
     #   - the timeout is long just to ensure we never deadlock for any reason
     shell: |
       exec 3>/tmp/ostree_metadata_commits_central.lock
-      flock --exclusive --timeout 180 3 || echo "ERROR: flock failed: $?"
+      flock --exclusive --timeout 180 3 || \
-      if [ ! -f "{{ ostree_metadata_commits_central }}" ] \
+      { echo "ERROR: $? - flock failed while trying to get the commits."; exit 1; }
-        || ! diff -q <(ostree --repo="{{ ostree_repo_release_feed }}" rev-parse "{{ ostree_rev }}") \
+      {{ role_path }}/files/ostree-metadata-sync.sh --sw-version "{{ software_version }}" \
-        <(tail --lines=1 "{{ ostree_metadata_commits_central }}" | awk -F':' '{ print $2; }') > /dev/null 2>&1 ; then
+      --output "{{ ostree_metadata_commits_central }}" get-commits
-        {{ role_path }}/files/ostree-metadata-sync.sh \
-        --sw-version "{{ software_version }}" --output "{{ ostree_metadata_commits_central }}" get-commits
-      else
-        cat "{{ ostree_metadata_commits_central }}"
-      fi
       exec 3>&-  # release the lock
     register: system_controller_software_metadata_commits
     delegate_to: localhost
@@ -212,27 +226,6 @@
     debug:
       var: subcloud_software_metadata_commits
 
-  - debug:
-      msg:
-        - "ostree revision from {{ ostree_repo_release_feed }}:"
-        - "system controller: {{ ostree_commit_system_controller.stdout }}"
-        - "subcloud:          {{ ostree_commit_subcloud.stdout }}"
-        - "Software list:"
-        - "system controller:"
-        - "{{ system_controller_software_list.stdout }}"
-        - "{{ system_controller_software_metadata_commits.stdout }}"
-        - "subcloud:"
-        - "{{ subcloud_software_list.stdout }}"
-        - "{{ subcloud_software_metadata_commits.stdout }}"
-
-  - name: Gather system controller deployed release list
-    shell: |
-      while IFS= read -r line; do
-        echo "$line" | awk -F'|' '{split($2,p,"-"); print p[2]}'
-      done <<< "{{ system_controller_software_list.stdout }}"
-    register: minor_releases_list
-    delegate_to: localhost
-
   # It's necessary to temporarily change the owner to sysadmin so that
   # the system controller can push the files to the subcloud, since the files
   # are in the folders only the root can access but the synchronize only
@@ -245,21 +238,6 @@
       recurse: yes
     become: yes
 
-  - name: Copy system controller deployed release directory to subcloud
-    synchronize:
-      mode: "push"
-      src: "{{ usm_software_dir }}/rel-{{ item }}"
-      dest: "{{ usm_software_dir }}"
-    with_items: "{{ minor_releases_list.stdout_lines }}"
-    failed_when: false
-
-  - name: Copy system controller {{ usm_software_dir }}/software-scripts to subcloud
-    synchronize:
-      mode: "push"
-      src: "{{ usm_software_dir }}/software-scripts"
-      dest: "{{ usm_software_dir }}"
-    failed_when: false
-
   - name: Copy system controller {{ usm_metadata_dir }} to subcloud {{ tmp_metadata_sync_dir }}
     copy:
       src: "{{ usm_metadata_dir }}"
@@ -278,7 +256,7 @@
   # on the system controller as it does not exist on N-1 release e.g. 22.12
   - name: Synchronizing system controller ostree commits on subcloud
     script: "{{ role_path }}/files/ostree-metadata-sync.sh --sw-version {{ software_version }}
-            sync-subcloud 2>&1 | tee /tmp/sync-ostree-commits.log"
+            sync-subcloud"
     register: sync_software_commits
     become: true
 
@@ -290,6 +268,21 @@
         - "stderr: {{ sync_software_commits.stderr }}"
         - "stdout: {{ sync_software_commits.stdout }}"
 
+  - name: Copy system controller deployed release directory to subcloud
+    synchronize:
+      mode: "push"
+      src: "{{ usm_software_dir }}/rel-{{ item }}"
+      dest: "{{ usm_software_dir }}"
+    with_items: "{{ system_controller_releases }}"
+    failed_when: false
+
+  - name: Copy system controller {{ usm_software_dir }}/software-scripts to subcloud
+    synchronize:
+      mode: "push"
+      src: "{{ usm_software_dir }}/software-scripts"
+      dest: "{{ usm_software_dir }}"
+    failed_when: false
+
   always:
     - name: Restore the ownership of {{ usm_software_dir }}
       file: