diff --git a/playbookconfig/src/playbooks/roles/prestage/prepare-env/tasks/prepare-sw-packages-prestage-env.yml b/playbookconfig/src/playbooks/roles/prestage/prepare-env/tasks/prepare-sw-packages-prestage-env.yml index d0b40fd67..d2d87d152 100644 --- a/playbookconfig/src/playbooks/roles/prestage/prepare-env/tasks/prepare-sw-packages-prestage-env.yml +++ b/playbookconfig/src/playbooks/roles/prestage/prepare-env/tasks/prepare-sw-packages-prestage-env.yml @@ -65,6 +65,7 @@ force_ostree_dir_sync: "{{ 'true' if (prestage_source == 'remote' and host_software_version is version('22.12', '=') and not host_ostree_dir_exist) else 'false' }}" + - debug: msg: | Prestaging type: {{ prestage_type }} diff --git a/playbookconfig/src/playbooks/roles/prestage/sync-software-metadata/files/ostree-metadata-sync.sh b/playbookconfig/src/playbooks/roles/prestage/sync-software-metadata/files/ostree-metadata-sync.sh index daff0c75b..8a19738bc 100755 --- a/playbookconfig/src/playbooks/roles/prestage/sync-software-metadata/files/ostree-metadata-sync.sh +++ b/playbookconfig/src/playbooks/roles/prestage/sync-software-metadata/files/ostree-metadata-sync.sh @@ -38,6 +38,9 @@ SW_VERSION=${SW_VERSION:-} DEBUG=${DEBUG:-} DRY_RUN=${DRY_RUN:-} +USM_SOFTWARE_DIR=/opt/software +USM_METADATA_DIR=${USM_SOFTWARE_DIR}/metadata + help() { cat< are on the same line. + # Get the value of given XML attribute. + # We embed Python to parse the XML within script. + # Params: + # $1 = the metadata file path + # $2 = the xml attribute to find. can be a path that + # specifies the element. For example: + # /root/item/subitem1 + # Returns the value of the element if it exists. local meta_file=$1 local attrib=$2 local val - val=$(sed -n 's|<'"${attrib}"'>\(.*\)|\1|p' "${meta_file}") - val=$(trim "${val}") + +val=$(python - </,/<\/base>/d' "${meta_file}" | sed --quiet 's|\(.*\)|\1|p') + local commit_path="contents/ostree/commit1/commit" + + from_metadata_commit_hashes=$(get_simple_xml_attrib_from_metadata "${meta_file}" "${commit_path}") } get_usm_state_from_path() { @@ -318,12 +334,14 @@ get_usm_state_from_path() { } ostree_commit_exists() { - # Does given commit exist in ostree? i.e. has it been pulled into our repo yet? - # Note: this only works for locally defined ostree repositories. - # i.e. it can't get status from a remote server + # To ensure that the patch is entirely deployed, it must be verified in both repositories. + # This ensures that when checking whether the commit of a given patch exists on the system, + # it considers both repos (ostree repo and sysroot) local commit_hash=$1 local ref=${2:-${OSTREE_LOCAL_REF}} - ostree --repo="${OSTREE_REPO}" log "${ref}" | grep '^commit ' | grep --quiet "${commit_hash}" + ostree --repo="${OSTREE_REPO}" log "${ref}" | grep '^commit ' | grep --quiet "${commit_hash}" && \ + ostree --repo="${OSTREE_SYSROOT_REPO}" log "${ref}" | grep '^commit ' | grep --quiet "${commit_hash}" + return $? } translate_central_metadata_path() { @@ -333,33 +351,24 @@ translate_central_metadata_path() { } get_metadata_files_unique_to_central() { - # TODO ISSUE: - # This gets flagged for removal which it shouldn't - it's just a stage change: - # - # [sysadmin@controller-0 ~(keystone_admin)]$ diff -s /opt/software/tmp/metadata-sync/ostree-metadata-commits.* - # 1d0 - # < /opt/software/metadata/deployed/starlingx-24.09.1-metadata.xml:db313865837f9512b024a2356bd76106140ebcea783f8183e5fcc8d5cd28783b - # 2a2 - # > /opt/software/metadata/available/starlingx-24.09.1-metadata.xml:db313865837f9512b024a2356bd76106140ebcea783f8183e5fcc8d5cd28783b - - diff "${METADATA_SYNC_DIR}"/ostree-metadata-commits.{central,subcloud} | awk '/^/ {print $2;}' | awk -F ':' '{print $1;}' -} - -pull_ostree_commit_to_subcloud() { - # Pulls given commit into subcloud feed repo - # - local commit_hash=$1 - if ostree_commit_exists "${commit_hash}"; then - log_info "ostree commit ${commit_hash}: already exists in ${OSTREE_LOCAL_REF}" - else - log_info "Pulling ostree commit from system controller: ${commit_hash}" - run_cmd ostree --repo="${OSTREE_REPO}" pull --mirror "${OSTREE_REMOTE_REF}" "${OSTREE_BRANCH}@${commit_hash}" - check_rc_die $? "ostree pull failed" +sync_ostree_repo() { + # Synchronizes the remote ostree repository (system controller) to the local subcloud. + local tmp_ostree_sync_file="/tmp/sync-ostree-commits.log" + run_cmd ostree --repo="${OSTREE_REPO}" pull --mirror --depth=-1 "${OSTREE_REMOTE_REF}" > "${tmp_ostree_sync_file}" 2>&1 + rc=$? + # To avoid showing all ostree pull progress, which generates a very large output + # in Ansible, we show only the report line in case of success. + # In case of error only the last 10 lines. + if [ "$rc" != "0" ]; then + tail -10 "${tmp_ostree_sync_file}" + rm -f "${tmp_ostree_sync_file}" + check_rc_die $rc "Unable to synchronize ostree repository." fi + tail -1 "${tmp_ostree_sync_file}" + rm -f "${tmp_ostree_sync_file}" } configure_ostree_repo_for_central_pull() { @@ -436,7 +445,6 @@ find_all_ostree_commits_for_release() { get_commit_hashes_from_metadata commit_hashes "${metadata_file}" if [ "${#commit_hashes[@]}" -gt 0 ]; then # We only need to supply the first commit here. - # See how the sync_subcloud_metadata algorithm works - it only uses the first commit echo "${metadata_file}:${commit_hashes[0]}" fi fi @@ -456,19 +464,15 @@ sync_metadata_on_subcloud() { # - this will include the 'ostree-commit-id' and 'committed' ATTRIBUTES from systemController # * this has already been done by ansible # + # We are deleting all files of the release before synchronization to ensure that the + # metadata will be exactly the same as the SC. It is not necessary to update the commits since + # the subcloud repository is a mirror of the SC repository. + # # IF RELEASE does NOT EXIST on subcloud # IF 'ostree-commit-id' == NULL - # Create it with STATE = unavailable - # ELSE # Create it with STATE = available - # ELSE // RELEASE exists on subcloud - # IF subcloud STATE == deployed - # Leave it as deployed - # ELSE IF subcloud STATE == available or unavailable - # IF ‘ostree-commit-id’ == NULL - # Set STATE = unavailable - # ELSE - # Set STATE = available + # ELSE + # Create it with STATE = deployed # # For each RELEASE on SUBCLOUD but NOT synchronized from systemController # REMOVE RELEASE FROM SUBCLOUD @@ -506,50 +510,16 @@ sync_metadata_on_subcloud() { if [ -z "${subcloud_metadata_file}" ]; then # Not found: RELEASE does NOT EXIST on subcloud if ostree_commit_exists "${commit_hash}"; then - # Create it with STATE = available + # Create it with STATE = deployed log_debug_l "sync_metadata_on_subcloud: commit exists in local ${OSTREE_LOCAL_REF}"\ "ref: ${commit_hash}" - new_state="available" + new_state="deployed" else - # Create it with STATE = unavailable - new_state="unavailable" + # Create it with STATE = available + new_state="available" fi log_info "${log_hdr} does not exist on subcloud, setting to ${new_state}" run_cmd cp "${central_metadata_file}" "${METADATA_DIR}/${new_state}" - else - # RELEASE exists on subcloud - local subcloud_state - subcloud_state=$(get_usm_state_from_path "${subcloud_metadata_file}") - case "${subcloud_state}" in - 'deployed') - # Leave it as deployed - log_info "${log_hdr} is in sync (subcloud state: deployed)" - ;; - 'available'|'unavailable') - # Not found: RELEASE does NOT EXIST on subcloud - if ostree_commit_exists "${commit_hash}"; then - # Set STATE = available - log_debug_l "sync_metadata_on_subcloud: commit exists in local ${OSTREE_LOCAL_REF}"\ - "ref: ${commit_hash}" - new_state=available - else - # Set STATE = unavailable - new_state=unavailable - fi - log_info "${log_hdr} exists on subcloud, setting subcloud state: ${new_state}" - run_cmd rm "${subcloud_metadata_file}" - run_cmd cp "${central_metadata_file}" "${METADATA_DIR}/${new_state}" - ;; - 'committed') - log_info "${log_hdr} subcloud state is ${subcloud_state} - ignoring" - ;; - 'deploying'|'removing') - log_info "${log_hdr} subcloud state is ${subcloud_state} - transitional, ignoring" - ;; - *) - log_error "${log_hdr} subcloud state is unexpected: ${subcloud_state} - ignoring" - ;; - esac fi } @@ -560,30 +530,37 @@ sync_subcloud_metadata() { # # When this is invoked, we have the following in place (via ansible): # - # - "${METADATA_SYNC_DIR}"/ostree-metadata-commits.{central,subcloud} - # - these files summarizing the metadata files / ostree commits matching our given release + # - "${METADATA_SYNC_DIR}"/ostree-metadata-commits.central # - "${METADATA_SYNC_DIR}/metadata # - is a direct copy of the system controller /opt/software/medatada directory - # - we use this to calculate the new subcloud state of the release - # and to pull the missing ostree commits to the subcloud + # - we use this to ensure that the metadata files exist in the subcloud # # Synchronization is done on a per-major-release basis. # For given major release: - # 1) Get a list of all update metadata files needing to be synchronized - # (this is done by comparing (using diff) the central and subcloud file in - # "${METADATA_SYNC_DIR}"/ostree-metadata-commits.{central,subcloud}). - # 2) Ensure any ostree commit(s) for the update are pulled from central - # controller if necessary. - # 3) Synchronize the update metadata file into the proper state-based location + # 1) Syncronize ostree repo + # 2) Remove the metadata from the specified release to ensure that when synchronized, + # it is in the right directory, based on the state. + # 3) Get a list of all update metadata files needing to be synchronized + # 4) Synchronize the update metadata file into the proper state-based location # on the subcloud # local metadata_file commit_hash central_metadata_file + local sw_version=${1:-$SW_VERSION} + # Configure and sync ostree feed repo + # This will be able to ostree at the same level between System Controller and subcloud. configure_ostree_repo_for_central_pull + sync_ostree_repo local commit_hashes=() local commit_hash - # 1) Get list of metadata files requiring sync + + # Remove current files for specified release + log_info "Removing directories for release ${sw_version}" + rm -Rf ${USM_SOFTWARE_DIR}/rel-${sw_version}.* + find ${USM_METADATA_DIR} -type f -name "*${sw_version}*" | xargs rm -f + + # Get list of metadata files requiring sync for metadata_file in $(get_metadata_files_unique_to_central); do log_info "sync_subcloud_metadata: processing ${metadata_file} from central (sync)" central_metadata_file=$(translate_central_metadata_path "${metadata_file}") @@ -595,44 +572,9 @@ sync_subcloud_metadata() { "commit_hashes: ${commit_hashes[*]}" if [ "${#commit_hashes[@]}" -gt 0 ]; then - for commit_hash in "${commit_hashes[@]}"; do - # 2) Pull from central controller if necessary - - # TODO(kmacleod): check if previous_commit exists from metadata, fail - - pull_ostree_commit_to_subcloud "${commit_hash}" - done - # 3) Synchronize the metadata file + # Synchronize the metadata file sync_metadata_on_subcloud "${metadata_file}" "${central_metadata_file}" commit_hashes - fi - done - for metadata_file in $(get_metadata_files_unique_to_subcloud); do - log_info "sync_subcloud_metadata: processing ${metadata_file} from subcloud (check remove)" - commit_hashes=() - get_commit_hashes_from_metadata commit_hashes "${central_metadata_file}" - log_debug_l "sync_subcloud_metadata from subcloud (check remove): "\ - "metadata_file: ${metadata_file}"\ - "commit_hashes: ${commit_hashes[*]}" - local removed= - if [ "${#commit_hashes[@]}" -gt 0 ]; then - for commit_hash in "${commit_hashes[@]}"; do - if ! ostree_commit_exists "${commit_hash}"; then - log_info "sync_subcloud_metadata from subcloud: commit '${commit_hash}' does not exist, removing '${metadata_file}'" - removed=1 - fi - done - if [ -n "${removed}" ]; then - rm "${metadata_file}" - fi - fi - if [ -n "${removed}" ]; then - log_info_l "sync_subcloud_metadata from subcloud, removed file for non-existing commit(s): "\ - "metadata_file: ${metadata_file}"\ - "commit_hashes: ${commit_hashes[*]}" - else - log_info_l "sync_subcloud_metadata from subcloud, commit is in use, not removing: "\ - "metadata_file: ${metadata_file}"\ - "commit_hashes: ${commit_hashes[*]}" + commit_hashes=() fi done } diff --git a/playbookconfig/src/playbooks/roles/prestage/sync-software-metadata/tasks/main.yml b/playbookconfig/src/playbooks/roles/prestage/sync-software-metadata/tasks/main.yml index 37d78bb4f..c501984a7 100644 --- a/playbookconfig/src/playbooks/roles/prestage/sync-software-metadata/tasks/main.yml +++ b/playbookconfig/src/playbooks/roles/prestage/sync-software-metadata/tasks/main.yml @@ -150,33 +150,52 @@ subcloud_releases: "{{ subcloud_software_list.stdout_lines | \ map('regex_replace', '.*?-([0-9\\.]+).*', '\\1') | list }}" -- name: Verify patch levels between system controller and subcloud - debug: - msg: > - {% if system_controller_releases | length > subcloud_releases | length %} - The system controller has a patch level higher than the subcloud - {% elif system_controller_releases | length == subcloud_releases | length %} +# The subcloud must have at least one release for the given software version to be able +# to compare with the system controller version. Otherwise, it is assumed that the +# release does not exist in the subcloud and the prestage is executed. +- block: + # It is necessary to compare the current patch level between the SC and the subcloud + # to determine if prestage should be executed. In this way all scenarios are covered, + # including pre-patched scenario. + - name: Extract current patch number from system controller release list + set_fact: + system_controller_current_patch: "{{ system_controller_releases[-1] | \ + regex_replace('.*\\.', '') | int }}" + + - name: Extract current patch number from subcloud release list + set_fact: + subcloud_current_patch: "{{ subcloud_releases[-1] | regex_replace('.*\\.', '') | int }}" + + # We need to compare the patch level, since we must take into account even the + # pre-patched scenario. + - debug: + msg: | + ostree revision from: {{ ostree_repo_release_feed }}: + system controller current patch: {{ system_controller_releases[-1] }} + subcloud current patch: {{ subcloud_releases[-1] }} + system controller current commit: {{ ostree_commit_system_controller.stdout }} + subcloud current commit: {{ ostree_commit_subcloud.stdout }} + + {% if system_controller_current_patch == subcloud_current_patch %} Skipping for software deploy prestage as the subcloud has the same patch level than the system controller. - {% else %} + {% elif subcloud_current_patch > system_controller_current_patch %} Skipping for software deploy prestage as the subcloud has a higher patch level than the system controller. - {% endif %} + {% endif %} -- name: "Subcloud does not require software prestage. Exiting..." - meta: end_play - when: subcloud_releases | length >= system_controller_releases | length + - name: "Subcloud does not require software prestage. Exiting..." + meta: end_play + when: subcloud_current_patch >= system_controller_current_patch + when: subcloud_releases | length > 0 - block: + - debug: + msg: The system controller has a patch level higher than the subcloud + - name: Gather system controller metadata commits - # - # Use the existing ostree_metadata_commits_central file if: - # 1) It exists, and - # 2) There are no new commits. Compare last line of the ostree_metadata_commits_central - # file against current ostree repo commit. - # Otherwise, we generate a new ostree_metadata_commits_central file using the - # get-commits target to our script. - # + # Get the commits from the metadata, in the same way that software list does, + # avoiding ambiguous queries. # Parallel operations (for orchestration): # We use flock here because there may be many prestaging operations running # in parallel on system controller. flock behaviour: @@ -184,15 +203,10 @@ # - the timeout is long just to ensure we never deadlock for any reason shell: | exec 3>/tmp/ostree_metadata_commits_central.lock - flock --exclusive --timeout 180 3 || echo "ERROR: flock failed: $?" - if [ ! -f "{{ ostree_metadata_commits_central }}" ] \ - || ! diff -q <(ostree --repo="{{ ostree_repo_release_feed }}" rev-parse "{{ ostree_rev }}") \ - <(tail --lines=1 "{{ ostree_metadata_commits_central }}" | awk -F':' '{ print $2; }') > /dev/null 2>&1 ; then - {{ role_path }}/files/ostree-metadata-sync.sh \ - --sw-version "{{ software_version }}" --output "{{ ostree_metadata_commits_central }}" get-commits - else - cat "{{ ostree_metadata_commits_central }}" - fi + flock --exclusive --timeout 180 3 || \ + { echo "ERROR: $? - flock failed while trying to get the commits."; exit 1; } + {{ role_path }}/files/ostree-metadata-sync.sh --sw-version "{{ software_version }}" \ + --output "{{ ostree_metadata_commits_central }}" get-commits exec 3>&- # release the lock register: system_controller_software_metadata_commits delegate_to: localhost @@ -212,27 +226,6 @@ debug: var: subcloud_software_metadata_commits - - debug: - msg: - - "ostree revision from {{ ostree_repo_release_feed }}:" - - "system controller: {{ ostree_commit_system_controller.stdout }}" - - "subcloud: {{ ostree_commit_subcloud.stdout }}" - - "Software list:" - - "system controller:" - - "{{ system_controller_software_list.stdout }}" - - "{{ system_controller_software_metadata_commits.stdout }}" - - "subcloud:" - - "{{ subcloud_software_list.stdout }}" - - "{{ subcloud_software_metadata_commits.stdout }}" - - - name: Gather system controller deployed release list - shell: | - while IFS= read -r line; do - echo "$line" | awk -F'|' '{split($2,p,"-"); print p[2]}' - done <<< "{{ system_controller_software_list.stdout }}" - register: minor_releases_list - delegate_to: localhost - # It's necessary to temporarily change the owner to sysadmin so that # the system controller can push the files to the subcloud, since the files # are in the folders only the root can access but the synchronize only @@ -245,21 +238,6 @@ recurse: yes become: yes - - name: Copy system controller deployed release directory to subcloud - synchronize: - mode: "push" - src: "{{ usm_software_dir }}/rel-{{ item }}" - dest: "{{ usm_software_dir }}" - with_items: "{{ minor_releases_list.stdout_lines }}" - failed_when: false - - - name: Copy system controller {{ usm_software_dir }}/software-scripts to subcloud - synchronize: - mode: "push" - src: "{{ usm_software_dir }}/software-scripts" - dest: "{{ usm_software_dir }}" - failed_when: false - - name: Copy system controller {{ usm_metadata_dir }} to subcloud {{ tmp_metadata_sync_dir }} copy: src: "{{ usm_metadata_dir }}" @@ -278,7 +256,7 @@ # on the system controller as it does not exist on N-1 release e.g. 22.12 - name: Synchronizing system controller ostree commits on subcloud script: "{{ role_path }}/files/ostree-metadata-sync.sh --sw-version {{ software_version }} - sync-subcloud 2>&1 | tee /tmp/sync-ostree-commits.log" + sync-subcloud" register: sync_software_commits become: true @@ -290,6 +268,21 @@ - "stderr: {{ sync_software_commits.stderr }}" - "stdout: {{ sync_software_commits.stdout }}" + - name: Copy system controller deployed release directory to subcloud + synchronize: + mode: "push" + src: "{{ usm_software_dir }}/rel-{{ item }}" + dest: "{{ usm_software_dir }}" + with_items: "{{ system_controller_releases }}" + failed_when: false + + - name: Copy system controller {{ usm_software_dir }}/software-scripts to subcloud + synchronize: + mode: "push" + src: "{{ usm_software_dir }}/software-scripts" + dest: "{{ usm_software_dir }}" + failed_when: false + always: - name: Restore the ownership of {{ usm_software_dir }} file: