Adapt the prestage to support subcloud rehoming

When a subcloud is rehomed from one DC to a new DC, It has to
be able to reach the release level of the new DC.

The prestage for software deploy prepares the release level
that the subcloud should have deployed based on the release
level of the new DC.

This change affects the following scenarios:
- Subcloud with the same patch level as the system
  controller.
- Subcloud with a patch level lower than that of the
  system controller.
- System controller with a pre-patched load and
  subcloud with the same patch level applied.
- Subcloud with release N-1 and USM support.

Test Plan:
Setup 1: - Bring up a DC#1 with 24.09 and at least 1 subcloud.
	 - Bring up a DC#2 with 24.09.

PASS: - Rehome subcloud from DC#1 to DC#2.
      - Verify that the rehome is completed.
      - On DC#2 prestage the subcloud for software deploy.
      - Verify that the prestage is skipped because subcloud
        has the same release as the system controller.

PASS: - Upload and deploy patch 24.09.1 on DC#2.
      - Prestage the subcloud for software deploy.
      - Verify that the 24.09.1 patch is prestaged on the subcloud.

PASS: - Rehome subcloud from DC#2 to DC#1.
      - Verify that the rehome is completed.
      - On DC#1 prestage the subcloud for software deploy.
      - Verify that the prestage is skipped because the subcloud
        has a higher patch level than the system controller.

Setup 2: - Bring up a DC#1 with 22.12p5 (without USM feature)
           and at least 1 subcloud with the same release.
         - Bring up a DC#2 with 24.09.

PASS: - Rehome subcloud from DC#1 to DC#2.
      - Verify that the rehome is completed.
      - On DC#2 prestage the subcloud for software deploy.
      - Verify that the prestage is rejected because the USM
        service is not available on the subcloud.

PASS: - Upload and apply USM patch on the subcloud.
      - Verify that the USM service is now available.
      - On DC#2 prestage the subcloud for software deploy.
      - Verify that the 24.09 release is prestaged on the subcloud.

PASS: - Bring up a DC#1 with 24.09 and at least 1 subcloud.
      - Bring up a DC#2 with 24.09 pre-patched load.
      - Rehome subcloud from DC#1 to DC#2.
      - On DC#2 prestage the subcloud for software deploy.
      - Verify that the 24.09 release is prestaged on the subcloud.

Story: 2010676
Task: 51037

Change-Id: I7c2a985972df57b18b9ddcff4ac9d6ffd0d10e8e
Signed-off-by: Cristian Mondo <cristian.mondo@windriver.com>
This commit is contained in:
Cristian Mondo 2024-09-06 15:35:53 -03:00
parent e281cf7e3c
commit be476cc56b
3 changed files with 144 additions and 208 deletions

View File

@ -65,6 +65,7 @@
force_ostree_dir_sync: "{{ 'true' if (prestage_source == 'remote' and
host_software_version is version('22.12', '=') and
not host_ostree_dir_exist) else 'false' }}"
- debug:
msg: |
Prestaging type: {{ prestage_type }}

View File

@ -38,6 +38,9 @@ SW_VERSION=${SW_VERSION:-}
DEBUG=${DEBUG:-}
DRY_RUN=${DRY_RUN:-}
USM_SOFTWARE_DIR=/opt/software
USM_METADATA_DIR=${USM_SOFTWARE_DIR}/metadata
help() {
cat<<EOF
ostree metadata synchronization utilities.
@ -164,6 +167,7 @@ initialize_env() {
export MAJOR_SW_VERSION
export OSTREE_REPO="/var/www/pages/feed/rel-${MAJOR_SW_VERSION}/ostree_repo"
export OSTREE_SYSROOT_REPO="/sysroot/ostree/repo"
export OSTREE_REMOTE=starlingx
export OSTREE_BRANCH=starlingx
export OSTREE_LOCAL_REF="${OSTREE_REMOTE}"
@ -174,6 +178,7 @@ initialize_env() {
log_debug_l "SW_VERSION: ${SW_VERSION}"\
"MAJOR_SW_VERSION: ${MAJOR_SW_VERSION}"\
"OSTREE_REPO: ${OSTREE_REPO}"\
"OSTREE_SYSROOT_REPO: ${OSTREE_SYSROOT_REPO}"\
"OSTREE_LOCAL_REF: ${OSTREE_LOCAL_REF}"\
"OSTREE_REMOTE_REF: ${OSTREE_REMOTE_REF}"\
"METADATA_DIR: ${METADATA_DIR}"\
@ -255,35 +260,46 @@ find_metadata_file_for_attrib_val() {
}
get_simple_xml_attrib_from_metadata() {
# Retrieve the value of given attribute.
# WARNING: This function performs very basic parsing:
# It only works if the opening and closing
# <attrib> </attrib> are on the same line.
# Get the value of given XML attribute.
# We embed Python to parse the XML within script.
# Params:
# $1 = the metadata file path
# $2 = the xml attribute to find. can be a path that
# specifies the element. For example:
# /root/item/subitem1
# Returns the value of the element if it exists.
local meta_file=$1
local attrib=$2
local val
val=$(sed -n 's|<'"${attrib}"'>\(.*\)</'"${attrib}"'>|\1|p' "${meta_file}")
val=$(trim "${val}")
val=$(python - <<END
import xml.etree.ElementTree as et
import sys
try:
tree = et.parse('$meta_file')
root = tree.getroot()
tag = root.find(".//$attrib")
print(tag.text)
except:
sys.exit()
END
)
log_debug "metadata GET ${attrib}: ${val}"
echo "${val}"
}
get_commit_hashes_from_metadata() {
# Retrieves the commit hashes of given metadata file.
# The base commit and empty commits are ignored.
# Only package commits are taken into account.
#
# Using a nameref to update the passed-in array,
# see https://mywiki.wooledge.org/BashProgramming?highlight=%28nameref%29#Functions
# Retrieves the commit hash of given metadata file
# Currently the metadata file supports only a single commit.
# We use the commit1 path to find the hash.
local -n from_metadata_commit_hashes=$1
local meta_file=$2
local commit
while IFS= read -r commit; do
commit=$(trim "${commit}")
if [ ! -z "$commit" ]; then
from_metadata_commit_hashes+=( "${commit}" )
fi
done < <(sed '/<base>/,/<\/base>/d' "${meta_file}" | sed --quiet 's|<commit[0-9]*>\(.*\)</commit[0-9]*>|\1|p')
local commit_path="contents/ostree/commit1/commit"
from_metadata_commit_hashes=$(get_simple_xml_attrib_from_metadata "${meta_file}" "${commit_path}")
}
get_usm_state_from_path() {
@ -318,12 +334,14 @@ get_usm_state_from_path() {
}
ostree_commit_exists() {
# Does given commit exist in ostree? i.e. has it been pulled into our repo yet?
# Note: this only works for locally defined ostree repositories.
# i.e. it can't get status from a remote server
# To ensure that the patch is entirely deployed, it must be verified in both repositories.
# This ensures that when checking whether the commit of a given patch exists on the system,
# it considers both repos (ostree repo and sysroot)
local commit_hash=$1
local ref=${2:-${OSTREE_LOCAL_REF}}
ostree --repo="${OSTREE_REPO}" log "${ref}" | grep '^commit ' | grep --quiet "${commit_hash}"
ostree --repo="${OSTREE_REPO}" log "${ref}" | grep '^commit ' | grep --quiet "${commit_hash}" && \
ostree --repo="${OSTREE_SYSROOT_REPO}" log "${ref}" | grep '^commit ' | grep --quiet "${commit_hash}"
return $?
}
translate_central_metadata_path() {
@ -333,33 +351,24 @@ translate_central_metadata_path() {
}
get_metadata_files_unique_to_central() {
# TODO ISSUE:
# This gets flagged for removal which it shouldn't - it's just a stage change:
#
# [sysadmin@controller-0 ~(keystone_admin)]$ diff -s /opt/software/tmp/metadata-sync/ostree-metadata-commits.*
# 1d0
# < /opt/software/metadata/deployed/starlingx-24.09.1-metadata.xml:db313865837f9512b024a2356bd76106140ebcea783f8183e5fcc8d5cd28783b
# 2a2
# > /opt/software/metadata/available/starlingx-24.09.1-metadata.xml:db313865837f9512b024a2356bd76106140ebcea783f8183e5fcc8d5cd28783b
diff "${METADATA_SYNC_DIR}"/ostree-metadata-commits.{central,subcloud} | awk '/^</ {print $2;}' | awk -F ':' '{print $1;}'
cat "${METADATA_SYNC_DIR}"/ostree-metadata-commits.central | awk -F ':' '{print $1;}'
}
get_metadata_files_unique_to_subcloud() {
diff "${METADATA_SYNC_DIR}"/ostree-metadata-commits.{central,subcloud} | awk '/^>/ {print $2;}' | awk -F ':' '{print $1;}'
}
pull_ostree_commit_to_subcloud() {
# Pulls given commit into subcloud feed repo
#
local commit_hash=$1
if ostree_commit_exists "${commit_hash}"; then
log_info "ostree commit ${commit_hash}: already exists in ${OSTREE_LOCAL_REF}"
else
log_info "Pulling ostree commit from system controller: ${commit_hash}"
run_cmd ostree --repo="${OSTREE_REPO}" pull --mirror "${OSTREE_REMOTE_REF}" "${OSTREE_BRANCH}@${commit_hash}"
check_rc_die $? "ostree pull failed"
sync_ostree_repo() {
# Synchronizes the remote ostree repository (system controller) to the local subcloud.
local tmp_ostree_sync_file="/tmp/sync-ostree-commits.log"
run_cmd ostree --repo="${OSTREE_REPO}" pull --mirror --depth=-1 "${OSTREE_REMOTE_REF}" > "${tmp_ostree_sync_file}" 2>&1
rc=$?
# To avoid showing all ostree pull progress, which generates a very large output
# in Ansible, we show only the report line in case of success.
# In case of error only the last 10 lines.
if [ "$rc" != "0" ]; then
tail -10 "${tmp_ostree_sync_file}"
rm -f "${tmp_ostree_sync_file}"
check_rc_die $rc "Unable to synchronize ostree repository."
fi
tail -1 "${tmp_ostree_sync_file}"
rm -f "${tmp_ostree_sync_file}"
}
configure_ostree_repo_for_central_pull() {
@ -436,7 +445,6 @@ find_all_ostree_commits_for_release() {
get_commit_hashes_from_metadata commit_hashes "${metadata_file}"
if [ "${#commit_hashes[@]}" -gt 0 ]; then
# We only need to supply the first commit here.
# See how the sync_subcloud_metadata algorithm works - it only uses the first commit
echo "${metadata_file}:${commit_hashes[0]}"
fi
fi
@ -456,19 +464,15 @@ sync_metadata_on_subcloud() {
# - this will include the 'ostree-commit-id' and 'committed' ATTRIBUTES from systemController
# * this has already been done by ansible
#
# We are deleting all files of the release before synchronization to ensure that the
# metadata will be exactly the same as the SC. It is not necessary to update the commits since
# the subcloud repository is a mirror of the SC repository.
#
# IF RELEASE does NOT EXIST on subcloud
# IF 'ostree-commit-id' == NULL
# Create it with STATE = unavailable
# ELSE
# Create it with STATE = available
# ELSE // RELEASE exists on subcloud
# IF subcloud STATE == deployed
# Leave it as deployed
# ELSE IF subcloud STATE == available or unavailable
# IF ostree-commit-id == NULL
# Set STATE = unavailable
# ELSE
# Set STATE = available
# Create it with STATE = deployed
#
# For each RELEASE on SUBCLOUD but NOT synchronized from systemController
# REMOVE RELEASE FROM SUBCLOUD
@ -506,50 +510,16 @@ sync_metadata_on_subcloud() {
if [ -z "${subcloud_metadata_file}" ]; then
# Not found: RELEASE does NOT EXIST on subcloud
if ostree_commit_exists "${commit_hash}"; then
# Create it with STATE = available
# Create it with STATE = deployed
log_debug_l "sync_metadata_on_subcloud: commit exists in local ${OSTREE_LOCAL_REF}"\
"ref: ${commit_hash}"
new_state="available"
new_state="deployed"
else
# Create it with STATE = unavailable
new_state="unavailable"
# Create it with STATE = available
new_state="available"
fi
log_info "${log_hdr} does not exist on subcloud, setting to ${new_state}"
run_cmd cp "${central_metadata_file}" "${METADATA_DIR}/${new_state}"
else
# RELEASE exists on subcloud
local subcloud_state
subcloud_state=$(get_usm_state_from_path "${subcloud_metadata_file}")
case "${subcloud_state}" in
'deployed')
# Leave it as deployed
log_info "${log_hdr} is in sync (subcloud state: deployed)"
;;
'available'|'unavailable')
# Not found: RELEASE does NOT EXIST on subcloud
if ostree_commit_exists "${commit_hash}"; then
# Set STATE = available
log_debug_l "sync_metadata_on_subcloud: commit exists in local ${OSTREE_LOCAL_REF}"\
"ref: ${commit_hash}"
new_state=available
else
# Set STATE = unavailable
new_state=unavailable
fi
log_info "${log_hdr} exists on subcloud, setting subcloud state: ${new_state}"
run_cmd rm "${subcloud_metadata_file}"
run_cmd cp "${central_metadata_file}" "${METADATA_DIR}/${new_state}"
;;
'committed')
log_info "${log_hdr} subcloud state is ${subcloud_state} - ignoring"
;;
'deploying'|'removing')
log_info "${log_hdr} subcloud state is ${subcloud_state} - transitional, ignoring"
;;
*)
log_error "${log_hdr} subcloud state is unexpected: ${subcloud_state} - ignoring"
;;
esac
fi
}
@ -560,30 +530,37 @@ sync_subcloud_metadata() {
#
# When this is invoked, we have the following in place (via ansible):
#
# - "${METADATA_SYNC_DIR}"/ostree-metadata-commits.{central,subcloud}
# - these files summarizing the metadata files / ostree commits matching our given release
# - "${METADATA_SYNC_DIR}"/ostree-metadata-commits.central
# - "${METADATA_SYNC_DIR}/metadata
# - is a direct copy of the system controller /opt/software/medatada directory
# - we use this to calculate the new subcloud state of the release
# and to pull the missing ostree commits to the subcloud
# - we use this to ensure that the metadata files exist in the subcloud
#
# Synchronization is done on a per-major-release basis.
# For given major release:
# 1) Get a list of all update metadata files needing to be synchronized
# (this is done by comparing (using diff) the central and subcloud file in
# "${METADATA_SYNC_DIR}"/ostree-metadata-commits.{central,subcloud}).
# 2) Ensure any ostree commit(s) for the update are pulled from central
# controller if necessary.
# 3) Synchronize the update metadata file into the proper state-based location
# 1) Syncronize ostree repo
# 2) Remove the metadata from the specified release to ensure that when synchronized,
# it is in the right directory, based on the state.
# 3) Get a list of all update metadata files needing to be synchronized
# 4) Synchronize the update metadata file into the proper state-based location
# on the subcloud
#
local metadata_file commit_hash central_metadata_file
local sw_version=${1:-$SW_VERSION}
# Configure and sync ostree feed repo
# This will be able to ostree at the same level between System Controller and subcloud.
configure_ostree_repo_for_central_pull
sync_ostree_repo
local commit_hashes=()
local commit_hash
# 1) Get list of metadata files requiring sync
# Remove current files for specified release
log_info "Removing directories for release ${sw_version}"
rm -Rf ${USM_SOFTWARE_DIR}/rel-${sw_version}.*
find ${USM_METADATA_DIR} -type f -name "*${sw_version}*" | xargs rm -f
# Get list of metadata files requiring sync
for metadata_file in $(get_metadata_files_unique_to_central); do
log_info "sync_subcloud_metadata: processing ${metadata_file} from central (sync)"
central_metadata_file=$(translate_central_metadata_path "${metadata_file}")
@ -595,44 +572,9 @@ sync_subcloud_metadata() {
"commit_hashes: ${commit_hashes[*]}"
if [ "${#commit_hashes[@]}" -gt 0 ]; then
for commit_hash in "${commit_hashes[@]}"; do
# 2) Pull from central controller if necessary
# TODO(kmacleod): check if previous_commit exists from metadata, fail
pull_ostree_commit_to_subcloud "${commit_hash}"
done
# 3) Synchronize the metadata file
# Synchronize the metadata file
sync_metadata_on_subcloud "${metadata_file}" "${central_metadata_file}" commit_hashes
fi
done
for metadata_file in $(get_metadata_files_unique_to_subcloud); do
log_info "sync_subcloud_metadata: processing ${metadata_file} from subcloud (check remove)"
commit_hashes=()
get_commit_hashes_from_metadata commit_hashes "${central_metadata_file}"
log_debug_l "sync_subcloud_metadata from subcloud (check remove): "\
"metadata_file: ${metadata_file}"\
"commit_hashes: ${commit_hashes[*]}"
local removed=
if [ "${#commit_hashes[@]}" -gt 0 ]; then
for commit_hash in "${commit_hashes[@]}"; do
if ! ostree_commit_exists "${commit_hash}"; then
log_info "sync_subcloud_metadata from subcloud: commit '${commit_hash}' does not exist, removing '${metadata_file}'"
removed=1
fi
done
if [ -n "${removed}" ]; then
rm "${metadata_file}"
fi
fi
if [ -n "${removed}" ]; then
log_info_l "sync_subcloud_metadata from subcloud, removed file for non-existing commit(s): "\
"metadata_file: ${metadata_file}"\
"commit_hashes: ${commit_hashes[*]}"
else
log_info_l "sync_subcloud_metadata from subcloud, commit is in use, not removing: "\
"metadata_file: ${metadata_file}"\
"commit_hashes: ${commit_hashes[*]}"
fi
done
}

View File

@ -150,33 +150,52 @@
subcloud_releases: "{{ subcloud_software_list.stdout_lines | \
map('regex_replace', '.*?-([0-9\\.]+).*', '\\1') | list }}"
- name: Verify patch levels between system controller and subcloud
debug:
msg: >
{% if system_controller_releases | length > subcloud_releases | length %}
The system controller has a patch level higher than the subcloud
{% elif system_controller_releases | length == subcloud_releases | length %}
# The subcloud must have at least one release for the given software version to be able
# to compare with the system controller version. Otherwise, it is assumed that the
# release does not exist in the subcloud and the prestage is executed.
- block:
# It is necessary to compare the current patch level between the SC and the subcloud
# to determine if prestage should be executed. In this way all scenarios are covered,
# including pre-patched scenario.
- name: Extract current patch number from system controller release list
set_fact:
system_controller_current_patch: "{{ system_controller_releases[-1] | \
regex_replace('.*\\.', '') | int }}"
- name: Extract current patch number from subcloud release list
set_fact:
subcloud_current_patch: "{{ subcloud_releases[-1] | regex_replace('.*\\.', '') | int }}"
# We need to compare the patch level, since we must take into account even the
# pre-patched scenario.
- debug:
msg: |
ostree revision from: {{ ostree_repo_release_feed }}:
system controller current patch: {{ system_controller_releases[-1] }}
subcloud current patch: {{ subcloud_releases[-1] }}
system controller current commit: {{ ostree_commit_system_controller.stdout }}
subcloud current commit: {{ ostree_commit_subcloud.stdout }}
{% if system_controller_current_patch == subcloud_current_patch %}
Skipping for software deploy prestage as the subcloud has the same patch level
than the system controller.
{% else %}
{% elif subcloud_current_patch > system_controller_current_patch %}
Skipping for software deploy prestage as the subcloud has a higher patch level
than the system controller.
{% endif %}
- name: "Subcloud does not require software prestage. Exiting..."
meta: end_play
when: subcloud_releases | length >= system_controller_releases | length
when: subcloud_current_patch >= system_controller_current_patch
when: subcloud_releases | length > 0
- block:
- debug:
msg: The system controller has a patch level higher than the subcloud
- name: Gather system controller metadata commits
#
# Use the existing ostree_metadata_commits_central file if:
# 1) It exists, and
# 2) There are no new commits. Compare last line of the ostree_metadata_commits_central
# file against current ostree repo commit.
# Otherwise, we generate a new ostree_metadata_commits_central file using the
# get-commits target to our script.
#
# Get the commits from the metadata, in the same way that software list does,
# avoiding ambiguous queries.
# Parallel operations (for orchestration):
# We use flock here because there may be many prestaging operations running
# in parallel on system controller. flock behaviour:
@ -184,15 +203,10 @@
# - the timeout is long just to ensure we never deadlock for any reason
shell: |
exec 3>/tmp/ostree_metadata_commits_central.lock
flock --exclusive --timeout 180 3 || echo "ERROR: flock failed: $?"
if [ ! -f "{{ ostree_metadata_commits_central }}" ] \
|| ! diff -q <(ostree --repo="{{ ostree_repo_release_feed }}" rev-parse "{{ ostree_rev }}") \
<(tail --lines=1 "{{ ostree_metadata_commits_central }}" | awk -F':' '{ print $2; }') > /dev/null 2>&1 ; then
{{ role_path }}/files/ostree-metadata-sync.sh \
--sw-version "{{ software_version }}" --output "{{ ostree_metadata_commits_central }}" get-commits
else
cat "{{ ostree_metadata_commits_central }}"
fi
flock --exclusive --timeout 180 3 || \
{ echo "ERROR: $? - flock failed while trying to get the commits."; exit 1; }
{{ role_path }}/files/ostree-metadata-sync.sh --sw-version "{{ software_version }}" \
--output "{{ ostree_metadata_commits_central }}" get-commits
exec 3>&- # release the lock
register: system_controller_software_metadata_commits
delegate_to: localhost
@ -212,27 +226,6 @@
debug:
var: subcloud_software_metadata_commits
- debug:
msg:
- "ostree revision from {{ ostree_repo_release_feed }}:"
- "system controller: {{ ostree_commit_system_controller.stdout }}"
- "subcloud: {{ ostree_commit_subcloud.stdout }}"
- "Software list:"
- "system controller:"
- "{{ system_controller_software_list.stdout }}"
- "{{ system_controller_software_metadata_commits.stdout }}"
- "subcloud:"
- "{{ subcloud_software_list.stdout }}"
- "{{ subcloud_software_metadata_commits.stdout }}"
- name: Gather system controller deployed release list
shell: |
while IFS= read -r line; do
echo "$line" | awk -F'|' '{split($2,p,"-"); print p[2]}'
done <<< "{{ system_controller_software_list.stdout }}"
register: minor_releases_list
delegate_to: localhost
# It's necessary to temporarily change the owner to sysadmin so that
# the system controller can push the files to the subcloud, since the files
# are in the folders only the root can access but the synchronize only
@ -245,21 +238,6 @@
recurse: yes
become: yes
- name: Copy system controller deployed release directory to subcloud
synchronize:
mode: "push"
src: "{{ usm_software_dir }}/rel-{{ item }}"
dest: "{{ usm_software_dir }}"
with_items: "{{ minor_releases_list.stdout_lines }}"
failed_when: false
- name: Copy system controller {{ usm_software_dir }}/software-scripts to subcloud
synchronize:
mode: "push"
src: "{{ usm_software_dir }}/software-scripts"
dest: "{{ usm_software_dir }}"
failed_when: false
- name: Copy system controller {{ usm_metadata_dir }} to subcloud {{ tmp_metadata_sync_dir }}
copy:
src: "{{ usm_metadata_dir }}"
@ -278,7 +256,7 @@
# on the system controller as it does not exist on N-1 release e.g. 22.12
- name: Synchronizing system controller ostree commits on subcloud
script: "{{ role_path }}/files/ostree-metadata-sync.sh --sw-version {{ software_version }}
sync-subcloud 2>&1 | tee /tmp/sync-ostree-commits.log"
sync-subcloud"
register: sync_software_commits
become: true
@ -290,6 +268,21 @@
- "stderr: {{ sync_software_commits.stderr }}"
- "stdout: {{ sync_software_commits.stdout }}"
- name: Copy system controller deployed release directory to subcloud
synchronize:
mode: "push"
src: "{{ usm_software_dir }}/rel-{{ item }}"
dest: "{{ usm_software_dir }}"
with_items: "{{ system_controller_releases }}"
failed_when: false
- name: Copy system controller {{ usm_software_dir }}/software-scripts to subcloud
synchronize:
mode: "push"
src: "{{ usm_software_dir }}/software-scripts"
dest: "{{ usm_software_dir }}"
failed_when: false
always:
- name: Restore the ownership of {{ usm_software_dir }}
file: