From 35a2f1c2961b98adc7b1416288716929c6555d78 Mon Sep 17 00:00:00 2001 From: Kyle MacLeod Date: Sat, 10 Dec 2022 19:18:49 -0500 Subject: [PATCH] Validate prestaged ostree_repo via checksum For installs using prestage data (prestaging or prestage ISO), an md5 directory-based checksum is now included at the same directory level as ostree_repo (via related commits). This commit adds a validation check for any prestaged /opt/platform-backup/ostree_repo. The validation check consists of the following: - If a checksum file exists, use it for validation - Otherwise, print a warning and fall back to using ostree fsck - The ostree fsck command takes much longer to complete. If the validation check fails, the prestage data is removed, and the remote install falls back to doing a fresh ostree pull from the system controller. If the validation fails for a local prestage ISO install, the installation will fail during boot. This is unlikely; it would only happen if the USB is somehow corrupt. Test Plan PASS: Remote installs - Boot subcloud using prestage ISO. Perform remote install. Verify the checksum is validated as part of a successful install and bootstrap. - Boot subcloud using prestage ISO. Manually corrupt the /opt/platform-backup/ostree_repo. Perform remote install. Verify the following: 1) the checksum validation fails, 2) the corrupt /opt/platform-backup/ostree_repo directory is removed 3) the installation continues via remote ostree pull. PASS: Local Install - Boot subcloud using prestage ISO. Perform local install. Verify the checksum is validated as part of a successful install and bootstrap. PASS: Pre-corrupted ISO - Boot subcloud using a prestage ISO with a pre-corrupted ostree_repo Verify the boot fails due to the checksum validation failure. Depends-On: https://review.opendev.org/c/starlingx/utilities/+/867179 Depends-On: https://review.opendev.org/c/starlingx/ansible-playbooks/+/867178 Closes-Bug: 1999306 Signed-off-by: Kyle MacLeod Change-Id: I1fb69b76de4b7fa5bc49cb4b182297b3bb94ba78 --- kickstart/files/kickstart.cfg | 38 +++++++++++++++++++++++++++++++ kickstart/files/miniboot.cfg | 42 +++++++++++++++++++++++++++++++---- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/kickstart/files/kickstart.cfg b/kickstart/files/kickstart.cfg index 22515fd5..c7a24f42 100644 --- a/kickstart/files/kickstart.cfg +++ b/kickstart/files/kickstart.cfg @@ -976,6 +976,36 @@ if check_prestage -eq 0 ; then report_failure_with_msg "Unable to mount ${ISO_DEV} Error rc=${rc}" fi + local_repo_check_fail= + # Calculate local checksum and compare + checksum_file=${ISODIR}/.ostree_repo_checksum + if [ -f "${checksum_file}" ]; then + checksum_from_file=$(cat "${checksum_file}") + ilog "Verifying checksum for prestaged ${ISODIR}/ostree_repo" + pushd "${ISODIR}" > /dev/null + checksum=$(find ostree_repo -type f -exec md5sum {} + | LC_ALL=C sort | md5sum | awk '{ print $1; }') + popd > /dev/null + if [ "${checksum}" = "${checksum_from_file}" ]; then + ilog "Verified ostree checksum: ${checksum}" + else + elog "ostree checksum failed on ${ISODIR}/ostree_repo" + elog "Calculated checksum: ${checksum}" + elog "File checksum: ${checksum_from_file}" + local_repo_check_fail=true + fi + else + # No prestage checksum file is available. Use ostree fsck instead. + # The only problem with this is the length of time required for fsck to complete. + wlog "No ostree checksum file at ${checksum_file}. Performing ostree fsck instead." + if ! ostree --repo="${ISODIR}/ostree_repo" fsck; then + elog "ostree fsck failed on prestaged ${ISODIR}/ostree_repo: reverting to remote pull" + local_repo_check_fail=true + fi + fi + if [ -n "${local_repo_check_fail}" ]; then + report_failure_with_msg "ostree integrity check failed on ISO ${ISODIR}/ostree_repo" + fi + if [ -e "${ISODIR}/ks-setup.cfg" ]; then source "${ISODIR}/ks-setup.cfg" fi @@ -2814,6 +2844,14 @@ if [ "${controller}" = true ] ; then report_failure_with_msg "Unable to copy repo to /opt/platform-backup [rc=${rc}]" fi + # The summary file is not transferred on an ostree pull, so we need to + # regenerate the checksum here based on the new local repo contents + ilog "Calculating new checksum for prestaged ${backup_mount}/ostree_repo" + pushd "${backup_mount}" > /dev/null + find ostree_repo -type f -exec md5sum {} + | LC_ALL=C sort | md5sum | awk '{ print $1; }' > .ostree_repo_checksum + ilog "Calculated checksum: $(cat .ostree_repo_checksum)" + popd > /dev/null + if [ -e "/instboot/opt/platform-backup/${sw_release}" ]; then ilog "Copying images and patches to ${backup_mount}" cp -a /instboot/opt/platform-backup/${sw_release} ${backup_mount} diff --git a/kickstart/files/miniboot.cfg b/kickstart/files/miniboot.cfg index bfea210e..99c45b09 100644 --- a/kickstart/files/miniboot.cfg +++ b/kickstart/files/miniboot.cfg @@ -2057,11 +2057,45 @@ else # then set the ostree url to its location. remote_insturl= if [ -e ${backup_mount}/ostree_repo ]; then - # Preserve remote_insturl for use in 2nd ostree pull below - remote_insturl=${insturl} + local_repo_check_fail= + # Calculate local checksum and compare + checksum_file=${backup_mount}/.ostree_repo_checksum + if [ -f "${checksum_file}" ]; then + checksum_from_file=$(cat "${checksum_file}") + ilog "Verifying checksum for prestaged ${backup_mount}/ostree_repo" + pushd ${backup_mount} > /dev/null + checksum=$(find ostree_repo -type f -exec md5sum {} + | LC_ALL=C sort | md5sum | awk '{ print $1; }') + popd > /dev/null + if [ "${checksum}" = "${checksum_from_file}" ]; then + ilog "Verified ostree checksum: ${checksum}" + else + elog "ostree checksum failed on ${backup_mount}/ostree_repo" + elog "Calulated checksum: ${checksum}" + elog "File checksum: ${checksum_from_file}" + local_repo_check_fail=true + fi + else + # No prestage checksum file is available. Use ostree fsck instead. + # The only problem with this is the length of time required for fsck to complete. + wlog "No ostree checksum file at ${checksum_file}. Performing ostree fsck instead." + if ! ostree --repo="${backup_mount}/ostree_repo" fsck; then + elog "ostree fsck failed on prestaged ${backup_mount}/ostree_repo: reverting to remote pull" + local_repo_check_fail=true + fi + fi + if [ -z "${local_repo_check_fail}" ]; then + # Preserve remote_insturl for use in 2nd ostree pull below + remote_insturl=${insturl} - insturl="file:///${backup_mount}/ostree_repo" - ilog "Setting insturl to ${insturl} to use prestaged ostree_repo" + insturl="file:///${backup_mount}/ostree_repo" + ilog "Setting insturl to ${insturl} to use prestaged ostree_repo" + else + # Remove the corrupted ostree_repo. + # Avoid setting insturl which will revert to using a remote pull + elog "ostree integrity check failed: removing prestaged ${backup_mount}/ostree_repo" + rm -rf "${backup_mount}/ostree_repo" + elog "ostree integrity check failed: reverting to remote pull" + fi fi # Tell LAT to install from this local stage