starlingx/root
Code Issues Proposed changes

patch-iso-debian: refactor script

Browse Source 
- Remove need for STX_BUILD_HOME and MY_REPO env variables

- Get base ostree repo from Input ISO, not from /localdisk/deploy.
  This detaches the script from the build env
  and guarantees the ostree repo matches the input ISO.

- Fixed shellcheck errors and warnings

- No longer uses mount/guestmount to get Input ISO contents,
  as these require sudo privileges [1, 2]. Now uses 7z instead.

- To produce output ISO's ostree repo, use 'ostree prune'
  instead of 'ostree pull'. This generates "tombstone" commits,
  which signal that the missing commits were intentionally removed.
  This prevents some ostree errors, such as running
  "ostree pull --depth=-1" to pull from the resulting repo.

- If the Input ISO already has a "patches" folder, delete it before
  trying to add patches metadata. This allows pre-patched ISOs
  to be used as input, which is useful for testing.

With these changes, the only requirement left is that
the script be placed in the root repo so that it can find
the .pem file needed for signing.

IMPORTANT: Only works if ISO has ostree version 2022.2 or later.
But this requirement was introduced in an earlier commit,
when "ostree pull --depth=-1" was changed to "--depth=0",
as ostree had a blocking bug before that version:
https://review.opendev.org/c/starlingx/root/+/903888

Refs:
[1] https://bugs.launchpad.net/ubuntu/+source/libguestfs/+bug/1673431
[2] https://bugs.launchpad.net/fuel/+bug/1467579

Test Plan:
pass - create pre-patched ISO
pass - install pre-patched ISO
pass - apply and remove a patch on the installed system
pass - system shows patch metadata on "sw-patch" cmd

Story: 2011098
Task: 50534

Depends-On: https://review.opendev.org/c/starlingx/tools/+/926443

Change-Id: I7e5cb6865b715ab789f489dff44a7d39327a01c1
Signed-off-by: Leonardo Fagundes Luz Serrano <Leonardo.FagundesLuzSerrano@windriver.com>
This commit is contained in:
Leonardo Fagundes Luz Serrano
2024-07-09 13:01:08 -03:00
parent c188f6211b
commit 615bf2f250
1 changed files with 154 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

317
build-tools/patch-iso-debian
View File

@@ -1,37 +1,37 @@
#!/bin/bash #!/bin/bash
# #
# Copyright (c) 2023 Wind River Systems, Inc. # Copyright (c) 2024 Wind River Systems, Inc.
# #
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# #
# Utility for adding patch metadata to the iso # This script takes as input an ISO and one or more patches
# Debian patches are sequential and uses ostree # and generates as output an ISO with the following changes:
# so any patch will produce an updated iso, this #
# utility injects the patch metadata into the iso. # - Contains only the latest ostree commit from the input ISO
# During install the kickstart will copy the metadata # - ISO has a "patches" folder with the patches' metadata files.
# into the right location and the sw-patch query # This folder is processed by kickstart during install, so that
# command will output the correct patch level # 'sw-patch query' has access to this info.
#
# The intent is for the system to have record of the patches that are
# already pre-installed in the system.
# #
source "$(dirname $0)/image-utils.sh" BUILD_TOOLS_DIR="$(dirname "$0")"
if [ -z "${STX_BUILD_HOME}" ]; then # shellcheck source="./build-tools/image-utils.sh"
echo "Required environment variable STX_BUILD_HOME is not set" source "${BUILD_TOOLS_DIR}/image-utils.sh"
exit 1
fi
# Define MY_REPO, which is the path to the 'root' repo. Eg.: $REPO_ROOT/cgcs_root
# Value is used to locate the following file for ISO signing:
# ${MY_REPO}/build-tools/signing/dev-private-key.pem
if [ -z "${MY_REPO}" ]; then if [ -z "${MY_REPO}" ]; then
echo "Required environment variable MY_REPO is not set" MY_REPO="$(dirname "${BUILD_TOOLS_DIR}")"
exit 1
fi fi
DEPLOY_DIR="${STX_BUILD_HOME}/localdisk/deploy"
OSTREE_REPO="${DEPLOY_DIR}/ostree_repo"
function usage() { function usage() {
echo "" echo ""
echo "Usage: " echo "Usage: "
echo " $(basename $0) -i <input filename.iso> -o <output filename.iso> [ -p ] <patch> ..." echo " $(basename "$0") -i <input filename.iso> -o <output filename.iso> [ -p ] <patch> ..."
echo " -i <file>: Specify input ISO file" echo " -i <file>: Specify input ISO file"
echo " -o <file>: Specify output ISO file" echo " -o <file>: Specify output ISO file"
echo " -p <file>: Patch files. You can call it multiple times." echo " -p <file>: Patch files. You can call it multiple times."
@@ -44,28 +44,33 @@ function extract_ostree_commit_from_metadata_xml() {
# Check if xmllint is available. Otherwise, use python's xml standard lib # Check if xmllint is available. Otherwise, use python's xml standard lib
if (which xmllint &>/dev/null); then if (which xmllint &>/dev/null); then
xmllint --xpath "string(${XPATH})" ${XML_PATH} xmllint --xpath "string(${XPATH})" "${XML_PATH}"
else else
python3 -c "import xml.etree.ElementTree as ET ; print(ET.parse('${XML_PATH}').find('.${XPATH}').text, end='')" python3 -c "import xml.etree.ElementTree as ET ; print(ET.parse('${XML_PATH}').find('.${XPATH}').text, end='')"
fi fi
} }
function extract_metadata() { function extract_metadata() {
local patchesdir=${BUILDDIR}/patches local patchesdir
local patchfile=$1 local patchfile
local patchid=$(basename $patchfile .patch) local patchid
local ostree_log=$(ostree --repo=${OSTREE_REPO} log starlingx) local ostree_log
local ostree_commit
patchesdir="${BUILDDIR}/patches"
patchfile="$1"
patchid="$(basename "$patchfile" .patch)"
ostree_log="$(ostree --repo="${2}" log starlingx)"
echo "Extracting ${patchfile}" echo "Extracting ${patchfile}"
# Extract it # Extract it
tar xf ${patchfile} -O metadata.tar | tar x -O > ${patchesdir}/${patchid}-metadata.xml if ! tar xf "${patchfile}" -O metadata.tar | tar x -O > "${patchesdir}/${patchid}-metadata.xml"; then
if [ $? -ne 0 ]; then echo "ERROR: Failed to extract metadata from ${patchfile}"
echo "Failed to extract metadata from ${patchfile}"
exit 1 exit 1
fi fi
# Verify if top commit from metadata exist in ostree log # Verify if top commit from metadata exist in ostree log
patch_ostree_commit1=$(xmllint --xpath "string(//contents/ostree/commit1/commit)" ${patchesdir}/${patchid}-metadata.xml) patch_ostree_commit1="$(extract_ostree_commit_from_metadata_xml "${patchesdir}/${patchid}-metadata.xml")"
if [[ "$ostree_log" != *"$patch_ostree_commit1"* ]]; then if [[ "$ostree_log" != *"$patch_ostree_commit1"* ]]; then
echo "WARNING: Patch ostree commit 1 not found in input ISO." echo "WARNING: Patch ostree commit 1 not found in input ISO."
echo "Patch ostree commit 1: ${patch_ostree_commit1}" echo "Patch ostree commit 1: ${patch_ostree_commit1}"
@@ -74,8 +79,6 @@ function extract_metadata() {
declare INPUT_ISO= declare INPUT_ISO=
declare OUTPUT_ISO= declare OUTPUT_ISO=
declare ORIG_PWD=$PWD
declare DO_UPGRADES=1
while getopts "i:o:p:" opt; do while getopts "i:o:p:" opt; do
case $opt in case $opt in
@@ -86,7 +89,7 @@ while getopts "i:o:p:" opt; do
OUTPUT_ISO=$OPTARG OUTPUT_ISO=$OPTARG
;; ;;
p) p)
PATCH_FILES+=($OPTARG) PATCH_FILES+=("$OPTARG")
;; ;;
*) *)
usage usage
@@ -95,183 +98,162 @@ while getopts "i:o:p:" opt; do
esac esac
done done
if [ -z "$INPUT_ISO" -o -z "$OUTPUT_ISO" ]; then if [ -z "$INPUT_ISO" ] || [ -z "$OUTPUT_ISO" ]; then
usage usage
exit 1 exit 1
fi fi
if [ ! -f ${INPUT_ISO} ]; then if [ ! -f "${INPUT_ISO}" ]; then
echo "Input file does not exist: ${INPUT_ISO}" echo "ERROR: Input file does not exist: ${INPUT_ISO}"
exit 1 exit 1
fi fi
if [ -f ${OUTPUT_ISO} ]; then if [ -f "${OUTPUT_ISO}" ]; then
echo "Output file already exists: ${OUTPUT_ISO}" echo "ERROR: Output file already exists: ${OUTPUT_ISO}"
exit 1 exit 1
fi fi
for PATCH in "${PATCH_FILES[@]}"; for PATCH in "${PATCH_FILES[@]}";
do do
if [ ! -f ${PATCH} ]; then if [ ! -f "${PATCH}" ]; then
echo "Patch file dos not exists: ${PATCH}" echo "ERROR: Patch file dos not exists: ${PATCH}"
exit 1 exit 1
fi fi
if [[ ! ${PATCH} =~ \.patch$ ]]; then if [[ ! "${PATCH}" =~ \.patch$ ]]; then
echo "Specified file ${PATCH} does not have .patch extension" echo "ERROR: Specified file ${PATCH} does not have .patch extension"
exit 1 exit 1
fi fi
done done
shift $((OPTIND-1)) shift $((OPTIND-1))
declare MNTDIR=
declare BUILDDIR= declare BUILDDIR=
function check_requirements { function check_requirements {
# Next to each requirement is the deb package which provides the command listed.
# Run "apt install ..."
# Declare "require reqA or reqB" as "reqA__reqB" # Declare "require reqA or reqB" as "reqA__reqB"
local -a required_utils=( local -a required_utils=(
rsync 7z # p7zip-full
mkisofs mkisofs # genisoimage
isohybrid isohybrid # syslinux-utils
implantisomd5 implantisomd5 # isomd5sum
ostree ostree # ostree
xmllint__python3 xmllint__python3 # libxml2-utils
) )
if [ $UID -ne 0 ]; then
# If running as non-root user, additional utils are required
required_utils+=(
guestmount
guestunmount
)
fi
local -i missing=0 local -i missing=0
local reqA local reqA
local reqB local reqB
for req in ${required_utils[@]}; do for req in "${required_utils[@]}"; do
if [[ "$req" = *"__"* ]]; then if [[ "$req" = *"__"* ]]; then
reqA="${req%__*}" # select everything before "__" reqA="${req%__*}" # select everything before "__"
reqB="${req#*__}" # select everything after "__" reqB="${req#*__}" # select everything after "__"
if ! (which ${reqA} &>/dev/null) && ! (which ${reqB} &>/dev/null); then if ! (which "${reqA}" &>/dev/null) && ! (which "${reqB}" &>/dev/null); then
echo "Unable to find required utility: either ${reqA} or ${reqB}" >&2 echo "Unable to find required utility: either ${reqA} or ${reqB}" >&2
let missing++ missing=$(( missing+1 ))
fi fi
else else
if ! (which ${req} &>/dev/null); then if ! (which "${req}" &>/dev/null); then
echo "Unable to find required utility: ${req}" >&2 echo "Unable to find required utility: ${req}" >&2
let missing++ missing=$(( missing+1 ))
fi fi
fi fi
done done
if [ ${missing} -gt 0 ]; then if [ "${missing}" -gt 0 ]; then
echo "One or more required utilities are missing. Aborting..." >&2 echo "ERROR: One or more required utilities are missing" >&2
exit 1 exit 1
fi fi
} }
function mount_iso {
if [ $UID -eq 0 ]; then
# Mount the ISO
mount -o loop ${INPUT_ISO} ${MNTDIR}
if [ $? -ne 0 ]; then
echo "Failed to mount ${INPUT_ISO}" >&2
exit 1
fi
else
# As non-root user, mount the ISO using guestmount
guestmount -a ${INPUT_ISO} -m /dev/sda1 --ro ${MNTDIR}
rc=$?
if [ $rc -ne 0 ]; then
# Add a retry
echo "Call to guestmount failed with rc=$rc. Retrying once..."
guestmount -a ${INPUT_ISO} -m /dev/sda1 --ro ${MNTDIR}
rc=$?
if [ $rc -ne 0 ]; then
echo "Call to guestmount failed with rc=$rc. Aborting..."
exit $rc
fi
fi
fi
}
function unmount_iso {
if [ $UID -eq 0 ]; then
umount ${MNTDIR}
else
guestunmount ${MNTDIR}
fi
rmdir ${MNTDIR}
}
function cleanup() { function cleanup() {
if [ -n "$MNTDIR" -a -d "$MNTDIR" ]; then # Delete temporary build directory
unmount_iso if [ -n "$BUILDDIR" ] && [ -d "$BUILDDIR" ]; then
chmod -R +w "$BUILDDIR"
\rm -rf "$BUILDDIR"
fi fi
if [ -n "$BUILDDIR" -a -d "$BUILDDIR" ]; then
chmod -R +w $BUILDDIR
\rm -rf $BUILDDIR
fi
} }
check_requirements check_requirements
# Run cleanup() when finishing/interrupting execution
trap cleanup EXIT trap cleanup EXIT
MNTDIR=$(mktemp -d -p $PWD patchiso_mnt_XXXXXX) # Create temporary build directory
if [ -z "${MNTDIR}" -o ! -d ${MNTDIR} ]; then BUILDDIR=$(mktemp -d -p "$PWD" patchiso_build_XXXXXX)
echo "Failed to create mntdir. Aborting..." if [ -z "${BUILDDIR}" ] || [ ! -d "${BUILDDIR}" ]; then
exit $rc echo "ERROR: Failed to create temporary build directory"
exit 1
fi fi
BUILDDIR=$(mktemp -d -p $PWD patchiso_build_XXXXXX)
if [ -z "${BUILDDIR}" -o ! -d ${BUILDDIR} ]; then
echo "Failed to create builddir. Aborting..."
exit $rc
fi
# Mount the ISO
mount_iso
rsync -a --exclude 'ostree_repo' ${MNTDIR}/ ${BUILDDIR}/
rc=$?
if [ $rc -ne 0 ]; then
echo "Call to rsync ISO content. Aborting..."
exit $rc
fi
unmount_iso
# Fix for permission denied if not running as root # Fix for permission denied if not running as root
chmod +w ${BUILDDIR} chmod +w "${BUILDDIR}"
chmod -R +w ${BUILDDIR}/isolinux if [ -d "${BUILDDIR}/isolinux" ]; then
chmod -R +w "${BUILDDIR}/isolinux"
fi
# Create the directory where metadata will be stored # Erase current patch metadata from ISO if it exists
mkdir -p ${BUILDDIR}/patches # This way, this script can be used on pre-patched ISOs
chmod -R +w ${BUILDDIR}/patches if [ -d "${BUILDDIR}/patches" ]; then
rm -rf "${BUILDDIR}/patches"
fi
echo "Copying only the latest commit from ostree_repo..." # Create the directory where patch metadata will be stored
ostree --repo=${BUILDDIR}/ostree_repo init --mode=archive-z2 mkdir -p "${BUILDDIR}/patches"
ostree --repo=${BUILDDIR}/ostree_repo pull-local --depth=0 ${OSTREE_REPO} starlingx chmod -R +w "${BUILDDIR}/patches"
ostree --repo=${BUILDDIR}/ostree_repo summary --update
echo "Updated iso ostree commit:"
ostree --repo=${BUILDDIR}/ostree_repo log starlingx
echo "Extracting patch metadata" echo "Extracting Input ISO contents..."
7z x "${INPUT_ISO}" -o"${BUILDDIR}" 1>/dev/null
# Delete boot directory. It will be re-created when packing the output ISO
if [ -d "${BUILDDIR}/[BOOT]" ]; then
rm -rf "${BUILDDIR}/[BOOT]"
fi
echo "Extracting patch metadata..."
for PATCH in "${PATCH_FILES[@]}"; for PATCH in "${PATCH_FILES[@]}";
do do
extract_metadata $PATCH extract_metadata "$PATCH" "${BUILDDIR}/ostree_repo"
done done
echo "Original ostree repo history:"
echo "----------------------------------------------------------------------------------"
ostree --repo="${BUILDDIR}/ostree_repo" log starlingx
echo "----------------------------------------------------------------------------------"
echo "Clean up all commits from ostree repo except the latest one..."
function clean_ostree(){
# Create array of ostree commit IDs
mapfile -t ostree_commits < <(ostree --repo="${BUILDDIR}/ostree_repo" log starlingx | grep '^commit' | cut -d ' ' -f 2)
# Delete each commit except the latest one
for ostree_commit in "${ostree_commits[@]:1}"; do
echo "Removing commit: ${ostree_commit}"
ostree --repo="${BUILDDIR}/ostree_repo" prune --delete-commit="${ostree_commit}"
done
ostree --repo="${BUILDDIR}/ostree_repo" summary --update
}
if ! clean_ostree; then
echo "ERROR: Failed to clean ostree repo!"
exit 1
fi
echo "Output ISO ostree history:"
echo "----------------------------------------------------------------------------------"
ostree --repo="${BUILDDIR}/ostree_repo" log starlingx
echo "----------------------------------------------------------------------------------"
echo "Packing iso..." echo "Packing iso..."
# get the install label # get the install label
ISO_LABEL=$(grep -ri instiso "${BUILDDIR}"/isolinux/isolinux.cfg | head -1 | xargs -n1 | awk -F= /instiso/'{print $2}') ISO_LABEL=$(grep -ri instiso "${BUILDDIR}"/isolinux/isolinux.cfg | head -1 | xargs -n1 | awk -F= /instiso/'{print $2}')
if [ -z "${ISO_LABEL}" ] ; then if [ -z "${ISO_LABEL}" ] ; then
@@ -279,30 +261,39 @@ if [ -z "${ISO_LABEL}" ] ; then
fi fi
echo "ISO Label: ${ISO_LABEL}" echo "ISO Label: ${ISO_LABEL}"
# Repack the ISO function pack_iso(){
mkisofs -o "${OUTPUT_ISO}" \ # Repack the ISO
-A "${ISO_LABEL}" -V "${ISO_LABEL}" \ mkisofs -o "${OUTPUT_ISO}" \
-quiet -U -J -joliet-long -r -iso-level 2 \ -A "${ISO_LABEL}" -V "${ISO_LABEL}" \
-b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot \ -quiet -U -J -joliet-long -r -iso-level 2 \
-boot-load-size 4 -boot-info-table \ -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot \
-eltorito-alt-boot \ -boot-load-size 4 -boot-info-table \
-e efi.img \ -eltorito-alt-boot \
-no-emul-boot \ -e efi.img \
"${BUILDDIR}" -no-emul-boot \
"${BUILDDIR}"
isohybrid --uefi ${OUTPUT_ISO} isohybrid --uefi "${OUTPUT_ISO}"
implantisomd5 ${OUTPUT_ISO} implantisomd5 "${OUTPUT_ISO}"
}
# Sign the .iso with the developer private key if ! pack_iso; then
openssl dgst -sha256 \ echo "ERROR: Failed to build output ISO!"
-sign ${MY_REPO}/build-tools/signing/dev-private-key.pem \ exit 1
-binary \
-out ${OUTPUT_ISO/%.iso/.sig} \
${OUTPUT_ISO}
rc=$?
if [ $rc -ne 0 ]; then
echo "Call to $(basename ${SETUP_PATCH_REPO}) failed with rc=$rc. Aborting..."
exit $rc
fi fi
echo "Patched ISO: ${OUTPUT_ISO}" echo "Signing the .iso with the developer private key.."
function sign_iso(){
openssl dgst -sha256 \
-sign "${MY_REPO}/build-tools/signing/dev-private-key.pem" \
-binary \
-out "${OUTPUT_ISO/%.iso/.sig}" \
"${OUTPUT_ISO}"
}
if ! sign_iso; then
echo "ERROR: Failed to sign ISO!"
exit 1
fi
echo ""
echo "Output ISO: $(realpath "${OUTPUT_ISO}")"