From 3908e68fb325ffebd873a82ba37933ee42af93d7 Mon Sep 17 00:00:00 2001 From: Davlet Panech Date: Thu, 17 Mar 2022 17:25:23 -0400 Subject: [PATCH] debian: loci: allow tweaking of apt/yum repos build-stx-images.sh: - auto-detect OS - allow wheels tar to be a local file - patch loci Dockerfile: * add a script to tweak package manager configuration * copy wheels tar into docker file, instead of downloading it over HTTP; this allows wheels to be a local file * call our own custom script when building, instead of whats provided with loci. The custom script: tweaks apt or yum repos if necessary, calls the original Loci build script, then deletes the wheels tarball. - new parameter DIST_REPOS in Loci docker config files. Allows one to enable or disable upstream (bullseye etc) repositories when building. TESTS ============= 1) CentOS: build 2 representative images: stx-audit (Dockerfile builder) stx-heat (Loci builder) Compare filesystems in docker images before and after the patch Make sure the difference is trivial/expected 2) Debian: build 2 representative images: stx-audit (Dockerfile builder) stx-heat (Loci builder) stx-heat fails to compile due to openstack requirements file being incompatible with debian/bullseye. This will addressed as a separate patch. Story: 2009897 Task: 44693 Signed-off-by: Davlet Panech Change-Id: I70ea968d343d1f59e96eee73907e6999bf4050ec --- .../build-docker-images/build-stx-images.sh | 115 +++++++++++++++- .../loci/docker/Dockerfile.part | 5 + .../loci/docker/stx-scripts/cleanup.sh | 6 + .../loci/docker/stx-scripts/install.sh | 14 ++ .../docker/stx-scripts/setup-package-repos.sh | 126 ++++++++++++++++++ 5 files changed, 263 insertions(+), 3 deletions(-) create mode 100644 build-tools/build-docker-images/loci/docker/Dockerfile.part create mode 100755 build-tools/build-docker-images/loci/docker/stx-scripts/cleanup.sh create mode 100755 build-tools/build-docker-images/loci/docker/stx-scripts/install.sh create mode 100755 build-tools/build-docker-images/loci/docker/stx-scripts/setup-package-repos.sh diff --git a/build-tools/build-docker-images/build-stx-images.sh b/build-tools/build-docker-images/build-stx-images.sh index 53f1176b..549d7d57 100755 --- a/build-tools/build-docker-images/build-stx-images.sh +++ b/build-tools/build-docker-images/build-stx-images.sh @@ -20,7 +20,7 @@ fi source ${MY_REPO}/build-tools/git-utils.sh SUPPORTED_OS_ARGS=('centos' 'debian' 'distroless') -OS=centos +OS= BUILD_STREAM=stable IMAGE_VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp PREFIX=dev @@ -100,6 +100,27 @@ function is_empty { test $# -eq 0 } +function url_basename { + # http://foo/bar.tar?xxx#yyy => bar.tar + echo "$1" | sed -r -e 's/[?#].*//' -e 's#.*/##' +} + +# Usage: download $URL $OUTPUT_FILE +function download_file { + local url="$1" + local out_file="$2" + if echo "$url" | grep -E -q -e '^https?:' -e '^ftp:' ; then + \rm -f "$out_file.tmp" || return 1 + with_retries 5 wget -O "$out_file.tmp" "$url" || return 1 + \mv -f "$out_file.tmp" "$out_file" || exit 1 + else + local src_file + src_file="$(echo "$url" | sed -e 's#^file:/+##')" + \cp -a "$src_file" "$out_file" || return 1 + fi + return 0 +} + # # get_git: Clones a git into a subdirectory of ${WORKDIR}, and # leaves you in that directory. On error the directory @@ -196,6 +217,60 @@ function get_loci { return 0 } +function patch_loci { + # Loci contains a Docker file, which runs a script that installs + # Python modules etc based on build parameters. We replace the call + # to Loci's install script with our own, so that we can perform + # additional actions within the Dockerfile before Loci does its thing. + # + # loci/ <-- clone of Loci git repo + # Dockerfile + # ... + # Dockerfile.stx <-- create this by replacing "RUN..." in Dockerfile + # with the contents of loci/docker/Dockerfile.part + # stx-scripts/ <-- copy of build-docker-images/loci/docker/stx-scripts + # stx-wheels/ <-- copy of wheels tarball(s) specified on cmd line + # + + echo "Patching ${WORKDIR}/loci/Dockerfile" >&2 + + # Make a copy of Dockerfile + \cp -f "${WORKDIR}/loci/Dockerfile" "${WORKDIR}/loci/Dockerfile.stx" || exit 1 + + # Replace "RUN .../install.sh" with our own commands + sed -i -r "${WORKDIR}/loci/Dockerfile.stx" -e " + \\%^\\s*(RUN|run)\\s+/opt/loci/scripts/install.sh\\s*\$% { + s/^(.*)$/#\\1/ + r ${MY_SCRIPT_DIR}/loci/docker/Dockerfile.part + } + " || exit 1 + if diff -q "${WORKDIR}/loci/Dockerfile" "${WORKDIR}/loci/Dockerfile.stx" >/dev/null ; then + echo "${WORKDIR}/loci/Dockerfile: failed to patch Loci Dockerfile" >&2 + exit 1 + fi + + # Copy stx-scripts to Loci dir + \rm -rf "${WORKDIR}/loci/stx-scripts" || exit 1 + \cp -ar "${MY_SCRIPT_DIR}/loci/docker/stx-scripts" "${WORKDIR}/loci/" || exit 1 + + #diff -u "${WORKDIR}/loci/Dockerfile" "${WORKDIR}/loci/Dockerfile.stx" >&2 + + # clear wheels dir + \rm -rf "${WORKDIR}/loci/stx-wheels" || exit 1 + mkdir -p "${WORKDIR}/loci/stx-wheels" || exit 1 + +} + +function download_loci_wheels { + local url="$1" + out_file="${WORKDIR}/loci/stx-wheels/$(url_basename "$url")" + if [[ ! -f "$out_file" ]] ; then + echo "Downloading $url => $out_file" >&2 + download_file "$url" "$out_file" || return 1 + fi + return 0 +} + function update_image_record { # Update the image record file with a new/updated entry local LABEL=$1 @@ -345,6 +420,8 @@ function build_image_loci { SPICE_REPO=$(source ${image_build_file} && echo ${SPICE_REPO}) local SPICE_REF SPICE_REF=$(source ${image_build_file} && echo ${SPICE_REF}) + local DIST_REPOS + DIST_REPOS=$(source ${image_build_file} && echo ${DIST_REPOS}) echo "Building ${LABEL}" @@ -404,10 +481,14 @@ function build_image_loci { if [ "${PYTHON3}" == "no" ] ; then echo "Python2 service ${LABEL}" - BUILD_ARGS+=(--build-arg WHEELS=${WHEELS_PY2}) + download_loci_wheels "$WHEELS_PY2" || exit 1 + #BUILD_ARGS+=(--build-arg WHEELS=${WHEELS_PY2}) + BUILD_ARGS+=(--build-arg WHEELS="/opt/loci/stx-wheels/$(url_basename "$WHEELS_PY2")") else echo "Python3 service ${LABEL}" - BUILD_ARGS+=(--build-arg WHEELS=${WHEELS}) + download_loci_wheels "$WHEELS" || exit 1 + #BUILD_ARGS+=(--build-arg WHEELS=${WHEELS}) + BUILD_ARGS+=(--build-arg WHEELS="/opt/loci/stx-wheels/$(url_basename "$WHEELS")") fi if [ ! -z "$HTTP_PROXY" ]; then @@ -452,6 +533,13 @@ function build_image_loci { BUILD_ARGS+=(--build-arg SPICE_REF="${SPICE_REF}") fi + if [ -n "${DIST_REPOS}" ]; then + BUILD_ARGS+=(--build-arg DIST_REPOS="${DIST_REPOS}") + fi + + # Use patched docker file + BUILD_ARGS+=(--file "${WORKDIR}/loci/Dockerfile.stx") + local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}" with_retries ${MAX_ATTEMPTS} docker build ${WORKDIR}/loci --no-cache \ @@ -810,6 +898,13 @@ while true; do done # Validate the OS option +if [ -z "$OS" ] ; then + OS="$(ID= && source /etc/os-release 2>/dev/null && echo $ID || true)" + if [[ -z "$OS" ]] ; then + echo "Unable to determine OS, please re-run with \`--os' option" >&2 + exit 1 + fi +fi VALID_OS=1 for supported_os in ${SUPPORTED_OS_ARGS[@]}; do if [ "$OS" = "${supported_os}" ]; then @@ -838,6 +933,18 @@ if [[ -z "$WHEELS_PY2" && -n "$WHEELS" ]]; then fi fi +# Resolve local wheel file names to absolute paths +for var in WHEELS WHEELS_PY2 ; do + # skip empty vars + [[ -n "${!var}" ]] || continue + # skip network urls + echo "${!var}" | grep -E -q -e '^https?:' -e '^ftp:' && continue + # remove file:/ prefix if any + declare "$var=$(echo "${!var}" | sed -r 's#^file:/+##')" + # resolve it to an absolute path + declare "$var=$(readlink -f "${!var}")" || exit 1 +done + # Find the directives files IMAGE_BUILD_FILES=() function find_image_build_files { @@ -952,6 +1059,8 @@ if [ $? -ne 0 ]; then exit 1 fi +patch_loci + # Replace mod_wsgi dependency and add rh_python36_mod_wsgi in loci/bindep.txt for python3 package # refer to patch https://review.opendev.org/#/c/718603/ sed -i 's/mod_wsgi \[platform\:rpm apache\]/mod_wsgi \[platform\:rpm apache \!python3\]/g' ${WORKDIR}/loci/bindep.txt diff --git a/build-tools/build-docker-images/loci/docker/Dockerfile.part b/build-tools/build-docker-images/loci/docker/Dockerfile.part new file mode 100644 index 00000000..c172f68b --- /dev/null +++ b/build-tools/build-docker-images/loci/docker/Dockerfile.part @@ -0,0 +1,5 @@ +ARG DIST_REPOS +COPY stx-scripts /opt/loci/stx-scripts +COPY stx-wheels /opt/loci/stx-wheels +RUN /opt/loci/stx-scripts/install.sh + diff --git a/build-tools/build-docker-images/loci/docker/stx-scripts/cleanup.sh b/build-tools/build-docker-images/loci/docker/stx-scripts/cleanup.sh new file mode 100755 index 00000000..6ac890f0 --- /dev/null +++ b/build-tools/build-docker-images/loci/docker/stx-scripts/cleanup.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -ex + +rm -rf /opt/loci/stx-wheels/* + diff --git a/build-tools/build-docker-images/loci/docker/stx-scripts/install.sh b/build-tools/build-docker-images/loci/docker/stx-scripts/install.sh new file mode 100755 index 00000000..033bdb90 --- /dev/null +++ b/build-tools/build-docker-images/loci/docker/stx-scripts/install.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -ex + +LOCI_DIR="/opt/loci" + +# configure apt/yum repos +"$LOCI_DIR/stx-scripts/setup-package-repos.sh" + +# run Loci installer +"$LOCI_DIR/scripts/install.sh" "$@" + +# delete wheel tarball etc +"$LOCI_DIR/stx-scripts/cleanup.sh" diff --git a/build-tools/build-docker-images/loci/docker/stx-scripts/setup-package-repos.sh b/build-tools/build-docker-images/loci/docker/stx-scripts/setup-package-repos.sh new file mode 100755 index 00000000..dd43612f --- /dev/null +++ b/build-tools/build-docker-images/loci/docker/stx-scripts/setup-package-repos.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +set -ex + +# +# This script enables or disables package repos specified +# by the DIST_REPOS environment variable, which must contain +# a space-separated list of repos (in CentOS) or list files +# (Debian) to enable or disable. +# +# In CentOS repo names refer to the names in square brackets +# in any repo files under /etc/yum.repos.d. +# +# In Debian repo names refer to individual files under +# /etc/apt/sources.list.d/$NAME.list. +# +# Repo names may be prefixed with +# a "+" (enable) or a "-" (disable). The leading "+" may be +# omitted. +# +# Additionally, the following keywords are treated specially: +# +# STX - enable or disable all StarlingX repos, ie +# the locally-built package repos, the mirror/download +# repo, and any repo's passed on the command-line +# to "build-stx-image.sh" script. +# +# OS - same as "base updates extras" in CentOS +# same as "debian" in Debian +# +# +# These keywords have the same meaning in all distros, while actual +# repo names are distro-specific. +# +# Any repos not included in $DIST_REPOS will remain unchanged (ie +# they will remain enabled or disabled as defined in the base image). +# +# If a repo doesn't match an existing repository, this script will +# fail. +# +# CentOS Example +# ============== +# DIST_REPOS="-base -updates" +# disable "base" and "updates" repos normally defined +# in /etc/yum.repos.d/CentOS-Base.repo +# +# DIST_REPOS="-STX +OS -updates" +# disable all local repos, enable core OS repos, except "updates" +# +# Debian Example +# ============== +# DIST_REPOS="debian" +# enable core OS repos (ie /etc/apt/sources.list.d/debian.list) +# +# DIST_REPOS="OS -STX" +# enable core OS repos (ie /etc/apt/sources.list.d/debian.list), +# disable STX repos (ie /etc/apt/sources.list.d/stx.list) +# +# + +if [[ -n "$DIST_REPOS" ]] ; then + # basenames of files under /etc/apt/sources.list.d + declare -A DEBIAN_REPO_GROUPS=( + [OS]="debian" + [STX]="stx" + ) + # yum repo IDs + declare -A CENTOS_REPO_GROUPS=( + [OS]="base updates extras" + [STX]="/etc/yum.repos.d/stx.repo" # ie, all repos defined in this file + ) + + distro=$(awk -F= '/^ID=/ {gsub(/\"/, "", $2); print $2}' /etc/*release) + # enable or disable each repo + for base in $DIST_REPOS ; do + # starts with "-": disable this repo + if [[ "${base#-}" != "$base" ]] ; then + base="${base#-}" + enable=0 + # starts with "+": enable this repo + elif [[ "${base#+}" != "$base" ]] ; then + base="${base#+}" + enable=1 + # doesn't start with +/-: assume "+" + else + enable=1 + fi + + # enable or disable a repo + case ${distro} in + debian) + list_files="${DEBIAN_REPO_GROUPS[$base]:-$base}" + for list_file in $list_files ; do + if [[ $enable -eq 1 ]] ; then + cp -f /etc/apt/sources.list.d/${list_file}.list.disabled /etc/apt/sources.list.d/${list_file}.list + else + rm /etc/apt/sources.list.d/${list_file}.list + fi + done + ;; + centos) + specs="${CENTOS_REPO_GROUPS[$base]:-$base}" + for spec in $specs ; do + # repo id begins with a "/" - assume its a full path to a .repo file + # and enable/disable all repos defined in that file + if [[ "${spec#/}" != "$spec" ]] ; then + repos=$(sed -r -n 's/^\s*[[]([^]]+)[]]\s*$/\1/gp' "$spec") + else + repos=$spec + fi + for repo in $repos ; do + if [[ $enable -eq 1 ]] ; then + yum-config-manager --enable "$repo" + else + yum-config-manager --disable "$repo" + fi + done + done + ;; + *) + echo "error: unsupported OS \"$distro\"" >&2 + exit 1 + esac + done +fi +