debian: loci: allow tweaking of apt/yum repos
build-stx-images.sh:
- auto-detect OS
- allow wheels tar to be a local file
- patch loci Dockerfile:
* add a script to tweak package manager configuration
* copy wheels tar into docker file, instead of downloading
it over HTTP; this allows wheels to be a local file
* call our own custom script when building, instead of whats provided
with loci. The custom script: tweaks apt or yum repos if necessary,
calls the original Loci build script, then deletes the wheels
tarball.
- new parameter DIST_REPOS in Loci docker config files. Allows one to
enable or disable upstream (bullseye etc) repositories when building.
TESTS
=============
1) CentOS: build 2 representative images:
stx-audit (Dockerfile builder)
stx-heat (Loci builder)
Compare filesystems in docker images before and after the patch
Make sure the difference is trivial/expected
2) Debian: build 2 representative images:
stx-audit (Dockerfile builder)
stx-heat (Loci builder)
stx-heat fails to compile due to openstack requirements file being
incompatible with debian/bullseye. This will addressed as a separate
patch.
Story: 2009897
Task: 44693
Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
Change-Id: I70ea968d343d1f59e96eee73907e6999bf4050ec
This commit is contained in:
Davlet Panech
parent
075b71dcf9
commit
3908e68fb3
@ -20,7 +20,7 @@ fi
|
||||
source ${MY_REPO}/build-tools/git-utils.sh
|
||||
|
||||
SUPPORTED_OS_ARGS=('centos' 'debian' 'distroless')
|
||||
OS=centos
|
||||
OS=
|
||||
BUILD_STREAM=stable
|
||||
IMAGE_VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp
|
||||
PREFIX=dev
|
||||
@ -100,6 +100,27 @@ function is_empty {
|
||||
test $# -eq 0
|
||||
}
|
||||
|
||||
function url_basename {
|
||||
# http://foo/bar.tar?xxx#yyy => bar.tar
|
||||
echo "$1" | sed -r -e 's/[?#].*//' -e 's#.*/##'
|
||||
}
|
||||
|
||||
# Usage: download $URL $OUTPUT_FILE
|
||||
function download_file {
|
||||
local url="$1"
|
||||
local out_file="$2"
|
||||
if echo "$url" | grep -E -q -e '^https?:' -e '^ftp:' ; then
|
||||
\rm -f "$out_file.tmp" || return 1
|
||||
with_retries 5 wget -O "$out_file.tmp" "$url" || return 1
|
||||
\mv -f "$out_file.tmp" "$out_file" || exit 1
|
||||
else
|
||||
local src_file
|
||||
src_file="$(echo "$url" | sed -e 's#^file:/+##')"
|
||||
\cp -a "$src_file" "$out_file" || return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
#
|
||||
# get_git: Clones a git into a subdirectory of ${WORKDIR}, and
|
||||
# leaves you in that directory. On error the directory
|
||||
@ -196,6 +217,60 @@ function get_loci {
|
||||
return 0
|
||||
}
|
||||
|
||||
function patch_loci {
|
||||
# Loci contains a Docker file, which runs a script that installs
|
||||
# Python modules etc based on build parameters. We replace the call
|
||||
# to Loci's install script with our own, so that we can perform
|
||||
# additional actions within the Dockerfile before Loci does its thing.
|
||||
#
|
||||
# loci/ <-- clone of Loci git repo
|
||||
# Dockerfile
|
||||
# ...
|
||||
# Dockerfile.stx <-- create this by replacing "RUN..." in Dockerfile
|
||||
# with the contents of loci/docker/Dockerfile.part
|
||||
# stx-scripts/ <-- copy of build-docker-images/loci/docker/stx-scripts
|
||||
# stx-wheels/ <-- copy of wheels tarball(s) specified on cmd line
|
||||
#
|
||||
|
||||
echo "Patching ${WORKDIR}/loci/Dockerfile" >&2
|
||||
|
||||
# Make a copy of Dockerfile
|
||||
\cp -f "${WORKDIR}/loci/Dockerfile" "${WORKDIR}/loci/Dockerfile.stx" || exit 1
|
||||
|
||||
# Replace "RUN .../install.sh" with our own commands
|
||||
sed -i -r "${WORKDIR}/loci/Dockerfile.stx" -e "
|
||||
\\%^\\s*(RUN|run)\\s+/opt/loci/scripts/install.sh\\s*\$% {
|
||||
s/^(.*)$/#\\1/
|
||||
r ${MY_SCRIPT_DIR}/loci/docker/Dockerfile.part
|
||||
}
|
||||
" || exit 1
|
||||
if diff -q "${WORKDIR}/loci/Dockerfile" "${WORKDIR}/loci/Dockerfile.stx" >/dev/null ; then
|
||||
echo "${WORKDIR}/loci/Dockerfile: failed to patch Loci Dockerfile" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Copy stx-scripts to Loci dir
|
||||
\rm -rf "${WORKDIR}/loci/stx-scripts" || exit 1
|
||||
\cp -ar "${MY_SCRIPT_DIR}/loci/docker/stx-scripts" "${WORKDIR}/loci/" || exit 1
|
||||
|
||||
#diff -u "${WORKDIR}/loci/Dockerfile" "${WORKDIR}/loci/Dockerfile.stx" >&2
|
||||
|
||||
# clear wheels dir
|
||||
\rm -rf "${WORKDIR}/loci/stx-wheels" || exit 1
|
||||
mkdir -p "${WORKDIR}/loci/stx-wheels" || exit 1
|
||||
|
||||
}
|
||||
|
||||
function download_loci_wheels {
|
||||
local url="$1"
|
||||
out_file="${WORKDIR}/loci/stx-wheels/$(url_basename "$url")"
|
||||
if [[ ! -f "$out_file" ]] ; then
|
||||
echo "Downloading $url => $out_file" >&2
|
||||
download_file "$url" "$out_file" || return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
function update_image_record {
|
||||
# Update the image record file with a new/updated entry
|
||||
local LABEL=$1
|
||||
@ -345,6 +420,8 @@ function build_image_loci {
|
||||
SPICE_REPO=$(source ${image_build_file} && echo ${SPICE_REPO})
|
||||
local SPICE_REF
|
||||
SPICE_REF=$(source ${image_build_file} && echo ${SPICE_REF})
|
||||
local DIST_REPOS
|
||||
DIST_REPOS=$(source ${image_build_file} && echo ${DIST_REPOS})
|
||||
|
||||
echo "Building ${LABEL}"
|
||||
|
||||
@ -404,10 +481,14 @@ function build_image_loci {
|
||||
|
||||
if [ "${PYTHON3}" == "no" ] ; then
|
||||
echo "Python2 service ${LABEL}"
|
||||
BUILD_ARGS+=(--build-arg WHEELS=${WHEELS_PY2})
|
||||
download_loci_wheels "$WHEELS_PY2" || exit 1
|
||||
#BUILD_ARGS+=(--build-arg WHEELS=${WHEELS_PY2})
|
||||
BUILD_ARGS+=(--build-arg WHEELS="/opt/loci/stx-wheels/$(url_basename "$WHEELS_PY2")")
|
||||
else
|
||||
echo "Python3 service ${LABEL}"
|
||||
BUILD_ARGS+=(--build-arg WHEELS=${WHEELS})
|
||||
download_loci_wheels "$WHEELS" || exit 1
|
||||
#BUILD_ARGS+=(--build-arg WHEELS=${WHEELS})
|
||||
BUILD_ARGS+=(--build-arg WHEELS="/opt/loci/stx-wheels/$(url_basename "$WHEELS")")
|
||||
fi
|
||||
|
||||
if [ ! -z "$HTTP_PROXY" ]; then
|
||||
@ -452,6 +533,13 @@ function build_image_loci {
|
||||
BUILD_ARGS+=(--build-arg SPICE_REF="${SPICE_REF}")
|
||||
fi
|
||||
|
||||
if [ -n "${DIST_REPOS}" ]; then
|
||||
BUILD_ARGS+=(--build-arg DIST_REPOS="${DIST_REPOS}")
|
||||
fi
|
||||
|
||||
# Use patched docker file
|
||||
BUILD_ARGS+=(--file "${WORKDIR}/loci/Dockerfile.stx")
|
||||
|
||||
local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}"
|
||||
|
||||
with_retries ${MAX_ATTEMPTS} docker build ${WORKDIR}/loci --no-cache \
|
||||
@ -810,6 +898,13 @@ while true; do
|
||||
done
|
||||
|
||||
# Validate the OS option
|
||||
if [ -z "$OS" ] ; then
|
||||
OS="$(ID= && source /etc/os-release 2>/dev/null && echo $ID || true)"
|
||||
if [[ -z "$OS" ]] ; then
|
||||
echo "Unable to determine OS, please re-run with \`--os' option" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
VALID_OS=1
|
||||
for supported_os in ${SUPPORTED_OS_ARGS[@]}; do
|
||||
if [ "$OS" = "${supported_os}" ]; then
|
||||
@ -838,6 +933,18 @@ if [[ -z "$WHEELS_PY2" && -n "$WHEELS" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# Resolve local wheel file names to absolute paths
|
||||
for var in WHEELS WHEELS_PY2 ; do
|
||||
# skip empty vars
|
||||
[[ -n "${!var}" ]] || continue
|
||||
# skip network urls
|
||||
echo "${!var}" | grep -E -q -e '^https?:' -e '^ftp:' && continue
|
||||
# remove file:/ prefix if any
|
||||
declare "$var=$(echo "${!var}" | sed -r 's#^file:/+##')"
|
||||
# resolve it to an absolute path
|
||||
declare "$var=$(readlink -f "${!var}")" || exit 1
|
||||
done
|
||||
|
||||
# Find the directives files
|
||||
IMAGE_BUILD_FILES=()
|
||||
function find_image_build_files {
|
||||
@ -952,6 +1059,8 @@ if [ $? -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
patch_loci
|
||||
|
||||
# Replace mod_wsgi dependency and add rh_python36_mod_wsgi in loci/bindep.txt for python3 package
|
||||
# refer to patch https://review.opendev.org/#/c/718603/
|
||||
sed -i 's/mod_wsgi \[platform\:rpm apache\]/mod_wsgi \[platform\:rpm apache \!python3\]/g' ${WORKDIR}/loci/bindep.txt
|
||||
|
||||
@ -0,0 +1,5 @@
|
||||
ARG DIST_REPOS
|
||||
COPY stx-scripts /opt/loci/stx-scripts
|
||||
COPY stx-wheels /opt/loci/stx-wheels
|
||||
RUN /opt/loci/stx-scripts/install.sh
|
||||
|
||||
6
build-tools/build-docker-images/loci/docker/stx-scripts/cleanup.sh
Executable file
6
build-tools/build-docker-images/loci/docker/stx-scripts/cleanup.sh
Executable file
@ -0,0 +1,6 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
rm -rf /opt/loci/stx-wheels/*
|
||||
|
||||
14
build-tools/build-docker-images/loci/docker/stx-scripts/install.sh
Executable file
14
build-tools/build-docker-images/loci/docker/stx-scripts/install.sh
Executable file
@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
LOCI_DIR="/opt/loci"
|
||||
|
||||
# configure apt/yum repos
|
||||
"$LOCI_DIR/stx-scripts/setup-package-repos.sh"
|
||||
|
||||
# run Loci installer
|
||||
"$LOCI_DIR/scripts/install.sh" "$@"
|
||||
|
||||
# delete wheel tarball etc
|
||||
"$LOCI_DIR/stx-scripts/cleanup.sh"
|
||||
126
build-tools/build-docker-images/loci/docker/stx-scripts/setup-package-repos.sh
Executable file
126
build-tools/build-docker-images/loci/docker/stx-scripts/setup-package-repos.sh
Executable file
@ -0,0 +1,126 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
#
|
||||
# This script enables or disables package repos specified
|
||||
# by the DIST_REPOS environment variable, which must contain
|
||||
# a space-separated list of repos (in CentOS) or list files
|
||||
# (Debian) to enable or disable.
|
||||
#
|
||||
# In CentOS repo names refer to the names in square brackets
|
||||
# in any repo files under /etc/yum.repos.d.
|
||||
#
|
||||
# In Debian repo names refer to individual files under
|
||||
# /etc/apt/sources.list.d/$NAME.list.
|
||||
#
|
||||
# Repo names may be prefixed with
|
||||
# a "+" (enable) or a "-" (disable). The leading "+" may be
|
||||
# omitted.
|
||||
#
|
||||
# Additionally, the following keywords are treated specially:
|
||||
#
|
||||
# STX - enable or disable all StarlingX repos, ie
|
||||
# the locally-built package repos, the mirror/download
|
||||
# repo, and any repo's passed on the command-line
|
||||
# to "build-stx-image.sh" script.
|
||||
#
|
||||
# OS - same as "base updates extras" in CentOS
|
||||
# same as "debian" in Debian
|
||||
#
|
||||
#
|
||||
# These keywords have the same meaning in all distros, while actual
|
||||
# repo names are distro-specific.
|
||||
#
|
||||
# Any repos not included in $DIST_REPOS will remain unchanged (ie
|
||||
# they will remain enabled or disabled as defined in the base image).
|
||||
#
|
||||
# If a repo doesn't match an existing repository, this script will
|
||||
# fail.
|
||||
#
|
||||
# CentOS Example
|
||||
# ==============
|
||||
# DIST_REPOS="-base -updates"
|
||||
# disable "base" and "updates" repos normally defined
|
||||
# in /etc/yum.repos.d/CentOS-Base.repo
|
||||
#
|
||||
# DIST_REPOS="-STX +OS -updates"
|
||||
# disable all local repos, enable core OS repos, except "updates"
|
||||
#
|
||||
# Debian Example
|
||||
# ==============
|
||||
# DIST_REPOS="debian"
|
||||
# enable core OS repos (ie /etc/apt/sources.list.d/debian.list)
|
||||
#
|
||||
# DIST_REPOS="OS -STX"
|
||||
# enable core OS repos (ie /etc/apt/sources.list.d/debian.list),
|
||||
# disable STX repos (ie /etc/apt/sources.list.d/stx.list)
|
||||
#
|
||||
#
|
||||
|
||||
if [[ -n "$DIST_REPOS" ]] ; then
|
||||
# basenames of files under /etc/apt/sources.list.d
|
||||
declare -A DEBIAN_REPO_GROUPS=(
|
||||
[OS]="debian"
|
||||
[STX]="stx"
|
||||
)
|
||||
# yum repo IDs
|
||||
declare -A CENTOS_REPO_GROUPS=(
|
||||
[OS]="base updates extras"
|
||||
[STX]="/etc/yum.repos.d/stx.repo" # ie, all repos defined in this file
|
||||
)
|
||||
|
||||
distro=$(awk -F= '/^ID=/ {gsub(/\"/, "", $2); print $2}' /etc/*release)
|
||||
# enable or disable each repo
|
||||
for base in $DIST_REPOS ; do
|
||||
# starts with "-": disable this repo
|
||||
if [[ "${base#-}" != "$base" ]] ; then
|
||||
base="${base#-}"
|
||||
enable=0
|
||||
# starts with "+": enable this repo
|
||||
elif [[ "${base#+}" != "$base" ]] ; then
|
||||
base="${base#+}"
|
||||
enable=1
|
||||
# doesn't start with +/-: assume "+"
|
||||
else
|
||||
enable=1
|
||||
fi
|
||||
|
||||
# enable or disable a repo
|
||||
case ${distro} in
|
||||
debian)
|
||||
list_files="${DEBIAN_REPO_GROUPS[$base]:-$base}"
|
||||
for list_file in $list_files ; do
|
||||
if [[ $enable -eq 1 ]] ; then
|
||||
cp -f /etc/apt/sources.list.d/${list_file}.list.disabled /etc/apt/sources.list.d/${list_file}.list
|
||||
else
|
||||
rm /etc/apt/sources.list.d/${list_file}.list
|
||||
fi
|
||||
done
|
||||
;;
|
||||
centos)
|
||||
specs="${CENTOS_REPO_GROUPS[$base]:-$base}"
|
||||
for spec in $specs ; do
|
||||
# repo id begins with a "/" - assume its a full path to a .repo file
|
||||
# and enable/disable all repos defined in that file
|
||||
if [[ "${spec#/}" != "$spec" ]] ; then
|
||||
repos=$(sed -r -n 's/^\s*[[]([^]]+)[]]\s*$/\1/gp' "$spec")
|
||||
else
|
||||
repos=$spec
|
||||
fi
|
||||
for repo in $repos ; do
|
||||
if [[ $enable -eq 1 ]] ; then
|
||||
yum-config-manager --enable "$repo"
|
||||
else
|
||||
yum-config-manager --disable "$repo"
|
||||
fi
|
||||
done
|
||||
done
|
||||
;;
|
||||
*)
|
||||
echo "error: unsupported OS \"$distro\"" >&2
|
||||
exit 1
|
||||
esac
|
||||
done
|
||||
fi
|
||||
|
||||
Loading…
Reference in New Issue
Block a user