debian: loci: allow tweaking of apt/yum repos

build-stx-images.sh:
- auto-detect OS
- allow wheels tar to be a local file
- patch loci Dockerfile:
  * add a script to tweak package manager configuration
  * copy wheels tar into docker file, instead of downloading
    it over HTTP; this allows wheels to be a local file
  * call our own custom script when building, instead of whats provided
    with loci. The custom script: tweaks apt or yum repos if necessary,
    calls the original Loci build script, then deletes the wheels
    tarball.
- new parameter DIST_REPOS in Loci docker config files. Allows one to
  enable or disable upstream (bullseye etc) repositories when building.

TESTS
=============
1) CentOS: build 2 representative images:
  stx-audit (Dockerfile builder)
  stx-heat (Loci builder)

  Compare filesystems in docker images before and after the patch

  Make sure the difference is trivial/expected

2) Debian: build 2 representative images:

  stx-audit (Dockerfile builder)
  stx-heat (Loci builder)

  stx-heat fails to compile due to openstack requirements file being
  incompatible with debian/bullseye. This will addressed as a separate
  patch.

Story: 2009897
Task: 44693

Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
Change-Id: I70ea968d343d1f59e96eee73907e6999bf4050ec

build-tools/build-docker-images/build-stx-images.sh

@@ -20,7 +20,7 @@ fi
 source ${MY_REPO}/build-tools/git-utils.sh
 
 SUPPORTED_OS_ARGS=('centos' 'debian' 'distroless')
-OS=centos
+OS=
 BUILD_STREAM=stable
 IMAGE_VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp
 PREFIX=dev
@@ -100,6 +100,27 @@ function is_empty {
     test $# -eq 0
 }
 
+function url_basename {
+    # http://foo/bar.tar?xxx#yyy => bar.tar
+    echo "$1" | sed -r -e 's/[?#].*//' -e 's#.*/##'
+}
+
+# Usage: download $URL $OUTPUT_FILE
+function download_file {
+    local url="$1"
+    local out_file="$2"
+    if echo "$url" | grep -E -q -e '^https?:' -e '^ftp:' ; then
+        \rm -f "$out_file.tmp" || return 1
+        with_retries 5 wget -O "$out_file.tmp" "$url" || return 1
+        \mv -f "$out_file.tmp" "$out_file" || exit 1
+    else
+        local src_file
+        src_file="$(echo "$url" | sed -e 's#^file:/+##')"
+        \cp -a "$src_file" "$out_file" || return 1
+    fi
+    return 0
+}
+
 #
 # get_git: Clones a git into a subdirectory of ${WORKDIR}, and
 #          leaves you in that directory.  On error the directory
@@ -196,6 +217,60 @@ function get_loci {
     return 0
 }
 
+function patch_loci {
+    # Loci contains a Docker file, which runs a script that installs
+    # Python modules etc based on build parameters. We replace the call
+    # to Loci's install script with our own, so that we can perform
+    # additional actions within the Dockerfile before Loci does its thing.
+    #
+    # loci/                <-- clone of Loci git repo
+    #   Dockerfile
+    #   ...
+    #   Dockerfile.stx     <-- create this by replacing "RUN..." in Dockerfile
+    #                          with the contents of loci/docker/Dockerfile.part
+    #   stx-scripts/       <-- copy of build-docker-images/loci/docker/stx-scripts
+    #   stx-wheels/        <-- copy of wheels tarball(s) specified on cmd line
+    #
+
+    echo "Patching ${WORKDIR}/loci/Dockerfile" >&2
+
+    # Make a copy of Dockerfile
+    \cp -f "${WORKDIR}/loci/Dockerfile" "${WORKDIR}/loci/Dockerfile.stx" || exit 1
+
+    # Replace "RUN .../install.sh" with our own commands
+    sed -i -r  "${WORKDIR}/loci/Dockerfile.stx" -e "
+        \\%^\\s*(RUN|run)\\s+/opt/loci/scripts/install.sh\\s*\$% {
+            s/^(.*)$/#\\1/
+            r ${MY_SCRIPT_DIR}/loci/docker/Dockerfile.part
+        }
+    " || exit 1
+    if diff -q "${WORKDIR}/loci/Dockerfile" "${WORKDIR}/loci/Dockerfile.stx" >/dev/null ; then
+        echo "${WORKDIR}/loci/Dockerfile: failed to patch Loci Dockerfile" >&2
+        exit 1
+    fi
+
+    # Copy stx-scripts to Loci dir
+    \rm -rf "${WORKDIR}/loci/stx-scripts" || exit 1
+    \cp -ar "${MY_SCRIPT_DIR}/loci/docker/stx-scripts" "${WORKDIR}/loci/" || exit 1
+
+    #diff -u "${WORKDIR}/loci/Dockerfile" "${WORKDIR}/loci/Dockerfile.stx" >&2
+
+    # clear wheels dir
+    \rm -rf "${WORKDIR}/loci/stx-wheels" || exit 1
+    mkdir -p "${WORKDIR}/loci/stx-wheels" || exit 1
+
+}
+
+function download_loci_wheels {
+    local url="$1"
+    out_file="${WORKDIR}/loci/stx-wheels/$(url_basename "$url")"
+    if [[ ! -f "$out_file" ]] ; then
+        echo "Downloading $url => $out_file" >&2
+        download_file "$url" "$out_file" || return 1
+    fi
+    return 0
+}
+
 function update_image_record {
     # Update the image record file with a new/updated entry
     local LABEL=$1
@@ -345,6 +420,8 @@ function build_image_loci {
     SPICE_REPO=$(source ${image_build_file} && echo ${SPICE_REPO})
     local SPICE_REF
     SPICE_REF=$(source ${image_build_file} && echo ${SPICE_REF})
+    local DIST_REPOS
+    DIST_REPOS=$(source ${image_build_file} && echo ${DIST_REPOS})
 
     echo "Building ${LABEL}"
 
@@ -404,10 +481,14 @@ function build_image_loci {
 
     if [ "${PYTHON3}" == "no" ] ; then
         echo "Python2 service ${LABEL}"
-        BUILD_ARGS+=(--build-arg WHEELS=${WHEELS_PY2})
+        download_loci_wheels "$WHEELS_PY2" || exit 1
+        #BUILD_ARGS+=(--build-arg WHEELS=${WHEELS_PY2})
+        BUILD_ARGS+=(--build-arg WHEELS="/opt/loci/stx-wheels/$(url_basename "$WHEELS_PY2")")
     else
         echo "Python3 service ${LABEL}"
-        BUILD_ARGS+=(--build-arg WHEELS=${WHEELS})
+        download_loci_wheels "$WHEELS" || exit 1
+        #BUILD_ARGS+=(--build-arg WHEELS=${WHEELS})
+        BUILD_ARGS+=(--build-arg WHEELS="/opt/loci/stx-wheels/$(url_basename "$WHEELS")")
     fi
 
     if [ ! -z "$HTTP_PROXY" ]; then
@@ -452,6 +533,13 @@ function build_image_loci {
         BUILD_ARGS+=(--build-arg SPICE_REF="${SPICE_REF}")
     fi
 
+    if [ -n "${DIST_REPOS}" ]; then
+        BUILD_ARGS+=(--build-arg DIST_REPOS="${DIST_REPOS}")
+    fi
+
+    # Use patched docker file
+    BUILD_ARGS+=(--file "${WORKDIR}/loci/Dockerfile.stx")
+
     local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}"
 
     with_retries ${MAX_ATTEMPTS} docker build ${WORKDIR}/loci --no-cache \
@@ -810,6 +898,13 @@ while true; do
 done
 
 # Validate the OS option
+if [ -z "$OS" ] ; then
+    OS="$(ID= && source /etc/os-release 2>/dev/null && echo $ID || true)"
+    if [[ -z "$OS" ]] ; then
+        echo "Unable to determine OS, please re-run with \`--os' option" >&2
+        exit 1
+    fi
+fi
 VALID_OS=1
 for supported_os in ${SUPPORTED_OS_ARGS[@]}; do
     if [ "$OS" = "${supported_os}" ]; then
@@ -838,6 +933,18 @@ if [[ -z "$WHEELS_PY2" && -n "$WHEELS" ]]; then
     fi
 fi
 
+# Resolve local wheel file names to absolute paths
+for var in WHEELS WHEELS_PY2 ; do
+    # skip empty vars
+    [[ -n "${!var}" ]] || continue
+    # skip network urls
+    echo "${!var}" | grep -E -q -e '^https?:' -e '^ftp:' && continue
+    # remove file:/ prefix if any
+    declare "$var=$(echo "${!var}" | sed -r 's#^file:/+##')"
+    # resolve it to an absolute path
+    declare "$var=$(readlink -f "${!var}")" || exit 1
+done
+
 # Find the directives files
 IMAGE_BUILD_FILES=()
 function find_image_build_files {
@@ -952,6 +1059,8 @@ if [ $? -ne 0 ]; then
     exit 1
 fi
 
+patch_loci
+
 # Replace mod_wsgi dependency and add rh_python36_mod_wsgi in loci/bindep.txt for python3 package
 # refer to patch https://review.opendev.org/#/c/718603/
 sed -i 's/mod_wsgi                    \[platform\:rpm apache\]/mod_wsgi                    \[platform\:rpm apache \!python3\]/g'  ${WORKDIR}/loci/bindep.txt

build-tools/build-docker-images/loci/docker/Dockerfile.part

@@ -0,0 +1,5 @@
+ARG DIST_REPOS
+COPY stx-scripts /opt/loci/stx-scripts
+COPY stx-wheels  /opt/loci/stx-wheels
+RUN /opt/loci/stx-scripts/install.sh
+

build-tools/build-docker-images/loci/docker/stx-scripts/cleanup.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -ex
+
+rm -rf /opt/loci/stx-wheels/*
+

build-tools/build-docker-images/loci/docker/stx-scripts/install.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -ex
+
+LOCI_DIR="/opt/loci"
+
+# configure apt/yum repos
+"$LOCI_DIR/stx-scripts/setup-package-repos.sh"
+
+# run Loci installer
+"$LOCI_DIR/scripts/install.sh" "$@"
+
+# delete wheel tarball etc
+"$LOCI_DIR/stx-scripts/cleanup.sh"

build-tools/build-docker-images/loci/docker/stx-scripts/setup-package-repos.sh

@@ -0,0 +1,126 @@
+#!/bin/bash
+
+set -ex
+
+#
+# This script enables or disables package repos specified
+# by the DIST_REPOS environment variable, which must contain
+# a space-separated list of repos (in CentOS) or list files
+# (Debian) to enable or disable.
+#
+# In CentOS repo names refer to the names in square brackets
+# in any repo files under /etc/yum.repos.d.
+#
+# In Debian repo names refer to individual files under
+# /etc/apt/sources.list.d/$NAME.list.
+#
+# Repo names may be prefixed with
+# a "+" (enable) or a "-" (disable). The leading "+" may be
+# omitted.
+#
+# Additionally, the following keywords are treated specially:
+#
+#   STX   - enable or disable all StarlingX repos, ie
+#           the locally-built package repos, the mirror/download
+#           repo, and any repo's passed on the command-line
+#           to "build-stx-image.sh" script.
+#
+#   OS    - same as "base updates extras" in CentOS
+#           same as "debian" in Debian
+#
+#
+# These keywords have the same meaning in all distros, while actual
+# repo names are distro-specific.
+#
+# Any repos not included in $DIST_REPOS will remain unchanged (ie
+# they will remain enabled or disabled as defined in the base image).
+#
+# If a repo doesn't match an existing repository, this script will
+# fail.
+#
+# CentOS Example
+# ==============
+#   DIST_REPOS="-base -updates"
+#      disable "base" and "updates" repos normally defined
+#      in /etc/yum.repos.d/CentOS-Base.repo
+#
+#   DIST_REPOS="-STX +OS -updates"
+#      disable all local repos, enable core OS repos, except "updates"
+#
+# Debian Example
+# ==============
+#   DIST_REPOS="debian"
+#      enable core OS repos (ie /etc/apt/sources.list.d/debian.list)
+#
+#   DIST_REPOS="OS -STX"
+#      enable core OS repos (ie /etc/apt/sources.list.d/debian.list),
+#      disable STX repos (ie /etc/apt/sources.list.d/stx.list)
+#
+#
+
+if [[ -n "$DIST_REPOS" ]] ; then
+    # basenames of files under /etc/apt/sources.list.d
+    declare -A DEBIAN_REPO_GROUPS=(
+        [OS]="debian"
+        [STX]="stx"
+    )
+    # yum repo IDs
+    declare -A CENTOS_REPO_GROUPS=(
+        [OS]="base updates extras"
+        [STX]="/etc/yum.repos.d/stx.repo"   # ie, all repos defined in this file
+    )
+
+    distro=$(awk -F= '/^ID=/ {gsub(/\"/, "", $2); print $2}' /etc/*release)
+    # enable or disable each repo
+    for base in $DIST_REPOS ; do
+        # starts with "-": disable this repo
+        if [[ "${base#-}" != "$base" ]] ; then
+            base="${base#-}"
+            enable=0
+        # starts with "+": enable this repo
+        elif [[ "${base#+}" != "$base" ]] ; then
+            base="${base#+}"
+            enable=1
+        # doesn't start with +/-: assume "+"
+        else
+            enable=1
+        fi
+
+        # enable or disable a repo
+        case ${distro} in
+            debian)
+                list_files="${DEBIAN_REPO_GROUPS[$base]:-$base}"
+                for list_file in $list_files ; do
+                    if [[ $enable -eq 1 ]] ; then
+                        cp -f /etc/apt/sources.list.d/${list_file}.list.disabled /etc/apt/sources.list.d/${list_file}.list
+                    else
+                        rm /etc/apt/sources.list.d/${list_file}.list
+                    fi
+                done
+                ;;
+            centos)
+                specs="${CENTOS_REPO_GROUPS[$base]:-$base}"
+                for spec in $specs ; do
+                    # repo id begins with a "/" - assume its a full path to a .repo file
+                    # and enable/disable all repos defined in that file
+                    if [[ "${spec#/}" != "$spec" ]] ; then
+                        repos=$(sed -r -n 's/^\s*[[]([^]]+)[]]\s*$/\1/gp' "$spec")
+                    else
+                        repos=$spec
+                    fi
+                    for repo in $repos ; do
+                        if [[ $enable -eq 1 ]] ; then
+                            yum-config-manager --enable "$repo"
+                        else
+                            yum-config-manager --disable "$repo"
+                        fi
+                    done
+                done
+                ;;
+            *)
+                echo "error: unsupported OS \"$distro\"" >&2
+                exit 1
+        esac
+    done
+fi
+