Enable build of docker images, charts and wheels from a layered build

1) Add config for the container layer.

2) Add support for the new config file 'required_layer_wheel_inc.cfg',
Which will specify the default urls for lower layer wheels.inc files.
These files must be downloaded by download_mirror.sh, and linked by
generate-cgcs-centos-repo.sh.

3) Add options to download_mirror.sh and generate-cgcs-centos-repo.sh
allowing overrides or additions to the set of url's for lower layer
wheels.inc

4) Changes to allow build scripts running within a docker container
to launch a sister (not daughter) container.  This will allow
build-wheels to run within a container.

Story: 2006166
Task: 38980
Depends-On: https://review.opendev.org/711773
Depends-On: https://review.opendev.org/717755
Change-Id: I95c71a0327dc9d919ed835d3661a6f1ec7ce30c7
Signed-off-by: Scott Little <scott.little@windriver.com>

Dockerfile

@@ -56,6 +56,7 @@ RUN groupadd -g 751 cgts && \
         cpanminus \
         createrepo \
         deltarpm \
+        docker-client \
         expat-devel \
         flex \
         isomd5sum \

centos-mirror-tools/config/centos/containers/required_layer_pkgs.cfg

@@ -0,0 +1,5 @@
+compiler,std,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/compiler/latest_build/outputs/RPMS/std/rpm.lst
+distro,std,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/RPMS/std/rpm.lst
+distro,rt,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/RPMS/rt/rpm.lst
+distro,installer,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/RPMS/installer/rpm.lst
+flock,std,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/flock/latest_build/outputs/RPMS/std/rpm.lst

centos-mirror-tools/config/centos/containers/required_layer_wheel_inc.cfg

@@ -0,0 +1,4 @@
+distro,stable,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/centos_stable_wheels.inc
+distro,dev,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/centos_dev_wheels.inc
+flock,stable,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/flock/latest_build/outputs/centos_stable_wheels.inc
+flock,dev,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/flock/latest_build/outputs/centos_dev_wheels.inc

centos-mirror-tools/config/centos/containers/yum.repos.d/StarlingX_cengn_compiler_layer.repo

@@ -0,0 +1,12 @@
+#####
+# StarlingX compiler layer at CENGN
+
+[Starlingx-cengn_compiler_std_layer]
+name=Starlingx-cengn_compiler_std_layer
+baseurl=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/compiler/latest_build/outputs/RPMS/std/
+enabled=1
+
+[Starlingx-cengn_compiler_installer_layer]
+name=Starlingx-cengn_compiler_installer_layer
+baseurl=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/compiler/latest_build/outputs/RPMS/installer/
+enabled=1

centos-mirror-tools/config/centos/containers/yum.repos.d/StarlingX_cengn_distro_layer.repo

@@ -0,0 +1,18 @@
+#####
+# StarlingX distro layer at CENGN
+
+[Starlingx-cengn_distro_std_layer]
+name=Starlingx-cengn_distro_std_layer
+baseurl=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/RPMS/std/
+enabled=1
+
+[Starlingx-cengn_distro_rt_layer]
+name=Starlingx-cengn_distro_rt_layer
+baseurl=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/RPMS/rt/
+enabled=1
+
+[Starlingx-cengn_distro_installer_layer]
+name=Starlingx-cengn_distro_installer_layer
+baseurl=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/RPMS/installer/
+enabled=1
+

centos-mirror-tools/config/centos/containers/yum.repos.d/StarlingX_cengn_flock_layer.repo

@@ -0,0 +1,8 @@
+#####
+# StarlingX flock layer at CENGN
+
+[Starlingx-cengn_flock_std_layer]
+name=Starlingx-cengn_flock_std_layer
+baseurl=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/flock/latest_build/outputs/RPMS/std/
+enabled=1
+

centos-mirror-tools/config/centos/flock/required_layer_wheel_inc.cfg

@@ -0,0 +1,2 @@
+distro,stable,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/centos_stable_wheels.inc
+distro,dev,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/centos_dev_wheels.inc

centos-mirror-tools/download_mirror.sh

@@ -43,6 +43,11 @@ usage() {
     echo "                   layer's build type.  Normally the url(s) is read from"
     echo "                   <config_dir>/<distro>/<layer>/required_layer_pkgs.cfg."
     echo "                   This option can be used more than once."
+    echo "  -W <lower_layer>,<stream>,<url>:"
+    echo "                   Override the url for the wheels.inc list of a lower"
+    echo "                   layer's build type.  Normally the url(s) is read from"
+    echo "                   <config_dir>/<distro>/<layer>/required_layer_wheel_inc.cfg."
+    echo "                   This option can be used more than once."
     echo
 }
 
@@ -105,6 +110,7 @@ rpms_from_centos_3rd_parties_template="rpms_centos3rdparties.lst"
 rpms_from_layer_build_dir=${DL_MIRROR_OUTPUT_DIR}/layer_pkg_lists
 rpms_from_layer_repos_dir=${DL_MIRROR_OUTPUT_DIR}/layer_repos
 image_inc_from_layer_build_dir=${DL_MIRROR_OUTPUT_DIR}/layer_image_inc
+wheels_inc_from_layer_build_dir=${DL_MIRROR_OUTPUT_DIR}/layer_wheels_inc
 build_info_from_layer_build_dir=${DL_MIRROR_OUTPUT_DIR}/layer_build_info
 tarball_downloads_template="tarball-dl.lst"
 other_downloads_template="other_downloads.lst"
@@ -151,7 +157,7 @@ multiple_dl_flag_check () {
 
 
 # Parse out optional arguments
-while getopts "c:Cd:ghI:sl:L:nSuU" o; do
+while getopts "c:Cd:ghI:sl:L:nSuUW:" o; do
     case "${o}" in
         c)
             # Pass -c ("use alternate yum.conf") to rpm downloader
@@ -173,6 +179,9 @@ while getopts "c:Cd:ghI:sl:L:nSuU" o; do
         I)
             set_layer_image_inc_urls "${OPTARG}"
             ;;
+        W)
+            set_layer_wheels_inc_urls "${OPTARG}"
+            ;;
         l)
             # layer
             set_and_validate_layer "${OPTARG}"
@@ -310,6 +319,22 @@ for key in "${!layer_image_inc_urls[@]}"; do
     fi
 done
 
+\rm -rf ${wheels_inc_from_layer_build_dir}
+mkdir -p ${wheels_inc_from_layer_build_dir}
+
+for key in "${!layer_wheels_inc_urls[@]}"; do
+    lower_layer="${key%,*}"
+    stream="${key#*,}"
+    url="${layer_wheels_inc_urls[${key}]}"
+    name_from_url=$(url_to_file_name $url)
+    list="${wheels_inc_from_layer_build_dir}/${name_from_url}"
+    curl --silent --fail ${url} > ${list} ||
+    if [ $? -ne 0 ]; then
+        echo "ERROR: Failed to download from url: ${url}"
+        exit 1
+    fi
+done
+
 \rm -rf ${build_info_from_layer_build_dir}
 mkdir -p ${build_info_from_layer_build_dir}
 
@@ -326,7 +351,7 @@ for key in "${!layer_image_inc_urls[@]}"; do
     url=$( echo ${layer_image_inc_urls[${key}]} | sed 's#image.inc$#BUILD_INFO#' )
     name_from_url=$(url_to_file_name $url)
     dest="${build_info_from_layer_build_dir}/${name_from_url}"
-    curl --silent --fail ${url} > ${dest}
+    curl --silent --fail ${url} > ${dest} ||
     if [ $? -ne 0 ]; then
         echo "ERROR: Failed to download from url: ${url}"
         exit 1

tb.sh

@@ -49,6 +49,7 @@ function run_container {
     docker run -it --rm \
         --name ${TC_CONTAINER_NAME} \
         --detach \
+        -v /var/run/docker.sock:/var/run/docker.sock \
         -v ${LOCALDISK}:/${GUEST_LOCALDISK} \
         -v ${HOST_MIRROR_DIR}:/import/mirrors:ro \
         -v /sys/fs/cgroup:/sys/fs/cgroup:ro \

toCOPY/generate-cgcs-centos-repo.sh

@@ -48,6 +48,11 @@ usage () {
     echo "                      layer's build type.  Normally the url(s) is read from"
     echo "                      <config_dir>/<distro>/<layer>/required_layer_pkgs.cfg."
     echo "                      This option can be used more than once."
+    echo "  --layer-wheels-inc-url=<lower_layer>,<stream>,<url>:"
+    echo "                      Override the url for the image include file of a lower"
+    echo "                      layer's build type.  Normally the url(s) is read from"
+    echo "                      <config_dir>/<distro>/<layer>/required_layer_wheel_inc.cfg"
+    echo "                      This option can be used more than once."
     echo "  --layer-dir=<dir>:  Look in provided dir for packages to link to."
     echo "                      This option can be used more than once."
     echo
@@ -72,7 +77,7 @@ if [ -z "$MY_REPO" ]; then
 fi
 
 
-TEMP=$(getopt -o h --long help,config-dir:,distro:,layer:,layer-dir:,layer-inc-url:,layer-pkg-url:,mirror-dir: -n 'generate-cgcs-centos-repo' -- "$@")
+TEMP=$(getopt -o h --long help,config-dir:,distro:,layer:,layer-dir:,layer-inc-url:,layer-pkg-url:,layer-wheels-inc-url:,mirror-dir: -n 'generate-cgcs-centos-repo' -- "$@")
 if [ $? -ne 0 ]; then
     echo "getopt error"
     usage
@@ -86,6 +91,7 @@ while true ; do
         --mirror-dir)     mirror_dir=$2 ; shift 2 ;;
         --layer-dir)      layer_dirs+=" ${2/,/ }" ; shift 2 ;;
         --layer-inc-url)  set_layer_image_inc_urls "${2}" ; shift 2 ;;
+        --layer-wheels-inc-url)  set_layer_wheels_inc_urls "${2}" ; shift 2 ;;
         --layer-pkg-url)  set_layer_pkg_urls "${2}" ; shift 2 ;;
         --config-dir)     set_and_validate_config_dir "${2}"; shift 2 ;;
         --distro)         set_and_validate_distro "${2}"; shift 2 ;;
@@ -114,6 +120,8 @@ echo "layer_pkg_urls=${layer_pkg_urls[@]}"
 echo
 echo "layer_image_inc_urls=${layer_image_inc_urls[@]}"
 echo
+echo "layer_wheels_inc_urls=${layer_wheels_inc_urls[@]}"
+echo
 
 dest_dir=$MY_REPO/cgcs-centos-repo
 timestamp="$(date +%F_%H%M)"
@@ -126,6 +134,7 @@ TMP_LST_DIR=$(mktemp -d /tmp/tmp_lst_dir_XXXXXX)
 mkdir -p $TMP_LST_DIR
 lst_file_dir="$TMP_LST_DIR"
 inc_file_dir="${dest_dir}/layer_image_inc"
+wheels_file_dir="${dest_dir}/layer_wheels_inc"
 build_info_file_dir="${dest_dir}/layer_build_info"
 
 rpm_lst_files="rpms_3rdparties.lst rpms_centos3rdparties.lst rpms_centos.lst"
@@ -208,6 +217,37 @@ for key in "${!layer_image_inc_urls[@]}"; do
     fi
 done
 
+#
+# Dowload wheels inc files from layer_wheels_inc_urls
+#
+\rm -rf ${wheels_file_dir}
+mkdir -p ${wheels_file_dir}
+for key in "${!layer_wheels_inc_urls[@]}"; do
+    lower_layer="${key%,*}"
+    stream="${key#*,}"
+    url="${layer_wheels_inc_urls[${key}]}"
+    name_from_url=$(url_to_file_name "${url}")
+
+    ideal_name="${lower_layer}_${distro}_${stream}_${wheels_inc_from_layer_build_template}"
+
+    list="${ideal_name}"
+
+    for f in $(find -L ${layer_dirs} ${mirror_dir} -type f -name "${name_from_url}"); do
+        cp $f ${wheels_file_dir}/${list}
+        if [ $? -ne 0 ]; then
+            echo "WARNING: Failed to copy from cached file '$f' to satisfy url '${url}'"
+        fi
+    done
+
+    if [ ! -f ${wheels_file_dir}/${list} ]; then
+        curl --silent --fail ${url} > ${wheels_file_dir}/${list}
+        if [ $? -ne 0 ]; then
+            echo "ERROR: Failed to download from url '${url}'"
+            exit 1
+        fi
+    fi
+done
+
 #
 # Dowload build info files
 #

toCOPY/lst_utils.sh

@@ -7,6 +7,7 @@
 rpms_from_layer_build_template="rpm.lst"
 image_inc_from_layer_build_template="image.inc"
 dev_image_inc_from_layer_build_template="image-dev.inc"
+wheels_inc_from_layer_build_template="wheels.inc"
 
 config_dir=${MY_REPO}/../stx-tools/centos-mirror-tools/config
 distro="centos"
@@ -18,6 +19,9 @@ declare -A layer_pkg_urls
 # Store urls for image include files of the various layers in an associative array
 declare -A layer_image_inc_urls
 
+# Store urls for wheel include files of the various layers in an associative array
+declare -A layer_wheels_inc_urls
+
 url_to_file_name () {
     echo "${1}" | sed 's#[:/ ]#-#g'
 }
@@ -74,13 +78,28 @@ set_layer_image_inc_urls () {
     fi
 
     local layer_and_inc_type="${option%,*}"
-    local layer="${layer_and_build_type%,*}"
+    local layer="${layer_and_inc_type%,*}"
     local inc_type="${layer_and_inc_type#*,}"
     local layer_image_inc_url="${option##*,}"
 
     layer_image_inc_urls["${layer_and_inc_type}"]="${layer_image_inc_url}"
 }
 
+set_layer_wheels_inc_urls () {
+    local option="${1}"
+
+    if [ "${option}" == "" ]; then
+        return
+    fi
+
+    local layer_and_stream="${option%,*}"
+    local layer="${layer_and_stream%,*}"
+    local stream="${layer_and_stream#*,}"
+    local layer_wheels_inc_url="${option##*,}"
+
+    layer_wheels_inc_urls["${layer_and_stream}"]="${layer_wheels_inc_url}"
+}
+
 set_layer_pkg_urls () {
     local option="${1}"
 
@@ -120,6 +139,30 @@ read_layer_image_inc_urls () {
     done < "${cfg}"
 }
 
+read_layer_wheels_inc_urls () {
+    local layer="${1}"
+    local cfg="${config_dir}/${distro}/${layer}/required_layer_wheel_inc.cfg"
+    local line=""
+    local key
+
+    if [ ! -f "${cfg}" ]; then
+        return 0;
+    fi
+
+    # Clear all pre-existing entries
+    for key in "${!layer_wheels_inc_urls[@]}"; do
+        unset layer_wheels_inc_urls[${key}]
+    done
+
+    while read line; do
+        line=$(echo "${line}" | sed 's/^[ \t]*//;s/[ \t]*$//' | grep '^[^#]')
+        if [ "${line}" == "" ]; then
+            continue
+        fi
+        set_layer_wheels_inc_urls "${line}"
+    done < "${cfg}"
+}
+
 read_layer_pkg_urls () {
     local layer="${1}"
     local cfg="${config_dir}/${distro}/${layer}/required_layer_pkgs.cfg"
@@ -172,6 +215,7 @@ set_and_validate_distro () {
     if [ -d ${config_dir}/${distro}/${layer} ]; then
         read_layer_pkg_urls ${layer}
         read_layer_image_inc_urls ${layer}
+        read_layer_wheels_inc_urls ${layer}
     else
         echo "Warning: layer ${layer} not defined for distro '${distro}', please provide a valid layer via '-l <layer>'"
     fi
@@ -191,6 +235,7 @@ set_and_validate_layer () {
 
     read_layer_pkg_urls ${layer}
     read_layer_image_inc_urls ${layer}
+    read_layer_wheels_inc_urls ${layer}
 }
 
 # Pick up value of the config_dir from environment if set
@@ -205,3 +250,4 @@ fi
 
 read_layer_pkg_urls ${layer}
 read_layer_image_inc_urls ${layer}
+read_layer_wheels_inc_urls ${layer}