starlingx/tools
Code Issues Proposed changes

Enable build of docker images, charts and wheels from a layered build

Browse Source 
1) Add config for the container layer.

2) Add support for the new config file 'required_layer_wheel_inc.cfg',
Which will specify the default urls for lower layer wheels.inc files.
These files must be downloaded by download_mirror.sh, and linked by
generate-cgcs-centos-repo.sh.

3) Add options to download_mirror.sh and generate-cgcs-centos-repo.sh
allowing overrides or additions to the set of url's for lower layer
wheels.inc

4) Changes to allow build scripts running within a docker container
to launch a sister (not daughter) container.  This will allow
build-wheels to run within a container.

Story: 2006166
Task: 38980
Depends-On: https://review.opendev.org/711773
Depends-On: https://review.opendev.org/717755
Change-Id: I95c71a0327dc9d919ed835d3661a6f1ec7ce30c7
Signed-off-by: Scott Little <scott.little@windriver.com>
This commit is contained in:
Scott Little 2020-03-04 12:24:47 -05:00
parent ce1e395bd9
commit f6755e179a
11 changed files with 166 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Dockerfile
View File

@ -56,6 +56,7 @@ RUN groupadd -g 751 cgts && \
cpanminus \
createrepo \
deltarpm \
docker-client \
expat-devel \
flex \
isomd5sum \

5
centos-mirror-tools/config/centos/containers/required_layer_pkgs.cfg Normal file
View File

@ -0,0 +1,5 @@
compiler,std,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/compiler/latest_build/outputs/RPMS/std/rpm.lst
distro,std,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/RPMS/std/rpm.lst
distro,rt,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/RPMS/rt/rpm.lst
distro,installer,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/RPMS/installer/rpm.lst
flock,std,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/flock/latest_build/outputs/RPMS/std/rpm.lst

4
centos-mirror-tools/config/centos/containers/required_layer_wheel_inc.cfg Normal file
View File

@ -0,0 +1,4 @@
distro,stable,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/centos_stable_wheels.inc
distro,dev,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/centos_dev_wheels.inc
flock,stable,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/flock/latest_build/outputs/centos_stable_wheels.inc
flock,dev,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/flock/latest_build/outputs/centos_dev_wheels.inc

12
centos-mirror-tools/config/centos/containers/yum.repos.d/StarlingX_cengn_compiler_layer.repo Normal file
View File

@ -0,0 +1,12 @@
#####
# StarlingX compiler layer at CENGN
[Starlingx-cengn_compiler_std_layer]
name=Starlingx-cengn_compiler_std_layer
baseurl=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/compiler/latest_build/outputs/RPMS/std/
enabled=1
[Starlingx-cengn_compiler_installer_layer]
name=Starlingx-cengn_compiler_installer_layer
baseurl=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/compiler/latest_build/outputs/RPMS/installer/
enabled=1

18
centos-mirror-tools/config/centos/containers/yum.repos.d/StarlingX_cengn_distro_layer.repo Normal file
View File

@ -0,0 +1,18 @@
#####
# StarlingX distro layer at CENGN
[Starlingx-cengn_distro_std_layer]
name=Starlingx-cengn_distro_std_layer
baseurl=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/RPMS/std/
enabled=1
[Starlingx-cengn_distro_rt_layer]
name=Starlingx-cengn_distro_rt_layer
baseurl=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/RPMS/rt/
enabled=1
[Starlingx-cengn_distro_installer_layer]
name=Starlingx-cengn_distro_installer_layer
baseurl=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/RPMS/installer/
enabled=1

8
centos-mirror-tools/config/centos/containers/yum.repos.d/StarlingX_cengn_flock_layer.repo Normal file
View File

@ -0,0 +1,8 @@
#####
# StarlingX flock layer at CENGN
[Starlingx-cengn_flock_std_layer]
name=Starlingx-cengn_flock_std_layer
baseurl=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/flock/latest_build/outputs/RPMS/std/
enabled=1

2
centos-mirror-tools/config/centos/flock/required_layer_wheel_inc.cfg Normal file
View File

@ -0,0 +1,2 @@
distro,stable,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/centos_stable_wheels.inc
distro,dev,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/centos_dev_wheels.inc

29
centos-mirror-tools/download_mirror.sh
View File

@ -43,6 +43,11 @@ usage() {
echo " layer's build type. Normally the url(s) is read from"
echo " <config_dir>/<distro>/<layer>/required_layer_pkgs.cfg."
echo " This option can be used more than once."
echo " -W <lower_layer>,<stream>,<url>:"
echo " Override the url for the wheels.inc list of a lower"
echo " layer's build type. Normally the url(s) is read from"
echo " <config_dir>/<distro>/<layer>/required_layer_wheel_inc.cfg."
echo " This option can be used more than once."
echo
}
@ -105,6 +110,7 @@ rpms_from_centos_3rd_parties_template="rpms_centos3rdparties.lst"
rpms_from_layer_build_dir=${DL_MIRROR_OUTPUT_DIR}/layer_pkg_lists
rpms_from_layer_repos_dir=${DL_MIRROR_OUTPUT_DIR}/layer_repos
image_inc_from_layer_build_dir=${DL_MIRROR_OUTPUT_DIR}/layer_image_inc
wheels_inc_from_layer_build_dir=${DL_MIRROR_OUTPUT_DIR}/layer_wheels_inc
build_info_from_layer_build_dir=${DL_MIRROR_OUTPUT_DIR}/layer_build_info
tarball_downloads_template="tarball-dl.lst"
other_downloads_template="other_downloads.lst"
@ -151,7 +157,7 @@ multiple_dl_flag_check () {
# Parse out optional arguments
while getopts "c:Cd:ghI:sl:L:nSuU" o; do
while getopts "c:Cd:ghI:sl:L:nSuUW:" o; do
case "${o}" in
c)
# Pass -c ("use alternate yum.conf") to rpm downloader
@ -173,6 +179,9 @@ while getopts "c:Cd:ghI:sl:L:nSuU" o; do
I)
set_layer_image_inc_urls "${OPTARG}"
;;
W)
set_layer_wheels_inc_urls "${OPTARG}"
;;
l)
# layer
set_and_validate_layer "${OPTARG}"
@ -310,6 +319,22 @@ for key in "${!layer_image_inc_urls[@]}"; do
fi
done
\rm -rf ${wheels_inc_from_layer_build_dir}
mkdir -p ${wheels_inc_from_layer_build_dir}
for key in "${!layer_wheels_inc_urls[@]}"; do
lower_layer="${key%,*}"
stream="${key#*,}"
url="${layer_wheels_inc_urls[${key}]}"
name_from_url=$(url_to_file_name $url)
list="${wheels_inc_from_layer_build_dir}/${name_from_url}"
curl --silent --fail ${url} > ${list} ||
if [ $? -ne 0 ]; then
echo "ERROR: Failed to download from url: ${url}"
exit 1
fi
done
\rm -rf ${build_info_from_layer_build_dir}
mkdir -p ${build_info_from_layer_build_dir}
@ -326,7 +351,7 @@ for key in "${!layer_image_inc_urls[@]}"; do
url=$( echo ${layer_image_inc_urls[${key}]} | sed 's#image.inc$#BUILD_INFO#' )
name_from_url=$(url_to_file_name $url)
dest="${build_info_from_layer_build_dir}/${name_from_url}"
curl --silent --fail ${url} > ${dest}
curl --silent --fail ${url} > ${dest} ||
if [ $? -ne 0 ]; then
echo "ERROR: Failed to download from url: ${url}"
exit 1

1
tb.sh
View File

@ -49,6 +49,7 @@ function run_container {
docker run -it --rm \
--name ${TC_CONTAINER_NAME} \
--detach \
-v /var/run/docker.sock:/var/run/docker.sock \
-v ${LOCALDISK}:/${GUEST_LOCALDISK} \
-v ${HOST_MIRROR_DIR}:/import/mirrors:ro \
-v /sys/fs/cgroup:/sys/fs/cgroup:ro \

42
toCOPY/generate-cgcs-centos-repo.sh
View File

@ -48,6 +48,11 @@ usage () {
echo " layer's build type. Normally the url(s) is read from"
echo " <config_dir>/<distro>/<layer>/required_layer_pkgs.cfg."
echo " This option can be used more than once."
echo " --layer-wheels-inc-url=<lower_layer>,<stream>,<url>:"
echo " Override the url for the image include file of a lower"
echo " layer's build type. Normally the url(s) is read from"
echo " <config_dir>/<distro>/<layer>/required_layer_wheel_inc.cfg"
echo " This option can be used more than once."
echo " --layer-dir=<dir>: Look in provided dir for packages to link to."
echo " This option can be used more than once."
echo
@ -72,7 +77,7 @@ if [ -z "$MY_REPO" ]; then
fi
TEMP=$(getopt -o h --long help,config-dir:,distro:,layer:,layer-dir:,layer-inc-url:,layer-pkg-url:,mirror-dir: -n 'generate-cgcs-centos-repo' -- "$@")
TEMP=$(getopt -o h --long help,config-dir:,distro:,layer:,layer-dir:,layer-inc-url:,layer-pkg-url:,layer-wheels-inc-url:,mirror-dir: -n 'generate-cgcs-centos-repo' -- "$@")
if [ $? -ne 0 ]; then
echo "getopt error"
usage
@ -86,6 +91,7 @@ while true ; do
--mirror-dir) mirror_dir=$2 ; shift 2 ;;
--layer-dir) layer_dirs+=" ${2/,/ }" ; shift 2 ;;
--layer-inc-url) set_layer_image_inc_urls "${2}" ; shift 2 ;;
--layer-wheels-inc-url) set_layer_wheels_inc_urls "${2}" ; shift 2 ;;
--layer-pkg-url) set_layer_pkg_urls "${2}" ; shift 2 ;;
--config-dir) set_and_validate_config_dir "${2}"; shift 2 ;;
--distro) set_and_validate_distro "${2}"; shift 2 ;;
@ -114,6 +120,8 @@ echo "layer_pkg_urls=${layer_pkg_urls[@]}"
echo
echo "layer_image_inc_urls=${layer_image_inc_urls[@]}"
echo
echo "layer_wheels_inc_urls=${layer_wheels_inc_urls[@]}"
echo
dest_dir=$MY_REPO/cgcs-centos-repo
timestamp="$(date +%F_%H%M)"
@ -126,6 +134,7 @@ TMP_LST_DIR=$(mktemp -d /tmp/tmp_lst_dir_XXXXXX)
mkdir -p $TMP_LST_DIR
lst_file_dir="$TMP_LST_DIR"
inc_file_dir="${dest_dir}/layer_image_inc"
wheels_file_dir="${dest_dir}/layer_wheels_inc"
build_info_file_dir="${dest_dir}/layer_build_info"
rpm_lst_files="rpms_3rdparties.lst rpms_centos3rdparties.lst rpms_centos.lst"
@ -208,6 +217,37 @@ for key in "${!layer_image_inc_urls[@]}"; do
fi
done
#
# Dowload wheels inc files from layer_wheels_inc_urls
#
\rm -rf ${wheels_file_dir}
mkdir -p ${wheels_file_dir}
for key in "${!layer_wheels_inc_urls[@]}"; do
lower_layer="${key%,*}"
stream="${key#*,}"
url="${layer_wheels_inc_urls[${key}]}"
name_from_url=$(url_to_file_name "${url}")
ideal_name="${lower_layer}_${distro}_${stream}_${wheels_inc_from_layer_build_template}"
list="${ideal_name}"
for f in $(find -L ${layer_dirs} ${mirror_dir} -type f -name "${name_from_url}"); do
cp $f ${wheels_file_dir}/${list}
if [ $? -ne 0 ]; then
echo "WARNING: Failed to copy from cached file '$f' to satisfy url '${url}'"
fi
done
if [ ! -f ${wheels_file_dir}/${list} ]; then
curl --silent --fail ${url} > ${wheels_file_dir}/${list}
if [ $? -ne 0 ]; then
echo "ERROR: Failed to download from url '${url}'"
exit 1
fi
fi
done
#
# Dowload build info files
#

48
toCOPY/lst_utils.sh
View File

@ -7,6 +7,7 @@
rpms_from_layer_build_template="rpm.lst"
image_inc_from_layer_build_template="image.inc"
dev_image_inc_from_layer_build_template="image-dev.inc"
wheels_inc_from_layer_build_template="wheels.inc"
config_dir=${MY_REPO}/../stx-tools/centos-mirror-tools/config
distro="centos"
@ -18,6 +19,9 @@ declare -A layer_pkg_urls
# Store urls for image include files of the various layers in an associative array
declare -A layer_image_inc_urls
# Store urls for wheel include files of the various layers in an associative array
declare -A layer_wheels_inc_urls
url_to_file_name () {
echo "${1}" | sed 's#[:/ ]#-#g'
}
@ -74,13 +78,28 @@ set_layer_image_inc_urls () {
fi
local layer_and_inc_type="${option%,*}"
local layer="${layer_and_build_type%,*}"
local layer="${layer_and_inc_type%,*}"
local inc_type="${layer_and_inc_type#*,}"
local layer_image_inc_url="${option##*,}"
layer_image_inc_urls["${layer_and_inc_type}"]="${layer_image_inc_url}"
}
set_layer_wheels_inc_urls () {
local option="${1}"
if [ "${option}" == "" ]; then
return
fi
local layer_and_stream="${option%,*}"
local layer="${layer_and_stream%,*}"
local stream="${layer_and_stream#*,}"
local layer_wheels_inc_url="${option##*,}"
layer_wheels_inc_urls["${layer_and_stream}"]="${layer_wheels_inc_url}"
}
set_layer_pkg_urls () {
local option="${1}"
@ -120,6 +139,30 @@ read_layer_image_inc_urls () {
done < "${cfg}"
}
read_layer_wheels_inc_urls () {
local layer="${1}"
local cfg="${config_dir}/${distro}/${layer}/required_layer_wheel_inc.cfg"
local line=""
local key
if [ ! -f "${cfg}" ]; then
return 0;
fi
# Clear all pre-existing entries
for key in "${!layer_wheels_inc_urls[@]}"; do
unset layer_wheels_inc_urls[${key}]
done
while read line; do
line=$(echo "${line}" | sed 's/^[ \t]*//;s/[ \t]*$//' | grep '^[^#]')
if [ "${line}" == "" ]; then
continue
fi
set_layer_wheels_inc_urls "${line}"
done < "${cfg}"
}
read_layer_pkg_urls () {
local layer="${1}"
local cfg="${config_dir}/${distro}/${layer}/required_layer_pkgs.cfg"
@ -172,6 +215,7 @@ set_and_validate_distro () {
if [ -d ${config_dir}/${distro}/${layer} ]; then
read_layer_pkg_urls ${layer}
read_layer_image_inc_urls ${layer}
read_layer_wheels_inc_urls ${layer}
else
echo "Warning: layer ${layer} not defined for distro '${distro}', please provide a valid layer via '-l <layer>'"
fi
@ -191,6 +235,7 @@ set_and_validate_layer () {
read_layer_pkg_urls ${layer}
read_layer_image_inc_urls ${layer}
read_layer_wheels_inc_urls ${layer}
}
# Pick up value of the config_dir from environment if set
@ -205,3 +250,4 @@ fi
read_layer_pkg_urls ${layer}
read_layer_image_inc_urls ${layer}
read_layer_wheels_inc_urls ${layer}