Improved local mirror support in tools

This change improves support for organizational or personal local
mirrors.  Download tools work in a wider variety of environmnets,
and the population of build environments is now driven by .lst
files.

The download_mirror.sh (and related files) now have options to
better support running in non-containerized environments.  This
includes options to run using local yum.conf files (rather than
system files) and not executing sudo or chgrp commands.  These
options are enabled by flags.  Default behaviour remains
unchanged.

The generate-cgcs-centos-repo.sh tool now populates build
environment as driven by lst files.  This allows a mirror to be
maintained (and grown) at an organizational level, while allowing
a developer to produce exactly the environment they need as per
the lst files they have checked out.

generate-cgcs-centos-repo.sh now also populates the downloads
directory by invoking the populate_downloads.sh script (also
driven by lst files)

Change-Id: I8aae32c776df7618a5ef545a0ce43903b41112d0
Story: 2003422
Task: 24586
Signed-off-by: jmckenna <jason.mckenna@windriver.com>
This commit is contained in:
jmckenna 2018-08-10 10:20:30 -04:00
parent b229cdac10
commit 13f92f4668
5 changed files with 270 additions and 76 deletions

View File

@ -1,19 +1,60 @@
#!/bin/bash -e
#
# SPDX-License-Identifier: Apache-2.0
#
# download RPMs/SRPMs from different sources.
# this script was originated by Brian Avery, and later updated by Yong Hu
usage() {
echo "$0 [-n] [-c <yum.conf>] <rpms_list> <match_level> <from_where>"
echo ""
echo "Options:"
echo " -n: Do not use sudo when performing operations"
echo " -c: Use an alternate yum.conf rather than the system file"
echo " rpm_list: a list of RPM files to be downloaded."
echo " match_level: value could be L1, L2 or L3:"
echo " L1: use name, major version and minor version:"
echo " vim-7.4.160-2.el7 to search vim-7.4.160-2.el7.src.rpm"
echo " L2: use name and major version:"
echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm"
echo " L3: use name:"
echo " using vim to search vim-7.4.160-2.el7.src.rpm"
echo " from_where: where to download the RPMs: 'centos'from CentOS Repos,"
echo " otherwise from 3rd-party websets"
echo ""
}
# By default, we use "sudo" and we don't use a local yum.conf. These can
# be overridden via flags.
SUDOCMD="sudo -E"
YUMCONFOPT=""
# Parse option flags
while getopts "c:nh" o; do
case "${o}" in
n)
# No-sudo
SUDOCMD=""
;;
c)
# Use an alternate yum.conf
YUMCONFOPT="-c $OPTARG"
;;
h)
# Help
usage
exit 0
;;
*)
usage
exit 1
;;
esac
done
shift $((OPTIND-1))
if [ $# -lt 3 ]; then
echo "$0 <rpms_list> <match_level> <from_where>"
echo "rpm_list: a list of RPM files to be downloaded."
echo "match_level: value could be L1, L2 or L3:"
echo " L1: use name, major version and minor version:"
echo " vim-7.4.160-2.el7 to search vim-7.4.160-2.el7.src.rpm"
echo " L2: use name and major version:"
echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm"
echo " L3: use name:"
echo " using vim to search vim-7.4.160-2.el7.src.rpm"
echo "from_where: where to download the RPMs: 'centos'from CentOS Repos,"
echo "otherwise from 3rd-party websets"
usage
exit -1
fi
@ -83,11 +124,11 @@ download () {
fi
echo " ------ using $SFILE to search $rpm_name ------"
if [ "$_type" == "src" ];then
download_cmd="sudo -E yumdownloader -q -C --source $SFILE"
download_url_cmd="sudo -E yumdownloader --urls -q -C --source $SFILE"
download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE"
download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE"
else
download_cmd="sudo -E yumdownloader -q -C $SFILE --archlist=noarch,x86_64"
download_url_cmd="sudo -E yumdownloader --urls -q -C $SFILE --archlist=noarch,x86_64"
download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
fi
else
rpm_name=`echo $ff | cut -d"#" -f1-1`
@ -95,13 +136,13 @@ download () {
download_cmd="wget $rpm_url"
SFILE=$rpm_name
fi
echo "--> run: $download_cmd"
echo "--> run: $download_cmd"
if [ "$_type" == "src" ]; then
if [ ! -e $MDIR_SRC/$rpm_name ]; then
echo "Looking for $rpm_name"
if $download_cmd ; then
# Success! Record download URL.
# Use 'sort --unique' because sometimes
# Use 'sort --unique' because sometimes
# yumdownloader reports the url twice
$download_url_cmd | sort --unique >> $URL_SRPMS
@ -144,7 +185,7 @@ download () {
}
# prime the cache
sudo -E yum makecache
${SUDOCMD} yum ${YUMCONFOPT} makecache
#go to download *.noarch.rpm files
noarch_rpms=`echo "$(cat $rpms_list | grep '.noarch.rpm')"`

View File

@ -93,6 +93,10 @@ for line in $(cat $tarball_file); do
if [[ "$line" =~ ^'!' ]]; then
tarball_name="${tarball_name//!/}"
if [ -e "$output_tarball/$tarball_name" ]; then
echo "Already have $tarball_name"
continue
fi
echo $tarball_name
pushd $output_tarball
if [ "$tarball_name" = "integrity-kmod-e6aef069.tar.gz" ]; then

View File

@ -1,4 +1,65 @@
#!/bin/bash -e
#
# SPDX-License-Identifier: Apache-2.0
#
usage() {
echo "$0 [-n] [-c <yum.conf>] [-g]"
echo ""
echo "Options:"
echo " -n: Do not use sudo when performing operations (option passed on to"
echo " subscripts when appropriate)"
echo " -c: Use an alternate yum.conf rather than the system file (option passed"
echo " on to subscripts when appropriate)"
echo " -g: do not change group IDs of downloaded artifacts"
echo ""
}
rpm_downloader="./dl_rpms.sh"
tarball_downloader="./dl_tarball.sh"
other_downloader="./dl_other_from_centos_repo.sh"
# track optional arguments
change_group_ids=1
use_system_yum_conf=1
rpm_downloader_extra_args=""
tarball_downloader_extra_args=""
# lst files to use as input
rpms_from_3rd_parties="./rpms_from_3rd_parties.lst"
rpms_from_centos_repo="./rpms_from_centos_repo.lst"
rpms_from_centos_3rd_parties="./rpms_from_centos_3rd_parties.lst"
other_downloads="./other_downloads.lst"
# Parse out optional -c or -n arguments
while getopts "c:ngh" o; do
case "${o}" in
n)
# Pass -n ("no-sudo") to rpm downloader
rpm_downloader_extra_args="${rpm_downloader_extra_args} -n"
;;
c)
# Pass -c ("use alternate yum.conf") to rpm downloader
use_system_yum_conf=0
rpm_downloader_extra_args="${rpm_downloader_extra_args} -c ${OPTARG}"
;;
g)
# Do not attempt to change group IDs on downloaded packages
change_group_ids=0
;;
h)
# Help
usage
exit 0
;;
*)
usage
exit 1
;;
esac
done
shift $((OPTIND-1))
echo "--------------------------------------------------------------"
echo "WARNING: this script HAS TO access internet (http/https/ftp),"
@ -16,56 +77,51 @@ need_file(){
}
# Check extistence of prerequisites files
need_file dl_rpms.sh dl_other_from_centos_repo.sh dl_tarball.sh
need_file rpms_from_3rd_parties.lst
need_file rpms_from_centos_3rd_parties.lst
need_file rpms_from_centos_repo.lst
need_file other_downloads.lst
need_file ${rpm_downloader} ${other_downloader} ${tarball_downloader}
need_file ${rpms_from_3rd_parties}
need_file ${rpms_from_centos_3rd_parties}
need_file ${rpms_from_centos_repo}
need_file ${other_downloads}
need_file tarball-dl.lst mvn-artifacts.lst
#download RPMs/SRPMs from 3rd_party websites (not CentOS repos) by "wget"
echo "step #1: start downloading RPMs/SRPMs from 3rd-party websites..."
# Restore StarlingX_3rd repos from backup
REPO_SOURCE_DIR=/localdisk/yum.repos.d
REPO_DIR=/etc/yum.repos.d
if [ -d $REPO_SOURCE_DIR ] && [ -d $REPO_DIR ]; then
\cp -f $REPO_SOURCE_DIR/*.repo $REPO_DIR/
if [ ${use_system_yum_conf} -eq 0 ]; then
# Restore StarlingX_3rd repos from backup
REPO_SOURCE_DIR=/localdisk/yum.repos.d
REPO_DIR=/etc/yum.repos.d
if [ -d $REPO_SOURCE_DIR ] && [ -d $REPO_DIR ]; then
\cp -f $REPO_SOURCE_DIR/*.repo $REPO_DIR/
fi
fi
rpm_downloader="./dl_rpms.sh"
$rpm_downloader ./rpms_from_3rd_parties.lst L1 3rd | tee ./logs/log_download_rpms_from_3rd_party.txt
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 3rd | tee ./logs/log_download_rpms_from_3rd_party.txt
retcode=${PIPESTATUS[0]}
if [ $retcode -ne 0 ];then
echo "ERROR: something wrong with downloading, please check the log!!"
fi
# download RPMs/SRPMs from 3rd_party repos by "yumdownloader"
$rpm_downloader ./rpms_from_centos_3rd_parties.lst L1 3rd-centos | tee ./logs/log_download_rpms_from_centos_3rd_parties_L1.txt
# deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo.
\rm -f $REPO_DIR/StarlingX_3rd*.repo
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 3rd-centos | tee ./logs/log_download_rpms_from_centos_3rd_parties_L1.txt
if [ ${use_system_yum_conf} -eq 1 ]; then
# deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo.
\rm -f $REPO_DIR/StarlingX_3rd*.repo
fi
echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..."
#download RPMs/SRPMs from CentOS repos by "yumdownloader"
$rpm_downloader ./rpms_from_centos_repo.lst L1 centos | tee ./logs/log_download_rpms_from_centos_L1.txt
retcode=${PIPESTATUS[0]}
if [ $retcode -eq 0 ]; then
if [ $retcode -ne 0 ]; then
echo "finish 1st round of RPM downloading successfully!"
if [ -e "./output/centos_rpms_missing_L1.txt" ]; then
missing_num=`wc -l ./output/centos_rpms_missing_L1.txt | cut -d " " -f1-1`
if [ "$missing_num" != "0" ];then
echo "ERROR: -------RPMs missing $missing_num in yumdownloader with L1 match ---------------"
fi
#echo "start 2nd round of downloading Binary RPMs with L2 match criteria..."
#$rpm_downloader ./output/centos_rpms_missing_L1.txt L2 centos | tee ./logs/log_download_rpms_from_centos_L2.txt
#if [ $? == 0 ]; then
# echo "finish 2nd round of RPM downloading successfully!"
# if [ -e "./output/rpms_missing_L2.txt" ]; then
# echo "WARNING: we're still missing RPMs listed in ./rpms_missing_L2.txt !"
# fi
#fi
fi
if [ -e "./output/centos_srpms_missing_L1.txt" ]; then
@ -73,14 +129,6 @@ if [ $retcode -eq 0 ]; then
if [ "$missing_num" != "0" ];then
echo "ERROR: --------- SRPMs missing $missing_num in yumdownloader with L1 match ---------------"
fi
#echo "start 2nd round of downloading Source RPMs with L2 match criteria..."
#$rpm_downloader ./output/centos_srpms_missing_L1.txt L2 centos | tee ./logs/log_download_srpms_from_centos_L2.txt
#if [ $? == 0 ]; then
# echo "finish 2nd round of SRPM downloading successfully!"
# if [ -e "./srpms_missing_L2.txt" ]; then
# echo "WARNING: we're still missing SRPMs listed in ./rpms_missing_L2.txt !"
# fi
#fi
fi
else
echo "finish 1st round with failures!"
@ -93,9 +141,10 @@ find ./output -type f -name "*.rpm" | xargs rpm -K | grep -i "MISSING KEYS" > ./
find ./output -name "*.i686.rpm" | tee ./output/all_i686.txt
find ./output -name "*.i686.rpm" | xargs rm -f
line1=`wc -l rpms_from_3rd_parties.lst | cut -d " " -f1-1`
line2=`wc -l rpms_from_centos_repo.lst | cut -d " " -f1-1`
line3=`wc -l rpms_from_centos_3rd_parties.lst | cut -d " " -f1-1`
line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1`
line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1`
line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1`
let total_line=$line1+$line2+$line3
echo "We expect to download $total_line RPMs."
num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1`
@ -104,14 +153,15 @@ if [ "$total_line" != "$num_of_downloaded_rpms" ]; then
echo "WARNING: Not the same number of RPMs in output as RPMs expected to be downloaded, need to check outputs and logs."
fi
# change "./output" and sub-folders to 751 (cgcs) group
chown 751:751 -R ./output
if [ $change_group_ids -eq 1 ]; then
# change "./output" and sub-folders to 751 (cgcs) group
chown 751:751 -R ./output
fi
echo "step #3: start downloading other files ..."
other_downloader="./dl_other_from_centos_repo.sh"
$other_downloader ./other_downloads.lst ./output/stx-r1/CentOS/pike/Binary/ | tee ./logs/log_download_other_files_centos.txt
${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ | tee ./logs/log_download_other_files_centos.txt
retcode=${PIPESTATUS[0]}
if [ $retcode -eq 0 ];then
echo "step #3: done successfully"
@ -120,7 +170,7 @@ fi
# StarlingX requires a group of source code pakages, in this section
# they will be downloaded.
echo "step #4: start downloading tarball compressed files"
./dl_tarball.sh
${tarball_downloader} ${tarball_downloader_extra_args}
echo "IMPORTANT: The following 3 files are just bootstrap versions. Based"
echo "on them, the workable images for StarlingX could be generated by"

View File

@ -27,37 +27,66 @@ comps_xml_file=$MY_REPO/build-tools/repo_files/comps.xml
mock_cfg_dest_file=$MY_REPO/cgcs-centos-repo/mock.cfg.proto
comps_xml_dest_file=$MY_REPO/cgcs-centos-repo/Binary/comps.xml
if [[ ( ! -d $mirror_dir/Binary ) || ( ! -d $mirror_dir/Source ) ]]; then
echo "The mirror $mirror_dir doesn't has the Binary and Source"
lst_file_dir="$MY_REPO_ROOT_DIR/stx-tools/centos-mirror-tools"
rpm_lst_files="rpms_from_3rd_parties.lst rpms_from_centos_3rd_parties.lst rpms_from_centos_repo.lst"
other_lst_file="other_downloads.lst"
missing_rpms_file=missing.txt
rm -f ${missing_rpms_file}
# Strip trailing / from mirror_dir if it was specified...
mirror_dir=$(echo ${mirror_dir} | sed "s%/$%%")
if [[ ( ! -d ${mirror_dir}/Binary ) || ( ! -d ${mirror_dir}/Source ) ]]; then
echo "The mirror ${mirror_dir} doesn't has the Binary and Source"
echo "folders. Please provide a valid mirror"
exit -1
fi
if [ ! -d "$dest_dir" ]; then
mkdir -p "$dest_dir"
if [ ! -d "${dest_dir}" ]; then
mkdir -p "${dest_dir}"
fi
for t in "Binary" "Source" ; do
target_dir=$dest_dir/$t
target_dir=${dest_dir}/$t
if [ ! -d "$target_dir" ]; then
mkdir -p "$target_dir"
else
mv -f "$target_dir" "$target_dir-backup-$timestamp"
mkdir -p "$target_dir"
fi
done
pushd "$mirror_dir/$t"|| exit 1
find . -type d -exec mkdir -p "${target_dir}"/{} \;
all_files=$(find . -type f -name "*")
popd || exit 1
unsuccessful_file=0
for lst_file in ${rpm_lst_files} ; do
grep -v "^#" ${lst_file_dir}/${lst_file} | while IFS="#" read rpmname extrafields; do
if [ -z "${rpmname}" ]; then
continue
fi
mirror_file=$(find ${mirror_dir} -name ${rpmname})
if [ -z "${mirror_file}" ]; then
echo "Error -- could not find requested ${rpmname} in ${mirror_dir}"
echo ${rpmname} >> ${missing_rpms_file}
unsuccessful_file=1
continue
fi
for ff in $all_files; do
# Great, we found the file! Let's strip the mirror_dir prefix from it...
ff=$(echo ${mirror_file} | sed "s%^${mirror_dir}/%%")
f_name=$(basename "$ff")
sub_dir=$(dirname "$ff")
ln -sf "$mirror_dir/$t/$ff" "$target_dir/$sub_dir"
echo "Creating symlink for $target_dir/$sub_dir/$f_name"
echo "------------------------------"
# Make sure we have a subdir (so we don't symlink the first file as
# the subdir name)
mkdir -p ${dest_dir}/${sub_dir}
# Link it!
echo "Creating symlink for ${dest_dir}/${sub_dir}/${f_name}"
ln -sf "${mirror_dir}/$ff" "${dest_dir}/${sub_dir}"
if [ $? -ne 0 ]; then
echo "Failed ${mirror_file}: ln -sf \"${mirror_dir}/$ff\" \"${dest_dir}/${sub_dir}\""
unsuccessful_file=1
fi
done
done
@ -75,13 +104,30 @@ fi
echo "Copying mock.cfg.proto and comps.xml files."
if [ -f "$mock_cfg_dest_file" ]; then
cp "$mock_cfg_dest_file" "$mock_cfg_dest_file-backup-$timestamp"
\cp -f "$mock_cfg_dest_file" "$mock_cfg_dest_file-backup-$timestamp"
fi
cp "$mock_cfg_file" "$mock_cfg_dest_file"
if [ -f "$comps_xml_dest_file" ]; then
cp "$comps_xml_dest_file" "$comps_xml_dest_file-backup-$timestamp"
\cp -f "$comps_xml_dest_file" "$comps_xml_dest_file-backup-$timestamp"
fi
cp "$comps_xml_file" "$comps_xml_dest_file"
echo "Done"
# Populate the contents from other list files
cat ${lst_file_dir}/${other_lst_file} | grep -v "#" | while IFS=":" read targettype item extrafields; do
if [ "${targettype}" == "folder" ]; then
echo "Creating folder ${item}"
mkdir -p $MY_REPO/cgcs-centos-repo/Binary/${item}
fi
if [ "${targettype}" == "file" ]; then
mkdir -p $MY_REPO/cgcs-centos-repo/Binary/$(dirname ${item})
echo "Creating symlink for $MY_REPO/cgcs-centos-repo/Binary/${item}"
ln -sf ${mirror_dir}/Binary/${item} $MY_REPO/cgcs-centos-repo/Binary/${item}
fi
done
echo "Done creating repo directory"
if [ ${unsuccessful_file} -ne 0 ]; then
echo "WARNING: Some targets could not be found. Your repo may be incomplete."
fi

53
toCOPY/populate_downloads.sh Executable file
View File

@ -0,0 +1,53 @@
#!/bin/bash
#
# SPDX-License-Identifier: Apache-2.0
#
usage () {
echo "$0 <mirror-path>"
}
if [ $# -ne 1 ]; then
usage
exit -1
fi
if [ -z "$MY_REPO" ]; then
echo "\$MY_REPO is not set. Ensure you are running this script"
echo "from the container and \$MY_REPO points to the root of"
echo "your folder tree."
exit -1
fi
mirror_dir=$1
tarball_lst=${MY_REPO_ROOT_DIR}/stx-tools/centos-mirror-tools/tarball-dl.lst
downloads_dir=${MY_REPO}/stx/downloads
extra_downloads="mlnx-ofa_kernel-4.3-OFED.4.3.3.0.2.1.gcf60532.src.rpm libibverbs-41mlnx1-OFED.4.2.1.0.6.42120.src.rpm rdma-core-43mlnx1-1.43302.src.rpm"
mkdir -p ${MY_REPO}/stx/downloads
grep -v "^#" ${tarball_lst} | while read x; do
if [ -z "$x" ]; then
continue
fi
# Get first element of item & strip leading ! if appropriate
tarball_file=$(echo $x | sed "s/#.*//" | sed "s/^!//")
# put the file in downloads
source_file=$(find ${mirror_dir}/downloads -name "${tarball_file}")
if [ -z ${source_file} ]; then
echo "Could not find ${tarball_file}"
else
rel_path=$(echo ${source_file} | sed "s%^${mirror_dir}/downloads/%%")
rel_dir_name=$(dirname ${rel_path})
if [ ! -e ${downloads_dir}/${rel_dir_name}/${tarball_file} ]; then
mkdir -p ${downloads_dir}/${rel_dir_name}
ln -sf ${source_file} ${downloads_dir}/${rel_dir_name}/
fi
fi
done
for x in ${extra_downloads}; do
ln -sf ${mirror_dir}/downloads/$x ${downloads_dir}
done