From 1a539ac2a5e663d199b5ea9ac1b2a1c9541b478e Mon Sep 17 00:00:00 2001 From: Marcela Rosales Date: Wed, 8 Aug 2018 10:40:00 -0500 Subject: [PATCH] Change lists names and downloading scripts -By standardizing the list files name it's possible to eliminate one parameter in the download function in dl_rpms and reduce complexity. -Now the download function does not receive it as a parameter anymore, and instead it gets it from the RPM name. -Also the download function now decides if build a wget command or a yumdownloader commnad based on the content of the list (by identifying the # character) instead of using the name of the list. I adapted download_mirror.sh. Change-Id: I041fc9c704156215f06149e5b4c16cd92990e17c Signed-off-by: Marcela Rosales --- centos-mirror-tools/dl_rpms.sh | 64 +++++++++---------- centos-mirror-tools/download_mirror.sh | 22 +++---- ...om_3rd_parties.lst => rpms_3rdparties.lst} | 0 ...s_from_centos_repo.lst => rpms_centos.lst} | 0 ..._parties.lst => rpms_centos3rdparties.lst} | 0 5 files changed, 42 insertions(+), 44 deletions(-) rename centos-mirror-tools/{rpms_from_3rd_parties.lst => rpms_3rdparties.lst} (100%) rename centos-mirror-tools/{rpms_from_centos_repo.lst => rpms_centos.lst} (100%) rename centos-mirror-tools/{rpms_from_centos_3rd_parties.lst => rpms_centos3rdparties.lst} (100%) diff --git a/centos-mirror-tools/dl_rpms.sh b/centos-mirror-tools/dl_rpms.sh index ac51ea6e..303bf94e 100755 --- a/centos-mirror-tools/dl_rpms.sh +++ b/centos-mirror-tools/dl_rpms.sh @@ -6,7 +6,7 @@ # this script was originated by Brian Avery, and later updated by Yong Hu usage() { - echo "$0 [-n] [-c ] " + echo "$0 [-n] [-c ] " echo "" echo "Options:" echo " -n: Do not use sudo when performing operations" @@ -19,11 +19,16 @@ usage() { echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm" echo " L3: use name:" echo " using vim to search vim-7.4.160-2.el7.src.rpm" - echo " from_where: where to download the RPMs: 'centos'from CentOS Repos," - echo " otherwise from 3rd-party websets" echo "" } +get_from() { + list=$1 + base=$(basename $list .lst) + from=$(echo $base | cut -d'_' -f2-2) + echo $from +} + # By default, we use "sudo" and we don't use a local yum.conf. These can # be overridden via flags. SUDOCMD="sudo -E" @@ -53,7 +58,7 @@ while getopts "c:nh" o; do done shift $((OPTIND-1)) -if [ $# -lt 3 ]; then +if [ $# -lt 2 ]; then usage exit -1 fi @@ -72,7 +77,6 @@ if [ ! -z "$2" -a "$2" != " " ];then match_level=$2 fi -from=$3 timestamp=$(date +%F_%H%M) echo $timestamp @@ -83,6 +87,8 @@ mkdir -p $MDIR_SRC MDIR_BIN=$DESTDIR/stx-r1/CentOS/pike/Binary mkdir -p $MDIR_BIN + +from=$(get_from $rpms_list) FAIL_MOVE_SRPMS="$DESTDIR/${from}_srpms_fail_move_${match_level}.txt" FOUND_SRPMS="$DESTDIR/${from}_srpms_found_${match_level}.txt" MISSING_SRPMS="$DESTDIR/${from}_srpms_missing_${match_level}.txt" @@ -103,18 +109,21 @@ cat /dev/null > $FOUND_RPMS cat /dev/null > $MISSING_RPMS cat /dev/null > $URL_RPMS -#function to download different type of RPMs in different ways +# Function to download different types of RPMs in different ways download () { - _list=$1 + _file=$1 _level=$2 - _from=$3 - _type=$4 - + _list=$(cat $_file) + _from=$(get_from $_file) echo "now the rpm will come from: $_from" for ff in $_list; do - ## download RPM from CentOS repos - if [ "$_from" == "centos" -o "$_from" == "3rd-centos" ]; then + _type=$(echo $ff | rev | cut -d'.' -f2-2 | rev) + + # Decide if the list will be downloaded using yumdownloader or wget + if [[ $ff != *"#"* ]]; then rpm_name=$ff + + # Cut the rpm name for the specified level (L1, L2 or L3) if [ $_level == "L1" ]; then SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev` elif [ $match_level == "L2" ];then @@ -123,6 +132,7 @@ download () { SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev` fi echo " ------ using $SFILE to search $rpm_name ------" + # Yumdownloader with the appropriate flag for src, noarch or x86_64 if [ "$_type" == "src" ];then download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE" download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE" @@ -131,12 +141,15 @@ download () { download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" fi else + # Buid wget command rpm_name=`echo $ff | cut -d"#" -f1-1` rpm_url=`echo $ff | cut -d"#" -f2-2` download_cmd="wget $rpm_url" SFILE=$rpm_name fi + echo "--> run: $download_cmd" + # Put the RPM in the Binary or Source directory if [ "$_type" == "src" ]; then if [ ! -e $MDIR_SRC/$rpm_name ]; then echo "Looking for $rpm_name" @@ -163,7 +176,7 @@ download () { echo "Looking for $rpm_name..." if $download_cmd ; then # Success! Record download URL. - # Use 'sort --unique' because sometimes + # Use 'sort --unique' because sometimes # yumdownloader reports the url twice $download_url_cmd | sort --unique >> $URL_RPMS @@ -184,28 +197,13 @@ download () { done } -# prime the cache +# Prime the cache ${SUDOCMD} yum ${YUMCONFOPT} makecache -#go to download *.noarch.rpm files -noarch_rpms=`echo "$(cat $rpms_list | grep '.noarch.rpm')"` -if [ ! -z "$noarch_rpms" ];then - echo "--> start searching noarch RPMs ....." - download "$noarch_rpms" $match_level $from "noarch" -fi - -#go to download *.x86_64.rpm files -x86_64_rpms=`echo "$(cat $rpms_list | grep '.x86_64.rpm')"` -if [ ! -z "$x86_64_rpms" ];then - echo "--> start searching x86_64 RPMs ....." - download "$x86_64_rpms" $match_level $from "x86_64" -fi - -#go to download *.src.rpm files -src_rpms=`echo "$(cat $rpms_list | grep '.src.rpm')"` -if [ ! -z "$src_rpms" ];then - echo "--> start searching source RPMs ....." - download "$src_rpms" $match_level $from "src" +# Download files +if [ -s "$rpms_list" ];then + echo "--> start searching "$rpms_list + download $rpms_list $match_level fi echo "done!!" diff --git a/centos-mirror-tools/download_mirror.sh b/centos-mirror-tools/download_mirror.sh index 52075c67..963f7661 100755 --- a/centos-mirror-tools/download_mirror.sh +++ b/centos-mirror-tools/download_mirror.sh @@ -26,9 +26,9 @@ rpm_downloader_extra_args="" tarball_downloader_extra_args="" # lst files to use as input -rpms_from_3rd_parties="./rpms_from_3rd_parties.lst" -rpms_from_centos_repo="./rpms_from_centos_repo.lst" -rpms_from_centos_3rd_parties="./rpms_from_centos_3rd_parties.lst" +rpms_from_3rd_parties="./rpms_3rdparties.lst" +rpms_from_centos_repo="./rpms_centos.lst" +rpms_from_centos_3rd_parties="./rpms_centos3rdparties.lst" other_downloads="./other_downloads.lst" # Parse out optional -c or -n arguments @@ -96,14 +96,17 @@ if [ ${use_system_yum_conf} -ne 0 ]; then fi fi -$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 3rd | tee ./logs/log_download_rpms_from_3rd_party.txt +logfile="log_download_3rdparties_L1.txt" +$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 | tee ./logs/$logfile retcode=${PIPESTATUS[0]} -if [ $retcode -ne 0 ];then +if [ $retcode -ne 0 ]; then echo "ERROR: something wrong with downloading, please check the log!!" fi # download RPMs/SRPMs from 3rd_party repos by "yumdownloader" -$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 3rd-centos | tee ./logs/log_download_rpms_from_centos_3rd_parties_L1.txt +logfile="log_download_centos3rdparties_L1.txt" +$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 | tee ./logs/$logfile + if [ ${use_system_yum_conf} -eq 1 ]; then # deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo. \rm -f $REPO_DIR/StarlingX_3rd*.repo @@ -111,9 +114,8 @@ fi echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..." #download RPMs/SRPMs from CentOS repos by "yumdownloader" - -$rpm_downloader ./rpms_from_centos_repo.lst L1 centos | tee ./logs/log_download_rpms_from_centos_L1.txt - +logfile="log_download_centos_L1.txt" +$rpm_downloader ${rpms_from_centos_repo} L1 | tee ./logs/$logfile retcode=${PIPESTATUS[0]} if [ $retcode -ne 0 ]; then echo "finish 1st round of RPM downloading successfully!" @@ -144,7 +146,6 @@ find ./output -name "*.i686.rpm" | xargs rm -f line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1` line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1` line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1` - let total_line=$line1+$line2+$line3 echo "We expect to download $total_line RPMs." num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1` @@ -178,4 +179,3 @@ echo "running \"update-pxe-network-installer\" command after \"build-iso\"" echo " - out/stx-r1/CentOS/pike/Binary/LiveOS/squashfs.img" echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/initrd.img" echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/vmlinuz" - diff --git a/centos-mirror-tools/rpms_from_3rd_parties.lst b/centos-mirror-tools/rpms_3rdparties.lst similarity index 100% rename from centos-mirror-tools/rpms_from_3rd_parties.lst rename to centos-mirror-tools/rpms_3rdparties.lst diff --git a/centos-mirror-tools/rpms_from_centos_repo.lst b/centos-mirror-tools/rpms_centos.lst similarity index 100% rename from centos-mirror-tools/rpms_from_centos_repo.lst rename to centos-mirror-tools/rpms_centos.lst diff --git a/centos-mirror-tools/rpms_from_centos_3rd_parties.lst b/centos-mirror-tools/rpms_centos3rdparties.lst similarity index 100% rename from centos-mirror-tools/rpms_from_centos_3rd_parties.lst rename to centos-mirror-tools/rpms_centos3rdparties.lst