Use kojipkgs.fedoraproject.org as a backup rpm source.

EPEL7 rpms builds are visible at kojipkgs.fedoraproject.org
and often remain available there long after they age out
of the official repo.

Use kojipkgs.fedoraproject.org as a backup source for
packages when not found in the repo.  Log seperately
so that these dropouts can be identified and corrected
in the lst files.

Also improved the tests for already downloaded packages
in several places to accelerate the test cycle.

Change-Id: Iaaa9fe05ac605a3604acf0048c571c6e88303692
Story: 2003157
Task: 23292
Signed-off-by: Scott Little <scott.little@windriver.com>

centos-mirror-tools/dl_other_from_centos_repo.sh

@@ -31,6 +31,11 @@ for ff in $all; do
     if [ "$_type" == "folder" ];then
         mkdir -p $save_path/$_name
     else
+        if [ -e "$save_path/$_name" ]; then
+            echo "Already have $save_path/$_name"
+            continue
+        fi
+
         echo "remote path: $url_prefix/$_name"
         echo "local path: $save_path/$_name"
         if wget $url_prefix/$_name; then

centos-mirror-tools/dl_rpms.sh

@@ -11,6 +11,7 @@ usage() {
     echo "Options:"
     echo "  -n: Do not use sudo when performing operations"
     echo "  -c: Use an alternate yum.conf rather than the system file"
+    echo "  -x: Clean log files only, do not run."
     echo "  rpm_list: a list of RPM files to be downloaded."
     echo "  match_level: value could be L1, L2 or L3:"
     echo "    L1: use name, major version and minor version:"
@@ -19,6 +20,12 @@ usage() {
     echo "        using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm"
     echo "    L3: use name:"
     echo "        using vim to search vim-7.4.160-2.el7.src.rpm"
+    echo "    K1: Use Koji rather than yum repos as a source."
+    echo "        Koji has a longer retention period than epel mirrors."
+    echo ""
+    echo "Returns: 0 = All files downloaded successfully"
+    echo "         1 = Some files could not be downloaded"
+    echo "         2 = Bad arguements or other error"
     echo ""
 }
 
@@ -34,13 +41,20 @@ get_from() {
 SUDOCMD="sudo -E"
 YUMCONFOPT=""
 
+CLEAN_LOGS_ONLY=0
+dl_rc=0
+
 # Parse option flags
-while getopts "c:nh" o; do
+while getopts "c:nxh" o; do
     case "${o}" in
         n)
             # No-sudo
             SUDOCMD=""
             ;;
+        x)
+            # Clean only
+            CLEAN_LOGS_ONLY=1
+            ;;
         c)
             # Use an alternate yum.conf
             YUMCONFOPT="-c $OPTARG"
@@ -52,7 +66,7 @@ while getopts "c:nh" o; do
             ;;
         *)
             usage
-            exit 1
+            exit 2
             ;;
     esac
 done
@@ -60,12 +74,12 @@ shift $((OPTIND-1))
 
 if [ $# -lt 2 ]; then
     usage
-    exit -1
+    exit 2
 fi
 
 if [ "$1" == "" ]; then
     echo "Need to supply the rpm file list"
-    exit -1;
+    exit 2;
 else
     rpms_list=$1
     echo "using $rpms_list as the download name lists"
@@ -109,50 +123,121 @@ cat /dev/null > $FOUND_RPMS
 cat /dev/null > $MISSING_RPMS
 cat /dev/null > $URL_RPMS
 
+if [ $CLEAN_LOGS_ONLY -eq 1 ];then
+    exit 0
+fi
+
+# Function to split an rpm filename into parts.
+#
+# Returns a space seperated list containing:
+#    <NAME> <VERSION> <RELEASE> <ARCH> <EPOCH>
+#
+split_filename () {
+    local rpm_filename=$1
+
+    local RPM=""
+    local SFILE=""
+    local ARCH=""
+    local RELEASE=""
+    local VERSION=""
+    local NAME=""
+
+    RPM=$(echo $rpm_filename | rev | cut -d'.' -f-1 | rev)
+    SFILE=$(echo $rpm_filename | rev | cut -d'.' -f2- | rev)
+    ARCH=$(echo $SFILE | rev | cut -d'.' -f-1 | rev)
+    SFILE=$(echo $SFILE | rev | cut -d'.' -f2- | rev)
+    RELEASE=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
+    SFILE=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
+    VERSION=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
+    NAME=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
+
+    if [[ $NAME = *":"* ]]; then
+        EPOCH=$(echo $NAME | cut -d':' -f-1)
+        NAME=$(echo $NAME | cut -d':' -f2-)
+    fi
+
+    echo "$NAME" "$VERSION" "$RELEASE" "$ARCH" "$EPOCH"
+}
+
+# Function to predict the URL where a rpm might be found.
+# Assumes the rpm was compile for EPEL by fedora's koji.
+koji_url () {
+    local rpm_filename=$1
+
+    local arr=( $(split_filename $rpm_filename) )
+
+    local n=${arr[0]}
+    local v=${arr[1]}
+    local r=${arr[2]}
+    local a=${arr[3]}
+    local e=${arr[4]}
+
+    echo "https://kojipkgs.fedoraproject.org/packages/$n/$v/$r/$a/$n-$v-$r.$a.rpm"
+}
+
 # Function to download different types of RPMs in different ways
 download () {
-    _file=$1
+    local _file=$1
-    _level=$2
+    local _level=$2
-    _list=$(cat $_file)
+
-    _from=$(get_from $_file)
+    local _list=$(cat $_file)
+    local _from=$(get_from $_file)
+    local _type=""
+
+    local rc=0
+    local download_cmd=""
+    local download_url_cmd=""
+    local rpm_name=""
+    local rpm_url=""
+    local SFILE=""
+
     echo "now the rpm will come from: $_from"
     for ff in $_list; do
+        download_cmd=""
+        download_url_cmd=""
         _type=$(echo $ff | rev | cut -d'.' -f2-2 | rev)
 
         # Decide if the list will be downloaded using yumdownloader or wget
         if [[ $ff != *"#"* ]]; then
             rpm_name=$ff
 
-            # Cut the rpm name for the specified level (L1, L2 or L3)
+            if [ $_level == "K1" ]; then
-            if [ $_level == "L1" ]; then
                 SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev`
-            elif [ $match_level == "L2" ];then
+                rpm_url=$(koji_url $rpm_name)
-                SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev`
+                download_cmd="wget $rpm_url)"
+                download_url_cmd="echo $rpm_url)"
             else
-                SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev`
+                if [ $_level == "L1" ]; then
-            fi
+                    SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev`
-            echo " ------ using $SFILE to search $rpm_name ------"
+                elif [ $match_level == "L2" ];then
-            # Yumdownloader with the appropriate flag for src, noarch or x86_64
+                    SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev`
-            if [ "$_type" == "src" ];then
+                else
-                download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE"
+                    SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev`
-                download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE"
+                fi
-            else
+                echo " ------ using $SFILE to search $rpm_name ------"
-                download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
+                # Yumdownloader with the appropriate flag for src, noarch or x86_64
-                download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
+                if [ "$_type" == "src" ];then
+                    download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE"
+                    download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE"
+                else
+                    download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
+                    download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
+                fi
             fi
         else
             # Buid wget command
             rpm_name=`echo $ff | cut -d"#" -f1-1`
             rpm_url=`echo $ff | cut -d"#" -f2-2`
             download_cmd="wget $rpm_url"
+            download_url_cmd="echo $rpm_url"
             SFILE=$rpm_name
         fi
 
-        echo "--> run: $download_cmd"
         # Put the RPM in the Binary or Source directory
         if [ "$_type" == "src" ]; then
             if [ ! -e $MDIR_SRC/$rpm_name ]; then
                 echo "Looking for $rpm_name"
+                echo "--> run: $download_cmd"
                 if $download_cmd ; then
                     # Success!   Record download URL.
                     # Use 'sort --unique' because sometimes
@@ -165,15 +250,18 @@ download () {
                     fi
                     echo $rpm_name >> $FOUND_SRPMS
                 else
+                    echo "Warning: $rpm_name not found"
                     echo $rpm_name >> $MISSING_SRPMS
+                    rc=1
                 fi
             else
-                echo "Already have ${MDIR_BIN}/${_type}/$rpm_name"
+                echo "Already have ${MDIR_SRC}/${_type}/$rpm_name"
                 echo $rpm_name >> $FOUND_SRPMS
             fi
         else  ## noarch or x86_64
             if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then
                 echo "Looking for $rpm_name..."
+                echo "--> run: $download_cmd"
                 if $download_cmd ; then
                     # Success!   Record download URL.
                     # Use 'sort --unique' because sometimes
@@ -187,7 +275,9 @@ download () {
                     fi
                     echo $rpm_name >> $FOUND_RPMS
                 else
+                    echo "Warning: $rpm_name not found"
                     echo $rpm_name >> $MISSING_RPMS
+                    rc=1
                 fi
             else
                 echo "Already have ${MDIR_BIN}/${_type}/$rpm_name"
@@ -195,6 +285,8 @@ download () {
             fi
         fi
     done
+
+    return $rc
 }
 
 # Prime the cache
@@ -204,8 +296,11 @@ ${SUDOCMD} yum ${YUMCONFOPT} makecache
 if [ -s "$rpms_list" ];then
     echo "--> start searching "$rpms_list
     download $rpms_list $match_level
+    if [ $? -ne 0 ]; then
+        dl_rc=1
+    fi
 fi
 
 echo "done!!"
 
-exit 0
+exit $dl_rc

centos-mirror-tools/dl_tarball.sh

@@ -75,6 +75,9 @@ for line in $(cat $tarball_file); do
     directory_name=$(echo $line | cut -d"#" -f2-2)
     tarball_url=$(echo $line | cut -d"#" -f3-3)
 
+    # Remove leading '!' if present
+    tarball_name="${tarball_name//!/}"
+
     # - For the General category and the Puppet category:
     #   - Packages have a common process: download, decompressed,
     #     change the directory path and compressed.
@@ -87,16 +90,16 @@ for line in $(cat $tarball_file); do
         download_directory=$output_tarball
     fi
 
+    if [ -e $download_path ]; then
+        echo "Already have $download_path"
+        continue
+    fi
+
     # We have 6 packages from the text file starting with the character "!":
     # they require special handling besides the common process: remove directory,
     # remove text from some files, clone a git repository, etc.
 
     if [[ "$line" =~ ^'!' ]]; then
-        tarball_name="${tarball_name//!/}"
-        if [ -e "$output_tarball/$tarball_name" ]; then
-            echo "Already have $tarball_name"
-            continue
-        fi
         echo $tarball_name
         pushd $output_tarball
         if [ "$tarball_name" = "integrity-kmod-e6aef069.tar.gz" ]; then
@@ -157,7 +160,10 @@ for line in $(cat $tarball_file); do
             else
                 echo "$pkg_version : unknown version"
             fi
-            rm -f "$tarball_name"
+            # Don't delete the original MLNX_OFED_LINUX tarball.
+            # We don't use it, but it will prevent re-downloading this file.
+            #   rm -f "$tarball_name"
+
             rm -rf "MLNX_OFED_SRC-${pkg_version}"
             rm -rf "$directory_name"
         elif [ "$tarball_name" = "qat1.7.upstream.l.1.0.3-42.tar.gz" ]; then

centos-mirror-tools/download_mirror.sh

@@ -31,6 +31,9 @@ rpms_from_centos_repo="./rpms_centos.lst"
 rpms_from_centos_3rd_parties="./rpms_centos3rdparties.lst"
 other_downloads="./other_downloads.lst"
 
+# Overall success
+success=1
+
 # Parse out optional -c or -n arguments
 while getopts "c:ngh" o; do
     case "${o}" in
@@ -71,7 +74,7 @@ need_file(){
     for f in $*; do
         if [ ! -e $f ]; then
             echo "ERROR: $f does not exist."
-            exit -1
+            exit 1
         fi
     done
 }
@@ -84,6 +87,7 @@ need_file ${rpms_from_centos_repo}
 need_file ${other_downloads}
 need_file tarball-dl.lst mvn-artifacts.lst
 
+
 #download RPMs/SRPMs from 3rd_party websites (not CentOS repos) by "wget"
 echo "step #1: start downloading RPMs/SRPMs from 3rd-party websites..."
 
@@ -97,43 +101,90 @@ if [ ${use_system_yum_conf} -ne 0 ]; then
 fi
 
 logfile="log_download_3rdparties_L1.txt"
-$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 | tee ./logs/$logfile
+$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 |& tee ./logs/$logfile
 retcode=${PIPESTATUS[0]}
-if [ $retcode -ne 0 ]; then
+if [ $retcode -ne 0 ];then
-    echo "ERROR: something wrong with downloading, please check the log!!"
+    echo "ERROR: Something wrong with downloading files listed in ${rpms_from_3rd_parties}."
+    echo "   Please check the log at $(pwd)/logs/$logfile !"
+    echo ""
+    success=0
 fi
 
 # download RPMs/SRPMs from 3rd_party repos by "yumdownloader"
 logfile="log_download_centos3rdparties_L1.txt"
-$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 | tee ./logs/$logfile
+$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 |& tee ./logs/$logfile
+retcode=${PIPESTATUS[0]}
+if [ $retcode -ne 0 ];then
+    echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_3rd_parties}."
+    echo "   Please check the log at $(pwd)/logs/$logfile !"
+    echo ""
+    success=0
+fi
 
 if [ ${use_system_yum_conf} -eq 1 ]; then
     # deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo.
     \rm -f $REPO_DIR/StarlingX_3rd*.repo
 fi
 
+
 echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..."
 #download RPMs/SRPMs from CentOS repos by "yumdownloader"
 logfile="log_download_centos_L1.txt"
-$rpm_downloader ${rpms_from_centos_repo} L1 | tee ./logs/$logfile
+$rpm_downloader ${rpms_from_centos_repo} L1 |& tee ./logs/$logfile
 retcode=${PIPESTATUS[0]}
-if [ $retcode -ne 0 ]; then
+
+K1_logfile="log_download_rpms_from_centos_K1.txt"
+if [ $retcode -ne 1 ]; then
+    # K1 step not needed. Clear any K1 logs from previous download attempts.
+    $rpm_downloader -x ./output/centos_rpms_missing_L1.txt K1 |& tee ./logs/$K1_logfile
+fi
+
+if [ $retcode -eq 0 ]; then
     echo "finish 1st round of RPM downloading successfully!"
+elif [ $retcode -eq 1 ]; then
+    echo "finish 1st round of RPM downloading with missing files!"
     if [ -e "./output/centos_rpms_missing_L1.txt" ]; then
-        missing_num=`wc -l ./output/centos_rpms_missing_L1.txt | cut -d " " -f1-1`
+
+        echo "start 2nd round of downloading Binary RPMs with K1 match criteria..."
+        $rpm_downloader ./output/centos_rpms_missing_L1.txt K1 centos |& tee ./logs/$K1_logfile
+        retcode=${PIPESTATUS[0]}
+        if [ $retcode -eq 0 ]; then
+            echo "finish 2nd round of RPM downloading successfully!"
+        elif [ $retcode -eq 1 ]; then
+            echo "finish 2nd round of RPM downloading with missing files!"
+            if [ -e "./output/rpms_missing_K1.txt" ]; then
+                echo "WARNING: missing RPMs listed in ./output/centos_rpms_missing_K1.txt !"
+            fi
+        fi
+
+        # Remove files found by K1 download from centos_rpms_missing_L1.txt to prevent
+        # false reporting of missing files.
+        grep -v -x -F -f ./output/centos_rpms_found_K1.txt ./output/centos_rpms_missing_L1.txt  > ./output/centos_rpms_missing_L1.tmp
+        mv -f ./output/centos_rpms_missing_L1.tmp ./output/centos_rpms_missing_L1.txt
+
+
+        missing_num=`wc -l ./output/centos_rpms_missing_K1.txt | cut -d " " -f1-1`
         if [ "$missing_num" != "0" ];then
-            echo "ERROR:  -------RPMs missing $missing_num in yumdownloader with L1 match ---------------"
+            echo "ERROR:  -------RPMs missing: $missing_num ---------------"
+            retcode=1
         fi
     fi
 
     if [ -e "./output/centos_srpms_missing_L1.txt" ]; then
         missing_num=`wc -l ./output/centos_srpms_missing_L1.txt | cut -d " " -f1-1`
         if [ "$missing_num" != "0" ];then
-            echo "ERROR: --------- SRPMs missing $missing_num in yumdownloader with L1 match ---------------"
+            echo "ERROR: --------- SRPMs missing: $missing_num ---------------"
+            retcode=1
         fi
     fi
-else
+fi
-    echo "finish 1st round with failures!"
+
+if [ $retcode -ne 0 ]; then
+    echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_repo}."
+    echo "   Please check the logs at $(pwd)/logs/$logfile"
+    echo "   and $(pwd)/logs/$K1_logfile !"
+    echo ""
+    success=0
 fi
 
 ## verify all RPMs SRPMs we download for the GPG keys
@@ -147,7 +198,7 @@ line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1`
 line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1`
 line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1`
 let total_line=$line1+$line2+$line3
-echo "We expect to download $total_line RPMs."
+echo "We expected to download $total_line RPMs."
 num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1`
 echo "There are $num_of_downloaded_rpms RPMs in output directory."
 if [ "$total_line" != "$num_of_downloaded_rpms" ]; then
@@ -162,16 +213,33 @@ fi
 
 echo "step #3: start downloading other files ..."
 
-${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ | tee ./logs/log_download_other_files_centos.txt
+${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee ./logs/log_download_other_files_centos.txt
 retcode=${PIPESTATUS[0]}
 if [ $retcode -eq 0 ];then
     echo "step #3: done successfully"
+else
+    echo "step #3: finished with errors"
+    echo "ERROR: Something wrong with downloading from ${other_downloads}."
+    echo "   Please check the log at $(pwd)/logs/log_download_other_files_centos.txt !"
+    echo ""
+    success=0
 fi
 
+
 # StarlingX requires a group of source code pakages, in this section
 # they will be downloaded.
 echo "step #4: start downloading tarball compressed files"
-${tarball_downloader} ${tarball_downloader_extra_args}
+${tarball_downloader} ${tarball_downloader_extra_args}  |& tee ./logs/log_download_tarballs.txt
+retcode=${PIPESTATUS[0]}
+if [ $retcode -eq 0 ];then
+    echo "step #4: done successfully"
+else
+    echo "step #4: finished with errors"
+    echo "ERROR: Something wrong with downloading tarballs."
+    echo "   Please check the log at $(pwd)/logs/log_download_tarballs.txt !"
+    echo ""
+    success=0
+fi
 
 echo "IMPORTANT: The following 3 files are just bootstrap versions. Based"
 echo "on them, the workable images for StarlingX could be generated by"
@@ -179,3 +247,12 @@ echo "running \"update-pxe-network-installer\" command after \"build-iso\""
 echo "    - out/stx-r1/CentOS/pike/Binary/LiveOS/squashfs.img"
 echo "    - out/stx-r1/CentOS/pike/Binary/images/pxeboot/initrd.img"
 echo "    - out/stx-r1/CentOS/pike/Binary/images/pxeboot/vmlinuz"
+
+echo ""
+if [ $success -ne 1 ]; then
+    echo "Warning: Not all download steps succeeded.  You are likely missing files."
+    exit 1
+fi
+
+echo "Success"
+exit 0