Refactor on dl_rpms.sh

A refactor was made in the dl_rpms.sh script to simplify the
maintenance and development. A new file utils.sh was created to
store all the common functions that could be used by another scripts.

Also, some unit testing was added to the util.sh file.

Story: 2002736
Task: 26194

Change-Id: Iaac2e74a84abbfc9f40ef51b1da819702d040655
Signed-off-by: Marcela Rosales <marcela.a.rosales.jimenez@intel.com>
Signed-off-by: Erich Cordoba <erich.cordoba.malibran@intel.com>

centos-mirror-tools/dl_rpms.sh

@@ -5,6 +5,18 @@
 # download RPMs/SRPMs from different sources.
 # this script was originated by Brian Avery, and later updated by Yong Hu
 
+set -o errexit
+set -o nounset
+
+# By default, we use "sudo" and we don't use a local yum.conf. These can
+# be overridden via flags.
+
+SUDOCMD="sudo -E"
+RELEASEVER="--releasever=7"
+YUMCONFOPT=""
+
+source utils.sh
+
 usage() {
     echo "$0 [-n] [-c <yum.conf>] <rpms_list> <match_level> "
     echo ""
@@ -29,19 +41,6 @@ usage() {
     echo ""
 }
 
-get_from() {
-    list=$1
-    base=$(basename $list .lst) # removing lst extension 
-    base=$(basename $base .log) # removing log extension
-    from=$(echo $base | rev | cut -d'_' -f1-1 | rev)
-    echo $from
-}
-
-# By default, we use "sudo" and we don't use a local yum.conf. These can
-# be overridden via flags.
-SUDOCMD="sudo -E"
-RELEASEVER="--releasever=7"
-YUMCONFOPT=""
 
 CLEAN_LOGS_ONLY=0
 dl_rc=0
@@ -94,12 +93,9 @@ if [ ! -z "$2" -a "$2" != " " ];then
     match_level=$2
 fi
 
-
 timestamp=$(date +%F_%H%M)
 echo $timestamp
 
-
-
 DESTDIR="output"
 MDIR_SRC=$DESTDIR/stx-r1/CentOS/pike/Source
 mkdir -p $MDIR_SRC
@@ -124,134 +120,35 @@ if [ $CLEAN_LOGS_ONLY -eq 1 ];then
     exit 0
 fi
 
-# Function to split an rpm filename into parts.
-#
-# Returns a space seperated list containing:
-#    <NAME> <VERSION> <RELEASE> <ARCH> <EPOCH>
-#
-split_filename () {
-    local rpm_filename=$1
-
-    local RPM=""
-    local SFILE=""
-    local ARCH=""
-    local RELEASE=""
-    local VERSION=""
-    local NAME=""
-
-    RPM=$(echo $rpm_filename | rev | cut -d'.' -f-1 | rev)
-    SFILE=$(echo $rpm_filename | rev | cut -d'.' -f2- | rev)
-    ARCH=$(echo $SFILE | rev | cut -d'.' -f-1 | rev)
-    SFILE=$(echo $SFILE | rev | cut -d'.' -f2- | rev)
-    RELEASE=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
-    SFILE=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
-    VERSION=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
-    NAME=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
-
-    if [[ $NAME = *":"* ]]; then
-        EPOCH=$(echo $NAME | cut -d':' -f-1)
-        NAME=$(echo $NAME | cut -d':' -f2-)
-    fi
-
-    echo "$NAME" "$VERSION" "$RELEASE" "$ARCH" "$EPOCH"
-}
-
-# Function to predict the URL where a rpm might be found.
-# Assumes the rpm was compile for EPEL by fedora's koji.
-koji_url () {
-    local rpm_filename=$1
-
-    local arr=( $(split_filename $rpm_filename) )
-
-    local n=${arr[0]}
-    local v=${arr[1]}
-    local r=${arr[2]}
-    local a=${arr[3]}
-    local e=${arr[4]}
-
-    echo "https://kojipkgs.fedoraproject.org/packages/$n/$v/$r/$a/$n-$v-$r.$a.rpm"
-}
-
 # Function to download different types of RPMs in different ways
 download () {
     local _file=$1
     local _level=$2
+    local _list=$(cat $_file)
+    local _from=$(get_from $_file)
 
-    local _list
+    local _arch=""
-    local _from
-    local _type=""
 
     local rc=0
     local download_cmd=""
-    local download_url_cmd=""
+    local download_url=""
     local rpm_name=""
-    local rpm_url=""
     local SFILE=""
 
-    _list=$(cat $_file)
-    _from=$(get_from $_file)
-
     echo "now the rpm will come from: $_from"
     for ff in $_list; do
-        download_cmd=""
+        _arch=$(get_arch_from_rpm $ff)
-        download_url_cmd=""
+        rpm_name="$(get_rpm_name $ff)"
-        _type=$(echo $ff | rev | cut -d'.' -f2-2 | rev)
+        download_cmd="$(get_download_cmd $ff $_level)"
+        dest_dir="$(get_dest_directory $_arch)"
 
-        # Decide if the list will be downloaded using yumdownloader or wget
+        if [ ! -e $dest_dir/$rpm_name ]; then
-        if [[ $ff != *"#"* ]]; then
-            rpm_name=$ff
-
-            if [ $_level == "K1" ]; then
-                SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev`
-                rpm_url=$(koji_url $rpm_name)
-                download_cmd="wget $rpm_url)"
-                download_url_cmd="echo $rpm_url)"
-            else
-                if [ $_level == "L1" ]; then
-                    SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev`
-                elif [ $match_level == "L2" ];then
-                    SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev`
-                else
-                    SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev`
-                fi
-                echo " ------ using $SFILE to search $rpm_name ------"
-                # Yumdownloader with the appropriate flag for src, noarch or x86_64
-                if [ "$_type" == "src" ];then
-                    download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} ${RELEASEVER} -C --source $SFILE"
-                    download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT} ${RELEASEVER} -C --source $SFILE"
-                else
-                    download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} ${RELEASEVER} $SFILE --archlist=noarch,x86_64"
-                    download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} ${RELEASEVER} $SFILE --archlist=noarch,x86_64"
-                fi
-            fi
-        else
-            # Buid wget command
-            rpm_name=`echo $ff | cut -d"#" -f1-1`
-            rpm_url=`echo $ff | cut -d"#" -f2-2`
-            download_cmd="wget $rpm_url"
-            download_url_cmd="echo $rpm_url"
-            SFILE=$rpm_name
-        fi
-
-        # Put the RPM in the Binary or Source directory
-        if [ "$_type" == "src" ]; then
-            if [ ! -e $MDIR_SRC/$rpm_name ]; then
             echo "Looking for $rpm_name"
             echo "--> run: $download_cmd"
             if $download_cmd ; then
-                    # Success!   Record download URL.
+                download_url="$(get_url $ff $_level)"
-                    # Use 'sort --unique' because sometimes
+                SFILE="$(get_rpm_level_name $rpm_name $_level)"
-                    # yumdownloader reports the url twice
+                process_result "$_arch" "$dest_dir" "$download_url" "$SFILE"
-                    URL=$($download_url_cmd | sort --unique)
-                    echo "The url is: $URL"
-                    echo "url_srpm:$URL" >> $LOG
-
-                    if ! mv -f $SFILE* $MDIR_SRC ; then
-                        echo "FAILED to move $rpm_name"
-                        echo "fail_move_srpm:$rpm_name" >> $LOG
-                    fi
-                    echo "found_srpm:$rpm_name" >> $LOG
-                    echo $rpm_name >> $FOUND_SRPMS
             else
                 echo "Warning: $rpm_name not found"
                 echo "missing_srpm:$rpm_name" >> $LOG
@@ -259,44 +156,16 @@ download () {
                 rc=1
             fi
         else
-                echo "Already have ${MDIR_SRC}/${_type}/$rpm_name"
+            echo "Already have $dest_dir/$rpm_name"
             echo "already_there_srpm:$rpm_name" >> $LOG
         fi
-        else  ## noarch or x86_64
+        echo
-            if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then
-                echo "Looking for $rpm_name..."
-                echo "--> run: $download_cmd"
-                if $download_cmd ; then
-                    # Success!   Record download URL.
-                    # Use 'sort --unique' because sometimes
-                    # yumdownloader reports the url twice
-                    URL=$($download_url_cmd | sort --unique)
-                    echo "The url is: $URL"
-                    echo "url_rpm:$URL" >> $LOG
-
-                    mkdir -p $MDIR_BIN/${_type}
-                    if ! mv -f $SFILE* $MDIR_BIN/${_type}/ ; then
-                        echo "FAILED to move $rpm_name"
-                        echo "fail_move_rpm:$rpm_name" >> $LOG
-                    fi
-                    echo "found_rpm:$rpm_name" >> $LOG
-                    echo $rpm_name >> $FOUND_RPMS
-                else
-                    echo "Warning: $rpm_name not found"
-                    echo "missing_rpm:$rpm_name" >> $LOG
-                    echo $rpm_name >> $MISSING_RPMS
-                    rc=1
-                fi
-            else
-                echo "Already have ${MDIR_BIN}/${_type}/$rpm_name"
-                echo "already_there_rpm:$rpm_name" >> $LOG
-            fi
-        fi
     done
 
     return $rc
 }
 
+
 # Prime the cache
 ${SUDOCMD} yum ${YUMCONFOPT} ${RELEASEVER} makecache
 
@@ -309,7 +178,6 @@ if [ -s "$rpms_list" ];then
     fi
 fi
 
-echo "done!!"
+echo "Done!"
 
 exit $dl_rc
-

centos-mirror-tools/utils.sh

@@ -0,0 +1,196 @@
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+get_yum_command() {
+    local _file=$1
+    local _level=$2
+    local rpm_name=""
+    local arr=( $(split_filename $_file) )
+    local arch=${arr[3]}
+    local yumdownloader_extra_opts=""
+    rpm_name="$(get_rpm_level_name $_file $_level)"
+
+    if [ "$arch" == "src" ]; then
+        yumdownloader_extra_opts="--source"
+    else
+        yumdownloader_extra_opts="--archlist=noarch,x86_64"
+    fi
+
+    echo "yumdownloader -q -C ${YUMCONFOPT} ${RELEASEVER} $yumdownloader_extra_opts $rpm_name"
+}
+
+get_wget_command() {
+    local _name="$1"
+    local _ret=""
+    if [[ "$_name" == http?(s)://* ]]; then
+        _ret="wget -q $_name"
+    else
+        _ret="wget -q $(koji_url $_name)"
+    fi
+    echo "$_ret"
+}
+
+get_rpm_level_name() {
+    local _rpm_name=$1
+    local _level=$2
+    if [ $_level == "L1" ]; then
+        SFILE=`echo $_rpm_name | rev | cut -d'.' -f3- | rev`
+    elif [ $_level == "L2" ];then
+        SFILE=`echo $_rpm_name | rev | cut -d'-' -f2- | rev`
+    else
+        SFILE=`echo $_rpm_name | rev | cut -d'-' -f3- | rev`
+    fi
+    echo "$SFILE"
+}
+
+get_url() {
+    local _name="$1"
+    local _level="$2"
+    local _ret=""
+
+    if [ "$_level" == "K1" ]; then
+        _ret="$(koji_url $_name)"
+    elif [[ "$_name" == *"#"* ]]; then
+        _ret="$(echo $_name | cut -d'#' -f2-2)"
+    else
+        _url_cmd="$(get_yum_command $_name $_level)"
+        _ret="$($_url_cmd --url)"
+    fi
+    echo "$_ret"
+}
+
+# Function to split an rpm filename into parts.
+#
+# Returns a space seperated list containing:
+#    <NAME> <VERSION> <RELEASE> <ARCH> <EPOCH>
+#
+split_filename () {
+    local rpm_filename=$1
+
+    local RPM=""
+    local SFILE=""
+    local ARCH=""
+    local RELEASE=""
+    local VERSION=""
+    local NAME=""
+    local EPOCH=""
+
+    RPM=$(echo $rpm_filename | rev | cut -d'.' -f-1 | rev)
+    SFILE=$(echo $rpm_filename | rev | cut -d'.' -f2- | rev)
+    ARCH=$(echo $SFILE | rev | cut -d'.' -f-1 | rev)
+    SFILE=$(echo $SFILE | rev | cut -d'.' -f2- | rev)
+    RELEASE=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
+    SFILE=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
+    VERSION=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
+    NAME=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
+
+    if [[ $NAME = *":"* ]]; then
+        EPOCH=$(echo $NAME | cut -d':' -f-1)
+        NAME=$(echo $NAME | cut -d':' -f2-)
+    fi
+
+    echo "$NAME" "$VERSION" "$RELEASE" "$ARCH" "$EPOCH"
+}
+
+# Function to predict the URL where a rpm might be found.
+# Assumes the rpm was compile for EPEL by fedora's koji.
+koji_url () {
+    local rpm_filename=$1
+
+    local arr=( $(split_filename $rpm_filename) )
+
+    local n=${arr[0]}
+    local v=${arr[1]}
+    local r=${arr[2]}
+    local a=${arr[3]}
+
+    echo "https://kojipkgs.fedoraproject.org/packages/$n/$v/$r/$a/$n-$v-$r.$a.rpm"
+}
+
+get_dest_directory() {
+    local _type=$1
+    local _dest=""
+    if [ "$_type" == "src" ]; then
+        _dest="$MDIR_SRC"
+    else
+        _dest="$MDIR_BIN/$_type"
+    fi
+    echo "$_dest"
+}
+
+process_result() {
+    local _type="$1"
+    local dest_dir="$2"
+    local url="$3"
+    local sfile="$4"
+
+    if [ "$_type" != "src" ] && [ ! -d $dest_dir ]; then
+        mkdir -p $dest_dir
+    fi
+
+    echo "url_srpm:$url" >> $LOG
+
+    if ! mv -f $sfile* $dest_dir ; then
+        echo "FAILED to move $rpm_name"
+        echo "fail_move_srpm:$rpm_name" >> $LOG
+    fi
+    echo "found_srpm:$rpm_name" >> $LOG
+    echo $rpm_name >> $FOUND_SRPMS
+}
+
+
+get_download_cmd() {
+    local ff="$1"
+    local _level="$2"
+
+    # Decide if the list will be downloaded using yumdownloader or wget
+    if [[ $ff != *"#"* ]]; then
+        rpm_name=$ff
+        if [ $_level == "K1" ]; then
+            download_cmd="$(get_wget_command $rpm_name)"
+        else
+            # yumdownloader with the appropriate flag for src, noarch or x86_64
+            download_cmd="${SUDOCMD} $(get_yum_command $rpm_name $_level)"
+        fi
+    else
+        # Build wget command
+        rpm_url=$(echo $ff | cut -d"#" -f2-2)
+        download_cmd="$(get_wget_command $rpm_url)"
+    fi
+
+    echo "$download_cmd"
+}
+
+get_rpm_name() {
+    local ret=""
+
+    if [[ "$1" != *"#"* ]]; then
+        ret="$1"
+    else
+        ret="$(echo $1 | cut -d"#" -f1-1)"
+    fi
+    echo "$ret"
+}
+
+get_arch_from_rpm() {
+    local _file=$1
+    local _split=()
+    local _arch=""
+    if [[ "$1" == *"#"* ]]; then
+        _file=$(echo $_file | cut -d"#" -f1-1)
+    fi
+
+    _split=( $(split_filename $_file) )
+    _arch=${_split[3]}
+
+    echo "$_arch"
+}
+
+get_from() {
+    list=$1
+    base=$(basename $list .lst) # removing lst extension
+    base=$(basename $base .log) # removing log extension
+    from=$(echo $base | rev | cut -d'_' -f1-1 | rev)
+    echo $from
+}

centos-mirror-tools/utils_tests.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# Set of unit tests for dl_rpms.sh
+
+set -o errexit
+set -o nounset
+
+YUMCONFOPT=""
+RELEASEVER="--releasever=7"
+
+source utils.sh
+
+check_result() {
+    local _res="$1"
+    local _expect="$2"
+    if [ "$_res" != "$_expect" ]; then
+        echo "Fail"
+        echo "expected $_expect"
+        echo "returned $_res"
+        exit 1
+    fi
+    echo "Success"
+}
+
+# get_wget_command
+
+res=$(get_wget_command "https://libvirt.org/sources/python/libvirt-python-3.5.0-1.fc24.src.rpm")
+expect="wget -q https://libvirt.org/sources/python/libvirt-python-3.5.0-1.fc24.src.rpm"
+check_result "$res" "$expect"
+
+res=$(get_wget_command "python2-httpbin-0.5.0-6.el7.noarch.rpm")
+expect="wget -q https://kojipkgs.fedoraproject.org/packages/python2-httpbin/0.5.0/6.el7/noarch/python2-httpbin-0.5.0-6.el7.noarch.rpm"
+check_result "$res" "$expect"
+
+# get_url
+
+res=$(get_url "acpid-2.0.19-9.el7.x86_64.rpm" "L1")
+expect="http://vault.centos.org/centos/7.4.1708/cr/x86_64/Packages/acpid-2.0.19-9.el7.x86_64.rpm"
+check_result "$res" "$expect"
+
+res=$(get_url "python2-httpbin-0.5.0-6.el7.noarch.rpm#http://cbs.centos.org/kojifiles/packages/python-httpbin/0.5.0/6.el7/noarch/python2-httpbin-0.5.0-6.el7.noarch.rpm" "L1")
+expect="http://cbs.centos.org/kojifiles/packages/python-httpbin/0.5.0/6.el7/noarch/python2-httpbin-0.5.0-6.el7.noarch.rpm"
+check_result "$res" "$expect"
+
+res=$(get_url "python2-httpbin-0.5.0-6.el7.noarch.rpm" "K1")
+expect="https://kojipkgs.fedoraproject.org/packages/python2-httpbin/0.5.0/6.el7/noarch/python2-httpbin-0.5.0-6.el7.noarch.rpm"
+check_result "$res" "$expect"
+
+# get_yum_command
+
+res=$(get_yum_command "anaconda-21.48.22.121-1.el7.centos.src.rpm" "L1")
+expect="yumdownloader -q -C  --releasever=7 --source anaconda-21.48.22.121-1.el7.centos"
+check_result "$res" "$expect"
+
+res=$(get_yum_command "acpid-2.0.19-9.el7.x86_64.rpm" "L1")
+expect="yumdownloader -q -C  --releasever=7 --archlist=noarch,x86_64 acpid-2.0.19-9.el7"
+check_result "$res" "$expect"
+
+# get_rpm_level_name
+
+res=$(get_rpm_level_name "acl-2.2.51-12.el7.x86_64.rpm" "L1")
+expect="acl-2.2.51-12.el7"
+check_result "$res" "$expect"
+
+res=$(get_rpm_level_name "acl-2.2.51-12.el7.x86_64.rpm" "L3")
+expect="acl"
+check_result "$res" "$expect"
+
+res=$(get_rpm_level_name "anaconda-21.48.22.121-1.el7.centos.src.rpm" "L2")
+expect="anaconda-21.48.22.121"
+check_result "$res" "$expect"
+
+res=$(get_arch_from_rpm "acl-2.2.51-12.el7.x86_64.rpm")
+expect="x86_64"
+check_result "$res" "$expect"
+
+res=$(get_arch_from_rpm "acl-2.2.51-12.el7.noarch.rpm")
+expect="noarch"
+check_result "$res" "$expect"
+
+res=$(get_arch_from_rpm "acl-2.2.51-12.el7.src.rpm")
+expect="src"
+check_result "$res" "$expect"
+
+res=$(get_arch_from_rpm "acl-2.2.51-12.el7.src.rpm#https://someurl.com/acl-2.2.51-12.el7.src.rpm")
+expect="src"
+check_result "$res" "$expect"