starlingx
/
tools
Code Issues Proposed changes

Merge "Reduce number of log files in dl_rpms.sh and keep all logs in logs dir"

This commit is contained in:
Zuul 2018-08-30 16:37:25 +00:00 committed by Gerrit Code Review
parent bd4b59dde2 cceb3ee9d0
commit bd39de7fb9
2 changed files with 89 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
centos-mirror-tools/dl_rpms.sh
View File

@ -31,8 +31,9 @@ usage() {
get_from() {
list=$1
base=$(basename $list .lst)
from=$(echo $base | cut -d'_' -f2-2)
base=$(basename $list .lst) # removing lst extension
base=$(basename $base .log) # removing log extension
from=$(echo $base | rev | cut -d'_' -f1-1 | rev)
echo $from
}
@ -95,33 +96,27 @@ fi
timestamp=$(date +%F_%H%M)
echo $timestamp
DESTDIR="output"
MDIR_SRC=$DESTDIR/stx-r1/CentOS/pike/Source
mkdir -p $MDIR_SRC
MDIR_BIN=$DESTDIR/stx-r1/CentOS/pike/Binary
mkdir -p $MDIR_BIN
LOGSDIR="logs"
from=$(get_from $rpms_list)
FAIL_MOVE_SRPMS="$DESTDIR/${from}_srpms_fail_move_${match_level}.txt"
FOUND_SRPMS="$DESTDIR/${from}_srpms_found_${match_level}.txt"
MISSING_SRPMS="$DESTDIR/${from}_srpms_missing_${match_level}.txt"
URL_SRPMS="$DESTDIR/${from}_srpms_urls_${match_level}.txt"
cat /dev/null > $FAIL_MOVE_SRPMS
cat /dev/null > $FOUND_SRPMS
LOG="$LOGSDIR/${match_level}_failmoved_url_${from}.log"
MISSING_SRPMS="$LOGSDIR/${match_level}_srpms_missing_${from}.log"
MISSING_RPMS="$LOGSDIR/${match_level}_rpms_missing_${from}.log"
FOUND_SRPMS="$LOGSDIR/${match_level}_srpms_found_${from}.log"
FOUND_RPMS="$LOGSDIR/${match_level}_rpms_found_${from}.log"
cat /dev/null > $LOG
cat /dev/null > $MISSING_SRPMS
cat /dev/null > $URL_SRPMS
FAIL_MOVE_RPMS="$DESTDIR/${from}_rpms_fail_move_${match_level}.txt"
FOUND_RPMS="$DESTDIR/${from}_rpms_found_${match_level}.txt"
MISSING_RPMS="$DESTDIR/${from}_rpms_missing_${match_level}.txt"
URL_RPMS="$DESTDIR/${from}_rpms_urls_${match_level}.txt"
cat /dev/null > $FAIL_MOVE_RPMS
cat /dev/null > $FOUND_RPMS
cat /dev/null > $MISSING_RPMS
cat /dev/null > $URL_RPMS
cat /dev/null > $FOUND_SRPMS
cat /dev/null > $FOUND_RPMS
if [ $CLEAN_LOGS_ONLY -eq 1 ];then
exit 0
@ -245,21 +240,25 @@ download () {
# Success! Record download URL.
# Use 'sort --unique' because sometimes
# yumdownloader reports the url twice
$download_url_cmd | sort --unique >> $URL_SRPMS
URL=$($download_url_cmd | sort --unique)
echo "The url is: $URL"
echo "url_srpm:$URL" >> $LOG
if ! mv -f $SFILE* $MDIR_SRC ; then
echo "FAILED to move $rpm_name"
echo $rpm_name >> $FAIL_MOVE_SRPMS
echo "fail_move_srpm:$rpm_name" >> $LOG
fi
echo "found_srpm:$rpm_name" >> $LOG
echo $rpm_name >> $FOUND_SRPMS
else
echo "Warning: $rpm_name not found"
echo "missing_srpm:$rpm_name" >> $LOG
echo $rpm_name >> $MISSING_SRPMS
rc=1
fi
else
echo "Already have ${MDIR_SRC}/${_type}/$rpm_name"
echo $rpm_name >> $FOUND_SRPMS
echo "already_there_srpm:$rpm_name" >> $LOG
fi
else ## noarch or x86_64
if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then
@ -269,22 +268,26 @@ download () {
# Success! Record download URL.
# Use 'sort --unique' because sometimes
# yumdownloader reports the url twice
$download_url_cmd | sort --unique >> $URL_RPMS
URL=$($download_url_cmd | sort --unique)
echo "The url is: $URL"
echo "url_rpm:$URL" >> $LOG
mkdir -p $MDIR_BIN/${_type}
if ! mv -f $SFILE* $MDIR_BIN/${_type}/ ; then
echo "FAILED to move $rpm_name"
echo $rpm_name >> $FAIL_MOVE_RPMS
echo "fail_move_rpm:$rpm_name" >> $LOG
fi
echo "found_rpm:$rpm_name" >> $LOG
echo $rpm_name >> $FOUND_RPMS
else
echo "Warning: $rpm_name not found"
echo "missing_rpm:$rpm_name" >> $LOG
echo $rpm_name >> $MISSING_RPMS
rc=1
fi
else
echo "Already have ${MDIR_BIN}/${_type}/$rpm_name"
echo $rpm_name >> $FOUND_RPMS
echo "already_there_rpm:$rpm_name" >> $LOG
fi
fi
done
@ -297,7 +300,7 @@ ${SUDOCMD} yum ${YUMCONFOPT} makecache
# Download files
if [ -s "$rpms_list" ];then
echo "--> start searching "$rpms_list
echo "--> start searching $rpms_list"
download $rpms_list $match_level
if [ $? -ne 0 ]; then
dl_rc=1
@ -307,3 +310,4 @@ fi
echo "done!!"
exit $dl_rc

96
centos-mirror-tools/download_mirror.sh
View File

@ -15,6 +15,23 @@ usage() {
echo ""
}
generate_log_name() {
filename=$1
level=$2
base=$(basename $filename .lst)
echo $LOGSDIR"/"$base"_download_"$level".log"
}
need_file(){
for f in $*; do
if [ ! -e $f ]; then
echo "ERROR: $f does not exist."
exit 1
fi
done
}
# Downloader scripts
rpm_downloader="./dl_rpms.sh"
tarball_downloader="./dl_tarball.sh"
other_downloader="./dl_other_from_centos_repo.sh"
@ -68,16 +85,10 @@ echo "--------------------------------------------------------------"
echo "WARNING: this script HAS TO access internet (http/https/ftp),"
echo "so please make sure your network working properly!!"
mkdir -p ./logs
need_file(){
for f in $*; do
if [ ! -e $f ]; then
echo "ERROR: $f does not exist."
exit 1
fi
done
}
LOGSDIR="logs"
mkdir -p $LOGSDIR
# Check extistence of prerequisites files
need_file ${rpm_downloader} ${other_downloader} ${tarball_downloader}
@ -100,23 +111,27 @@ if [ ${use_system_yum_conf} -ne 0 ]; then
fi
fi
logfile="log_download_3rdparties_L1.txt"
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 |& tee ./logs/$logfile
list=${rpms_from_3rd_parties}
level=L1
logfile=$(generate_log_name $list $level)
$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
retcode=${PIPESTATUS[0]}
if [ $retcode -ne 0 ];then
echo "ERROR: Something wrong with downloading files listed in ${rpms_from_3rd_parties}."
echo " Please check the log at $(pwd)/logs/$logfile !"
echo "ERROR: Something wrong with downloading files listed in $list."
echo " Please check the log at $(pwd)/$logfile !"
echo ""
success=0
fi
# download RPMs/SRPMs from 3rd_party repos by "yumdownloader"
logfile="log_download_centos3rdparties_L1.txt"
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 |& tee ./logs/$logfile
list=${rpms_from_centos_3rd_parties}
level=L1
logfile=$(generate_log_name $list $level)
$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
retcode=${PIPESTATUS[0]}
if [ $retcode -ne 0 ];then
echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_3rd_parties}."
echo " Please check the log at $(pwd)/logs/$logfile !"
echo "ERROR: Something wrong with downloading files listed in $list."
echo " Please check the log at $(pwd)/$logfile !"
echo ""
success=0
fi
@ -129,49 +144,52 @@ fi
echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..."
#download RPMs/SRPMs from CentOS repos by "yumdownloader"
logfile="log_download_centos_L1.txt"
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_repo} L1 |& tee ./logs/$logfile
list=${rpms_from_centos_repo}
level=L1
logfile=$(generate_log_name $list $level)
$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
retcode=${PIPESTATUS[0]}
K1_logfile="log_download_rpms_from_centos_K1.txt"
K1_logfile=$(generate_log_name ${rpms_from_centos_repo} K1)
if [ $retcode -ne 1 ]; then
# K1 step not needed. Clear any K1 logs from previous download attempts.
$rpm_downloader ${rpm_downloader_extra_args} -x ./output/centos_rpms_missing_L1.txt K1 |& tee ./logs/$K1_logfile
$rpm_downloader -x $LOGSDIR/L1_rpms_missing_centos.log K1 |& tee $K1_logfile
fi
if [ $retcode -eq 0 ]; then
echo "finish 1st round of RPM downloading successfully!"
elif [ $retcode -eq 1 ]; then
echo "finish 1st round of RPM downloading with missing files!"
if [ -e "./output/centos_rpms_missing_L1.txt" ]; then
if [ -e "$LOGSDIR/L1_rpms_missing_centos.log" ]; then
echo "start 2nd round of downloading Binary RPMs with K1 match criteria..."
$rpm_downloader ${rpm_downloader_extra_args} ./output/centos_rpms_missing_L1.txt K1 centos |& tee ./logs/$K1_logfile
$rpm_downloader $LOGSDIR/L1_rpms_missing_centos.log K1 centos |& tee $K1_logfile
retcode=${PIPESTATUS[0]}
if [ $retcode -eq 0 ]; then
echo "finish 2nd round of RPM downloading successfully!"
elif [ $retcode -eq 1 ]; then
echo "finish 2nd round of RPM downloading with missing files!"
if [ -e "./output/rpms_missing_K1.txt" ]; then
echo "WARNING: missing RPMs listed in ./output/centos_rpms_missing_K1.txt !"
if [ -e "$LOGSDIR/rpms_missing_K1.log" ]; then
echo "WARNING: missing RPMs listed in $LOGSDIR/centos_rpms_missing_K1.log !"
fi
fi
# Remove files found by K1 download from centos_rpms_missing_L1.txt to prevent
# Remove files found by K1 download from L1_rpms_missing_centos.txt to prevent
# false reporting of missing files.
grep -v -x -F -f ./output/centos_rpms_found_K1.txt ./output/centos_rpms_missing_L1.txt > ./output/centos_rpms_missing_L1.tmp
mv -f ./output/centos_rpms_missing_L1.tmp ./output/centos_rpms_missing_L1.txt
grep -v -x -F -f $LOGSDIR/K1_rpms_found_centos.log $LOGSDIR/L1_rpms_missing_centos.log > $LOGSDIR/L1_rpms_missing_centos.tmp
mv -f $LOGSDIR/L1_rpms_missing_centos.tmp $LOGSDIR/L1_rpms_missing_centos.log
missing_num=`wc -l ./output/centos_rpms_missing_K1.txt | cut -d " " -f1-1`
missing_num=`wc -l $LOGSDIR/K1_rpms_missing_centos.log | cut -d " " -f1-1`
if [ "$missing_num" != "0" ];then
echo "ERROR: -------RPMs missing: $missing_num ---------------"
retcode=1
fi
fi
if [ -e "./output/centos_srpms_missing_L1.txt" ]; then
missing_num=`wc -l ./output/centos_srpms_missing_L1.txt | cut -d " " -f1-1`
if [ -e "$LOGSDIR/L1_srpms_missing_centos.log" ]; then
missing_num=`wc -l $LOGSDIR/L1_srpms_missing_centos.log | cut -d " " -f1-1`
if [ "$missing_num" != "0" ];then
echo "ERROR: --------- SRPMs missing: $missing_num ---------------"
retcode=1
@ -181,17 +199,17 @@ fi
if [ $retcode -ne 0 ]; then
echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_repo}."
echo " Please check the logs at $(pwd)/logs/$logfile"
echo " Please check the logs at $(pwd)/$logfile"
echo " and $(pwd)/logs/$K1_logfile !"
echo ""
success=0
fi
## verify all RPMs SRPMs we download for the GPG keys
find ./output -type f -name "*.rpm" | xargs rpm -K | grep -i "MISSING KEYS" > ./rpm-gpg-key-missing.txt
find ./output -type f -name "*.rpm" | xargs rpm -K | grep -i "MISSING KEYS" > $LOGSDIR/rpm-gpg-key-missing.txt
# remove all i686.rpms to avoid pollute the chroot dep chain
find ./output -name "*.i686.rpm" | tee ./output/all_i686.txt
find ./output -name "*.i686.rpm" | tee $LOGSDIR/all_i686.txt
find ./output -name "*.i686.rpm" | xargs rm -f
line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1`
@ -213,14 +231,15 @@ fi
echo "step #3: start downloading other files ..."
${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee ./logs/log_download_other_files_centos.txt
logfile=$LOGSDIR"/otherfiles_centos_download.log"
${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee $logfile
retcode=${PIPESTATUS[0]}
if [ $retcode -eq 0 ];then
echo "step #3: done successfully"
else
echo "step #3: finished with errors"
echo "ERROR: Something wrong with downloading from ${other_downloads}."
echo " Please check the log at $(pwd)/logs/log_download_other_files_centos.txt !"
echo " Please check the log at $(pwd)/$logfile!"
echo ""
success=0
fi
@ -229,14 +248,15 @@ fi
# StarlingX requires a group of source code pakages, in this section
# they will be downloaded.
echo "step #4: start downloading tarball compressed files"
${tarball_downloader} ${tarball_downloader_extra_args} |& tee ./logs/log_download_tarballs.txt
logfile=$LOGSDIR"/tarballs_download.log"
${tarball_downloader} ${tarball_downloader_extra_args} |& tee $logfile
retcode=${PIPESTATUS[0]}
if [ $retcode -eq 0 ];then
echo "step #4: done successfully"
else
echo "step #4: finished with errors"
echo "ERROR: Something wrong with downloading tarballs."
echo " Please check the log at $(pwd)/logs/log_download_tarballs.txt !"
echo " Please check the log at $(pwd)/$logfile !"
echo ""
success=0
fi