tools/centos-mirror-tools/utils.sh
Scott Little bbd79fcd38 Fast download of lower layer rpms
The default yumdownloader approach to downloading files
does so one at a time, and is very slow.  Too slow for layered
build which is built around the assumption that files can
be downloaded faster than they can be built.

This update will switch from yumdownloader to reposync
for the download of lower layer rpms.  It exploits the
fact that each layer and build type publishes it's own lst file
of rpms to download.  The lst file is transformed into an'
includepkgs directive in a custom yum.conf which is passed
to reposync, so we only sync the desited rpms.

Reposync won't redownload rpms that it already has, even if the
upstream repodata indicates that the files checksum has changed.
Forceing the redownload of these rpms requires that we manually
download the upstream repodata and that we use verifytree to
identify and delete the new obsolete rpms.

Also including two small bug fixes found while investivgating an
alternative solution to launchpad  1864221.
- incorrect userid in a chown
- Bug in get_url, --url pass being passed in the wrong place

Story: 2006166
Task: 39307
Closes-Bug: 1864221
Change-Id: If12b98ff4f5f24d9318250356920f397419f0f80
Signed-off-by: Scott Little <scott.little@windriver.com>
2020-04-14 16:14:01 -04:00

219 lines
5.3 KiB
Bash

#
# SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2019 Intel Corporation
#
UTILS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
source $UTILS_DIR/url_utils.sh
get_yum_command() {
local _file=$1
local _level=$2
local rpm_name=""
local arr=( $(split_filename $_file) )
local arch=${arr[3]}
local yumdownloader_extra_opts=""
rpm_name="$(get_rpm_level_name $_file $_level)"
if [ "$arch" == "src" ]; then
yumdownloader_extra_opts="--source"
else
yumdownloader_extra_opts="--exclude='*.i686' --archlist=noarch,x86_64"
fi
echo "yumdownloader -q -C ${YUMCONFOPT} ${RELEASEVER} $yumdownloader_extra_opts $rpm_name"
}
get_wget_command() {
local _name="$1"
local _ret=""
if [[ "$_name" == http?(s)://* ]]; then
_ret="wget -q $_name"
else
_ret="wget -q $(koji_url $_name)"
fi
echo "$_ret"
}
get_rpm_level_name() {
local _rpm_name=$1
local _level=$2
if [ $_level == "L1" ]; then
SFILE=`echo $_rpm_name | rev | cut -d'.' -f3- | rev`
elif [ $_level == "$dl_from_stx_mirror" ];then
# stx mirror uses L1 matches
SFILE=`echo $_rpm_name | rev | cut -d'.' -f3- | rev`
elif [ $_level == "L2" ];then
SFILE=`echo $_rpm_name | rev | cut -d'-' -f2- | rev`
else
SFILE=`echo $_rpm_name | rev | cut -d'-' -f3- | rev`
fi
echo "$SFILE"
}
get_url() {
local _name="$1"
local _level="$2"
local _ret=""
if [ "$_level" == "K1" ]; then
_ret="$(koji_url $_name)"
elif [[ "$_name" == *"#"* ]]; then
_ret="$(echo $_name | cut -d'#' -f2-2)"
if [ $_level == "stx_mirror" ]; then
_ret="$(url_to_stx_mirror_url $_ret $distro)"
fi
else
_url_cmd="$(get_yum_command $_name $_level) --url"
# When we add --url to the yum download command,
# --archlist is no longer enforced. Multiple
# url's might be returned. So use grep to
# filter urls for the desitered arch.
local arr=( $(split_filename $_name) )
local arch=${arr[3]}
_ret="$($_url_cmd | grep "[.]$arch[.]rpm$" | head -n 1)"
fi
echo "$_ret"
}
# Function to split an rpm filename into parts.
#
# Returns a space seperated list containing:
# <NAME> <VERSION> <RELEASE> <ARCH> <EPOCH>
#
split_filename () {
local rpm_filename=$1
local RPM=""
local SFILE=""
local ARCH=""
local RELEASE=""
local VERSION=""
local NAME=""
local EPOCH=""
RPM=$(echo $rpm_filename | rev | cut -d'.' -f-1 | rev)
SFILE=$(echo $rpm_filename | rev | cut -d'.' -f2- | rev)
ARCH=$(echo $SFILE | rev | cut -d'.' -f-1 | rev)
SFILE=$(echo $SFILE | rev | cut -d'.' -f2- | rev)
RELEASE=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
SFILE=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
VERSION=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
NAME=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
if [[ $NAME = *":"* ]]; then
EPOCH=$(echo $NAME | cut -d':' -f-1)
NAME=$(echo $NAME | cut -d':' -f2-)
fi
echo "$NAME" "$VERSION" "$RELEASE" "$ARCH" "$EPOCH"
}
# Function to predict the URL where a rpm might be found.
# Assumes the rpm was compile for EPEL by fedora's koji.
koji_url () {
local rpm_filename=$1
local arr=( $(split_filename $rpm_filename) )
local n=${arr[0]}
local v=${arr[1]}
local r=${arr[2]}
local a=${arr[3]}
echo "https://kojipkgs.fedoraproject.org/packages/$n/$v/$r/$a/$n-$v-$r.$a.rpm"
}
get_dest_directory() {
local _type=$1
local _dest=""
if [ "$_type" == "src" ]; then
_dest="$MDIR_SRC"
else
_dest="$MDIR_BIN/$_type"
fi
echo "$_dest"
}
process_result() {
local _type="$1"
local dest_dir="$2"
local url="$3"
local sfile="$4"
if [ "$_type" != "src" ] && [ ! -d $dest_dir ]; then
mkdir -p $dest_dir
fi
echo "url_srpm:$url"
if ! mv -f $sfile* $dest_dir ; then
echo "FAILED to move $rpm_name"
echo "fail_move_srpm:$rpm_name" >> $LOG
return 1
fi
echo "found_srpm:$rpm_name"
echo $rpm_name >> $FOUND_SRPMS
return 0
}
get_download_cmd() {
local ff="$1"
local _level="$2"
# Decide if the list will be downloaded using yumdownloader or wget
if [[ $ff != *"#"* ]]; then
rpm_name=$ff
if [ $_level == "K1" ]; then
download_cmd="$(get_wget_command $rpm_name)"
else
# yumdownloader with the appropriate flag for src, noarch or x86_64
download_cmd="${SUDOCMD} $(get_yum_command $rpm_name $_level)"
fi
else
# Build wget command
rpm_url=$(get_url "$ff" "$_level")
download_cmd="$(get_wget_command $rpm_url)"
fi
echo "$download_cmd"
}
get_rpm_name() {
local ret=""
if [[ "$1" != *"#"* ]]; then
ret="$1"
else
ret="$(echo $1 | cut -d"#" -f1-1)"
fi
echo "$ret"
}
get_arch_from_rpm() {
local _file=$1
local _split=()
local _arch=""
if [[ "$1" == *"#"* ]]; then
_file=$(echo $_file | cut -d"#" -f1-1)
fi
_split=( $(split_filename $_file) )
_arch=${_split[3]}
echo "$_arch"
}
get_from() {
list=$1
base=$(basename $list .lst) # removing lst extension
base=$(basename $base .log) # removing log extension
from=$(echo $base | rev | cut -d'_' -f1-1 | rev)
echo $from
}