Fix starlingx_add_pkgs.sh for use in a layered environment.
1) Make starlingx_add_pkgs.sh aware of the extra rpm repositories required for a layered build. 2) Don't spew errors if we come accross a dependency on rpmlib(*). These aren't dependencies on packages, but rather internal features of rpm. 3) Don't get baffled by epochs. feature_to_pkg returns a package version that might included an epoch. The queries within pkg_in_cgcsrepo were baffled by this. I add a second query stripping the epoch from the query. 4) Favor packages we already have over newer content from upstream. pkg_to_dependencies favors upstream packages if they look to be a higher version, so I dropped it's use. Instead I do a recursive search, with a modified feature_to_pkg that prefers existing sources. Story: 2006166 Task: 37103 Change-Id: I2b199c809267904fef08d9e93383dd2475adcb94 Signed-off-by: Scott Little <scott.little@windriver.com>
This commit is contained in:
parent
451088b757
commit
faea86cf16
@ -84,16 +84,11 @@ if [ -z "$MY_REPO_ROOT_DIR" ]; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
STARLINGX_ADD_PKGS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
|
||||||
|
|
||||||
|
source $STARLINGX_ADD_PKGS_DIR/../toCOPY/lst_utils.sh
|
||||||
|
|
||||||
STXTOOLS=${MY_REPO_ROOT_DIR}/stx-tools
|
STXTOOLS=${MY_REPO_ROOT_DIR}/stx-tools
|
||||||
REPO_DIR=${STXTOOLS}/centos-mirror-tools/yum.repos.d
|
|
||||||
REPOCFG_STD_FILES=$(ls ${REPO_DIR}/StarlingX*.repo | grep -v StarlingX_3rd)
|
|
||||||
REPOCFG_3RD_FILES=${REPO_DIR}/StarlingX_3rd*.repo
|
|
||||||
REPOCFG_STD_MERGED=$(mktemp /tmp/REPOCFG_STD_MERGED_XXXXXX)
|
|
||||||
cat $REPOCFG_STD_FILES > $REPOCFG_STD_MERGED
|
|
||||||
REPOCFG_3RD_MERGED=$(mktemp /tmp/REPOCFG_3RD_MERGED_XXXXXX)
|
|
||||||
cat $REPOCFG_3RD_FILES > $REPOCFG_3RD_MERGED
|
|
||||||
REPOCFG_ALL_MERGED=$(mktemp /tmp/REPOCFG_ALL_MERGED_XXXXXX)
|
|
||||||
cat $REPOCFG_STD_FILES $REPOCFG_3RD_FILES > $REPOCFG_ALL_MERGED
|
|
||||||
|
|
||||||
CGCSREPO_PATH=$MY_REPO/cgcs-centos-repo/Binary
|
CGCSREPO_PATH=$MY_REPO/cgcs-centos-repo/Binary
|
||||||
TISREPO_PATH=$MY_WORKSPACE/std/rpmbuild/RPMS
|
TISREPO_PATH=$MY_WORKSPACE/std/rpmbuild/RPMS
|
||||||
@ -110,18 +105,9 @@ FAILED_LOG=failed.log
|
|||||||
RPMLIST=
|
RPMLIST=
|
||||||
DOWNLOAD_LIST=
|
DOWNLOAD_LIST=
|
||||||
|
|
||||||
# It seems we have to manually disable the repos from /etc/yum.repos.d,
|
|
||||||
# even though we're specifying a config file
|
|
||||||
REPOQUERY_ARGS=$(grep -h '^\[' /etc/yum.repos.d/* | sed 's/[][]//g' | sed 's/^/--disablerepo=/')
|
|
||||||
|
|
||||||
REPOQUERY_CMD="repoquery --archlist=x86_64,noarch $REPOQUERY_ARGS"
|
|
||||||
REPOQUERY_STD_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_STD_MERGED"
|
|
||||||
REPOQUERY_3RD_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_3RD_MERGED"
|
|
||||||
REPOQUERY_ALL_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_ALL_MERGED"
|
|
||||||
REPOQUERY_LOCAL_CMD="$REPOQUERY_CMD --quiet --repofrompath cgcs,$CGCSREPO_PATH $TISREPO_PATH_ARGS"
|
|
||||||
|
|
||||||
function cleanup {
|
function cleanup {
|
||||||
rm -f $REPOCFG_STD_MERGED $REPOCFG_3RD_MERGED $REPOCFG_ALL_MERGED
|
rm -f $REPOCFG_STD_MERGED $REPOCFG_3RD_MERGED $REPOCFG_LOWER_LAYER_MERGED $REPOCFG_ALL_MERGED
|
||||||
}
|
}
|
||||||
|
|
||||||
trap cleanup EXIT
|
trap cleanup EXIT
|
||||||
@ -129,7 +115,7 @@ trap cleanup EXIT
|
|||||||
function show_usage {
|
function show_usage {
|
||||||
cat >&2 <<EOF
|
cat >&2 <<EOF
|
||||||
Usage:
|
Usage:
|
||||||
$(basename $0) [ -d <pkgname> ] ... [ <rpmfile> ] ...
|
$(basename $0) [ -C <config_dir> ] [ -l <layer> ] [ -d <pkgname> ] ... [ <rpmfile> ] ...
|
||||||
|
|
||||||
This utility uses the cgcs-centos-repo repo, and optionally the rpmbuild/RPMS
|
This utility uses the cgcs-centos-repo repo, and optionally the rpmbuild/RPMS
|
||||||
repo from \$MY_WORKSPACE/std, as a baseline, downloading packages required
|
repo from \$MY_WORKSPACE/std, as a baseline, downloading packages required
|
||||||
@ -145,7 +131,7 @@ In addition, this utility will record a list of downloaded RPMs in the $RESULTS_
|
|||||||
or $RESULTS_3RD_LOG files, with failures recorded in $FAILED_LOG or $NOTFOUND_LOG.
|
or $RESULTS_3RD_LOG files, with failures recorded in $FAILED_LOG or $NOTFOUND_LOG.
|
||||||
|
|
||||||
The resulting download list can then be added to the appropriate .lst file in
|
The resulting download list can then be added to the appropriate .lst file in
|
||||||
\$MY_REPO_ROOT_DIR/stx-tools/centos-mirror-tools
|
\$MY_REPO_ROOT_DIR/stx-tools/centos-mirror-tools/config/centos/flock/
|
||||||
|
|
||||||
Example:
|
Example:
|
||||||
$(basename $0) -d linuxptp -d zlib puppet-gnocchi-11.3.0-1.el7.src.rpm
|
$(basename $0) -d linuxptp -d zlib puppet-gnocchi-11.3.0-1.el7.src.rpm
|
||||||
@ -155,11 +141,19 @@ EOF
|
|||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
while getopts "d:h" opt; do
|
while getopts "C:d:l:h" opt; do
|
||||||
case $opt in
|
case $opt in
|
||||||
|
C)
|
||||||
|
# Alternate config directory
|
||||||
|
set_and_validate_config_dir "${OPTARG}"
|
||||||
|
;;
|
||||||
d)
|
d)
|
||||||
DOWNLOAD_LIST="$DOWNLOAD_LIST $OPTARG"
|
DOWNLOAD_LIST="$DOWNLOAD_LIST $OPTARG"
|
||||||
;;
|
;;
|
||||||
|
l)
|
||||||
|
# Set layer
|
||||||
|
set_and_validate_layer "${OPTARG}"
|
||||||
|
;;
|
||||||
h)
|
h)
|
||||||
show_usage
|
show_usage
|
||||||
;;
|
;;
|
||||||
@ -173,31 +167,82 @@ done
|
|||||||
shift $((OPTIND-1))
|
shift $((OPTIND-1))
|
||||||
RPMLIST="${RPMLIST} $@"
|
RPMLIST="${RPMLIST} $@"
|
||||||
|
|
||||||
|
REPO_DIR=${STXTOOLS}/centos-mirror-tools/yum.repos.d
|
||||||
|
LAYER_REPO_DIR=${config_dir}/centos/${layer}/yum.repos.d
|
||||||
|
|
||||||
|
REPOCFG_STD_FILES=$(ls ${REPO_DIR}/StarlingX*.repo | grep -v StarlingX_3rd)
|
||||||
|
REPOCFG_3RD_FILES=${REPO_DIR}/StarlingX_3rd*.repo
|
||||||
|
REPOCFG_LOWER_LAYER_FILES=""
|
||||||
|
if [ -d ${LAYER_REPO_DIR} ]; then
|
||||||
|
REPOCFG_LOWER_LAYER_FILES=${LAYER_REPO_DIR}/StarlingX*.repo
|
||||||
|
fi
|
||||||
|
|
||||||
|
REPOCFG_STD_MERGED=$(mktemp /tmp/REPOCFG_STD_MERGED_XXXXXX)
|
||||||
|
cat $REPOCFG_STD_FILES > $REPOCFG_STD_MERGED
|
||||||
|
|
||||||
|
REPOCFG_3RD_MERGED=$(mktemp /tmp/REPOCFG_3RD_MERGED_XXXXXX)
|
||||||
|
cat $REPOCFG_3RD_FILES > $REPOCFG_3RD_MERGED
|
||||||
|
|
||||||
|
REPOCFG_LOWER_LAYER_MERGED=$(mktemp /tmp/REPOCFG_LOWER_LAYER_MERGED_XXXXXX)
|
||||||
|
if [ "$REPOCFG_LOWER_LAYER_FILES" != "" ]; then
|
||||||
|
cat $REPOCFG_LOWER_LAYER_FILES > $REPOCFG_LOWER_LAYER_MERGED
|
||||||
|
fi
|
||||||
|
|
||||||
|
REPOCFG_ALL_MERGED=$(mktemp /tmp/REPOCFG_ALL_MERGED_XXXXXX)
|
||||||
|
cat $REPOCFG_STD_FILES $REPOCFG_3RD_FILES $REPOCFG_LOWER_LAYER_FILES > $REPOCFG_ALL_MERGED
|
||||||
|
|
||||||
|
# It seems we have to manually disable the repos from /etc/yum.repos.d,
|
||||||
|
# even though we're specifying a config file
|
||||||
|
REPOQUERY_ARGS=$(grep -h '^\[' /etc/yum.repos.d/* | sed 's/[][]//g' | sed 's/^/--disablerepo=/')
|
||||||
|
|
||||||
|
REPOQUERY_CMD="repoquery --archlist=x86_64,noarch $REPOQUERY_ARGS"
|
||||||
|
REPOQUERY_STD_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_STD_MERGED"
|
||||||
|
REPOQUERY_3RD_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_3RD_MERGED"
|
||||||
|
REPOQUERY_LOWER_LAYER_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_LOWER_LAYER_MERGED"
|
||||||
|
REPOQUERY_ALL_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_ALL_MERGED"
|
||||||
|
REPOQUERY_LOCAL_CMD="$REPOQUERY_CMD --quiet --repofrompath cgcs,$CGCSREPO_PATH $TISREPO_PATH_ARGS"
|
||||||
|
|
||||||
|
|
||||||
function rpmfile_requires {
|
function rpmfile_requires {
|
||||||
#
|
#
|
||||||
# Map a specified rpm file to its dependency list
|
# Map a specified rpm file to its dependency list
|
||||||
#
|
#
|
||||||
local rpmfile=$1
|
local rpmfile=$1
|
||||||
|
|
||||||
rpm -qp --requires $rpmfile | awk '{print $1}'
|
# Never process rpmlib(*) requirements.
|
||||||
|
# They are not regular requirements, but rather internal features of rpm.
|
||||||
|
rpm -qp --requires $rpmfile | grep -v '^rpmlib[(]'
|
||||||
}
|
}
|
||||||
|
|
||||||
function feature_to_pkg {
|
function feature_to_pkg {
|
||||||
#
|
#
|
||||||
# Map a feature/capability to the package that provides it
|
# Map a feature/capability to the package that provides it
|
||||||
#
|
#
|
||||||
local feature=$1
|
local feature="$1"
|
||||||
local pkg=
|
local pkg=
|
||||||
pkg=$($REPOQUERY_STD_CMD $feature | head -1)
|
|
||||||
|
pkg=$($REPOQUERY_LOCAL_CMD "$feature" | head -1)
|
||||||
if [ -z $pkg ]; then
|
if [ -z $pkg ]; then
|
||||||
pkg=$($REPOQUERY_STD_CMD --qf='%{name}' --whatprovides $feature | head -1)
|
pkg=$($REPOQUERY_LOCAL_CMD --qf='%{name}' --whatprovides "$feature" | head -1)
|
||||||
if [ -z $pkg ]; then
|
if [ -z $pkg ]; then
|
||||||
pkg=$($REPOQUERY_3RD_CMD $feature | head -1)
|
pkg=$($REPOQUERY_LOWER_LAYER_CMD "$feature" | head -1)
|
||||||
if [ -z $pkg ]; then
|
if [ -z $pkg ]; then
|
||||||
pkg=$($REPOQUERY_3RD_CMD --qf='%{name}' --whatprovides $feature | head -1)
|
pkg=$($REPOQUERY_LOWER_LAYER_CMD --qf='%{name}' --whatprovides "$feature" | head -1)
|
||||||
if [ -z "$pkg" ]; then
|
if [ -z $pkg ]; then
|
||||||
echo "Could not find in repo: $feature" >&2
|
pkg=$($REPOQUERY_STD_CMD "$feature" | head -1)
|
||||||
echo "Could not find in repo: $feature" >> $NOTFOUND_LOG
|
if [ -z $pkg ]; then
|
||||||
|
pkg=$($REPOQUERY_STD_CMD --qf='%{name}' --whatprovides "$feature" | head -1)
|
||||||
|
if [ -z $pkg ]; then
|
||||||
|
pkg=$($REPOQUERY_3RD_CMD "$feature" | head -1)
|
||||||
|
if [ -z $pkg ]; then
|
||||||
|
pkg=$($REPOQUERY_3RD_CMD --qf='%{name}' --whatprovides "$feature" | head -1)
|
||||||
|
if [ -z "$pkg" ]; then
|
||||||
|
echo "Could not find in repo: $feature" >&2
|
||||||
|
echo "Could not find in repo: $feature" >> $NOTFOUND_LOG
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
@ -214,6 +259,15 @@ function pkg_to_dependencies {
|
|||||||
$REPOQUERY_ALL_CMD --resolve --requires --qf='%{name}' $pkg
|
$REPOQUERY_ALL_CMD --resolve --requires --qf='%{name}' $pkg
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function simplified_pkg {
|
||||||
|
#
|
||||||
|
# drop the epoch
|
||||||
|
#
|
||||||
|
local pkg=$1
|
||||||
|
|
||||||
|
echo $pkg | sed 's/-[0-9]*:/-/'
|
||||||
|
}
|
||||||
|
|
||||||
function pkg_in_cgcsrepo {
|
function pkg_in_cgcsrepo {
|
||||||
#
|
#
|
||||||
# Check whether the specified package is already in the downloaded (or built) repo
|
# Check whether the specified package is already in the downloaded (or built) repo
|
||||||
@ -223,14 +277,20 @@ function pkg_in_cgcsrepo {
|
|||||||
local results=
|
local results=
|
||||||
results=$($REPOQUERY_LOCAL_CMD --whatprovides $pkg)
|
results=$($REPOQUERY_LOCAL_CMD --whatprovides $pkg)
|
||||||
if [ -n "$results" ]; then
|
if [ -n "$results" ]; then
|
||||||
return 0
|
results=$($REPOQUERY_LOCAL_CMD $(simplified_pkg $pkg))
|
||||||
|
if [ -n "$results" ]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
local pkgname=
|
local pkgname=
|
||||||
pkgname=$($REPOQUERY_ALL_CMD --quiet $REPOCFG_ARGS --qf='%{name}' --whatprovides $pkg | head -1)
|
pkgname=$($REPOQUERY_ALL_CMD --quiet $REPOCFG_ARGS --qf='%{name}' --whatprovides $pkg | head -1)
|
||||||
if [ -z "$pkgname" ]; then
|
if [ -z "$pkgname" ]; then
|
||||||
echo "Failed to find a package providing $pkg" >&2
|
pkgname=$($REPOQUERY_ALL_CMD --quiet $REPOCFG_ARGS --qf='%{name}' $(simplified_pkg $pkg) | head -1)
|
||||||
return 1
|
if [ -z "$pkgname" ]; then
|
||||||
|
echo "Failed to find a package providing $pkg" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
results=$($REPOQUERY_LOCAL_CMD $pkgname)
|
results=$($REPOQUERY_LOCAL_CMD $pkgname)
|
||||||
|
|
||||||
@ -241,9 +301,9 @@ function download_pkg {
|
|||||||
#
|
#
|
||||||
# Download the specified package and its dependencies
|
# Download the specified package and its dependencies
|
||||||
#
|
#
|
||||||
local feature=$1
|
local feature="$1"
|
||||||
local pkg=
|
local pkg=
|
||||||
pkg=$(feature_to_pkg $feature)
|
pkg=$(feature_to_pkg "$feature")
|
||||||
if [ -z "$pkg" ]; then
|
if [ -z "$pkg" ]; then
|
||||||
# Error should already be to stderr
|
# Error should already be to stderr
|
||||||
return 1
|
return 1
|
||||||
@ -254,25 +314,25 @@ function download_pkg {
|
|||||||
local relativepath=
|
local relativepath=
|
||||||
local arch=
|
local arch=
|
||||||
local deps=
|
local deps=
|
||||||
|
local rpm_path=
|
||||||
|
|
||||||
repoid=$($REPOQUERY_STD_CMD --qf='%{repoid}' $pkg | head -1)
|
repoid=$($REPOQUERY_STD_CMD --qf='%{repoid}' $pkg | head -1)
|
||||||
if [ -n "$repoid" ]; then
|
if [ -n "$repoid" ]; then
|
||||||
url=$($REPOQUERY_STD_CMD --location $pkg | head -1)
|
url=$($REPOQUERY_STD_CMD --location $pkg | head -1)
|
||||||
relativepath=$($REPOQUERY_STD_CMD --qf='%{relativepath}' $pkg | head -1)
|
relativepath=$($REPOQUERY_STD_CMD --qf='%{relativepath}' $pkg | head -1)
|
||||||
arch=$($REPOQUERY_STD_CMD --qf='%{arch}' $pkg | head -1)
|
arch=$($REPOQUERY_STD_CMD --qf='%{arch}' $pkg | head -1)
|
||||||
deps=$($REPOQUERY_STD_CMD --requires --qf='%{name}' $pkg | awk '{print $1}')
|
|
||||||
LOG=$RESULTS_LOG
|
LOG=$RESULTS_LOG
|
||||||
else
|
else
|
||||||
repoid=$($REPOQUERY_3RD_CMD --qf='%{repoid}' $pkg | head -1)
|
repoid=$($REPOQUERY_3RD_CMD --qf='%{repoid}' $pkg | head -1)
|
||||||
url=$($REPOQUERY_3RD_CMD --location $pkg | head -1)
|
url=$($REPOQUERY_3RD_CMD --location $pkg | head -1)
|
||||||
relativepath=$($REPOQUERY_3RD_CMD --qf='%{relativepath}' $pkg | head -1)
|
relativepath=$($REPOQUERY_3RD_CMD --qf='%{relativepath}' $pkg | head -1)
|
||||||
arch=$($REPOQUERY_3RD_CMD --qf='%{arch}' $pkg | head -1)
|
arch=$($REPOQUERY_3RD_CMD --qf='%{arch}' $pkg | head -1)
|
||||||
deps=$($REPOQUERY_3RD_CMD --requires --qf='%{name}' $pkg | awk '{print $1}')
|
|
||||||
LOG=$RESULTS_3RD_LOG
|
LOG=$RESULTS_3RD_LOG
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "Downloading $url"
|
echo "Downloading $url"
|
||||||
wget -q -O $CGCSREPO_PATH/$arch/$(basename $relativepath) $url
|
rpm_path=$CGCSREPO_PATH/$arch/$(basename $relativepath)
|
||||||
|
wget -q -O $rpm_path $url
|
||||||
|
|
||||||
if [ $? -ne 0 ]; then
|
if [ $? -ne 0 ]; then
|
||||||
echo "Failed to download $url" >&2
|
echo "Failed to download $url" >&2
|
||||||
@ -293,29 +353,39 @@ function download_pkg {
|
|||||||
echo "${pkg},${repoid},$url" >> $LOG
|
echo "${pkg},${repoid},$url" >> $LOG
|
||||||
|
|
||||||
# Now check its dependencies
|
# Now check its dependencies
|
||||||
local dep=
|
download_rpm_dependencies $rpm_path
|
||||||
for dep in $deps; do
|
}
|
||||||
pkg_in_cgcsrepo $dep && continue
|
|
||||||
download_pkg $dep
|
|
||||||
|
function download_rpm_dependencies {
|
||||||
|
local rf=$1
|
||||||
|
|
||||||
|
rpmfile_requires $rf | while read feature; do
|
||||||
|
|
||||||
|
local pkg=
|
||||||
|
pkg=$(feature_to_pkg "$feature")
|
||||||
|
if [ -z "$pkg" ]; then
|
||||||
|
# Already msged to stderr
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
pkg_in_cgcsrepo $pkg && continue
|
||||||
|
download_pkg $pkg
|
||||||
done || exit $?
|
done || exit $?
|
||||||
}
|
}
|
||||||
|
|
||||||
if [ -n "$RPMLIST" ]; then
|
if [ -n "$RPMLIST" ]; then
|
||||||
for rf in $RPMLIST; do
|
for rf in $RPMLIST; do
|
||||||
rpmfile_requires $rf | while read feature; do
|
rpmfile_requires $rf | while read feature; do
|
||||||
pkg=$(feature_to_pkg $feature)
|
pkg=$(feature_to_pkg "$feature")
|
||||||
if [ -z "$pkg" ]; then
|
if [ -z "$pkg" ]; then
|
||||||
# Already msged to stderr
|
# Already msged to stderr
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
dependencies=$(pkg_to_dependencies $pkg)
|
pkg_in_cgcsrepo $pkg && continue
|
||||||
|
download_pkg $pkg
|
||||||
|
|
||||||
for dep in $feature $dependencies; do
|
|
||||||
pkg_in_cgcsrepo $dep && continue
|
|
||||||
|
|
||||||
download_pkg $dep
|
|
||||||
done
|
|
||||||
done || exit $?
|
done || exit $?
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
Loading…
Reference in New Issue
Block a user