diff --git a/centos-mirror-tools/starlingx_add_pkgs.sh b/centos-mirror-tools/starlingx_add_pkgs.sh index 35b96585..4245e3ee 100755 --- a/centos-mirror-tools/starlingx_add_pkgs.sh +++ b/centos-mirror-tools/starlingx_add_pkgs.sh @@ -84,16 +84,11 @@ if [ -z "$MY_REPO_ROOT_DIR" ]; then exit 1 fi +STARLINGX_ADD_PKGS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +source $STARLINGX_ADD_PKGS_DIR/../toCOPY/lst_utils.sh + STXTOOLS=${MY_REPO_ROOT_DIR}/stx-tools -REPO_DIR=${STXTOOLS}/centos-mirror-tools/yum.repos.d -REPOCFG_STD_FILES=$(ls ${REPO_DIR}/StarlingX*.repo | grep -v StarlingX_3rd) -REPOCFG_3RD_FILES=${REPO_DIR}/StarlingX_3rd*.repo -REPOCFG_STD_MERGED=$(mktemp /tmp/REPOCFG_STD_MERGED_XXXXXX) -cat $REPOCFG_STD_FILES > $REPOCFG_STD_MERGED -REPOCFG_3RD_MERGED=$(mktemp /tmp/REPOCFG_3RD_MERGED_XXXXXX) -cat $REPOCFG_3RD_FILES > $REPOCFG_3RD_MERGED -REPOCFG_ALL_MERGED=$(mktemp /tmp/REPOCFG_ALL_MERGED_XXXXXX) -cat $REPOCFG_STD_FILES $REPOCFG_3RD_FILES > $REPOCFG_ALL_MERGED CGCSREPO_PATH=$MY_REPO/cgcs-centos-repo/Binary TISREPO_PATH=$MY_WORKSPACE/std/rpmbuild/RPMS @@ -110,18 +105,9 @@ FAILED_LOG=failed.log RPMLIST= DOWNLOAD_LIST= -# It seems we have to manually disable the repos from /etc/yum.repos.d, -# even though we're specifying a config file -REPOQUERY_ARGS=$(grep -h '^\[' /etc/yum.repos.d/* | sed 's/[][]//g' | sed 's/^/--disablerepo=/') - -REPOQUERY_CMD="repoquery --archlist=x86_64,noarch $REPOQUERY_ARGS" -REPOQUERY_STD_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_STD_MERGED" -REPOQUERY_3RD_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_3RD_MERGED" -REPOQUERY_ALL_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_ALL_MERGED" -REPOQUERY_LOCAL_CMD="$REPOQUERY_CMD --quiet --repofrompath cgcs,$CGCSREPO_PATH $TISREPO_PATH_ARGS" function cleanup { - rm -f $REPOCFG_STD_MERGED $REPOCFG_3RD_MERGED $REPOCFG_ALL_MERGED + rm -f $REPOCFG_STD_MERGED $REPOCFG_3RD_MERGED $REPOCFG_LOWER_LAYER_MERGED $REPOCFG_ALL_MERGED } trap cleanup EXIT @@ -129,7 +115,7 @@ trap cleanup EXIT function show_usage { cat >&2 < ] ... [ ] ... + $(basename $0) [ -C ] [ -l ] [ -d ] ... [ ] ... This utility uses the cgcs-centos-repo repo, and optionally the rpmbuild/RPMS repo from \$MY_WORKSPACE/std, as a baseline, downloading packages required @@ -145,7 +131,7 @@ In addition, this utility will record a list of downloaded RPMs in the $RESULTS_ or $RESULTS_3RD_LOG files, with failures recorded in $FAILED_LOG or $NOTFOUND_LOG. The resulting download list can then be added to the appropriate .lst file in -\$MY_REPO_ROOT_DIR/stx-tools/centos-mirror-tools +\$MY_REPO_ROOT_DIR/stx-tools/centos-mirror-tools/config/centos/flock/ Example: $(basename $0) -d linuxptp -d zlib puppet-gnocchi-11.3.0-1.el7.src.rpm @@ -155,11 +141,19 @@ EOF exit 1 } -while getopts "d:h" opt; do +while getopts "C:d:l:h" opt; do case $opt in + C) + # Alternate config directory + set_and_validate_config_dir "${OPTARG}" + ;; d) DOWNLOAD_LIST="$DOWNLOAD_LIST $OPTARG" ;; + l) + # Set layer + set_and_validate_layer "${OPTARG}" + ;; h) show_usage ;; @@ -173,31 +167,82 @@ done shift $((OPTIND-1)) RPMLIST="${RPMLIST} $@" +REPO_DIR=${STXTOOLS}/centos-mirror-tools/yum.repos.d +LAYER_REPO_DIR=${config_dir}/centos/${layer}/yum.repos.d + +REPOCFG_STD_FILES=$(ls ${REPO_DIR}/StarlingX*.repo | grep -v StarlingX_3rd) +REPOCFG_3RD_FILES=${REPO_DIR}/StarlingX_3rd*.repo +REPOCFG_LOWER_LAYER_FILES="" +if [ -d ${LAYER_REPO_DIR} ]; then + REPOCFG_LOWER_LAYER_FILES=${LAYER_REPO_DIR}/StarlingX*.repo +fi + +REPOCFG_STD_MERGED=$(mktemp /tmp/REPOCFG_STD_MERGED_XXXXXX) +cat $REPOCFG_STD_FILES > $REPOCFG_STD_MERGED + +REPOCFG_3RD_MERGED=$(mktemp /tmp/REPOCFG_3RD_MERGED_XXXXXX) +cat $REPOCFG_3RD_FILES > $REPOCFG_3RD_MERGED + +REPOCFG_LOWER_LAYER_MERGED=$(mktemp /tmp/REPOCFG_LOWER_LAYER_MERGED_XXXXXX) +if [ "$REPOCFG_LOWER_LAYER_FILES" != "" ]; then + cat $REPOCFG_LOWER_LAYER_FILES > $REPOCFG_LOWER_LAYER_MERGED +fi + +REPOCFG_ALL_MERGED=$(mktemp /tmp/REPOCFG_ALL_MERGED_XXXXXX) +cat $REPOCFG_STD_FILES $REPOCFG_3RD_FILES $REPOCFG_LOWER_LAYER_FILES > $REPOCFG_ALL_MERGED + +# It seems we have to manually disable the repos from /etc/yum.repos.d, +# even though we're specifying a config file +REPOQUERY_ARGS=$(grep -h '^\[' /etc/yum.repos.d/* | sed 's/[][]//g' | sed 's/^/--disablerepo=/') + +REPOQUERY_CMD="repoquery --archlist=x86_64,noarch $REPOQUERY_ARGS" +REPOQUERY_STD_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_STD_MERGED" +REPOQUERY_3RD_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_3RD_MERGED" +REPOQUERY_LOWER_LAYER_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_LOWER_LAYER_MERGED" +REPOQUERY_ALL_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_ALL_MERGED" +REPOQUERY_LOCAL_CMD="$REPOQUERY_CMD --quiet --repofrompath cgcs,$CGCSREPO_PATH $TISREPO_PATH_ARGS" + + function rpmfile_requires { # # Map a specified rpm file to its dependency list # local rpmfile=$1 - rpm -qp --requires $rpmfile | awk '{print $1}' + # Never process rpmlib(*) requirements. + # They are not regular requirements, but rather internal features of rpm. + rpm -qp --requires $rpmfile | grep -v '^rpmlib[(]' } function feature_to_pkg { # # Map a feature/capability to the package that provides it # - local feature=$1 + local feature="$1" local pkg= - pkg=$($REPOQUERY_STD_CMD $feature | head -1) + + pkg=$($REPOQUERY_LOCAL_CMD "$feature" | head -1) if [ -z $pkg ]; then - pkg=$($REPOQUERY_STD_CMD --qf='%{name}' --whatprovides $feature | head -1) + pkg=$($REPOQUERY_LOCAL_CMD --qf='%{name}' --whatprovides "$feature" | head -1) if [ -z $pkg ]; then - pkg=$($REPOQUERY_3RD_CMD $feature | head -1) + pkg=$($REPOQUERY_LOWER_LAYER_CMD "$feature" | head -1) if [ -z $pkg ]; then - pkg=$($REPOQUERY_3RD_CMD --qf='%{name}' --whatprovides $feature | head -1) - if [ -z "$pkg" ]; then - echo "Could not find in repo: $feature" >&2 - echo "Could not find in repo: $feature" >> $NOTFOUND_LOG + pkg=$($REPOQUERY_LOWER_LAYER_CMD --qf='%{name}' --whatprovides "$feature" | head -1) + if [ -z $pkg ]; then + pkg=$($REPOQUERY_STD_CMD "$feature" | head -1) + if [ -z $pkg ]; then + pkg=$($REPOQUERY_STD_CMD --qf='%{name}' --whatprovides "$feature" | head -1) + if [ -z $pkg ]; then + pkg=$($REPOQUERY_3RD_CMD "$feature" | head -1) + if [ -z $pkg ]; then + pkg=$($REPOQUERY_3RD_CMD --qf='%{name}' --whatprovides "$feature" | head -1) + if [ -z "$pkg" ]; then + echo "Could not find in repo: $feature" >&2 + echo "Could not find in repo: $feature" >> $NOTFOUND_LOG + fi + fi + fi + fi fi fi fi @@ -214,6 +259,15 @@ function pkg_to_dependencies { $REPOQUERY_ALL_CMD --resolve --requires --qf='%{name}' $pkg } +function simplified_pkg { + # + # drop the epoch + # + local pkg=$1 + + echo $pkg | sed 's/-[0-9]*:/-/' +} + function pkg_in_cgcsrepo { # # Check whether the specified package is already in the downloaded (or built) repo @@ -223,14 +277,20 @@ function pkg_in_cgcsrepo { local results= results=$($REPOQUERY_LOCAL_CMD --whatprovides $pkg) if [ -n "$results" ]; then - return 0 + results=$($REPOQUERY_LOCAL_CMD $(simplified_pkg $pkg)) + if [ -n "$results" ]; then + return 0 + fi fi local pkgname= pkgname=$($REPOQUERY_ALL_CMD --quiet $REPOCFG_ARGS --qf='%{name}' --whatprovides $pkg | head -1) if [ -z "$pkgname" ]; then - echo "Failed to find a package providing $pkg" >&2 - return 1 + pkgname=$($REPOQUERY_ALL_CMD --quiet $REPOCFG_ARGS --qf='%{name}' $(simplified_pkg $pkg) | head -1) + if [ -z "$pkgname" ]; then + echo "Failed to find a package providing $pkg" >&2 + return 1 + fi fi results=$($REPOQUERY_LOCAL_CMD $pkgname) @@ -241,9 +301,9 @@ function download_pkg { # # Download the specified package and its dependencies # - local feature=$1 + local feature="$1" local pkg= - pkg=$(feature_to_pkg $feature) + pkg=$(feature_to_pkg "$feature") if [ -z "$pkg" ]; then # Error should already be to stderr return 1 @@ -254,25 +314,25 @@ function download_pkg { local relativepath= local arch= local deps= + local rpm_path= repoid=$($REPOQUERY_STD_CMD --qf='%{repoid}' $pkg | head -1) if [ -n "$repoid" ]; then url=$($REPOQUERY_STD_CMD --location $pkg | head -1) relativepath=$($REPOQUERY_STD_CMD --qf='%{relativepath}' $pkg | head -1) arch=$($REPOQUERY_STD_CMD --qf='%{arch}' $pkg | head -1) - deps=$($REPOQUERY_STD_CMD --requires --qf='%{name}' $pkg | awk '{print $1}') LOG=$RESULTS_LOG else repoid=$($REPOQUERY_3RD_CMD --qf='%{repoid}' $pkg | head -1) url=$($REPOQUERY_3RD_CMD --location $pkg | head -1) relativepath=$($REPOQUERY_3RD_CMD --qf='%{relativepath}' $pkg | head -1) arch=$($REPOQUERY_3RD_CMD --qf='%{arch}' $pkg | head -1) - deps=$($REPOQUERY_3RD_CMD --requires --qf='%{name}' $pkg | awk '{print $1}') LOG=$RESULTS_3RD_LOG fi echo "Downloading $url" - wget -q -O $CGCSREPO_PATH/$arch/$(basename $relativepath) $url + rpm_path=$CGCSREPO_PATH/$arch/$(basename $relativepath) + wget -q -O $rpm_path $url if [ $? -ne 0 ]; then echo "Failed to download $url" >&2 @@ -293,29 +353,39 @@ function download_pkg { echo "${pkg},${repoid},$url" >> $LOG # Now check its dependencies - local dep= - for dep in $deps; do - pkg_in_cgcsrepo $dep && continue - download_pkg $dep + download_rpm_dependencies $rpm_path +} + + +function download_rpm_dependencies { + local rf=$1 + + rpmfile_requires $rf | while read feature; do + + local pkg= + pkg=$(feature_to_pkg "$feature") + if [ -z "$pkg" ]; then + # Already msged to stderr + continue + fi + + pkg_in_cgcsrepo $pkg && continue + download_pkg $pkg done || exit $? } if [ -n "$RPMLIST" ]; then for rf in $RPMLIST; do rpmfile_requires $rf | while read feature; do - pkg=$(feature_to_pkg $feature) + pkg=$(feature_to_pkg "$feature") if [ -z "$pkg" ]; then # Already msged to stderr continue fi - dependencies=$(pkg_to_dependencies $pkg) + pkg_in_cgcsrepo $pkg && continue + download_pkg $pkg - for dep in $feature $dependencies; do - pkg_in_cgcsrepo $dep && continue - - download_pkg $dep - done done || exit $? done fi