starlingx/root
Code Issues Proposed changes
818830b25e
root/build-tools/build-rpms-parallel
Scott Little 78be59c758 Fix memory overcommit that caused OOM killer 
Parallel package builds use large ramdisks.
It's important not to commit too much memory to these
ram disks, or we may push the system into memory exhaustion.
At that stage the Kernel will invoke the OOM killer,
It will likely select our build, or worse someone else's build,
to sacrifice.

The current algorithm only considers free memory at the instant
the parallel build starts.  It does not consider how many
other builds are in flight, but might not have allocated their
ramdisk yet.  The other build intends to use the memory, we see
the memory as free and try to use the same memory.

Solution is to consider total memory, and number of builds
already running or which might foreseeably start in the near future
(share factor) to derive an alternate estimate of memory available.
We then allocate the lesser amount.

Also fixed some issues with cleaning up of child processes when
a newer mockchain-parallel is in use.

Closes-Bug: 1917525
Signed-off-by: Scott Little <scott.little@windriver.com>
Change-Id: Iab178c6f9acbd5a209d66d0da21f367911f34905
2021-03-10 10:05:20 -05:00

2508 lines
76 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
#
# Copyright (c) 2018-2020 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
#
# Builds rpm files from src.rpm files.
#
# This version compiles packages in parrallel if sufficient resources
# (cpu's and memory) are available.
#
# The location of packages to be built is
# $MY_WORKSPACE/<build-type>/rpmbuild/SRPMS.
#
# The build order is a derived from the BuildRequires in the
# spec files in the src.rpms. Note that the BuildRequires sometimes
# create dependency loops, so no correct order can be computed. In these
# cases we add a retry loop. As long as one new package builds, we
# keep retrying the loop, until all are built, or no progress is made.
# So please don't panic and CTRL-C just because you see a few error
# messages go by!
#
export ME=$(basename "$0")
CMDLINE="$ME $@"
BUILD_RPMS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
# Set PKG_MANAGER for our build environment.
source "${BUILD_RPMS_PARALLEL_DIR}/pkg-manager-utils.sh"
# Build for distribution. Currently 'centos' is only supported value.
export DISTRO="centos"
# Maximum number of parallel build environments
ABSOLUTE_MAX_WORKERS=4
# Maximum space in gb for each tmpfs based parallel build environment.
# Note: currently 11 gb is sufficient to build everything except ceph
MAX_MEM_PER_WORKER=11
# Minimum space in gb for each tmpfs based parallel build environment
# Note: tmpfs is typically 2.5 gb when compiling many small jobs
MIN_MEM_PER_WORKER=3
# Maximum number of disk based parallel build environments
MAX_DISK_BASED_WORKERS=2
# Minimum space in gb for each disk based parallel build environment
MIN_DISK_PER_WORKER=20
# How many srpms to build before we add another parallel build environment
MIN_TASKS_PER_CORE=3
# Max number of new concurrent builds to allow for
MAX_SHARE_FACTOR=4
# Always leave at least MEMORY_RESERVE gb of available mem for the system
MEMORY_RESERVE=1
# These two values will be reassigned in the 'compute_resources' subroutine
MOCKCHAIN_RESOURCE_ALLOCATION=""
MAX_WORKERS=$ABSOLUTE_MAX_WORKERS
CREATEREPO=$(which createrepo_c)
if [ $? -ne 0 ]; then
CREATEREPO="createrepo"
fi
# Old repo path or new?
LOCAL_REPO=${MY_REPO}/local-repo
if [ ! -d ${LOCAL_REPO} ]; then
LOCAL_REPO=${MY_REPO}/cgcs-tis-repo
if [ ! -d ${LOCAL_REPO} ]; then
# This one isn't fatal, LOCAL_REPO is not required
LOCAL_REPO=${MY_REPO}/local-repo
fi
fi
# Make sure we have a dependency cache
DEPENDANCY_DIR="${LOCAL_REPO}/dependancy-cache"
SRPM_DIRECT_REQUIRES_FILE="$DEPENDANCY_DIR/SRPM-direct-requires"
SRPM_TRANSITIVE_REQUIRES_FILE="$DEPENDANCY_DIR/SRPM-transitive-requires"
SRPM_TRANSITIVE_DESCENDANTS_FILE="$DEPENDANCY_DIR/SRPM-transitive-descendants"
SRPM_DIRECT_DESCENDANTS_FILE="$DEPENDANCY_DIR/SRPM-direct-descendants"
SRPM_RPM_DIRECT_REQUIRES_FILE="$DEPENDANCY_DIR/SRPM-direct-requires-rpm"
RPM_DIRECT_REQUIRES_FILE="$DEPENDANCY_DIR/RPM-direct-requires"
RPM_TO_SRPM_MAP_FILE="$DEPENDANCY_DIR/rpm-to-srpm"
SRPM_TO_RPM_MAP_FILE="$DEPENDANCY_DIR/srpm-to-rpm"
UNBUILT_PATTERN_FILE="$MY_REPO/build-data/unbuilt_rpm_patterns"
SIGN_SECURE_BOOT="sign-secure-boot"
SIGN_SECURE_BOOT_LOG="sign-secure-boot.log"
export MOCK=/usr/bin/mock
BUILD_RPMS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
source "${BUILD_RPMS_PARALLEL_DIR}/image-utils.sh"
source "${BUILD_RPMS_PARALLEL_DIR}/wheel-utils.sh"
source "${BUILD_RPMS_PARALLEL_DIR}/spec-utils"
source "${BUILD_RPMS_PARALLEL_DIR}/srpm-utils"
HOME=$(pwd)
usage () {
echo ""
echo "Usage: "
echo " $ME [ [--rt] [--no-required] [--no-descendants] [--no-build-info] [--no-autoclean] [--formal] <optional list of package names> ]"
echo " $ME --dep-test <package name>"
echo " $ME --clean [ [--no-descendants] <optional list of package names> ]"
echo " $ME --help"
echo ""
}
number_of_users () {
users | tr ' ' '\n' | sort --uniq | wc -l
}
total_mem_gb () {
free -g | grep 'Mem:' | awk '{ print $2 }'
}
available_mem_gb () {
free -g | grep 'Mem:' | awk '{ print $7 }'
}
available_disk_gb () {
df -BG $MY_WORKSPACE | grep -v '^Filesystem' | awk '{ print $4 }' | sed 's#G$##'
}
number_of_cpus () {
/usr/bin/nproc
}
number_of_builds_in_progress () {
local x
x=$(ps -ef | grep build-pkgs-parallel | wc -l)
x=$((x-1))
echo $x
}
sqrt () {
echo -e "sqrt($1)" | bc -q -i | head -2 | tail -1
}
join_by () { local IFS="$1"; shift; echo "$*"; }
create-no-clean-list () {
local MY_YUM_CONF=$(create-yum-conf)
local NO_CLEAN_LIST_FILE=$MY_WORKSPACE/no_clean_list.txt
local NEED_REBUILD=0
if [ ! -f $NO_CLEAN_LIST_FILE ]; then
NEED_REBUILD=1
else
if [ -f $MY_BUILD_CFG ]; then
find "$MY_BUILD_CFG" -not -newermm "$NO_CLEAN_LIST_FILE" | grep -q $(basename $MY_BUILD_CFG)
if [ $? -eq 0 ]; then
NEED_REBUILD=1
fi
fi
fi
if [ $NEED_REBUILD -eq 1 ]; then
local install_groups=""
local install_packages=""
local p
for p in $(grep "config_opts\['chroot_setup_cmd'\]" $MY_BUILD_CFG | tail -n1 | cut -d '=' -f 2 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e "s/^'//" -e "s/'$//" -e 's/^install //'); do
if [[ $p == @* ]] ; then
install_groups=$(join_by ' ' $install_groups $(echo $p | cut -c 2-))
else
install_packages=$(join_by ' ' $install_packages $p)
fi
done
local noclean_last_list_len=0
local noclean_list=""
local tmp_list=""
local g
for g in $install_groups; do
# Find mandatory packages in the group.
# Discard anything before (and including) 'Mandatory Packages:'
# and anything after (and including) 'Optional Packages:'.
# Also discard leading spaces or '+' characters.
tmp_list=$(${PKG_MANAGER} -c $MY_YUM_CONF groupinfo $g 2>> /dev/null \
| awk 'f;/Mandatory Packages:/{f=1}' \
| sed -n '/Optional Packages:/q;p' \
| sed 's#[ +]*##')
noclean_list=$(join_by ' ' $noclean_list $tmp_list)
done
noclean_list=$(join_by ' ' $noclean_list $install_packages)
noclean_list=$(echo $noclean_list | tr ' ' '\n' | sort --uniq)
noclean_list_len=$(echo $noclean_list | wc -w)
while [ $noclean_list_len -gt $noclean_last_list_len ]; do
noclean_last_list_len=$noclean_list_len
noclean_list=$( (${PKG_MANAGER} -c $MY_YUM_CONF deplist $noclean_list 2>> /dev/null | grep provider: | awk '{ print $2 }' | awk -F . '{ print $1 }'; for p in $noclean_list; do echo $p; done) | sort --uniq)
noclean_list_len=$(echo $noclean_list | wc -w)
done
echo $noclean_list > $NO_CLEAN_LIST_FILE
fi
cat $NO_CLEAN_LIST_FILE
}
str_lst_contains() {
TARGET="$1"
LST="$2"
if [[ $LST =~ (^|[[:space:]])$TARGET($|[[:space:]]) ]] ; then
return 0
else
return 1
fi
}
compute_resources () {
local weight=0
local b
echo ""
for f in $@; do
b=$(basename $f)
if [ -f $SOURCES_DIR/$b/BIG ] || [ ]; then
weight=$((weight+MIN_TASKS_PER_CORE))
else
weight=$((weight+1))
fi
done
weight=$((weight/MIN_TASKS_PER_CORE))
# gather data about the build machines resources
local users=$(number_of_users)
if [ $users -lt 1 ]; then users=1; fi
local mem=$(available_mem_gb)
local total_mem=$(total_mem_gb)
local disk=$(available_disk_gb)
local cpus=$(number_of_cpus)
local num_users=$(sqrt $users)
local num_build=$(number_of_builds_in_progress)
num_build=$((num_build+1))
echo "compute_resources: total: cpus=$cpus, total_mem=$total_mem, avail_mem=$mem, disk=$disk, weight=$weight, num_build=$num_build"
# What fraction of the machine will we use
local share_factor=$num_users
if [ $share_factor -gt $((MAX_SHARE_FACTOR+num_build-1)) ]; then share_factor=$((MAX_SHARE_FACTOR+num_build-1)); fi
if [ $share_factor -lt $num_build ]; then share_factor=$num_build; fi
# What fraction of free memory can we use.
# e.g.
# We intend to support 4 concurrent builds (share_factor)
# Two builds (excluding ours) are already underway (num_build-1)
# So we should be able to support 2 more builds (mem_share_factor)
local mem_share_factor=$((share_factor-(num_build-1)))
if [ $mem_share_factor -lt 1 ]; then mem_share_factor=1; fi
echo "compute_resources: share_factor=$share_factor mem_share_factor=$mem_share_factor"
# What resources are we permitted to use
# Continuing the example from above ... memory share is the lesser of
# - Half the available memory (mem/mem_share_factor)
# - A quarter of the total memory (total_mem/share_factor)
local mem_share=$(((mem-MEMORY_RESERVE)/mem_share_factor))
if [ $mem_share -lt 0 ]; then mem_share=0; fi
local total_mem_share=$(((total_mem-MEMORY_RESERVE)/share_factor))
if [ $total_mem_share -lt 0 ]; then total_mem_share=0; fi
if [ $mem_share -gt $total_mem_share ]; then mem_share=$total_mem_share; fi
local disk_share=$((disk/share_factor))
local cpus_share=$((cpus/share_factor))
echo "compute_resources: our share: cpus=$cpus_share, mem=$mem_share, disk=$disk_share"
# How many build jobs, how many jobs will use tmpfs, and how much mem for each tmpfs
local workers=$cpus_share
if [ $workers -gt $MAX_WORKERS ]; then workers=$MAX_WORKERS; fi
if [ $workers -gt $weight ]; then workers=$weight; fi
if [ $workers -lt 1 ]; then workers=1; fi
local max_mem_based_workers=$((mem_share/MIN_MEM_PER_WORKER))
if [ $max_mem_based_workers -lt 0 ]; then max_mem_based_workers=0; fi
local max_disk_based_workers=$((disk_share/MIN_DISK_PER_WORKER))
if [ $max_disk_based_workers -gt $MAX_DISK_BASED_WORKERS ]; then max_disk_based_workers=$MAX_DISK_BASED_WORKERS; fi
if [ $max_disk_based_workers -lt 1 ]; then max_disk_based_workers=1; fi
echo "max_disk_based_workers=$max_disk_based_workers, max_mem_based_workers=$max_mem_based_workers"
local mem_based_workers=$max_mem_based_workers
if [ $mem_based_workers -ge $workers ]; then mem_based_workers=$((workers-1)); fi
local disk_based_workers=$((workers-mem_based_workers))
if [ $disk_based_workers -gt $max_disk_based_workers ]; then disk_based_workers=$max_disk_based_workers; fi
if [ $disk_based_workers -lt 1 ]; then disk_based_workers=1; fi
echo "disk_based_workers=$disk_based_workers, mem_based_workers=$mem_based_workers"
if [ $workers -gt $((disk_based_workers+mem_based_workers)) ]; then workers=$((disk_based_workers+mem_based_workers)); fi
local mem_spoken_for=$((mem_based_workers*MIN_MEM_PER_WORKER))
local avail_mem=$((mem_share-mem_spoken_for))
local x=""
for i in $(seq 0 $((workers-1))); do
if [ $i -lt $disk_based_workers ]; then
x="$x:0"
else
extra_mem=$(($MAX_MEM_PER_WORKER-$MIN_MEM_PER_WORKER))
if [ $extra_mem -gt $avail_mem ]; then extra_mem=$avail_mem; fi
avail_mem=$((avail_mem-extra_mem))
mem_for_worker=$((MIN_MEM_PER_WORKER+extra_mem))
x="$x:$mem_for_worker"
fi
done
# Our output is saved in environment variables
MOCKCHAIN_RESOURCE_ALLOCATION=$(echo $x | sed 's#^:##')
MAX_WORKERS=$workers
echo "compute_resources: MAX_WORKERS=$MAX_WORKERS, MOCKCHAIN_RESOURCE_ALLOCATION=$MOCKCHAIN_RESOURCE_ALLOCATION"
echo ""
}
#
# Create a list of rpms in the directory
#
create_lst () {
local DIR=${1}
(cd $DIR
[ -f rpm.lst ] && \rm -rf rpm.lst
[ -f srpm.lst ] && \rm -rf srpm.lst
find . -name '*.rpm' -and -not -name '*.src.rpm' | sed 's#^[.][/]##' | sort > rpm.lst
find . -name '*.src.rpm' | sed 's#^[.][/]##' | sort > srpm.lst
)
}
#
# Delete old repodata and reate a new one
#
recreate_repodata () {
local DIR=${1}
(
mkdir -p $DIR
cd $DIR
if [ -f repodata/*comps*xml ]; then
\mv repodata/*comps*xml comps.xml
fi
\rm -rf repodata
\rm -rf .repodata
if [ -f comps.xml ]; then
$CREATEREPO -g comps.xml --workers $(number_of_cpus) $(pwd)
else
$CREATEREPO --workers $(number_of_cpus) $(pwd)
fi
)
}
#
# Update existing repodata
#
update_repodata () {
local DIR=${1}
(cd $DIR
TMP=$(mktemp /tmp/update_repodata_XXXXXX)
RC=0
if [ -f comps.xml ]; then
$CREATEREPO --update -g comps.xml --workers $(number_of_cpus) $(pwd) &> $TMP
RC=$?
else
$CREATEREPO --update --workers $(number_of_cpus) $(pwd) &> $TMP
RC=$?
fi
if [ $RC -ne 0 ]; then
cat $TMP
fi
\rm -f $TMP
)
}
#
# return array that is the intersection of two other arrays
#
# NEW_ARRAY=( $( intersection ARRAY1 ARRAY2 ) )
#
intersection () {
local Aname=$1[@]
local Bname=$2[@]
local A=("${!Aname}")
local B=("${!Bname}")
# echo "${A[@]}"
# echo "${B[@]}"
for a in "${A[@]}"; do
# echo "a=$a"
for b in "${B[@]}"; do
# echo "b=$b"
if [ "$a" == "$b" ]; then
echo "$a"
break
fi
done
done
}
#
# return array that is the union of two other arrays
#
# NEW_ARRAY=( $( union ARRAY1 ARRAY2 ) )
#
union () {
local Aname=$1[@]
local Bname=$2[@]
local A=("${!Aname}")
local B=("${!Bname}")
local a
local b
for a in "${A[@]}"; do
echo "$a"
done
for b in "${B[@]}"; do
local found=0
for a in "${A[@]}"; do
if [ "$a" == "$b" ]; then
found=1
break
fi
done
if [ $found -eq 0 ]; then
echo $b
fi
done
}
#
# returns 0 if element is in the array
#
# e.g. contains ARRAY $SEEKING && echo "$SEEKING is in 'ARRAY'"
#
contains () {
local Aname=$1[@]
local A=("${!Aname}")
local seeking=$2
local in=1
for a in "${A[@]}"; do
if [[ $a == $seeking ]]; then
in=0
break
fi
done
return $in
}
#
# Append element to array if not present
#
# ARRAY=( $( put ARRAY $ELEMENT ) )
#
put () {
local Aname=$1[@]
local A=("${!Aname}")
local element="$2"
for a in "${A[@]}"; do
echo "$a"
done
contains A "$element" || echo "$element"
}
build_order_recursive () {
local target=$1
local idx
local remainder_list
local needs
local needs_list
for((idx=0;idx<${#UNORDERED_LIST[@]};idx++)); do
if [ ${UNORDERED_LIST[idx]} == $target ]; then
remainder_list=( ${UNORDERED_LIST[@]:0:$idx} ${UNORDERED_LIST[@]:$((idx + 1))} )
UNORDERED_LIST=( ${remainder_list[@]} )
needs=( $(grep "^$target;" "$SRPM_DIRECT_REQUIRES_FILE" | sed "s/$target;//" | sed 's/,/ /g') )
needs_list=( $(intersection needs remainder_list) )
for((idx=0;idx<${#needs_list[@]};idx++)); do
build_order_recursive ${needs_list[idx]}
done
echo $target
break
fi
done
}
build_order () {
local Aname=$1[@]
local original_list=("${!Aname}")
local needs
local needs_list
local remainder_list
local idx
local element
local next_start=0
local old_next_start=0
local progress=1
while [ ${#original_list[@]} -gt 0 ] && [ $progress -gt 0 ]; do
progress=0
old_next_start=$next_start
for((idx=$next_start;idx<${#original_list[@]};idx++)); do
element=${original_list[idx]}
next_start=$idx
remainder_list=( ${original_list[@]:0:$idx} ${original_list[@]:$((idx + 1))} )
needs=( $(grep "^$element;" "$SRPM_DIRECT_REQUIRES_FILE" | sed "s/$element;//" | sed 's/,/ /g') )
needs_list=( $(intersection needs remainder_list) )
if [ ${#needs_list[@]} -eq 0 ]; then
echo "$element"
original_list=( "${remainder_list[@]}" )
if [ $next_start -ge ${#original_list[@]} ]; then
next_start=0
fi
progress=1
break
fi
done
if [ $old_next_start -ne 0 ]; then
progress=1
next_start=0
fi
done
if [ ${#original_list[@]} -gt 0 ]; then
# Had trouble calculating a build order for these remaining packages, so stick them at the end
UNORDERED_LIST=( ${original_list[@]} )
while [ ${#UNORDERED_LIST[@]} -gt 0 ]; do
element=${UNORDERED_LIST[0]}
build_order_recursive $element
done
fi
}
set_mock_symlinks () {
local LNK
local DEST
local CFG=$1
if [ -d /localdisk/loadbuild/mock ]; then
mkdir -p $MY_WORKSPACE
LNK=$(echo "/localdisk/loadbuild/mock/$(basename $CFG)" | sed 's/.cfg$//')
if [ ! -L $LNK ] && [ -d $LNK ]; then
echo "WARNING: Found directory at '$LNK' when symlink was expected. Fixing..."
\rm -rf $LNK
if [ -d $LNK ]; then
\mv $LNK $LNK.clean_me
fi
fi
if [ -L $LNK ]; then
DEST=$(readlink $LNK)
if [ "$DEST" != "$MY_WORKSPACE" ] || [ ! -d "$MY_WORKSPACE" ]; then
echo "WARNING: Found broken symlink at '$LNK'. Fixing..."
\rm -f $LNK
fi
fi
if [ ! -L $LNK ]; then
if [ ! -d "$MY_WORKSPACE" ]; then
echo "ERROR: Can't create symlink from $LNK to $MY_WORKSPACE as destination does not exist."
exit 1
fi
ln -s $MY_WORKSPACE $LNK
fi
fi
if [ -d /localdisk/loadbuild/mock-cache ]; then
mkdir -p $MY_WORKSPACE/cache
LNK=$(echo "/localdisk/loadbuild/mock-cache/$(basename $CFG)" | sed 's/.cfg$//')
if [ ! -L $LNK ] && [ -d $LNK ]; then
echo "WARNING: Found directory at '$LNK' when symlink was expected. Fixing..."
\rm -rf $LNK
if [ -d $LNK ]; then
\mv $LNK $LNK.clean_me
fi
fi
if [ -L $LNK ]; then
DEST=$(readlink $LNK)
if [ "$DEST" != "$MY_WORKSPACE/cache" ] || [ ! -d "$MY_WORKSPACE/cache" ]; then
echo "WARNING: Found broken symlink at '$LNK'. Fixing..."
\rm -f $LNK
fi
fi
if [ ! -L $LNK ]; then
if [ ! -d "$MY_WORKSPACE/cache" ]; then
echo "ERROR: Can't create symlink from $LNK to $MY_WORKSPACE/cache as destination does not exist."
exit 1
fi
ln -s $MY_WORKSPACE/cache $LNK
fi
fi
}
remove_mock_symlinks () {
local LNK
local CFG=$1
if [ -d /localdisk/loadbuild/mock ]; then
LNK=$(echo "/localdisk/loadbuild/mock/$(basename $CFG)" | sed 's/.cfg$//')
if [ -L $LNK ]; then
\rm -f $LNK
fi
if [ -d $LNK ]; then
\rm -rf $LNK
if [ $? -ne 0 ]; then
\mv -f $LNK $LNK.clean_me
fi
fi
fi
if [ -d /localdisk/loadbuild/mock-cache ]; then
LNK=$(echo "/localdisk/loadbuild/mock-cache/$(basename $MY_BUILD_CFG)" | sed 's/.cfg$//')
if [ -L $LNK ]; then
\rm -f $MY_WORKSPACE/cache $LNK
fi
if [ -d $LNK ]; then
\rm -rf $LNK
if [ $? -ne 0 ]; then
\mv -f $LNK $LNK.clean_me
fi
fi
fi
}
umount_mock_root_as_tmpfs_all () {
for i in $(seq 0 $((ABSOLUTE_MAX_WORKERS-1))); do
umount_mock_root_as_tmpfs $i
done
}
umount_mock_root_as_tmpfs_cfg () {
local CFG=$1
local build_idx=$(basename $CFG | sed 's#.*[.]b\([0-9]*\)[.]cfg#\1#')
if [ "$build_idx" != "" ]; then
umount_mock_root_as_tmpfs $build_idx
else
echo "umount_mock_root_as_tmpfs_cfg: Failed to map '$CFG' to a build_idx"
fi
}
umount_mock_root_as_tmpfs () {
local build_idx=$1
local mount_dir=$(readlink -f $MY_WORKSPACE/mock)/b${build_idx}/root
local rc
mount | grep tmpfs | grep $mount_dir &> /dev/null
if [ $? -ne 0 ]; then
return 0
fi
mock_tmpfs_umount $mount_dir &> /dev/null
rc=$?
if [ $rc -ne 0 ]; then
echo "FAILED: mock_tmpfs_umount $mount_dir"
fi
return $rc
}
kill_descendents ()
{
local kill_pid=$1
local kill_all=$2
local need_stop=$3
local iteration=$4
local ret=0
local rc=0
# echo "kill_descendents pid=$kill_pid, all=$kill_all stop=$need_stop, iteration=$iteration"
local relevant_recursive_children="$ME"
local relevant_recursive_promote_children="mock"
local relevant_other_children="mockchain-parallel mockchain-parallel-1.3.4 mockchain-parallel-1.4.16 mockchain-parallel-2.6 mockchain-parallel-2.7"
local recursive_promote_children=$(for relevant_child in $relevant_recursive_promote_children; do pgrep -P $kill_pid $relevant_child; done)
local recursive_children=$(for relevant_child in $relevant_recursive_children; do pgrep -P $kill_pid $relevant_child; done)
local other_children=""
if [ $kill_all -eq 1 ]; then
recursive_promote_children=""
recursive_children=$(pgrep -P $kill_pid)
fi
if [ $iteration -eq 0 ]; then
other_children=$(for relevant_child in $relevant_other_children; do pgrep -P $kill_pid $relevant_child; done)
if [ "$other_children" != "" ]; then
ret=1
fi
fi
if [ $need_stop -eq 1 ]; then
for pid in $recursive_children $recursive_promote_children; do
kill -SIGSTOP $pid &> /dev/null
done
fi
for pid in $recursive_children; do
kill_descendents "$pid" $kill_all $need_stop $((iteration + 1))
done
for pid in $recursive_promote_children; do
kill_descendents "$pid" 1 1 $((iteration + 1))
done
# echo "kill: $recursive_children $recursive_promote_children"
for pid in $recursive_children $recursive_promote_children; do
kill $pid &> /dev/null
rc=$?
if [ $need_stop -eq 1 ]; then
kill -SIGCONT $pid &> /dev/null
fi
if [ $rc -eq 0 ] && [ $iteration -eq 0 ]; then
wait $pid
fi
done
# echo "kill: $other_children"
for pid in $other_children; do
kill $pid &> /dev/null
rc=$?
if [ $rc -eq 0 ] && [ $iteration -eq 0 ]; then
wait $pid
fi
done
return $ret
}
function my_exit_n() {
local need_mock_cleanup
# echo "$BASHPID: $ME: my_exit: killing children"
local need_mock_cleanup
kill_descendents $BASHPID 0 0 0
need_mock_cleanup=$?
# echo "$BASHPID: $ME: my_exit: waiting"
wait
# echo "$BASHPID: $ME: my_exit: wait complete"
# echo "$BASHPID: $ME: my_exit: need_mock_cleanup=$need_mock_cleanup"
if [ $need_mock_cleanup -ne 0 ]; then
umount_mock_root_as_tmpfs_all
fi
}
function my_exit() {
local need_mock_cleanup
# echo "$BASHPID: $ME: my_exit: killing children"
local need_mock_cleanup
kill_descendents $BASHPID 0 0 0
need_mock_cleanup=$?
# echo "$BASHPID: $ME: my_exit: waiting"
wait
# echo "$BASHPID: $ME: my_exit: wait complete"
# echo "$BASHPID: $ME: my_exit: need_mock_cleanup=$need_mock_cleanup"
if [ $need_mock_cleanup -ne 0 ]; then
sleep 1
fi
umount_mock_root_as_tmpfs_all
}
function my_sigint_n() {
local ARG=$1
echo "$BASHPID: $ME: my_sigint_n: ARG=$ARG"
echo "$BASHPID: $ME: my_sigint_n: killing children"
local need_mock_cleanup
kill_descendents $BASHPID 0 0 0
need_mock_cleanup=$?
echo "$BASHPID: $ME: my_sigint_n: waiting"
wait
echo "$BASHPID: $ME: my_sigint_n: wait complete"
if [ $need_mock_cleanup -ne 0 ]; then
umount_mock_root_as_tmpfs_cfg $ARG
fi
exit 1
}
function my_sighup_n() {
local ARG=$1
echo "$BASHPID: $ME: my_sighup_n: ARG=$ARG"
echo "$BASHPID: $ME: my_sighup_n: killing children"
local need_mock_cleanup
kill_descendents $BASHPID 0 0 0
need_mock_cleanup=$?
echo "$BASHPID: $ME: my_sighup_n: waiting"
wait
echo "$BASHPID: $ME: my_sighup_n: wait complete"
if [ $need_mock_cleanup -ne 0 ]; then
umount_mock_root_as_tmpfs_cfg $ARG
fi
exit 1
}
function my_sigabrt_n() {
local ARG=$1
echo "$BASHPID: $ME: my_sigabrt_n: ARG=$ARG"
echo "$BASHPID: $ME: my_sigabrt_n: killing children"
local need_mock_cleanup
kill_descendents $BASHPID 0 0 0
need_mock_cleanup=$?
echo "$BASHPID: $ME: my_sigabrt_n: waiting"
wait
echo "$BASHPID: $ME: my_sigabrt_n: wait complete"
if [ $need_mock_cleanup -ne 0 ]; then
umount_mock_root_as_tmpfs_cfg $ARG
fi
exit 1
}
function my_sigterm_n() {
local ARG=$1
echo "$BASHPID: $ME: my_sigterm_n: ARG=$ARG"
echo "$BASHPID: $ME: my_sigterm_n: killing children"
local need_mock_cleanup
kill_descendents $BASHPID 0 0 0
need_mock_cleanup=$?
echo "$BASHPID: $ME: my_sigterm_n: waiting"
wait
echo "$BASHPID: $ME: my_sigterm_n: wait complete"
echo "$BASHPID: $ME: my_sigterm_n: need_mock_cleanup=$need_mock_cleanup"
if [ $need_mock_cleanup -ne 0 ]; then
umount_mock_root_as_tmpfs_cfg $ARG
fi
exit 1
}
function my_sigint() {
echo "$BASHPID: $ME: my_sigint: killing children"
local need_mock_cleanup
kill_descendents $BASHPID 0 0 0
need_mock_cleanup=$?
echo "$BASHPID: $ME: my_sigterm_n: waiting"
wait
echo "$BASHPID: $ME: my_sigterm_n: wait complete"
if [ $need_mock_cleanup -ne 0 ]; then
umount_mock_root_as_tmpfs_all
fi
exit 1
}
function my_sighup() {
echo "$BASHPID: $ME: my_sighup: killing children"
local need_mock_cleanup
kill_descendents $BASHPID 0 0 0
need_mock_cleanup=$?
echo "$BASHPID: $ME: my_sighup: waiting"
wait
echo "$BASHPID: $ME: my_sighup: wait complete"
if [ $need_mock_cleanup -ne 0 ]; then
umount_mock_root_as_tmpfs_all
fi
exit 1
}
function my_sigabrt() {
echo "$BASHPID: $ME: my_sigabrt: killing children"
local need_mock_cleanup
kill_descendents $BASHPID 0 0 0
need_mock_cleanup=$?
echo "$BASHPID: $ME: my_sigabrt: waiting"
wait
echo "$BASHPID: $ME: my_sigabrt: wait complete"
if [ $need_mock_cleanup -ne 0 ]; then
umount_mock_root_as_tmpfs_all
fi
exit 1
}
function my_sigterm() {
echo "$BASHPID: $ME: my_sigterm: killing children"
local need_mock_cleanup
kill_descendents $BASHPID 0 0 0
need_mock_cleanup=$?
echo "$BASHPID: $ME: my_sigterm: waiting"
wait
echo "$BASHPID: $ME: my_sigterm: wait complete"
echo "$BASHPID: $ME: my_sigterm: need_mock_cleanup=$need_mock_cleanup"
if [ $need_mock_cleanup -ne 0 ]; then
umount_mock_root_as_tmpfs_all
fi
exit 1
}
trapwrap() {
local WCMD=$1
shift
declare -i pid status=255
# set the trap for the foreground process
trap my_sigint INT
trap my_sighup HUP
trap my_sigabrt ABRT
trap my_sigterm TERM
# run the command in background
### "$@" & pid=$!
WARGS=()
x=0
for i in "$@"; do
WARGS[$x]="$i"
x=$((x+1))
done
echo "$WCMD ${WARGS[@]/#/}"
$WCMD "${WARGS[@]/#/}" & pid=$!
# wait until bg command finishes, handling interruptions by trapped signals
while (( status > 128 )); do
wait $pid
status=$?
done
# restore the trap
trap - INT
trap - HUP
trap - ABRT
trap - TERM
# return the command exit status
return $status
}
trapwrap_n() {
local ARG=$1
shift
local WCMD=$1
shift
declare -i pid status=255
# set the trap for the foreground process
trap my_exit_n EXIT
trap "my_sigint_n $ARG" INT
trap "my_sighup_n $ARG" HUP
trap "my_sigabrt_n $ARG" ABRT
trap "my_sigterm_n $ARG" TERM
# run the command in background
WARGS=()
x=0
for i in "$@"; do
WARGS[$x]="$i"
x=$((x+1))
done
echo "$WCMD ${WARGS[@]/#/}"
$WCMD "${WARGS[@]/#/}" & pid=$!
# wait until bg command finishes, handling interruptions by trapped signals
while (( status > 128 )); do
wait $pid
status=$?
done
# restore the trap
trap - INT
trap - HUP
trap - ABRT
trap - TERM
# return the command exit status
return $status
}
trap my_exit EXIT
mock_get_cache_dir () {
local CFG=$1
local CACHE_DIR="$MY_WORKSPACE/cache"
local CACHE_LINE=$(grep "config_opts[[][']cache_topdir['][]]" $CFG)
if [ $? -eq 0 ]; then
CACHE_DIR=$(echo "$CACHE_LINE" | awk -F \' '{ print $4 }')
fi
echo "$CACHE_DIR"
}
mock_get_root_dir () {
local CFG=$1
local ROOT_DIR="$MY_WORKSPACE/mock"
local ROOT_LINE=$(grep "config_opts[[][']root['][]]" $CFG)
if [ $? -eq 0 ]; then
ROOT_DIR="$MY_WORKSPACE/"$(echo "$ROOT_LINE" | awk -F \' '{ print $4 }')
fi
echo "$ROOT_DIR"
}
mock_clean_cfg () {
local CFG=$1
echo "${FUNCNAME[0]}: $CFG"
echo "=================================="
mock_clean_cache_cfg $CFG
echo "=================================="
echo "$MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all"
trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all
echo "=================================="
echo "$MOCK --root $CFG --configdir $(dirname $CFG) --clean"
trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --clean
### Note: this sometimes leaves behind a $MY_WORKSPACE/cache/mock/yum_cache/yumcache.lock
echo "=================================="
mock_clean_cache_all_cfg $CFG
echo "=================================="
}
mock_sub_configs () {
find $MY_WORKSPACE/configs/$MY_BUILD_ENVIRONMENT* -name "$MY_BUILD_ENVIRONMENT*b[0-9]*.cfg"
}
mock_clean () {
echo "${FUNCNAME[0]}: in"
echo "=================================="
remove_mock_symlinks $MY_BUILD_CFG
set_mock_symlinks $MY_BUILD_CFG
echo "=================================="
for SUB_CFG in $(mock_sub_configs); do
local PREFIX=$(echo $SUB_CFG | rev | cut -d . -f 2 | rev)
( mock_clean_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
done
wait
echo "=================================="
remove_mock_symlinks $MY_BUILD_CFG
echo "${FUNCNAME[0]}: out"
}
mock_partial_clean_cfg () {
local CFG=$1
local SRPMS_LIST="$2"
local RPMS_LIST="$3"
local CMD
local TMP
local RC
echo "${FUNCNAME[0]}: CFG=$CFG SRPMS_LIST='$SRPMS_LIST' RPMS_LIST='$RPMS_LIST'"
TMP=$(mktemp /tmp/mock_partial_clean_cfg_XXXXXX)
if [ $? -ne 0 ]; then
echo "${FUNCNAME[0]}: mktemp failed"
return 1
fi
local ROOT_DIR=$(mock_get_root_dir $CFG)
if [ -d $ROOT_DIR/root/builddir/build/SOURCES ]; then
echo "rm -rf $ROOT_DIR/root/builddir/build/SOURCES/*"
\rm -rf $ROOT_DIR/root/builddir/build/SOURCES/* 2>> /dev/null
fi
if [ -d $ROOT_DIR/root/builddir/build/SPECS ]; then
echo "rm -rf $ROOT_DIR/root/builddir/build/SPECS/*"
\rm -rf $ROOT_DIR/root/builddir/build/SPECS/* 2>> /dev/null
fi
for s in $SRPMS_LIST; do
f=$(basename $s)
if [ -f $ROOT_DIR/root/builddir/build/SRPMS/$f ]; then
\rm -f -v $ROOT_DIR/root/builddir/build/SRPMS/$f 2>> /dev/null
fi
if [ -f $ROOT_DIR/root/builddir/build/originals/$f ]; then
\rm -f -v $ROOT_DIR/root/builddir/build/originals/$f 2>> /dev/null
fi
done
for r in $RPMS_LIST; do
for d in $(find $ROOT_DIR/root/builddir/build/BUILD/ -maxdepth 1 -name '$r*' 2>> /dev/null); do
echo "rm -rf $d"
\rm -rf $d 2>> /dev/null
done
if [ -d $ROOT_DIR/root/builddir/build/RPMS ]; then
for f in $(find $ROOT_DIR/root/builddir/build/RPMS -maxdepth 1 -name "$r*rpm" 2>> /dev/null); do
\rm -f -v $f 2>> /dev/null
done
fi
done
local NO_CLEAN_LIST=$(create-no-clean-list)
echo "NO_CLEAN_LIST=$NO_CLEAN_LIST"
local RPMS_CLEAN_LIST=""
local NEED_FULL_MOCK_CLEAN=0
for r in $RPMS_LIST; do
if ! str_lst_contains $r "$NO_CLEAN_LIST" ; then
RPMS_CLEAN_LIST=$(join_by ' ' $RPMS_CLEAN_LIST $r)
else
echo "Can't remove '$r' from mock environment without a wipe";
NEED_FULL_MOCK_CLEAN=1
fi
done
if [ $NEED_FULL_MOCK_CLEAN -eq 1 ]; then
echo "Wipe the mock environment"
mock_clean_cfg $CFG
RC=$?
else
# Intent of following is for $RPMS_LIST to be expand now while the remaining $ varaibles are for bash inside mock to expand
echo "Try to uninstall from the mock environment these packages: $RPMS_CLEAN_LIST"
CMD='LST="'$RPMS_CLEAN_LIST'";
DELETE_LIST="";
for r in $LST; do
FOUND=$(rpm -q $r) ;
if [ $? -eq 0 ]; then
DELETE_LIST="$DELETE_LIST $FOUND";
fi;
done;
echo "uninstalling these packages: $DELETE_LIST";
if [ "$DELETE_LIST" != "" ]; then
rpm -e --nodeps $DELETE_LIST;
fi'
echo "$MOCK --root $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP
trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --chroot "bash -c '$CMD'" &>> $TMP
RC=$?
if [ $RC -ne 0 ]; then
cat $TMP
\rm -f $TMP
return $RC
fi
mock_clean_cache_cfg $CFG
RC=$?
\rm -f $TMP
fi
return $RC
}
mock_partial_clean () {
local SRPMS_LIST="$1"
local RPMS_LIST="$2"
echo "${FUNCNAME[0]}: in"
echo "${FUNCNAME[0]}: '$SRPMS_LIST' '$RPMS_LIST'"
echo "=================================="
local NO_CLEAN_LIST=$(create-no-clean-list)
echo "=================================="
for SUB_CFG in $(mock_sub_configs); do
local PREFIX=$(echo $SUB_CFG | rev | cut -d . -f 2 | rev)
( mock_partial_clean_cfg $SUB_CFG "$SRPMS_LIST" "$RPMS_LIST" 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
done
wait
echo "=================================="
echo "${FUNCNAME[0]}: out"
}
mock_clean_cache_cfg () {
local CFG=$1
local TMP
local RC
echo "${FUNCNAME[0]}: $CFG '$SRPMS_LIST' '$RPMS_LIST'"
TMP=$(mktemp /tmp/mock_clean_cache_cfg_XXXXXX)
if [ $? -ne 0 ]; then
echo "${FUNCNAME[0]}: mktemp failed"
return 1
fi
echo "${FUNCNAME[0]}: $CFG"
clean_yum_cache_cfg $CFG
echo "$MOCK --root $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache" &> $TMP
trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache &>> $TMP
RC=$?
if [ $RC -ne 0 ]; then
cat $TMP
fi
\rm -f $TPM
return $RC
}
mock_clean_cache () {
echo "${FUNCNAME[0]}: in"
for SUB_CFG in $(mock_sub_configs); do
local PREFIX=$(echo $SUB_CFG | rev | cut -d . -f 2 | rev)
( mock_clean_cache_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
done
wait
# mock_clean_cache_cfg $BUILD_CFG
echo "${FUNCNAME[0]}: out"
}
mock_clean_cache_all_cfg () {
local CFG=$1
echo "${FUNCNAME[0]}: $CFG"
echo "=================================="
clean_yum_cache_cfg $CFG
echo "=================================="
echo "$MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all"
trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all
echo "=================================="
}
mock_clean_cache_all () {
echo "${FUNCNAME[0]}: in"
for SUB_CFG in $(mock_sub_configs); do
local PREFIX=$(echo $SUB_CFG | rev | cut -d . -f 2 | rev)
( mock_clean_cache_all_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
done
wait
echo "${FUNCNAME[0]}: out"
}
mock_clean_metadata_cfg () {
local CFG=$1
local TMP
local RC
echo "${FUNCNAME[0]}: $CFG"
TMP=$(mktemp /tmp/mock_partial_clean_cfg_XXXXXX)
if [ $? -ne 0 ]; then
echo "${FUNCNAME[0]}: mktemp failed"
return 1
fi
#
# From mock config, extract the embedded yum/dnf config.
# Then extract the repo definitions,
# and convert to a series of yum commands to clean the
# metadata one repo at a time. e.g.
# CMD="yum --disablerepo=* --enablerepo=StxCentos7Distro clean metadata; \
# yum --disablerepo=* --enablerepo=StxCentos7Distro-rt clean metadata;
# ...
# "
#
CMD=$((grep -e config_opts\\[\'yum.conf\'\\\] $CFG \
-e config_opts\\[\'dnf.conf\'\\\] $CFG | \
sed 's#\\n#\n#g') | \
grep '^[[]' | \
grep -v main | \
sed -e 's/[][]//g' -e "s#^#${PKG_MANAGER} --disablerepo=* --enablerepo=#" -e 's#$# clean metadata#' | \
sort -u | \
tr '\n' ';')
echo "$MOCK --root $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP
trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --chroot "bash -c '($CMD)'" &>>$TMP
RC=$?
if [ $RC -ne 0 ]; then
cat $TMP
fi
\rm -f $TMP
return $RC
}
mock_clean_metadata () {
echo "${FUNCNAME[0]}: in"
for SUB_CFG in $(mock_sub_configs); do
local PREFIX=$(echo $SUB_CFG | rev | cut -d . -f 2 | rev)
( mock_clean_metadata_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
done
wait
echo "${FUNCNAME[0]}: out"
}
update_cgcs_repo () {
local REPO_NAME=$1
(
cd $MY_REPO/$REPO_NAME/
local CURR_HEAD=$(git rev-parse HEAD)
local LAST_HEAD_FILE="$MY_REPO/$REPO_NAME/.last_head"
local LAST_HEAD_FILE_OLD="$MY_WORKSPACE/$REPO_NAME.last_head"
local CHANGED
local NEW_UNTRACKED
local NEED_REBUILD
local NEED_MOCK_CLEAN=0
local d
if [ -f LAST_HEAD_FILE_OLD -a ! -f LAST_HEAD_FILE ]; then
\cp LAST_HEAD_FILE_OLD LAST_HEAD_FILE
fi
local LAST_HEAD=$(cat $LAST_HEAD_FILE | head -n 1)
for d in "Binary" "Source"; do
NEED_REBUILD=0
if [ ! -d $d/repodata ]; then
NEED_REBUILD=1
fi
if [ "$CURR_HEAD" != "$LAST_HEAD" ]; then
NEED_REBUILD=1
fi
CHANGED=$(git diff --name-only | grep $d)
if [ "x$CHANGED" != "x" ]; then
NEED_REBUILD=1
fi
NEW_UNTRACKED=$(git ls-files . --exclude-standard --others | grep $d)
if [ "x$NEW_UNTRACKED" != "x" ]; then
NEED_REBUILD=1
fi
if [ $NEED_REBUILD -eq 1 ]; then
NEED_MOCK_CLEAN=1
echo ""
echo "Need to recreate $REPO_NAME/$d/repodata"
mkdir -p $d
if [ -d $d/repodata ]; then
update_repodata "$d"
else
recreate_repodata "$d"
fi
create_lst "$d"
fi
done
echo "$CURR_HEAD" > $LAST_HEAD_FILE
\cp $LAST_HEAD_FILE $LAST_HEAD_FILE_OLD
if [ $NEED_MOCK_CLEAN -eq 1 ]; then
echo ""
echo "Need to clean mock"
mock_clean
set_mock_symlinks $MY_BUILD_CFG
fi
)
}
mock_clean_mounts_dir () {
local MOUNT=$1
local RC
if [ "$MOUNT" == "" ]; then
return 1
fi
mount | grep "$MOUNT" >> /dev/null
if [ $? -eq 0 ]; then
RC=1
which mock_cache_umount >> /dev/null
if [ $? -eq 0 ]; then
echo "umount '$MOUNT'"
mock_cache_umount "$MOUNT"
if [ $? -eq 0 ]; then
RC=0
fi
fi
if [ $RC -eq 1 ]; then
echo "ERROR: Directory '$MOUNT' is already mounted and will cause a build failure within mock."
echo "Ask your system administrator to umount '$MOUNT'."
exit 1
fi
fi
return 0
}
mock_clean_mounts_cfg () {
local CFG=$1
local ROOT_DIR=$(mock_get_root_dir $CFG)
local YUM_CACHE_MOUNT=$(readlink -f "$ROOT_DIR/root/var/cache/yum")
local PROC_MOUNT=$(readlink -f "$ROOT_DIR/root/proc")
local SYS_MOUNT=$(readlink -f "$ROOT_DIR/root/sys")
local SHM_MOUNT=$(readlink -f "$ROOT_DIR/root/dev/shm")
local PTS_MOUNT=$(readlink -f "$ROOT_DIR/root/dev/pts")
local MOUNT
echo "${FUNCNAME[0]}: $CFG"
for MOUNT in "$YUM_CACHE_MOUNT" "$PROC_MOUNT" "$SYS_MOUNT" "$SHM_MOUNT" "$PTS_MOUNT"; do
mock_clean_mounts_dir "$MOUNT"
done
}
mock_clean_mounts () {
echo "${FUNCNAME[0]}: in"
for SUB_CFG in $(mock_sub_configs); do
local PREFIX=$(echo $SUB_CFG | rev | cut -d . -f 2 | rev)
( mock_clean_mounts_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
done
wait
echo "${FUNCNAME[0]}: out"
}
clean_yum_cache_cfg () {
local CFG=$1
local CACHE_DIR=$(mock_get_cache_dir $CFG)
local ROOT_DIR=$(mock_get_root_dir $CFG)
local RC=0
echo "${FUNCNAME[0]}: $CFG"
local YUM_CACHE_MOUNT=$(readlink -f "$ROOT_DIR/root/var/cache/yum")
local YUM_CACHE_LOCK="$CACHE_DIR/mock/yum_cache/yumcache.lock"
# echo "clean_yum_cache YUM_CACHE_MOUNT='$YUM_CACHE_MOUNT' YUM_CACHE_LOCK='$YUM_CACHE_LOCK'"
if [ "$YUM_CACHE_MOUNT" != "" ]; then
mock_clean_mounts_dir "$YUM_CACHE_MOUNT"
fi
if [ -f "$YUM_CACHE_LOCK" ]; then
RC=1
which mock_cache_unlock >> /dev/null
if [ $? -eq 0 ]; then
mock_cache_unlock "$YUM_CACHE_LOCK"
if [ $? -eq 0 ]; then
RC=0
fi
fi
if [ $RC -eq 1 ]; then
echo "ERROR: File '$YUM_CACHE_LOCK' exists and will cause a build failure within mock."
echo "Ask your system administrator to delete '$YUM_CACHE_LOCK'."
exit 1
fi
fi
return $RC
}
clean_yum_cache () {
echo "${FUNCNAME[0]}: in"
for SUB_CFG in $(mock_sub_configs); do
local PREFIX=$(echo $SUB_CFG | rev | cut -d . -f 2 | rev)
( clean_yum_cache_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
done
wait
echo "${FUNCNAME[0]}: out"
}
mock_update_cfg () {
local CFG=$1
echo "${FUNCNAME[0]}: $CFG"
echo "=================================="
set_mock_symlinks $CFG
echo "$MOCK --root $CFG --configdir $(dirname $CFG) --update"
trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --update
echo "=================================="
}
mock_init_cfg () {
local CFG=$1
echo "${FUNCNAME[0]}: $CFG"
echo "=================================="
set_mock_symlinks $CFG
echo "$MOCK --root $CFG --configdir $(dirname $CFG) --init"
trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --init
echo "=================================="
}
mock_update_or_init_cfg () {
local CFG=$1
local TMP
local RC
echo "${FUNCNAME[0]}: $CFG"
local ROOT_DIR=$(mock_get_root_dir $CFG)
TMP=$(mktemp /tmp/mock_update_or_init_cfg_XXXXXX)
if [ $? -ne 0 ]; then
echo "${FUNCNAME[0]}: mktemp failed"
return 1
fi
if [ -d $ROOT_DIR/root ]; then
echo "Updating the mock environment"
set_mock_symlinks $CFG
echo "$MOCK --root $CFG --configdir $(dirname $CFG) --update"
trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --update &> $TMP
RC=$?
else
echo "Init the mock environment"
set_mock_symlinks $CFG
echo "$MOCK --root $CFG --configdir $(dirname $CFG) --init"
trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --init &> $TMP
RC=$?
fi
if [ $RC -ne 0 ]; then
cat $TMP
fi
\rm -f $TMP
return $RC
}
mock_update_or_init () {
echo "${FUNCNAME[0]}: in"
for SUB_CFG in $(mock_sub_configs); do
local PREFIX=$(echo $SUB_CFG | rev | cut -d . -f 2 | rev)
( mock_update_or_init_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
done
wait
echo "${FUNCNAME[0]}: out"
}
if [ "x$PROJECT" == "x" ]; then
echo "PROJECT environmnet variable is not defined."
exit 1
fi
if [ "x$SRC_BUILD_ENVIRONMENT" == "x" ]; then
echo "SRC_BUILD_ENVIRONMENT environmnet variable is not defined."
exit 1
fi
NO_DESCENDANTS=0
NO_REQUIRED=0
NO_AUTOCLEAN=0
NO_BUILD_INFO=0
HELP=0
CLEAN_FLAG=0
FORMAL_FLAG=0
CAREFUL=0
DEP_TEST_FLAG=0
# read the options
TEMP=$(getopt -o h --long parallel,std,rt,installer,containers,no-required,no-descendants,no-autoclean,no-build-info,dep-test,clean,tmpfs-clean,formal,careful,help,layer: -n "$ME" -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
fi
eval set -- "$TEMP"
export BUILD_TYPE=std
trap my_exit EXIT
# extract options and their arguments into variables.
while true ; do
case "$1" in
--careful) CAREFUL=1 ; shift ;;
--no-descendants) NO_DESCENDANTS=1 ; shift ;;
--no-required) NO_REQUIRED=1 ; shift ;;
--no-autoclean) NO_AUTOCLEAN=1; shift ;;
--no-build-info) NO_BUILD_INFO=1; shift ;;
--formal) FORMAL_FLAG=1; shift ;;
--std) BUILD_TYPE=std; shift ;;
--rt) BUILD_TYPE=rt; shift ;;
--installer) BUILD_TYPE=installer; shift ;;
--containers) BUILD_TYPE=containers; shift ;;
-h|--help) HELP=1 ; shift ;;
--clean) CLEAN_FLAG=1 ; shift ;;
--dep-test) DEP_TEST_FLAG=1 ; MAX_WORKERS=1; NO_DESCENDANTS=1; NO_REQUIRED=1; NO_BUILD_INFO=1; shift ;;
--tmpfs-clean) if [ -n "$MY_WORKSPACE" ]; then export MY_WORKSPACE=$MY_WORKSPACE/$BUILD_TYPE; exit 0; fi ;;
--parallel) shift ;;
--layer) export LAYER=$2 ; shift ; shift ;;
--) shift ; break ;;
*) echo "Internal error!" ; exit 1 ;;
esac
done
# Reset variables
if [ -n "$MY_WORKSPACE" ]; then
export MY_WORKSPACE_TOP=${MY_WORKSPACE_TOP:-$MY_WORKSPACE}
export MY_WORKSPACE=$MY_WORKSPACE_TOP/$BUILD_TYPE
else
export MY_PATCH_WORKSPACE_TOP=${MY_PATCH_WORKSPACE_TOP:-$MY_PATCH_WORKSPACE}
export MY_PATCH_WORKSPACE=$MY_PATCH_WORKSPACE_TOP/$BUILD_TYPE
fi
export MY_BUILD_DIR_TOP=${MY_BUILD_DIR_TOP:-$MY_BUILD_DIR}
export MY_BUILD_DIR=$MY_BUILD_DIR_TOP/$BUILD_TYPE
export MY_BUILD_ENVIRONMENT_TOP=${MY_BUILD_ENVIRONMENT_TOP:-$MY_BUILD_ENVIRONMENT}
export MY_BUILD_ENVIRONMENT=$MY_BUILD_ENVIRONMENT_TOP-$BUILD_TYPE
export MY_SRC_RPM_BUILD_DIR=$MY_BUILD_DIR/rpmbuild
export MY_BUILD_ENVIRONMENT_FILE=$MY_BUILD_ENVIRONMENT.cfg
export MY_BUILD_CFG=$MY_WORKSPACE/$MY_BUILD_ENVIRONMENT_FILE
export MY_MOCK_ROOT=$MY_WORKSPACE/mock/root
IMAGE_INC_FILE="${MY_WORKSPACE}/image.inc"
image_inc_list iso std ${DISTRO} > "${IMAGE_INC_FILE}"
DEV_IMAGE_INC_FILE="${MY_WORKSPACE}/image-dev.inc"
image_inc_list iso dev ${DISTRO} > "${DEV_IMAGE_INC_FILE}"
for STREAM in stable dev; do
WHEELS_INC_FILE="${MY_WORKSPACE}/${DISTRO}_${STREAM}_wheels.inc"
wheels_inc_list ${STREAM} ${DISTRO} > "${WHEELS_INC_FILE}"
done
LAST_PLATFORM_RELEASE_FILE="$MY_BUILD_DIR/.platform_release"
TARGETS=$@
if [ $HELP -eq 1 ]; then
usage
exit 0
fi
if [ $FORMAL_FLAG -eq 1 ]; then
export FORMAL_BUILD=1
fi
SRC_ROOT="$MY_REPO"
if [ "x$MY_REPO" == "x" ]; then
SRC_ROOT=$HOME
fi
BUILD_ROOT="$MY_WORKSPACE"
if [ "x$MY_WORKSPACE" == "x" ]; then
BUILD_ROOT="$MY_PATCH_WORKSPACE"
if [ "x$MY_PATCH_WORKSPACE" == "x" ]; then
echo "ERROR: reqiure one of MY_WORKSPACE or MY_PATCH_WORKSPACE be defined"
exit 1
fi
fi
export BUILD_BASE="$BUILD_ROOT"
export CCACHE_DIR="$BUILD_ROOT/.ccache"
export RESULT_DIR="$BUILD_BASE/results"
export SRC_BASE="$SRC_ROOT"
export STX_BASE=$SRC_BASE/stx
if [ "x$MY_SRC_RPM_BUILD_DIR" != "x" ]; then
RPM_BUILD_ROOT=$MY_SRC_RPM_BUILD_DIR
else
RPM_BUILD_ROOT=$BUILD_BASE/rpmbuild
fi
RELEASE_INFO_FILE="$(get_release_info)"
if [ -f "$RELEASE_INFO_FILE" ]; then
source "$RELEASE_INFO_FILE"
else
echo "Warning: failed to find RELEASE_INFO_FILE=$RELEASE_INFO_FILE"
fi
if [ "x$PLATFORM_RELEASE" == "x" ]; then
echo "Warning: PLATFORM_RELEASE is not defined in $RELEASE_INFO_FILE"
PLATFORM_RELEASE="00.00"
fi
export RPM_BUILD_BASE="$RPM_BUILD_ROOT"
export SRPM_OUT="$RPM_BUILD_BASE/SRPMS"
export RPM_DIR="$RPM_BUILD_BASE/RPMS"
export SPECS_DIR="$RPM_BUILD_BASE/SPECS"
export SOURCES_DIR="$RPM_BUILD_BASE/SOURCES"
export PLATFORM_RELEASE
if [ ! -d $BUILD_BASE ]; then
if [ $CLEAN_FLAG -eq 1 ]; then
exit 0
fi
echo "ERROR: expected to find directory at '$BUILD_BASE'"
exit 1
fi
mkdir -p $RPM_BUILD_BASE
if [ $? -ne 0 ]; then
echo "ERROR: Failed to create directory '$RPM_BUILD_BASE'"
exit 1
fi
mkdir -p $SRPM_OUT/repodata
if [ $? -ne 0 ]; then
echo "ERROR: Failed to create directory '$SRPM_OUT/repodata'"
exit 1
fi
mkdir -p $RPM_DIR/repodata
if [ $? -ne 0 ]; then
echo "ERROR: Failed to create directory '$RPM_DIR/repodata'"
exit 1
fi
if [ "x$MY_BUILD_CFG" == "x" ];then
echo "ERROR: reqiure MY_BUILD_CFG to be defined"
exit 1
fi
export BUILD_CFG="$MY_BUILD_CFG"
# Place build-time environement variables in mock environment
echo "FORMAL_BUILD=$FORMAL_BUILD"
echo "modify-build-cfg $BUILD_CFG"
${BUILD_RPMS_PARALLEL_DIR}/modify-build-cfg $BUILD_CFG
if [ $? -ne 0 ]; then
echo "Could not modifiy $BUILD_CFG";
exit 1
fi
if [ ! -f $BUILD_CFG ]; then
echo "ERROR: Mock config file not found at '$BUILD_CFG'"
exit 1
fi
# create temp dir
export TMPDIR="$MY_WORKSPACE/tmp"
mkdir -p "$TMPDIR"
# Create symlinks from /var/... to /localdisk/loadbuild/... if on a build server
set_mock_symlinks $MY_BUILD_CFG
if [ $CLEAN_FLAG -eq 1 ]; then
umount_mock_root_as_tmpfs_all
fi
if [ $CLEAN_FLAG -eq 0 ]; then
ls $SRPM_OUT/*.src.rpm &>> /dev/null
if [ $? -ne 0 ]; then
echo "Nothing to build in '$SRPM_OUT'"
exit 0
fi
fi
ALL=0
UNRESOLVED_TARGETS=" "
if [ $DEP_TEST_FLAG -eq 1 ]; then
# we expect exactly one package
if [ $(echo $TARGETS | wc -w) -ne 1 ]; then
echo "ERROR: dependency testing requires exactly one package"
usage
exit 1
fi
else
# we accept a list of packages, and no list implies all
if [ "x$TARGETS" == "x" ]; then
echo "make: all"
ALL=1
else
echo "make: $TARGETS"
UNRESOLVED_TARGETS="$TARGETS"
fi
fi
if [ "$BUILD_TYPE" != "std" ]; then
# This defines ...
# STD_SRPM_PKG_NAME_TO_PATH
# STD_SRPM_PKG_NAMES
srpm_build_std_dictionary $MY_WORKSPACE_TOP/std/rpmbuild/SRPMS
fi
# This defines ...
# SRPM_PKG_NAME_TO_PATH
# SRPM_PKG_NAMES
srpm_build_dictionary $SRPM_OUT
SRPMS_TO_COMPILE=()
SRPMS_LIST=""
RPMS_LIST=""
clean_list () {
local SRPMS_LIST="$1"
local RPMS_LIST="$2"
local ALL=$3
local TARGET
local b
local d
local f
local n
local p
local r
local s
local sn
local t
local SPEC_DIR
echo "${FUNCNAME[0]}: '$SRPMS_LIST' '$RPMS_LIST' '$ALL'"
if [ $ALL -eq 1 ]; then
for r in $(find $RPM_DIR -name "*.rpm"); do
\rm -f -v $r
done
if [ $CLEAN_FLAG -eq 1 ]; then
for d in $(find $SPECS_DIR -type d); do
echo "rm -rf $d"
\rm -rf "$d" 2>> /dev/null
done
fi
for d in $(find $RESULT_DIR/$USER-* -maxdepth 1 -type d 2>> /dev/null); do
echo "rm -rf $d"
\rm -rf "$d" 2>> /dev/null
done
else
for s in $SRPMS_LIST; do
(
SPEC_DIR=$(spec_cache_dir_from_srpm $s)
sn=$(rpm_get_name $s)
update_spec_cache $s
TARGET=$(rpm -qp --qf '%{NAME}-%{VERSION}\n' "$s")
for d in $(find $RESULT_DIR/$USER-* -maxdepth 1 -name "$TARGET*" 2>> /dev/null); do
echo "rm -rf $d"
\rm -rf "$d" 2>> /dev/null
done
for p in $(ls -1 $SPEC_DIR/BUILDS); do
for r in $(find $RESULT_DIR/$USER-* $RPM_DIR -name "$p-*.rpm" 2>> /dev/null); do
if [ -f $r ]; then
n=$(rpm_get_name $r)
if [ "$n" == "$p" ]; then
if [[ "$r" == *.src.rpm ]]; then
if [ "$n" != "$sn" ]; then
continue
fi
TARGET=$(rpm -qp --qf '%{NAME}-%{VERSION}\n' "$r")
for d in $(find $RESULT_DIR/$USER-* -maxdepth 1 -name "$TARGET*" 2>> /dev/null); do
echo "rm -rf $d"
\rm -rf "$d" 2>> /dev/null
done
else
rs=$(rpm_get_srpm $r)
if [[ "$rs" != "$sn"-[0-9]* ]]; then
continue
fi
fi
\rm -f -v $r
fi
fi
done
done
TARGET=$(rpm -qp --qf '%{NAME}-%{VERSION}\n' "$s")
if [ $CLEAN_FLAG -eq 1 ]; then
for d in $(find $SPECS_DIR -type d -name "$TARGET*" 2>> /dev/null); do
echo "rm -rf $d"
\rm -rf "$d" 2>> /dev/null
done
fi
for d in $(find $RESULT_DIR/$USER-* -maxdepth 1 -name "$TARGET*" 2>> /dev/null); do
echo "rm -rf $d"
\rm -rf "$d" 2>> /dev/null
done
) &
done
echo "waiting on file deletion"
wait
echo "wait complete"
fi
echo ""
echo "Cleaning repodata"
for d in $(find -L $MY_WORKSPACE/rpmbuild $MY_WORKSPACE/results -type d -name repodata); do
recreate_repodata $(dirname $d)
create_lst $(dirname $d)
done
echo ""
echo "Cleaning mock environment"
echo ""
if [ $ALL -eq 1 ]; then
# Wipe everything
if [ "x$RPM_DIR" != "x" ]; then
\rm -rf -v $RPM_DIR/* 2>> /dev/null
fi
\rm -f -v $RESULT_DIR/mockchain.log 2>> /dev/null
mock_clean
else
# If dependency test
if [ $DEP_TEST_FLAG -eq 1 ]; then
mock_clean
else
# Wipe only traces of what we built
mock_partial_clean "$SRPMS_LIST" "$RPMS_LIST"
fi
fi
}
echo "ALL=$ALL"
(
trap my_exit EXIT
trap my_sigint INT
trap my_sighup HUP
echo "$CMDLINE"
echo "ALL=$ALL"
if [ $CLEAN_FLAG -eq 0 ]; then
if [ -d $RESULT_DIR ]; then
# in case previous build recieved a ctrl-C and didn't get a change to copy it's successful work into RPM_DIR
for d in $(find $RESULT_DIR -name '*.rpm' | grep -v '[.]src[.]rpm' | xargs --no-run-if-empty --max-args=1 dirname | sort -u); do
rsync -u $d/*.rpm $RPM_DIR
done
for d in $(find -L $RESULT_DIR -type d -name repodata); do
update_repodata $(dirname $d)
done
fi
fi
spec_cache_dir_from_srpm () {
local SRPM=${1}
local SPEC_DIR=$(echo $SRPM | sed 's#/SRPMS/#/SPECS/#')
echo "$SPEC_DIR"
}
update_spec_cache () {
local SRPM=${1}
local SPEC_DIR=$(spec_cache_dir_from_srpm $SRPM)
local NEED_UPDATE=0
if [ ! -d $SPEC_DIR ]; then
mkdir -p $SPEC_DIR
NEED_UPDATE=1
else
find "$SPEC_DIR" -name '*.spec' | grep 'spec' >> /dev/null
if [ $? -ne 0 ]; then
# No spec file
NEED_UPDATE=1
fi
find "$SPEC_DIR" -not -newermm "$SRPM" -name '*.spec' | grep -q 'spec'
if [ $? -eq 0 ]; then
# spec is older than src.rpm
NEED_UPDATE=1
fi
fi
if [ $NEED_UPDATE -ne 0 ]; then
(
cd $SPEC_DIR
\rm -rf BUILDS BUILDS_VR *.spec 2>> /dev/null
mkdir -p BUILDS
mkdir -p NAMES
mkdir -p SERVICES
mkdir -p BUILDS_VR
rpm2cpio $SRPM | cpio -civ '*.spec'
if [ $? -ne 0 ]; then
echo "ERROR: no spec file found in '$SRPM'"
fi
for f in $(find . -name '*.spec' | sort -V); do
touch $f
for p in $(spec_list_ver_rel_packages $f); do
touch "BUILDS_VR/$p"
done
for p in $(spec_list_packages $f); do
touch "BUILDS/$p"
done
for p in $(spec_find_tag Name $f 2>> /dev/null); do
touch "NAMES/$p"
done
for p in $(spec_find_global service $f 2>> /dev/null); do
touch "SERVICES/$p"
done
done
)
fi
}
# Find the list of packages we must compile
echo "Find the list of packages we must compile"
mkdir -p $MY_WORKSPACE/tmp/
NEED_BUILD_DIR=$(mktemp -d $MY_WORKSPACE/tmp/$USER-$ME-need-build-XXXXXX)
if [ $? -ne 0 ] || [ "x$NEED_BUILD_DIR" == "x" ]; then
echo "Failed to create temp directory under $MY_WORKSPACE/tmp"
exit 1
fi
UNRESOLVED_TARGETS_DIR=$(mktemp -d $MY_WORKSPACE/tmp/$USER-$ME-unresolved-XXXXXX)
if [ $? -ne 0 ] || [ "x$UNRESOLVED_TARGETS_DIR" == "x" ]; then
echo "Failed to create temp directory under $MY_WORKSPACE/tmp"
exit 1
fi
for n in ${UNRESOLVED_TARGETS}; do
touch $UNRESOLVED_TARGETS_DIR/$n
done
PLATFORM_RELEASE_CHANGED=0
if [ -f $LAST_PLATFORM_RELEASE_FILE ]; then
LAST_PLATFORM_RELEASE=$(cat $LAST_PLATFORM_RELEASE_FILE)
if [ "$LAST_PLATFORM_RELEASE" != "$PLATFORM_RELEASE" ]; then
PLATFORM_RELEASE_CHANGED=1
fi
else
PLATFORM_RELEASE_CHANGED=1
fi
for n in "${SRPM_PKG_NAMES[@]}"; do
(
s=${SRPM_PKG_NAME_TO_PATH[$n]}
SPEC_DIR=$(spec_cache_dir_from_srpm $s)
update_spec_cache $s
# echo "$BASHPID: considering $n: $s, SPEC_DIR=$SPEC_DIR"
NEED_BUILD=0
if [ "x$TARGETS" == "x" ]; then
# We weren't given a list of build targets.
# Build anything missing or out of date.
NEED_BUILD=0
BN=$(basename ${s//.src.rpm/})
if [ -f $RESULT_DIR/$MY_BUILD_ENVIRONMENT/$BN/fail ]; then
echo "Found: $RESULT_DIR/$MY_BUILD_ENVIRONMENT/$BN/fail"
echo "Previous build of $BN failed"
NEED_BUILD=1
elif [ ! -f $RESULT_DIR/$MY_BUILD_ENVIRONMENT/$BN/success ]; then
echo "Not Found: $RESULT_DIR/$MY_BUILD_ENVIRONMENT/$BN/success"
echo "No previous build of $BN"
NEED_BUILD=1
else
LOCAL_RPMS_VRA_LIST=$(ls -1 $SPEC_DIR/BUILDS_VR | tr '\n' ' ')
for f in $LOCAL_RPMS_VRA_LIST; do
m=$(find $RPM_DIR/$f*rpm 2>> /dev/null | wc -l)
if [ $m -eq 0 ] && [ -f "$UNBUILT_PATTERN_FILE" ]; then
echo $f | grep -f "$UNBUILT_PATTERN_FILE" >> /dev/null && m=1
if [ $m -eq 1 ]; then
echo "Excluding '$f' due to match in UNBUILT_PATTERN_FILE '$UNBUILT_PATTERN_FILE'"
if [ -f "$IMAGE_INC_FILE" ] ; then
for t in $(grep -v '^#' "$IMAGE_INC_FILE"); do
ii=$(echo $f | grep "^$t-[0-9]" | wc -l)
if [ $ii -gt 0 ]; then
echo "Including '$f' due to match in IMAGE_INC_FILE '$IMAGE_INC_FILE' due to pattern '^$t-[0-9]'"
m=0
break
fi
done
fi
fi
fi
newer=$(find $RPM_DIR/$f*rpm -type f -not -newermm $s 2>> /dev/null | wc -l)
# echo "$m $newer=find $RPM_DIR/$f*rpm -type f -not -newermm $s 2>> /dev/null | wc -l"
if [ $m -eq 0 ] || [ $newer -gt 0 ] || [ $CLEAN_FLAG -eq 1 ]; then
if [ $newer -gt 0 ]; then
echo "Including '$f' due to newer code"
find $RPM_DIR/$f*rpm -type f -not -newermm $s
else
if [ $m -eq 0 ]; then
echo "Including '$f' due to m=0"
else
if [ $CLEAN_FLAG -eq 1 ]; then
echo "Including '$f' due to CLEAN_FLAG=1"
fi
fi
fi
NEED_BUILD=1
break
fi
done
fi
else
# We were given a list of build targets,
# try to find packages matching that list.
NEED_BUILD=0
for f in $(find $SPEC_DIR/NAMES $SPEC_DIR/SERVICES $SPEC_DIR/BUILDS -type f 2>> /dev/null); do
b=$(basename $f)
for t in $TARGETS; do
if [[ ( "$b" == "$t" ) || ( ( "$BUILD_TYPE" == "rt" ) && ( "$b" == "$t-rt" ) ) ]]; then
echo "Including named target '$f'"
TARGET_FOUND=$t
NEED_BUILD=1
# UNRESOLVED_TARGETS=$(echo "$UNRESOLVED_TARGETS" | sed "s/\(^\|[[:space:]]\)$TARGET_FOUND\([[:space:]]\|$\)/ /g")
if [ -f $UNRESOLVED_TARGETS_DIR/$TARGET_FOUND ]; then
\rm -f $UNRESOLVED_TARGETS_DIR/$TARGET_FOUND
fi
break
fi
done
done
fi
if [ $NO_BUILD_INFO -eq 0 ]; then
if [ "$n" == "build-info" ]; then
echo "Including '$n' by default"
NEED_BUILD=1
fi
fi
if [ $PLATFORM_RELEASE_CHANGED -eq 1 ]; then
grep '%{platform_release}' $SPEC_DIR/*.spec >> /dev/null
if [ $? -eq 0 ]; then
echo "Including '$n' due to changed platform_release"
NEED_BUILD=1
fi
fi
if [ $NEED_BUILD -eq 1 ]; then
echo "found $n: $s"
touch "$NEED_BUILD_DIR/$n"
# SRPMS_TO_COMPILE+=("$n")
fi
) &
done
echo "waiting"
wait
for n in $(ls -1 $NEED_BUILD_DIR); do
SRPMS_TO_COMPILE+=("$n")
done
UNRESOLVED_TARGETS=" "
for n in $(ls -1 $UNRESOLVED_TARGETS_DIR); do
UNRESOLVED_TARGETS="$UNRESOLVED_TARGETS $n"
done
\rm -rf $NEED_BUILD_DIR
\rm -rf $UNRESOLVED_TARGETS_DIR
ORIG_SRPMS_TO_COMPILE=( ${SRPMS_TO_COMPILE[@]} )
echo "SRPMS_TO_COMPILE = ${SRPMS_TO_COMPILE[@]}"
# adding dependant packages
if [ $CLEAN_FLAG -eq 0 ] && [ $NO_DESCENDANTS -eq 0 ] && [ -f $SRPM_DIRECT_DESCENDANTS_FILE ]; then
echo
echo "adding dependant packages"
# This array will accumulate a list of secondary build targets.
TRANSITIVE_SRPMS_TO_COMPILE=()
# Add packages that directly depend on the primary build targets in ORIG_SRPMS_TO_COMPILE
for n in ${ORIG_SRPMS_TO_COMPILE[@]}; do
needs=( $(grep "^$n;" "$SRPM_DIRECT_DESCENDANTS_FILE" | sed "s/$n;//" | sed 's/,/ /g'; alt_n=$(echo "$n" | sed 's#-rt$##'); if [ "$alt_n" != "$n" ]; then grep "^$alt_n;" "$SRPM_DIRECT_DESCENDANTS_FILE" | sed "s/$alt_n;//" | sed 's/,/ /g' | sed 's#\([^[:space:]]*\)#\1-rt#g'; fi ) )
# intersection of 'needs' and 'SRPM_PKG_NAMES' ... i.e. what should be compiled that we have source for
compilable_needs=( $(intersection needs SRPM_PKG_NAMES) )
TRANSITIVE_SRPMS_TO_COMPILE=( $(union compilable_needs TRANSITIVE_SRPMS_TO_COMPILE) )
done
# For non-std build, and if non specific build targets are named, then search all
# packages that we might build and check if they require a package that DID build
# in the std build. If so build the package as a secondary target, even though the
# primary target was from a different build_type.
if [ "$BUILD_TYPE" != "std" ] && [ $ALL -eq 1 ] && [ -f $SRPM_TO_RPM_MAP_FILE ] && [ -f $SRPM_RPM_DIRECT_REQUIRES_FILE ]; then
# Test all that we can build ...
for n in ${SRPM_PKG_NAMES[@]}; do
contains ORIG_SRPMS_TO_COMPILE $n
if [ $? -eq 0 ]; then
# Already on the primary build list, skip it.
echo "skip $n"
continue
fi
STD_NEEDS_BUILD=0
# Iterate over all binary rpms names produce by the candidate package
for b in $(grep "^$n;" "$SRPM_TO_RPM_MAP_FILE" | sed "s/$n;//" | sed 's/,/ /g'); do
# find an rpm file with the rpm name we seek
for bp in $(find $RPM_DIR -name "$b-[0-9]*.rpm" | grep -v '.src.rpm'); do
if [ "$b" != "$(rpm_get_name $bp)" ]; then
# rpm name doesn't match
continue
fi
# Iterate over binary rpms names required by the candidate package
for r in $(grep "^$n;" "$SRPM_RPM_DIRECT_REQUIRES_FILE" | sed "s/$n;//" | sed 's/,/ /g'); do
if [ $r == $n ]; then
# Ignore self dependency
continue
fi
# find a required rpm file with the rpm name we seek, AND is newer than the produced rpm file
for rp in $(find $(echo $RPM_DIR | sed "s#/$BUILD_TYPE/#/std/#") -name "$r-[0-9]*.rpm" -newermm $bp | grep -v '.src.rpm'); do
if [ "$r" != "$(rpm_get_name $rp)" ]; then
# rpm name doesn't match
continue
fi
# Ok, a required rpm is newer than a built rpm, we should rebuild!
echo "rebuild '$n' due to newer '$r'"
STD_NEEDS_BUILD=1
break
done
done
done
# Avoid pointless processing if we already have a positive result.
if [ $STD_NEEDS_BUILD -eq 1 ]; then
break
fi
done
if [ $STD_NEEDS_BUILD -eq 1 ]; then
# Compile is requires due to an updated required package in the std build.
# Add 'n' to array TRANSITIVE_SRPMS_TO_COMPILE.
TRANSITIVE_SRPMS_TO_COMPILE=( $(put TRANSITIVE_SRPMS_TO_COMPILE $n) )
fi
done
fi
# If the kernel or kernel-rt packages where absent from the primary build targets, but
# added as a secondary target, then make sure all out-of-tree kernel modules are also
# added.
for n in kernel kernel-rt; do
KERNEL_IN_ORIG=0
KERNEL_IN_TRANSITIVE=0
contains ORIG_SRPMS_TO_COMPILE "$n" && KERNEL_IN_ORIG=1
contains TRANSITIVE_SRPMS_TO_COMPILE "$n" && KERNEL_IN_TRANSITIVE=1
if [ $KERNEL_IN_TRANSITIVE -eq 1 ] && [ $KERNEL_IN_ORIG -eq 0 ]; then
needs=( $(grep "^$n;" "$SRPM_DIRECT_DESCENDANTS_FILE" | sed "s/$n;//" | sed 's/,/ /g'; alt_n=$(echo "$n" | sed 's#-rt$##'); if [ "$alt_n" != "$n" ]; then grep "^$alt_n;" "$SRPM_DIRECT_DESCENDANTS_FILE" | sed "s/$alt_n;//" | sed 's/,/ /g' | sed 's#\([^[:space:]]*\)#\1-rt#g'; fi ) )
# intersection of 'needs' and 'SRPM_PKG_NAMES' ... i.e. what should be compiled that we have source for
compilable_needs=( $(intersection needs SRPM_PKG_NAMES) )
TRANSITIVE_SRPMS_TO_COMPILE=( $(union compilable_needs TRANSITIVE_SRPMS_TO_COMPILE) )
fi
done
# Append the secondary targetc list to the primary list
SRPMS_TO_COMPILE=( $(union SRPMS_TO_COMPILE TRANSITIVE_SRPMS_TO_COMPILE) )
echo "SRPMS_TO_COMPILE = ${SRPMS_TO_COMPILE[@]}"
fi
MUST_SRPMS_TO_COMPILE=( ${SRPMS_TO_COMPILE[@]} )
# adding required packages
if [ $CLEAN_FLAG -eq 0 ] && [ "x$TARGETS" != "x" ] && [ $NO_REQUIRED -eq 0 ] && [ -f $SRPM_TRANSITIVE_REQUIRES_FILE ]; then
echo
echo "adding required packages"
TRANSITIVE_SRPMS_TO_COMPILE=()
for n in ${MUST_SRPMS_TO_COMPILE[@]}; do
needs=( $(grep "^$n;" "$SRPM_TRANSITIVE_REQUIRES_FILE" | sed "s/$n;//" | sed 's/,/ /g') )
# intersection of 'needs' and 'SRPM_PKG_NAMES' ... i.e. what should be compiled that we have source for
compilable_needs=( $(intersection needs SRPM_PKG_NAMES) )
TRANSITIVE_SRPMS_TO_COMPILE=( $(union compilable_needs TRANSITIVE_SRPMS_TO_COMPILE) )
for b in "${un[@]}"; do
echo $b
done
done
SRPMS_TO_COMPILE=( $(union TRANSITIVE_SRPMS_TO_COMPILE SRPMS_TO_COMPILE) )
echo "SRPMS_TO_COMPILE = ${SRPMS_TO_COMPILE[@]}"
fi
# Determine build order ... now done in mockchain4
SRPMS_TO_COMPILE=( $(echo ${SRPMS_TO_COMPILE[@]} | sed 's/ /\n/g' | sort -u) )
# convert pkg names to paths, clean work dirs if needed
echo
echo "Mapping packages to src rpm paths"
for n in ${SRPMS_TO_COMPILE[@]}; do
s=${SRPM_PKG_NAME_TO_PATH[$n]}
SPEC_DIR=$(spec_cache_dir_from_srpm $s)
update_spec_cache $s
SRPMS_LIST="$SRPMS_LIST $s"
# echo "SRPMS_LIST = $SRPMS_LIST"
TMP_RPMS_LIST=$(ls -1 $SPEC_DIR/BUILDS | tr '\n' ' ')
RPMS_LIST="$RPMS_LIST $TMP_RPMS_LIST"
done
echo
CENTOS_REPO=centos-repo
if [ ! -d ${MY_REPO}/${CENTOS_REPO} ]; then
CENTOS_REPO=cgcs-centos-repo
if [ ! -d ${MY_REPO}/${CENTOS_REPO} ]; then
echo "ERROR: directory ${MY_REPO}/centos-repo not found."
exit 1
fi
fi
if [ $CLEAN_FLAG -eq 0 ]; then
update_cgcs_repo ${CENTOS_REPO}
fi
mock_clean_mounts
# clean work dirs if needed
CLEAN_BEFORE_BUILD_SRPM_LIST=""
CLEAN_BEFORE_BUILD_RPM_LIST=""
if [ $CLEAN_FLAG -eq 0 ]; then
echo
echo "Calculating minimal clean list"
for nm in ${SRPMS_TO_COMPILE[@]}; do
MUST_CLEAN=0
contains MUST_SRPMS_TO_COMPILE $nm && MUST_CLEAN=1
s=${SRPM_PKG_NAME_TO_PATH[$nm]}
SPEC_DIR=$(spec_cache_dir_from_srpm $s)
update_spec_cache $s
LOCAL_RPMS_LIST=$(ls -1 $SPEC_DIR/BUILDS | tr '\n' ' ')
LOCAL_RPMS_VRA_LIST=$(ls -1 $SPEC_DIR/BUILDS_VR | tr '\n' ' ')
for f in $LOCAL_RPMS_VRA_LIST; do
m=$(find $RPM_DIR/$f*rpm 2>> /dev/null | wc -l)
if [ -f "$UNBUILT_PATTERN_FILE" ]; then
echo $f | grep -f "$UNBUILT_PATTERN_FILE" >> /dev/null && m=1
fi
n=$(find $RPM_DIR/$f*rpm -type f -not -newermm $s 2>> /dev/null | wc -l)
# echo "$n=find $RPM_DIR/$f*rpm -type f -not -newermm $s 2>> /dev/null | wc -l"
if [ $m -eq 0 ] || [ $n -gt 0 ] || [ $MUST_CLEAN -eq 1 ]; then
CLEAN_BEFORE_BUILD_SRPM_LIST="$CLEAN_BEFORE_BUILD_SRPM_LIST $s"
CLEAN_BEFORE_BUILD_RPM_LIST="$CLEAN_BEFORE_BUILD_RPM_LIST $LOCAL_RPMS_LIST"
break
fi
done
done
fi
if [ "$UNRESOLVED_TARGETS" != " " ]; then
if [ $CLEAN_FLAG -eq 0 ]; then
echo ""
echo "ERROR: failed to resolve build targets: $UNRESOLVED_TARGETS"
exit 1
fi
fi
echo "SRPMS_LIST = $SRPMS_LIST"
echo "RPMS_LIST = $RPMS_LIST"
echo
if [ $CLEAN_FLAG -eq 0 ]; then
# pre-create these directories as $USER,
# else mock will create them as root and fails to clean them.
# Note: keep these in sync with mockchain-parallel!
for i in $(seq 0 $((MAX_WORKERS-1))); do
mkdir -p $MY_WORKSPACE/mock/b$i
mkdir -p $MY_WORKSPACE/cache/b$i/mock
done
mock_update_or_init
fi
set_mock_symlinks $MY_BUILD_CFG
echo
echo "Cleaning"
if [ $CLEAN_FLAG -eq 1 ]; then
# Clean what the user asked for
echo "========= clean_list '$SRPMS_LIST' '$RPMS_LIST' $ALL"
\rm -r -f -v $MY_WORKSPACE/mock-$USER-*
clean_list "$SRPMS_LIST" "$RPMS_LIST" "$ALL"
exit 0
else
# Clean what we intend to build
if [ $NO_AUTOCLEAN -eq 1 ]; then
echo "no-autoclean was requested"
else
if [ "$CLEAN_BEFORE_BUILD_SRPM_LIST" != "" ]; then
echo "========= clean_list '$CLEAN_BEFORE_BUILD_SRPM_LIST' '$CLEAN_BEFORE_BUILD_RPM_LIST' 0"
clean_list "$CLEAN_BEFORE_BUILD_SRPM_LIST" "$CLEAN_BEFORE_BUILD_RPM_LIST" 0
fi
fi
fi
echo
echo "Cleaning repodata"
BUILD_ENVIRONMENT_DIR=$(basename $BUILD_CFG)
BUILD_ENVIRONMENT_DIR=${BUILD_ENVIRONMENT_DIR%.*}
LOCAL_URL=http://127.0.0.1:8088$BUILD_BASE/results/$BUILD_ENVIRONMENT_DIR/
LOCAL_SRC_URL=http://127.0.0.1:8088$BUILD_BASE/rpmbuild/SRPMS/
for d in $(find -L $RESULT_DIR -type d -name repodata); do
(cd $d/..
if [ -f repodata/*comps*xml ]; then
\mv repodata/*comps*xml comps.xml
fi
\rm -rf repodata
)
done
echo
echo "Cleaning Metadata"
MOCKCHAIN_LOG="$RESULT_DIR/mockchain.log"
mkdir -p $RESULT_DIR
touch $RESULT_DIR/build_start
\rm -rf $MOCKCHAIN_LOG
mock_clean_metadata
echo
echo "Building"
recreate_repodata $BUILD_BASE/results/$BUILD_ENVIRONMENT_DIR
CMD_PREFIX=""
if [ -x /bin/ionice ]; then
CMD_PREFIX="nice -n 20 ionice -c Idle /bin/ionice "
fi
REAL_MOCKCHAIN=0
MOCK_PASSTHROUGH="-m"
MOCKCHAIN="mockchain-parallel"
CHAIN_OPTION=""
if file $(which mockchain) | grep -q 'Python script'; then
REAL_MOCKCHAIN=1
fi
CMD_OPTIONS="$MOCK_PASSTHROUGH --no-clean $MOCK_PASSTHROUGH --no-cleanup-after"
if [ $CAREFUL -eq 1 ]; then
CMD_OPTIONS="$MOCK_PASSTHROUGH --no-cleanup-after"
fi
CMD_OPTIONS+=" $MOCK_PASSTHROUGH --enable-plugin=package_state"
CMD_OPTIONS+=" --log=$MOCKCHAIN_LOG"
echo "CAREFUL=$CAREFUL"
# Sets WORKERS and MOCKCHAIN_RESOURCE_ALLOCATION
compute_resources $SRPMS_LIST
if [ -f $SRPM_RPM_DIRECT_REQUIRES_FILE ]; then
CMD_OPTIONS+=" --srpm-dependency-file $SRPM_RPM_DIRECT_REQUIRES_FILE"
fi
if [ -f "$RPM_DIRECT_REQUIRES_FILE" ]; then
CMD_OPTIONS+=" --rpm-dependency-file $RPM_DIRECT_REQUIRES_FILE"
fi
if [ -f "$RPM_TO_SRPM_MAP_FILE" ]; then
CMD_OPTIONS+=" --rpm-to-srpm-map-file $RPM_TO_SRPM_MAP_FILE"
fi
for s in $SRPMS_LIST; do
d=$(echo "$s" | sed 's#/SRPMS/#/SOURCES/#')
if [ -f $d/BIG ]; then
BUILD_SIZE=$(cat $d/BIG | { read first rest ; echo $first ; })
CMD_OPTIONS="$CMD_OPTIONS --mark-big-path $BUILD_SIZE:$s"
fi
if [ -f $d/SLOW ]; then
BUILD_SPEED=$(cat $d/SLOW | { read first rest ; echo $first ; })
CMD_OPTIONS="$CMD_OPTIONS --mark-slow-path $BUILD_SPEED:$s"
fi
done
echo "CMD_OPTIONS=$CMD_OPTIONS"
echo "MAX_WORKERS=$MAX_WORKERS"
echo "MOCKCHAIN_RESOURCE_ALLOCATION=$MOCKCHAIN_RESOURCE_ALLOCATION"
CMD="$CMD_PREFIX $MOCKCHAIN --root $BUILD_CFG --localrepo $BUILD_BASE --recurse --workers=$MAX_WORKERS --worker-resources=$MOCKCHAIN_RESOURCE_ALLOCATION --basedir=$MY_WORKSPACE --tmp_prefix=$USER --addrepo=$LOCAL_URL --addrepo=$LOCAL_SRC_URL $CMD_OPTIONS $MOCK_PASSTHROUGH --rebuild"
CMD_BUILD_LIST="$CHAIN_OPTION $SRPMS_LIST"
echo ""
echo "$CMD $MOCK_PASSTHROUGH --define='_tis_dist .tis' $MOCK_PASSTHROUGH --define='platform_release $PLATFORM_RELEASE' $CMD_BUILD_LIST"
echo ""
trapwrap stdbuf -o0 $CMD $MOCK_PASSTHROUGH --define="_tis_dist .tis" $MOCK_PASSTHROUGH --define="platform_release $PLATFORM_RELEASE" $CMD_BUILD_LIST
MOCKCHAIN_RC=$?
echo $PLATFORM_RELEASE > $LAST_PLATFORM_RELEASE_FILE
if [ $CLEAN_FLAG -eq 0 ]; then
umount_mock_root_as_tmpfs_all
fi
for d in $(find $RESULT_DIR -name '*.rpm' | grep -v '[.]src[.]rpm' | xargs --max-args=1 dirname | sort -u); do
rsync -u $d/*.rpm $RPM_DIR
done
if [ $ALL -eq 1 ]; then
echo
echo "Auditing for obsolete srpms"
for r in $(find $RESULT_DIR $RPM_DIR -name '*.src.rpm'); do
(
f=$(basename $r)
if [ ! -f "$SRPM_OUT/$f" ]; then
\rm -fv $r
fi
) &
done
echo "waiting for srpm audit to complete"
wait
echo "Auditing for obsolete rpms"
for r in $(find $RESULT_DIR $RPM_DIR -name '*.rpm' | grep -v 'src.rpm'); do
(
s=$(rpm_get_srpm $r)
if [ ! -f "$SRPM_OUT/$s" ]; then
echo "Failed to find '$SRPM_OUT/$s'"
\rm -fv $r
fi
) &
done
echo "waiting for rpm audit to complete"
wait
echo "Audit complete"
echo ""
fi
if [ $MOCKCHAIN_RC -ne 0 ]; then
echo "ERROR: Failed to build rpms using '$CMD'"
exit 1
fi
echo "Recreate repodata"
for d in $(find -L $MY_WORKSPACE/rpmbuild $MY_WORKSPACE/results -type d -name repodata); do
update_repodata $(dirname "$d")
create_lst $(dirname "$d")
done
if [ -f $MOCKCHAIN_LOG ]; then
grep 'following pkgs could not be successfully built' $MOCKCHAIN_LOG >> /dev/null
if [ $? -eq 0 ]; then
FAILED_PKGS=""
for p in $(sed -n '/following pkgs could not be successfully built:/,/Results out to/p' $MOCKCHAIN_LOG | grep -v '*** Build Failed ***' | sed 1d | sed '$ d' | cut -d ':' -f2-); do
PKG=$(basename $p)
FAILED_PKGS="$PKG $FAILED_PKGS"
done
echo
echo "Failed to build packages: $FAILED_PKGS"
exit 1
fi
fi
# If we're doing a nightly or formal build (i.e. not a developer build) then we
# want to sign certain packages. Note that only certain users (i.e. jenkins)
# have the authority to requiest that packages be signed.
#
# Signing is not actually done on this server (the keys are kept safe on a
# different server with very limited access) but we can invoke a script to
# make calls to the signing server. Note that this will NOT work if you are
# not Jenkins and don't have access to the Jenkins cross server login keys.
#
# Note that both std and rt builds must be complete before invoking the signing
# script
if [ 0$FORMAL_BUILD -eq 1 ] && [ "$USER" == "jenkins" ]; then
if [ -e $MY_WORKSPACE_TOP/std ] && [ -e $MY_WORKSPACE_TOP/rt ]; then
# Create dir for log, if it doesn't exit
mkdir -p $MY_WORKSPACE_TOP/export
echo "We are jenkins, and we are trying to do a formal build -- calling signing server"
echo " to sign boot RPMs with secure boot keys"
MY_WORKSPACE=$MY_WORKSPACE_TOP ${SIGN_SECURE_BOOT} > $MY_WORKSPACE_TOP/export/${SIGN_SECURE_BOOT_LOG} 2>&1
if [ $? -ne 0 ]; then
echo "Signing of packages failed -- see $MY_WORKSPACE_TOP/export/${SIGN_SECURE_BOOT_LOG}"
exit 1
fi
fi
fi
exit 0
) 2>&1 | stdbuf -o0 awk '{ print strftime("%H:%M:%S"), $0; fflush(); }' | tee $(date "+$MY_WORKSPACE/build-rpms-parallel_%Y-%m-%d_%H-%M-%S.log") ; exit ${PIPESTATUS[0]}
View Git Blame Copy Permalink