root/build-tools/build-rpms-parallel

#!/bin/bash

#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#


export ME=$(basename "$0")
CMDLINE="$ME $@"



# Build for distribution.  Currently 'centos' is only supported value.
export DISTRO="centos"

# Maximum number of parallel build environments
ABSOLUTE_MAX_WORKERS=4

# Maximum space in gb for each tmpfs based parallel build environment.
#  Note: currently 11 gb is sufficient to build everything except ceph
MAX_MEM_PER_WORKER=11

# Minimum space in gb for each tmpfs based parallel build environment
#  Note: tmpfs is typically 2.5 gb when compiling many small jobs
MIN_MEM_PER_WORKER=3

# Maximum number of disk based parallel build environments 
MAX_DISK_BASED_WORKERS=2

# Minimum space in gb for each disk based parallel build environment
MIN_DISK_PER_WORKER=20

# How many srpms to build before we add another parallel build environment 
MIN_TASKS_PER_CORE=3

# Max number of new concurrent builds to allow for
MAX_SHARE_FACTOR=4

# Always leave at least MEMORY_RESERVE gb of available mem for the system 
MEMORY_RESERVE=1

# These two values will be reassigned in the 'compute_resources' subroutine 
MOCKCHAIN_RESOURCE_ALLOCATION=""
MAX_WORKERS=$ABSOLUTE_MAX_WORKERS


CREATEREPO=$(which createrepo_c)
if [ $? -ne 0 ]; then
   CREATEREPO="createrepo"
fi

DEPENDANCY_DIR="$MY_REPO/cgcs-tis-repo/dependancy-cache"
SRPM_DIRECT_REQUIRES_FILE="$DEPENDANCY_DIR/SRPM-direct-requires"
SRPM_TRANSITIVE_REQUIRES_FILE="$DEPENDANCY_DIR/SRPM-transitive-requires"
SRPM_TRANSITIVE_DESCENDANTS_FILE="$DEPENDANCY_DIR/SRPM-transitive-descendants"
SRPM_DIRECT_DESCENDANTS_FILE="$DEPENDANCY_DIR/SRPM-direct-descendants"
SRPM_RPM_DIRECT_REQUIRES_FILE="$DEPENDANCY_DIR/SRPM-direct-requires-rpm"
RPM_DIRECT_REQUIRES_FILE="$DEPENDANCY_DIR/RPM-direct-requires"
RPM_TO_SRPM_MAP_FILE="$DEPENDANCY_DIR/rpm-to-srpm"
SRPM_TO_RPM_MAP_FILE="$DEPENDANCY_DIR/srpm-to-rpm"

UNBUILT_PATTERN_FILE="$MY_REPO/build-data/unbuilt_rpm_patterns"

SIGN_SECURE_BOOT="sign-secure-boot"
SIGN_SECURE_BOOT_LOG="sign-secure-boot.log"

export MOCK=/usr/bin/mock

BUILD_RPMS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
source "${BUILD_RPMS_PARALLEL_DIR}/image-utils.sh"
source "${BUILD_RPMS_PARALLEL_DIR}/spec-utils"
source "${BUILD_RPMS_PARALLEL_DIR}/srpm-utils"

HOME=$(pwd)

usage () {
    echo ""
    echo "Usage: "
    echo "   $ME [ [--rt] [--no-required] [--no-descendants] [--no-build-info] [--no-autoclean] [--formal] <optional list of package names> ]"
    echo "   $ME --clean [ [--no-descendants] <optional list of package names> ]"
    echo "   $ME --help"
    echo ""
}


number_of_users () {
    users | tr ' ' '\n' | sort --uniq | wc -l
}

available_mem_gb () {
    free -g | grep 'Mem:' | awk '{ print $7 }'
}

available_disk_gb () {
    df -BG $MY_WORKSPACE | grep -v '^Filesystem' | awk '{ print $4 }' | sed 's#G$##'
}

number_of_cpus () {
    /usr/bin/nproc
}

number_of_builds_in_progress () {
    local x
    x=$(ps -ef | grep build-pkgs-parallel | wc -l)
    x=$((x-1))
    echo $x
}

sqrt () {
    echo -e "sqrt($1)" | bc -q -i | head -2 | tail -1
}

join_by () { local IFS="$1"; shift; echo "$*"; }

create-no-clean-list () {
   local MY_YUM_CONF=$(create-yum-conf)
   local NO_CLEAN_LIST_FILE=$MY_WORKSPACE/no_clean_list.txt
   local NEED_REBUILD=0

   if [ ! -f $NO_CLEAN_LIST_FILE ]; then
       NEED_REBUILD=1
   else
       if [ -f MY_BUILD_CFG ]; then
           if [ -f MY_BUILD_CFG ]; then
               find "$MY_BUILD_CFG" -not -newer "$NO_CLEAN_LIST_FILE" | grep $(basename $MY_BUILD_CFG) >> /dev/null
               if [ $? -eq 0 ]; then
                   NEED_REBUILD=1
               fi
           fi
       fi
   fi

   if [ $NEED_REBUILD -eq 1 ]; then
       local install_groups=""
       local install_packages=""
       local p
    
       for p in $(grep "config_opts\['chroot_setup_cmd'\]" $MY_BUILD_CFG | tail -n1 | cut -d '=' -f 2 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e "s/^'//" -e "s/'$//" -e 's/^install //'); do
          if [[ $p == @* ]] ; then
              install_groups=$(join_by ' ' $install_groups $(echo $p | cut -c 2-))
          else
              install_packages=$(join_by ' ' $install_packages $p)
          fi
       done
    
       local noclean_last_list_len=0
       local noclean_list=""
       local tmp_list=""
       local g
    
       for g in $install_groups; do
           tmp_list=$(yum -c $MY_YUM_CONF groupinfo $g 2>> /dev/null | awk 'f;/Mandatory Packages:/{f=1}' | cut -c 5-)
           noclean_list=$(join_by ' ' $noclean_list $tmp_list)
       done
    
       noclean_list=$(join_by ' ' $noclean_list $install_packages)
       noclean_list=$(echo $noclean_list | tr ' ' '\n' | sort --uniq)
       noclean_list_len=$(echo $noclean_list | wc -w)
    
       while [ $noclean_list_len -gt $noclean_last_list_len ]; do
           noclean_last_list_len=$noclean_list_len
           noclean_list=$( (yum -c $MY_YUM_CONF deplist $noclean_list 2>> /dev/null | grep provider: | awk '{ print $2 }' | awk -F . '{ print $1 }'; for p in $noclean_list; do echo $p; done) | sort --uniq)
           noclean_list_len=$(echo $noclean_list | wc -w)
       done
   
       echo $noclean_list > $NO_CLEAN_LIST_FILE
   fi

   cat $NO_CLEAN_LIST_FILE
}

str_lst_contains() { 
   TARGET="$1"
   LST="$2"
   if [[ $LST =~ (^|[[:space:]])$TARGET($|[[:space:]]) ]] ; then 
      return 0
   else
      return 1
   fi
}

compute_resources () {
    local weight=0
    local b

    echo ""
    for f in $@; do 
        b=$(basename $f)
        if [ -f $SOURCES_DIR/$b/BIG ] || [ ]; then
            weight=$((weight+MIN_TASKS_PER_CORE))
        else
            weight=$((weight+1))
        fi
    done
    weight=$((weight/MIN_TASKS_PER_CORE))

    # gather data about the build machines resources
    local users=$(number_of_users)
    if [ $users -lt 1 ]; then users=1; fi
    local mem=$(available_mem_gb)
    local disk=$(available_disk_gb)
    local cpus=$(number_of_cpus)
    local num_users=$(sqrt $users)
    local num_build=$(number_of_builds_in_progress)
    num_build=$((num_build+1))    
    echo "compute_resources: total: cpus=$cpus, mem=$mem, disk=$disk, weight=$weight, num_build=$num_build"

    # What fraction of the machine will we use
    local share_factor=$num_users
    if [ $share_factor -gt $((MAX_SHARE_FACTOR+num_build-1)) ]; then share_factor=$((MAX_SHARE_FACTOR+num_build-1)); fi
    if [ $share_factor -lt $num_build ]; then share_factor=$num_build; fi
    local mem_share_factor=$((share_factor-num_build))
    if [ $mem_share_factor -lt 1 ]; then mem_share_factor=1; fi
    echo "compute_resources: share_factor=$share_factor mem_share_factor=$mem_share_factor"

    # What resources are we permitted to use
    local mem_share=$(((mem-MEMORY_RESERVE)/mem_share_factor))
    if [ $mem_share -lt 0 ]; then mem_share=0; fi
    local disk_share=$((disk/share_factor))
    local cpus_share=$((cpus/share_factor))
    echo "compute_resources: our share: cpus=$cpus_share, mem=$mem_share, disk=$disk_share"

    # How many build jobs, how many jobs will use tmpfs, and how much mem for each tmpfs
    local workers=$cpus_share
    if [ $workers -gt $MAX_WORKERS ]; then workers=$MAX_WORKERS; fi
    if [ $workers -gt $weight ]; then workers=$weight; fi
    if [ $workers -lt 1 ]; then workers=1; fi
    local max_mem_based_workers=$((mem_share/MIN_MEM_PER_WORKER))
    if [ $max_mem_based_workers -lt 0 ]; then max_mem_based_workers=0; fi
    local max_disk_based_workers=$((disk_share/MIN_DISK_PER_WORKER))
    if [ $max_disk_based_workers -gt $MAX_DISK_BASED_WORKERS ]; then max_disk_based_workers=$MAX_DISK_BASED_WORKERS; fi
    if [ $max_disk_based_workers -lt 1 ]; then max_disk_based_workers=1; fi
    echo "max_disk_based_workers=$max_disk_based_workers, max_mem_based_workers=$max_mem_based_workers"
    local mem_based_workers=$max_mem_based_workers
    if [ $mem_based_workers -ge $workers ]; then mem_based_workers=$((workers-1)); fi
    local disk_based_workers=$((workers-mem_based_workers))
    if [ $disk_based_workers -gt $max_disk_based_workers ]; then disk_based_workers=$max_disk_based_workers; fi
    if [ $disk_based_workers -lt 1 ]; then disk_based_workers=1; fi
    echo "disk_based_workers=$disk_based_workers, mem_based_workers=$mem_based_workers"
    if [ $workers -gt $((disk_based_workers+mem_based_workers)) ]; then workers=$((disk_based_workers+mem_based_workers)); fi
    local mem_spoken_for=$((mem_based_workers*MIN_MEM_PER_WORKER))
    local avail_mem=$((mem_share-mem_spoken_for))
    local x=""
    for i in $(seq 0 $((workers-1))); do
        if [ $i -lt $disk_based_workers ]; then
            x="$x:0"
        else
            extra_mem=$(($MAX_MEM_PER_WORKER-$MIN_MEM_PER_WORKER))
            if [ $extra_mem -gt $avail_mem ]; then extra_mem=$avail_mem; fi
            avail_mem=$((avail_mem-extra_mem))
            mem_for_worker=$((MIN_MEM_PER_WORKER+extra_mem))
            x="$x:$mem_for_worker"
        fi
    done

    # Our output is saved in environmnet variables
    MOCKCHAIN_RESOURCE_ALLOCATION=$(echo $x | sed 's#^:##')
    MAX_WORKERS=$workers
    echo "compute_resources: MAX_WORKERS=$MAX_WORKERS, MOCKCHAIN_RESOURCE_ALLOCATION=$MOCKCHAIN_RESOURCE_ALLOCATION"
    echo ""
}


#
# Delete old repodata and reate a new one
#
recreate_repodata () {
   local DIR=${1}

       (cd $DIR
        if [ -f repodata/*comps*xml ]; then
           \mv repodata/*comps*xml comps.xml
        fi
        \rm -rf repodata
        \rm -rf .repodata
        if [ -f comps.xml ]; then
           $CREATEREPO -g comps.xml --workers $(number_of_cpus) $(pwd)
        else
           $CREATEREPO --workers $(number_of_cpus) $(pwd)
        fi
       )
}

#
# Update existing repodata
#
update_repodata () {
   local DIR=${1}

       (cd $DIR
        TMP=$(mktemp /tmp/update_repodata_XXXXXX)
        RC=0
        if [ -f comps.xml ]; then
           $CREATEREPO --update -g comps.xml --workers $(number_of_cpus) $(pwd) &> $TMP
           RC=$?
        else
           $CREATEREPO --update --workers $(number_of_cpus) $(pwd) &> $TMP
           RC=$?
        fi
        if [ $RC -ne 0 ]; then
           cat $TMP
        fi
        \rm -f $TMP
       )
}

#
# return array that is the intersection of two other arrays
#
# NEW_ARRAY=( $( intersection ARRAY1 ARRAY2 ) )
#
intersection () {
   local Aname=$1[@]
   local Bname=$2[@]
   local A=("${!Aname}")
   local B=("${!Bname}")

   # echo "${A[@]}"
   # echo "${B[@]}"
   for a in "${A[@]}"; do
      # echo "a=$a"
      for b in "${B[@]}"; do
         # echo "b=$b"
         if [ "$a" == "$b" ]; then
            echo "$a"
            break
         fi
      done
   done
}

#
# return array that is the union of two other arrays
#
# NEW_ARRAY=( $( union ARRAY1 ARRAY2 ) )
#
union () {
   local Aname=$1[@]
   local Bname=$2[@]
   local A=("${!Aname}")
   local B=("${!Bname}")
   local a
   local b

   for a in "${A[@]}"; do
      echo "$a"
   done

   for b in "${B[@]}"; do
      local found=0
      for a in "${A[@]}"; do
         if [ "$a" == "$b" ]; then
            found=1
            break
         fi
      done
      if [ $found -eq 0 ]; then
         echo $b
      fi
   done
} 

#
# returns 0 if element is in the array
# 
#  e.g.  contains ARRAY $SEEKING  && echo "$SEEKING is in 'ARRAY'"
#
contains () {
   local Aname=$1[@]
   local A=("${!Aname}")
   local seeking=$2
   local in=1

    for a in "${A[@]}"; do
        if [[ $a == $seeking ]]; then
            in=0
            break
        fi
    done
    return $in
}

#
# Append element to array if not present
#
# ARRAY=( $( put ARRAY $ELEMENT ) )
#
put () {
   local Aname=$1[@]
   local A=("${!Aname}")
   local element="$2"
   for a in "${A[@]}"; do
      echo "$a"
   done
   contains A "$element" || echo "$element"
}

build_order_recursive () {
   local target=$1
   local idx
   local remainder_list
   local needs
   local needs_list

   for((idx=0;idx<${#UNORDERED_LIST[@]};idx++)); do
      if [ ${UNORDERED_LIST[idx]} == $target ]; then
         remainder_list=( ${UNORDERED_LIST[@]:0:$idx} ${UNORDERED_LIST[@]:$((idx + 1))} )
         UNORDERED_LIST=( ${remainder_list[@]} )
         needs=( $(grep "^$target;" "$SRPM_DIRECT_REQUIRES_FILE" | sed "s/$target;//" | sed 's/,/ /g') )
         needs_list=( $(intersection needs remainder_list) )
         for((idx=0;idx<${#needs_list[@]};idx++)); do
             build_order_recursive ${needs_list[idx]} 
         done
         echo $target
         break
      fi
   done
}

build_order () {
   local Aname=$1[@]
   local original_list=("${!Aname}")
   local needs
   local needs_list
   local remainder_list
   local idx
   local element
   local next_start=0
   local old_next_start=0
   local progress=1

   while [ ${#original_list[@]} -gt 0 ] && [ $progress -gt 0 ]; do
      progress=0
      old_next_start=$next_start
      for((idx=$next_start;idx<${#original_list[@]};idx++)); do
         element=${original_list[idx]}
         next_start=$idx
         remainder_list=( ${original_list[@]:0:$idx} ${original_list[@]:$((idx + 1))} )
         needs=( $(grep "^$element;" "$SRPM_DIRECT_REQUIRES_FILE" | sed "s/$element;//" | sed 's/,/ /g') )
         needs_list=( $(intersection needs remainder_list) )
         if [ ${#needs_list[@]} -eq 0 ]; then
            echo "$element"
            original_list=( "${remainder_list[@]}" )
            if [ $next_start -ge ${#original_list[@]} ]; then
               next_start=0
            fi
            progress=1
            break
         fi
      done
      if [ $old_next_start -ne 0 ]; then
         progress=1
         next_start=0
      fi
   done

   if [ ${#original_list[@]} -gt 0 ]; then
      # Had trouble calculating a build order for these remaining packages, so stick them at the end
      UNORDERED_LIST=( ${original_list[@]} )
      while [ ${#UNORDERED_LIST[@]} -gt 0 ]; do
         element=${UNORDERED_LIST[0]}
         build_order_recursive $element 
      done
   fi
}

set_mock_symlinks () {
   local LNK
   local DEST
   local CFG=$1
   if [ -d /localdisk/loadbuild/mock ]; then
      mkdir -p $MY_WORKSPACE
      LNK=$(echo "/localdisk/loadbuild/mock/$(basename $CFG)" | sed 's/.cfg$//')
      if [ ! -L $LNK ] && [ -d $LNK ]; then
         echo "WARNING: Found directory at '$LNK' when symlink was expected. Fixing..."
         \rm -rf $LNK
         if [ -d $LNK ]; then
            \mv $LNK $LNK.clean_me
         fi
      fi
      if [ -L $LNK ]; then
         DEST=$(readlink $LNK)
         if [ "$DEST" != "$MY_WORKSPACE" ] || [ ! -d "$MY_WORKSPACE" ]; then
            echo "WARNING: Found broken symlink at '$LNK'. Fixing..."
            \rm -f $LNK
         fi
      fi
      if [ ! -L $LNK ]; then
         if [ ! -d "$MY_WORKSPACE" ]; then
            echo "ERROR: Can't create symlink from $LNK to $MY_WORKSPACE as destination does not exist."
            exit 1
         fi
         ln -s $MY_WORKSPACE $LNK
      fi
   fi

   if [ -d /localdisk/loadbuild/mock-cache ]; then
      mkdir -p $MY_WORKSPACE/cache
      LNK=$(echo "/localdisk/loadbuild/mock-cache/$(basename $CFG)" | sed 's/.cfg$//')
      if [ ! -L $LNK ] && [ -d $LNK ]; then
         echo "WARNING: Found directory at '$LNK' when symlink was expected. Fixing..."
         \rm -rf $LNK
         if [ -d $LNK ]; then
            \mv $LNK $LNK.clean_me
         fi
      fi
      if [ -L $LNK ]; then
         DEST=$(readlink $LNK)
         if [ "$DEST" != "$MY_WORKSPACE/cache" ] || [ ! -d "$MY_WORKSPACE/cache" ]; then
            echo "WARNING: Found broken symlink at '$LNK'. Fixing..."
            \rm -f $LNK
         fi
      fi
      if [ ! -L $LNK ]; then
         if [ ! -d "$MY_WORKSPACE/cache" ]; then
            echo "ERROR: Can't create symlink from $LNK to $MY_WORKSPACE/cache as destination does not exist."
            exit 1
         fi
         ln -s $MY_WORKSPACE/cache $LNK
      fi
   fi
}

remove_mock_symlinks () {
   local LNK
   local CFG=$1
   if [ -d /localdisk/loadbuild/mock ]; then
      LNK=$(echo "/localdisk/loadbuild/mock/$(basename $CFG)" | sed 's/.cfg$//')
      if [ -L $LNK ]; then
         \rm -f $LNK
      fi
      if [ -d $LNK ]; then
         \rm -rf $LNK
         if [ $? -ne 0 ]; then
            \mv -f $LNK $LNK.clean_me
         fi
      fi
   fi

   if [ -d /localdisk/loadbuild/mock-cache ]; then
      LNK=$(echo "/localdisk/loadbuild/mock-cache/$(basename $MY_BUILD_CFG)" | sed 's/.cfg$//')
      if [ -L $LNK ]; then
         \rm -f $MY_WORKSPACE/cache $LNK
      fi
      if [ -d $LNK ]; then
         \rm -rf $LNK
         if [ $? -ne 0 ]; then
            \mv -f $LNK $LNK.clean_me
         fi
      fi
   fi
}

umount_mock_root_as_tmpfs_all () {
    for i in $(seq 0 $((ABSOLUTE_MAX_WORKERS-1))); do
        umount_mock_root_as_tmpfs $i
    done
}

umount_mock_root_as_tmpfs_cfg () {
    local CFG=$1
    local build_idx=$(basename $CFG | sed 's#.*[.]b\([0-9]*\)[.]cfg#\1#')
    if [ "$build_idx" != "" ]; then
        umount_mock_root_as_tmpfs $build_idx
    else
        echo "umount_mock_root_as_tmpfs_cfg: Failed to map '$CFG' to a build_idx"
    fi
}

umount_mock_root_as_tmpfs () {
   local build_idx=$1
   local mount_dir=$(readlink -f $MY_WORKSPACE/mock)/b${build_idx}/root
   local rc

   mount | grep tmpfs | grep $mount_dir &> /dev/null
   if [ $? -ne 0 ]; then
      return 0
   fi
   mock_tmpfs_umount $mount_dir &> /dev/null
   
   rc=$?
   if [ $rc -ne 0 ]; then
      echo "FAILED: mock_tmpfs_umount $mount_dir"
   fi
   return $rc
}

kill_descendents ()
{
    local kill_pid=$1
    local kill_all=$2
    local need_stop=$3
    local iteration=$4
    local ret=0
    local rc=0

    # echo "kill_descendents pid=$kill_pid, all=$kill_all stop=$need_stop, iteration=$iteration"

    local relevant_recursive_children="$ME"
    local relevant_recursive_promote_children="mock"
    local relevant_other_children="mockchain-parallel"

    local recursive_promote_children=$(for relevant_child in $relevant_recursive_promote_children; do pgrep -P $kill_pid $relevant_child; done)
    local recursive_children=$(for relevant_child in $relevant_recursive_children; do pgrep -P $kill_pid $relevant_child; done)
    local other_children=""

    if [ $kill_all -eq 1 ]; then
        recursive_promote_children=""
        recursive_children=$(pgrep -P $kill_pid)
    fi
    
    if [ $iteration -eq 0 ]; then
        other_children=$(for relevant_child in $relevant_other_children; do pgrep -P $kill_pid $relevant_child; done)
        if [ "$other_children" != "" ]; then
            ret=1
        fi
    fi

    if [ $need_stop -eq 1 ]; then
        for pid in $recursive_children $recursive_promote_children; do
            kill -SIGSTOP $pid &> /dev/null
        done
    fi

    for pid in $recursive_children; do
        kill_descendents "$pid" $kill_all $need_stop $((iteration + 1)) 
    done
    for pid in $recursive_promote_children; do
        kill_descendents "$pid" 1 1 $((iteration + 1))
    done

    # echo "kill: $recursive_children $recursive_promote_children"
    for pid in $recursive_children $recursive_promote_children; do
        kill $pid &> /dev/null
        rc=$?
        if [ $need_stop -eq 1 ]; then
            kill -SIGCONT $pid &> /dev/null
        fi
        if [ $rc -eq 0 ] && [ $iteration -eq 0 ]; then
            wait $pid
        fi
    done

    # echo "kill: $other_children"
    for pid in $other_children; do
        kill $pid &> /dev/null
        rc=$?
        if [ $rc -eq 0 ] && [ $iteration -eq 0 ]; then
            wait $pid
        fi
    done

    return $ret
}

function my_exit_n() {
    local need_mock_cleanup
    # echo "$BASHPID: $ME: my_exit: killing children"
    local need_mock_cleanup
    kill_descendents  $BASHPID 0 0 0
    need_mock_cleanup=$?
    # echo "$BASHPID: $ME: my_exit: waiting"
    wait
    # echo "$BASHPID: $ME: my_exit: wait complete"
    # echo "$BASHPID: $ME: my_exit: need_mock_cleanup=$need_mock_cleanup"
    if [ $need_mock_cleanup -ne 0 ]; then
        umount_mock_root_as_tmpfs_all
    fi
}

function my_exit() {
    local need_mock_cleanup
    # echo "$BASHPID: $ME: my_exit: killing children"
    local need_mock_cleanup
    kill_descendents  $BASHPID 0 0 0
    need_mock_cleanup=$?
    # echo "$BASHPID: $ME: my_exit: waiting"
    wait
    # echo "$BASHPID: $ME: my_exit: wait complete"
    # echo "$BASHPID: $ME: my_exit: need_mock_cleanup=$need_mock_cleanup"
    if [ $need_mock_cleanup -ne 0 ]; then
        sleep 1
    fi
    umount_mock_root_as_tmpfs_all
}

function my_sigint_n() {
    local ARG=$1
    echo "$BASHPID: $ME: my_sigint_n: ARG=$ARG"
    echo "$BASHPID: $ME: my_sigint_n: killing children"
    local need_mock_cleanup
    kill_descendents  $BASHPID 0 0 0
    need_mock_cleanup=$?
    echo "$BASHPID: $ME: my_sigint_n: waiting"
    wait
    echo "$BASHPID: $ME: my_sigint_n: wait complete"
    if [ $need_mock_cleanup -ne 0 ]; then
        umount_mock_root_as_tmpfs_cfg $ARG
    fi
    exit 1
}

function my_sighup_n() {
    local ARG=$1
    echo "$BASHPID: $ME: my_sighup_n: ARG=$ARG"
    echo "$BASHPID: $ME: my_sighup_n: killing children"
    local need_mock_cleanup
    kill_descendents  $BASHPID 0 0 0
    need_mock_cleanup=$?
    echo "$BASHPID: $ME: my_sighup_n: waiting"
    wait
    echo "$BASHPID: $ME: my_sighup_n: wait complete"
    if [ $need_mock_cleanup -ne 0 ]; then
        umount_mock_root_as_tmpfs_cfg $ARG
    fi
    exit 1
}

function my_sigabrt_n() {
    local ARG=$1
    echo "$BASHPID: $ME: my_sigabrt_n: ARG=$ARG"
    echo "$BASHPID: $ME: my_sigabrt_n: killing children"
    local need_mock_cleanup
    kill_descendents  $BASHPID 0 0 0
    need_mock_cleanup=$?
    echo "$BASHPID: $ME: my_sigabrt_n: waiting"
    wait
    echo "$BASHPID: $ME: my_sigabrt_n: wait complete"
    if [ $need_mock_cleanup -ne 0 ]; then
        umount_mock_root_as_tmpfs_cfg $ARG
    fi
    exit 1
}

function my_sigterm_n() {
    local ARG=$1
    echo "$BASHPID: $ME: my_sigterm_n: ARG=$ARG"
    echo "$BASHPID: $ME: my_sigterm_n: killing children"
    local need_mock_cleanup
    kill_descendents  $BASHPID 0 0 0
    need_mock_cleanup=$?
    echo "$BASHPID: $ME: my_sigterm_n: waiting"
    wait
    echo "$BASHPID: $ME: my_sigterm_n: wait complete"
    echo "$BASHPID: $ME: my_sigterm_n: need_mock_cleanup=$need_mock_cleanup"
    if [ $need_mock_cleanup -ne 0 ]; then
        umount_mock_root_as_tmpfs_cfg $ARG
    fi
    exit 1
}

function my_sigint() {
    echo "$BASHPID: $ME: my_sigint: killing children"
    local need_mock_cleanup
    kill_descendents  $BASHPID 0 0 0
    need_mock_cleanup=$?
    echo "$BASHPID: $ME: my_sigterm_n: waiting"
    wait
    echo "$BASHPID: $ME: my_sigterm_n: wait complete"
    if [ $need_mock_cleanup -ne 0 ]; then
        umount_mock_root_as_tmpfs_all
    fi
    exit 1
}

function my_sighup() {
    echo "$BASHPID: $ME: my_sighup: killing children"
    local need_mock_cleanup
    kill_descendents  $BASHPID 0 0 0
    need_mock_cleanup=$?
    echo "$BASHPID: $ME: my_sighup: waiting"
    wait
    echo "$BASHPID: $ME: my_sighup: wait complete"
    if [ $need_mock_cleanup -ne 0 ]; then
        umount_mock_root_as_tmpfs_all
    fi
    exit 1
}

function my_sigabrt() {
    echo "$BASHPID: $ME: my_sigabrt: killing children"
    local need_mock_cleanup
    kill_descendents  $BASHPID 0 0 0
    need_mock_cleanup=$?
    echo "$BASHPID: $ME: my_sigabrt: waiting"
    wait
    echo "$BASHPID: $ME: my_sigabrt: wait complete"
    if [ $need_mock_cleanup -ne 0 ]; then
        umount_mock_root_as_tmpfs_all
    fi
    exit 1
}

function my_sigterm() {
    echo "$BASHPID: $ME: my_sigterm: killing children"
    local need_mock_cleanup
    kill_descendents  $BASHPID 0 0 0
    need_mock_cleanup=$?
    echo "$BASHPID: $ME: my_sigterm: waiting"
    wait
    echo "$BASHPID: $ME: my_sigterm: wait complete"
    echo "$BASHPID: $ME: my_sigterm: need_mock_cleanup=$need_mock_cleanup"
    if [ $need_mock_cleanup -ne 0 ]; then
        umount_mock_root_as_tmpfs_all
    fi
    exit 1
}

trapwrap() {
    local WCMD=$1
    shift
    declare -i pid status=255
    # set the trap for the foreground process
    trap my_sigint INT
    trap my_sighup HUP
    trap my_sigabrt ABRT
    trap my_sigterm TERM
    # run the command in background
    ### "$@" & pid=$!
    WARGS=()
    x=0
    for i in "$@"; do
        WARGS[$x]="$i"
        x=$((x+1))
    done
    echo "$WCMD ${WARGS[@]/#/}"
    $WCMD "${WARGS[@]/#/}" & pid=$!
    # wait until bg command finishes, handling interruptions by trapped signals
    while (( status > 128 )); do
        wait $pid
        status=$?
    done
    # restore the trap
    trap - INT
    trap - HUP
    trap - ABRT
    trap - TERM
    # return the command exit status
    return $status
}

trapwrap_n() {
    local ARG=$1
    shift
    local WCMD=$1
    shift
    declare -i pid status=255
    # set the trap for the foreground process
    trap my_exit_n EXIT
    trap "my_sigint_n $ARG" INT
    trap "my_sighup_n $ARG" HUP
    trap "my_sigabrt_n $ARG" ABRT
    trap "my_sigterm_n $ARG" TERM
    # run the command in background
    WARGS=()
    x=0
    for i in "$@"; do
        WARGS[$x]="$i"
        x=$((x+1))
    done
    echo "$WCMD ${WARGS[@]/#/}"
    $WCMD "${WARGS[@]/#/}" & pid=$!
    # wait until bg command finishes, handling interruptions by trapped signals
    while (( status > 128 )); do
        wait $pid
        status=$?
    done
    # restore the trap
    trap - INT
    trap - HUP
    trap - ABRT
    trap - TERM
    # return the command exit status
    return $status
}

trap my_exit EXIT

mock_get_cache_dir () {
      local CFG=$1
      local CACHE_DIR="$MY_WORKSPACE/cache"
      local CACHE_LINE=$(grep "config_opts[[][']cache_topdir['][]]" $CFG)
      if [ $? -eq 0 ]; then
         CACHE_DIR=$(echo "$CACHE_LINE" | awk -F \' '{ print $4 }')
      fi
      echo "$CACHE_DIR"
}

mock_get_root_dir () {
      local CFG=$1
      local ROOT_DIR="$MY_WORKSPACE/mock"
      local ROOT_LINE=$(grep "config_opts[[][']root['][]]" $CFG)
      if [ $? -eq 0 ]; then
         ROOT_DIR="$MY_WORKSPACE/"$(echo "$ROOT_LINE" | awk -F \' '{ print $4 }')
      fi
      echo "$ROOT_DIR"
}

mock_clean_cfg () {
      local CFG=$1
      echo "${FUNCNAME[0]}: $CFG"
      echo "=================================="
      mock_clean_cache_cfg $CFG
      echo "=================================="
      echo "$MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all"
      trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all
      echo "=================================="
      echo "$MOCK -r $CFG --configdir $(dirname $CFG) --clean"
      trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --clean
      ### Note:  this sometimes leaves behind a $MY_WORKSPACE/cache/mock/yum_cache/yumcache.lock
      echo "=================================="
      mock_clean_cache_all_cfg $CFG
      echo "=================================="
}

mock_sub_configs () {
   find $MY_WORKSPACE/configs/$MY_BUILD_ENVIRONMENT* -name "$MY_BUILD_ENVIRONMENT*b[0-9]*.cfg"
}

mock_clean () {
   echo "${FUNCNAME[0]}: in"
   echo "=================================="
   remove_mock_symlinks $MY_BUILD_CFG
   set_mock_symlinks $MY_BUILD_CFG
   echo "=================================="
   for SUB_CFG in $(mock_sub_configs); do
      local PREFIX=$(echo $SUB_CFG | rev | cut -d . -f 2 | rev)
      ( mock_clean_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
   done
   wait
   # mock_clean_cfg $BUILD_CFG
   echo "=================================="
   remove_mock_symlinks $MY_BUILD_CFG
   echo "${FUNCNAME[0]}: out"
}

mock_partial_clean_cfg () {
   local CFG=$1
   local SRPMS_LIST="$2"
   local RPMS_LIST="$3"
   local CMD
   local TMP
   local RC

   echo "${FUNCNAME[0]}: CFG=$CFG  SRPMS_LIST='$SRPMS_LIST'  RPMS_LIST='$RPMS_LIST'"

   TMP=$(mktemp /tmp/mock_partial_clean_cfg_XXXXXX)
   if [ $? -ne 0 ]; then
      echo "${FUNCNAME[0]}: mktemp failed"
      return 1
   fi

   local ROOT_DIR=$(mock_get_root_dir $CFG)

   if [ -d $ROOT_DIR/root/builddir/build/SOURCES ]; then
      echo "rm -rf $ROOT_DIR/root/builddir/build/SOURCES/*"
      \rm -rf $ROOT_DIR/root/builddir/build/SOURCES/* 2>> /dev/null
   fi

   if [ -d $ROOT_DIR/root/builddir/build/SPECS ]; then
      echo "rm -rf $ROOT_DIR/root/builddir/build/SPECS/*"
      \rm -rf $ROOT_DIR/root/builddir/build/SPECS/* 2>> /dev/null
   fi

   for s in $SRPMS_LIST; do
      f=$(basename $s)
      if [ -f $ROOT_DIR/root/builddir/build/SRPMS/$f ]; then
         \rm -f -v $ROOT_DIR/root/builddir/build/SRPMS/$f 2>> /dev/null
      fi
      if [ -f $ROOT_DIR/root/builddir/build/originals/$f ]; then
         \rm -f -v $ROOT_DIR/root/builddir/build/originals/$f 2>> /dev/null
      fi
   done

   for r in $RPMS_LIST; do
      for d in $(find $ROOT_DIR/root/builddir/build/BUILD/ -maxdepth 1 -name '$r*' 2>> /dev/null); do
         echo "rm -rf $d"
         \rm -rf $d 2>> /dev/null
      done
      if [ -d $ROOT_DIR/root/builddir/build/RPMS ]; then
         for f in $(find $ROOT_DIR/root/builddir/build/RPMS -maxdepth 1 -name "$r*rpm" 2>> /dev/null); do
            \rm -f -v $f 2>> /dev/null
         done
      fi
   done


   local NO_CLEAN_LIST=$(create-no-clean-list)
   echo "NO_CLEAN_LIST=$NO_CLEAN_LIST"

   local RPMS_CLEAN_LIST=""
   local NEED_FULL_MOCK_CLEAN=0
   for r in $RPMS_LIST; do
       if ! str_lst_contains $r "$NO_CLEAN_LIST" ; then 
           RPMS_CLEAN_LIST=$(join_by ' ' $RPMS_CLEAN_LIST $r)
       else 
           echo "Can't remove '$r' from mock environment without a wipe"; 
           NEED_FULL_MOCK_CLEAN=1
       fi
   done

   if [ $NEED_FULL_MOCK_CLEAN -eq 1 ]; then
       echo "Wipe the mock environment"
       mock_clean_cfg $CFG
       RC=$?
   else
       # Intent of following is for $RPMS_LIST to be expand now while the remaining $ varaibles are for bash inside mock to expand
       echo "Try to uninstall from the mock environment these packages: $RPMS_CLEAN_LIST"
       CMD='LST="'$RPMS_CLEAN_LIST'";
            DELETE_LIST=""; 
            for r in $LST; do 
                  FOUND=$(rpm  -q $r) ; 
                  if [ $? -eq 0 ]; then 
                     DELETE_LIST="$DELETE_LIST $FOUND"; 
                  fi; 
            done; 
            echo "uninstalling these packages: $DELETE_LIST"; 
            if [ "$DELETE_LIST" != "" ]; then
                rpm  -e --nodeps $DELETE_LIST;
            fi'
       echo "$MOCK -r $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP
       trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --chroot "bash -c '$CMD'" &>> $TMP
       RC=$?
       if [ $RC -ne 0 ]; then
           cat $TMP
           \rm -f $TMP
           return $RC
       fi

       mock_clean_cache_cfg $CFG
       RC=$?
       \rm -f $TMP
   fi

   return $RC
}

mock_partial_clean () {
   local SRPMS_LIST="$1"
   local RPMS_LIST="$2"
   echo "${FUNCNAME[0]}: in"
   echo "${FUNCNAME[0]}: '$SRPMS_LIST'  '$RPMS_LIST'"
   echo "=================================="
   local NO_CLEAN_LIST=$(create-no-clean-list)
   echo "=================================="
   for SUB_CFG in $(mock_sub_configs); do
      local PREFIX=$(echo $SUB_CFG | rev | cut -d . -f 2 | rev)
      ( mock_partial_clean_cfg $SUB_CFG "$SRPMS_LIST" "$RPMS_LIST" 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
   done
   wait
   # mock_partial_clean_cfg $BUILD_CFG "$SRPMS_LIST" "$RPMS_LIST"
   echo "=================================="
   echo "${FUNCNAME[0]}: out"
}

mock_clean_cache_cfg () {
   local CFG=$1
   local TMP
   local RC

   echo "${FUNCNAME[0]}: $CFG  '$SRPMS_LIST'  '$RPMS_LIST'"

   TMP=$(mktemp /tmp/mock_clean_cache_cfg_XXXXXX)
   if [ $? -ne 0 ]; then
      echo "${FUNCNAME[0]}: mktemp failed"
      return 1
   fi

   echo "${FUNCNAME[0]}: $CFG"

   clean_yum_cache_cfg $CFG

   echo "$MOCK -r $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache" &> $TMP
   trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache &>> $TMP
   RC=$?
   if [ $RC -ne 0 ]; then
      cat $TMP
   fi

   \rm -f $TPM
   return $RC
}

mock_clean_cache () {
   echo "${FUNCNAME[0]}: in"
   for SUB_CFG in $(mock_sub_configs); do
      local PREFIX=$(echo $SUB_CFG | rev | cut -d . -f 2 | rev)
      ( mock_clean_cache_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
   done
   wait
   # mock_clean_cache_cfg $BUILD_CFG
   echo "${FUNCNAME[0]}: out"
}

mock_clean_cache_all_cfg () {
   local CFG=$1

   echo "${FUNCNAME[0]}: $CFG"
   echo "=================================="
   clean_yum_cache_cfg $CFG
   echo "=================================="
   echo "$MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all"
   trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all
   echo "=================================="
}

mock_clean_cache_all () {
   echo "${FUNCNAME[0]}: in"
   for SUB_CFG in $(mock_sub_configs); do
      local PREFIX=$(echo $SUB_CFG | rev | cut -d . -f 2 | rev)
      ( mock_clean_cache_all_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
   done
   wait
   # mock_clean_cache_all_cfg $BUILD_CFG
   echo "${FUNCNAME[0]}: out"
}

mock_clean_metadata_cfg () {
   local CFG=$1
   local TMP
   local RC

   echo "${FUNCNAME[0]}: $CFG"

   TMP=$(mktemp /tmp/mock_partial_clean_cfg_XXXXXX)
   if [ $? -ne 0 ]; then
      echo "${FUNCNAME[0]}: mktemp failed"
      return 1
   fi

   CMD=$((cat $CFG; grep config_opts\\[\'yum.conf\'\\\] $CFG | sed 's#\\n#\n#g') | grep '^[[]' | grep -v main | sed 's/[][]//g' | sed 's#^#yum --enablerepo=#' | sed 's#$# clean metadata#' | sort -u | tr '\n' ';')
   echo "$MOCK -r $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP
   trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --chroot "bash -c '($CMD)'" &>>$TMP
   RC=$?
   if [ $RC -ne 0 ]; then
      cat $TMP
   fi
   \rm -f $TMP
   return $RC
}

mock_clean_metadata () {
   echo "${FUNCNAME[0]}: in"
   for SUB_CFG in $(mock_sub_configs); do
      local PREFIX=$(echo $SUB_CFG | rev | cut -d . -f 2 | rev)
      ( mock_clean_metadata_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
   done
   wait
   echo "${FUNCNAME[0]}: out"
}

update_cgcs_repo () {
   local REPO_NAME=$1
   (
    cd $MY_REPO/$REPO_NAME/

    local CURR_HEAD=$(git rev-parse HEAD)
    local LAST_HEAD_FILE="$MY_REPO/$REPO_NAME/.last_head"
    local LAST_HEAD_FILE_OLD="$MY_WORKSPACE/$REPO_NAME.last_head"
    local CHANGED
    local NEW_UNTRACKED
    local NEED_REBUILD
    local NEED_MOCK_CLEAN=0
    local d

    if [ -f LAST_HEAD_FILE_OLD -a ! -f LAST_HEAD_FILE ]; then
       \cp LAST_HEAD_FILE_OLD LAST_HEAD_FILE
    fi

    local LAST_HEAD=$(cat $LAST_HEAD_FILE | head -n 1)

    for d in "Binary" "Source"; do
       NEED_REBUILD=0
       if [ ! -d $d/repodata ]; then
          NEED_REBUILD=1
       fi
       if [ "$CURR_HEAD" != "$LAST_HEAD" ]; then
          NEED_REBUILD=1
       fi

       CHANGED=$(git diff --name-only | grep $d)
       if [ "x$CHANGED" != "x" ]; then
          NEED_REBUILD=1
       fi

       NEW_UNTRACKED=$(git ls-files . --exclude-standard --others | grep $d)
       if [ "x$NEW_UNTRACKED" != "x" ]; then
          NEED_REBUILD=1
       fi
    
       if [ $NEED_REBUILD -eq 1 ]; then
          NEED_MOCK_CLEAN=1
          echo ""
          echo "Need to recreate $REPO_NAME/$d/repodata"
          mkdir -p $d
          if [ -d $d/repodata ]; then
             update_repodata "$d"
          else
             recreate_repodata "$d"
          fi
       fi
    done
    echo "$CURR_HEAD" > $LAST_HEAD_FILE
    \cp $LAST_HEAD_FILE $LAST_HEAD_FILE_OLD
    if [ $NEED_MOCK_CLEAN -eq 1 ]; then
      echo ""
      echo "Need to clean mock"
      mock_clean
      set_mock_symlinks $MY_BUILD_CFG
    fi
   )
}

mock_clean_mounts_dir () {
   local MOUNT=$1
   local RC

   if [ "$MOUNT" == "" ]; then
      return 1
   fi
   mount | grep "$MOUNT" >> /dev/null
   if [ $? -eq 0 ]; then
      RC=1
      which mock_cache_umount >> /dev/null
      if [ $? -eq 0 ]; then
         echo "umount '$MOUNT'"
         mock_cache_umount "$MOUNT"
         if [ $? -eq 0 ]; then
            RC=0
         fi
      fi
      if [ $RC -eq 1 ]; then
         echo "ERROR: Directory '$MOUNT' is already mounted and will cause a build failure within mock." 
         echo "Ask your system administrator to umount '$MOUNT'."
         exit 1
      fi
   fi
   return 0
}

mock_clean_mounts_cfg () {
   local CFG=$1
   local ROOT_DIR=$(mock_get_root_dir $CFG)
   local YUM_CACHE_MOUNT=$(readlink -f "$ROOT_DIR/root/var/cache/yum")
   local PROC_MOUNT=$(readlink -f "$ROOT_DIR/root/proc")
   local SYS_MOUNT=$(readlink -f "$ROOT_DIR/root/sys")
   local SHM_MOUNT=$(readlink -f "$ROOT_DIR/root/dev/shm")
   local PTS_MOUNT=$(readlink -f "$ROOT_DIR/root/dev/pts")
   local MOUNT

   echo "${FUNCNAME[0]}: $CFG"
   for MOUNT in "$YUM_CACHE_MOUNT" "$PROC_MOUNT" "$SYS_MOUNT" "$SHM_MOUNT" "$PTS_MOUNT"; do
      mock_clean_mounts_dir "$MOUNT"
   done
}

mock_clean_mounts () {
   echo "${FUNCNAME[0]}: in"
   for SUB_CFG in $(mock_sub_configs); do
      local PREFIX=$(echo $SUB_CFG | rev | cut -d . -f 2 | rev)
      ( mock_clean_mounts_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
   done
   wait
   # mock_clean_mounts_cfg $BUILD_CFG
   echo "${FUNCNAME[0]}: out"
}

clean_yum_cache_cfg () {
   local CFG=$1
   local CACHE_DIR=$(mock_get_cache_dir $CFG)
   local ROOT_DIR=$(mock_get_root_dir $CFG)
   local RC=0

   echo "${FUNCNAME[0]}: $CFG"

   local YUM_CACHE_MOUNT=$(readlink -f "$ROOT_DIR/root/var/cache/yum")
   local YUM_CACHE_LOCK="$CACHE_DIR/mock/yum_cache/yumcache.lock"
   # echo "clean_yum_cache YUM_CACHE_MOUNT='$YUM_CACHE_MOUNT' YUM_CACHE_LOCK='$YUM_CACHE_LOCK'"
   
   if [ "$YUM_CACHE_MOUNT" != "" ]; then
      mock_clean_mounts_dir "$YUM_CACHE_MOUNT"
   fi

   if [ -f "$YUM_CACHE_LOCK" ]; then
      RC=1
      which mock_cache_unlock >> /dev/null
      if [ $? -eq 0 ]; then
         mock_cache_unlock "$YUM_CACHE_LOCK"
         if [ $? -eq 0 ]; then
            RC=0
         fi
      fi
      if [ $RC -eq 1 ]; then
         echo "ERROR: File '$YUM_CACHE_LOCK' exists and will cause a build failure within mock."
         echo "Ask your system administrator to delete '$YUM_CACHE_LOCK'."
         exit 1
      fi
   fi
   return $RC
}


clean_yum_cache () {
   echo "${FUNCNAME[0]}: in"
   for SUB_CFG in $(mock_sub_configs); do
      local PREFIX=$(echo $SUB_CFG | rev | cut -d . -f 2 | rev)
      ( clean_yum_cache_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
   done
   wait
   # clean_yum_cache_cfg $BUILD_CFG
   echo "${FUNCNAME[0]}: out"
}

mock_update_cfg () {
   local CFG=$1
   echo "${FUNCNAME[0]}: $CFG"
   echo "=================================="
   set_mock_symlinks $CFG
   echo "$MOCK -r $CFG --configdir $(dirname $CFG) --update"
   trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --update 
   echo "=================================="
}

mock_init_cfg () {
   local CFG=$1
   echo "${FUNCNAME[0]}: $CFG"
   echo "=================================="
   set_mock_symlinks $CFG
   echo "$MOCK -r $CFG --configdir $(dirname $CFG) --init"
   trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --init
   echo "=================================="
}

mock_update_or_init_cfg () {
   local CFG=$1
   local TMP
   local RC
   echo "${FUNCNAME[0]}: $CFG"
   local ROOT_DIR=$(mock_get_root_dir $CFG)

   TMP=$(mktemp /tmp/mock_update_or_init_cfg_XXXXXX)
   if [ $? -ne 0 ]; then
      echo "${FUNCNAME[0]}: mktemp failed"
      return 1
   fi
   if [ -d $ROOT_DIR/root ]; then
      echo "Updating the mock environment"
      set_mock_symlinks $CFG
      echo "$MOCK -r $CFG --configdir $(dirname $CFG) --update"
      trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --update  &> $TMP
      RC=$?
   else
      echo "Init the mock environment"
      set_mock_symlinks $CFG
      echo "$MOCK -r $CFG --configdir $(dirname $CFG) --init"
      trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --init &> $TMP
      RC=$?
   fi
   if [ $RC -ne 0 ]; then
      cat $TMP
   fi
   \rm -f $TMP
   return $RC
}

mock_update_or_init () {
   echo "${FUNCNAME[0]}: in"
   for SUB_CFG in $(mock_sub_configs); do
      local PREFIX=$(echo $SUB_CFG | rev | cut -d . -f 2 | rev)
      ( mock_update_or_init_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
   done
   wait 
   # mock_update_or_init_cfg $BUILD_CFG
   echo "${FUNCNAME[0]}: out"
}

if [ "x$PROJECT" == "x" ]; then
    echo "PROJECT environmnet variable is not defined."
    exit 1
fi

if [ "x$SRC_BUILD_ENVIRONMENT" == "x" ]; then
    echo "SRC_BUILD_ENVIRONMENT environmnet variable is not defined."
    exit 1
fi

NO_DESCENDANTS=0
NO_REQUIRED=0
NO_AUTOCLEAN=0
NO_BUILD_INFO=0
HELP=0 
CLEAN_FLAG=0
FORMAL_FLAG=0
CAREFUL=0
   
# read the options
TEMP=$(getopt -o ha::bc: --long std,rt,installer,no-required,no-descendants,no-autoclean,no-build-info,clean,tmpfs-clean,formal,careful,help,arga::,argb,argc: -n "$ME" -- "$@")

if [ $? -ne 0 ]; then
    usage
    exit 1
fi

eval set -- "$TEMP"

export BUILD_TYPE=std
trap my_exit EXIT

# extract options and their arguments into variables.
while true ; do
    case "$1" in
        -a|--arga)
            case "$2" in
                "") ARG_A='some default value' ; shift 2 ;;
                *) ARG_A=$2 ; shift 2 ;;
            esac ;;
        -b|--argb) ARG_B=1 ; shift ;;
        --careful) CAREFUL=1 ; shift ;;
        --no-descendants) NO_DESCENDANTS=1 ; shift ;;
        --no-required) NO_REQUIRED=1 ; shift ;;
        --no-autoclean) NO_AUTOCLEAN=1; shift ;;
        --no-build-info) NO_BUILD_INFO=1; shift ;;
        --formal) FORMAL_FLAG=1; shift ;;
        --std) BUILD_TYPE=std; shift ;;
        --rt) BUILD_TYPE=rt; shift ;;
        --installer) BUILD_TYPE=installer; shift ;;
        -h|--help) HELP=1 ; shift ;;
        --clean) CLEAN_FLAG=1 ; shift ;;
        --tmpfs-clean) if [ -n "$MY_WORKSPACE" ]; then export MY_WORKSPACE=$MY_WORKSPACE/$BUILD_TYPE; exit 0; fi ;;
        -c|--argc)
            case "$2" in
                "") shift 2 ;;
                *) ARG_C=$2 ; shift 2 ;;
            esac ;;
        --) shift ; break ;;
        *) echo "Internal error!" ; exit 1 ;;
    esac
done 


# Reset variables
if [ -n "$MY_WORKSPACE" ]; then
   export MY_WORKSPACE_TOP=${MY_WORKSPACE_TOP:-$MY_WORKSPACE}
   export MY_WORKSPACE=$MY_WORKSPACE_TOP/$BUILD_TYPE
else
   export MY_PATCH_WORKSPACE_TOP=${MY_PATCH_WORKSPACE_TOP:-$MY_PATCH_WORKSPACE}
   export MY_PATCH_WORKSPACE=$MY_PATCH_WORKSPACE_TOP/$BUILD_TYPE
fi

export MY_BUILD_DIR_TOP=${MY_BUILD_DIR_TOP:-$MY_BUILD_DIR}
export MY_BUILD_DIR=$MY_BUILD_DIR_TOP/$BUILD_TYPE

export MY_BUILD_ENVIRONMENT_TOP=${MY_BUILD_ENVIRONMENT_TOP:-$MY_BUILD_ENVIRONMENT}
export MY_BUILD_ENVIRONMENT=$MY_BUILD_ENVIRONMENT_TOP-$BUILD_TYPE

export MY_SRC_RPM_BUILD_DIR=$MY_BUILD_DIR/rpmbuild
export MY_BUILD_ENVIRONMENT_FILE=$MY_BUILD_ENVIRONMENT.cfg
export MY_BUILD_CFG=$MY_WORKSPACE/$MY_BUILD_ENVIRONMENT_FILE
export MY_MOCK_ROOT=$MY_WORKSPACE/mock/root

IMAGE_INC_FILE="${MY_WORKSPACE}/image.inc"
image_inc_list iso ${BUILD_TYPE} ${DISTRO} > "${IMAGE_INC_FILE}"

LAST_PLATFORM_RELEASE_FILE="$MY_BUILD_DIR/.platform_release"

TARGETS=$@

if [ $HELP -eq 1 ]; then
    usage
    exit 0
fi

if [ $FORMAL_FLAG -eq 1 ]; then
   export FORMAL_BUILD=1
fi

SRC_ROOT="$MY_REPO"
if [ "x$MY_REPO" == "x" ]; then
   SRC_ROOT=$HOME
fi

BUILD_ROOT="$MY_WORKSPACE"
if [ "x$MY_WORKSPACE" == "x" ]; then
   BUILD_ROOT="$MY_PATCH_WORKSPACE"

   if [ "x$MY_PATCH_WORKSPACE" == "x" ]; then
       echo "ERROR: reqiure one of MY_WORKSPACE or MY_PATCH_WORKSPACE be defined"
       exit 1
   fi
fi

export BUILD_BASE="$BUILD_ROOT"
export CCACHE_DIR="$BUILD_ROOT/.ccache"
export RESULT_DIR="$BUILD_BASE/results"
export SRC_BASE="$SRC_ROOT"
export STX_BASE=$SRC_BASE/stx

if [ "x$MY_SRC_RPM_BUILD_DIR" != "x" ]; then
    RPM_BUILD_ROOT=$MY_SRC_RPM_BUILD_DIR
else
    RPM_BUILD_ROOT=$BUILD_BASE/rpmbuild
fi

RELEASE_INFO_FILE=$STX_BASE/stx-integ/utilities/build-info/release-info.inc
if [ -f $RELEASE_INFO_FILE ]; then
   source $RELEASE_INFO_FILE
else
   echo "ERROR: failed to find RELEASE_INFO_FILE=$RELEASE_INFO_FILE"
   exit 1
fi

if [ "x$PLATFORM_RELEASE" == "x" ]; then
   echo "ERROR: PLATFORM_RELEASE is not defined in $RELEASE_INFO_FILE"
   exit 1
fi

export RPM_BUILD_BASE="$RPM_BUILD_ROOT"
export SRPM_OUT="$RPM_BUILD_BASE/SRPMS"
export RPM_DIR="$RPM_BUILD_BASE/RPMS"
export SPECS_DIR="$RPM_BUILD_BASE/SPECS"
export SOURCES_DIR="$RPM_BUILD_BASE/SOURCES"
export PLATFORM_RELEASE

if [ ! -d $BUILD_BASE ]; then
   echo "ERROR: expected to find directory at '$BUILD_BASE'"
   exit 1
fi


mkdir -p $RPM_BUILD_BASE
if [ $? -ne 0 ]; then
   echo "ERROR: Failed to create directory '$RPM_BUILD_BASE'"
   exit 1
fi

mkdir -p $SRPM_OUT/repodata
if [ $? -ne 0 ]; then
   echo "ERROR: Failed to create directory '$SRPM_OUT/repodata'"
   exit 1
fi

mkdir -p $RPM_DIR/repodata
if [ $? -ne 0 ]; then
   echo "ERROR: Failed to create directory '$RPM_DIR/repodata'"
   exit 1
fi

if [ "x$MY_BUILD_CFG" == "x" ];then
   echo "ERROR: reqiure MY_BUILD_CFG to be defined"
   exit 1
fi

export BUILD_CFG="$MY_BUILD_CFG"

# Place build-time environement variables in mock environment
echo "FORMAL_BUILD=$FORMAL_BUILD"
echo "modify-build-cfg $BUILD_CFG"
${BUILD_RPMS_PARALLEL_DIR}/modify-build-cfg $BUILD_CFG
if [ $? -ne 0 ]; then
       echo "Could not modifiy $BUILD_CFG";
       exit 1
fi

if [ ! -f $BUILD_CFG ]; then
   echo "ERROR: Mock config file not found at '$BUILD_CFG'"
   exit 1
fi

# Create symlinks from /var/... to /localdisk/loadbuild/... if on a build server

set_mock_symlinks $MY_BUILD_CFG

if [ $CLEAN_FLAG -eq 1 ]; then
    umount_mock_root_as_tmpfs_all
fi

if [ $CLEAN_FLAG -eq 0 ]; then
    ls $SRPM_OUT/*.src.rpm &>> /dev/null
    if [ $? -ne 0 ]; then
        echo "Nothing to build in '$SRPM_OUT'"
        exit 0
    fi
fi

ALL=0
UNRESOLVED_TARGETS=" "
if [ "x$TARGETS" == "x" ]; then
    echo "make: all"
    ALL=1
else
    echo "make: $TARGETS"
    UNRESOLVED_TARGETS="$TARGETS"
fi

if [ "$BUILD_TYPE" != "std" ]; then
    # This defines ...
    #    STD_SRPM_PKG_NAME_TO_PATH
    #    STD_SRPM_PKG_NAMES
    srpm_build_std_dictionary $MY_WORKSPACE_TOP/std/rpmbuild/SRPMS
fi

# This defines ...
#    SRPM_PKG_NAME_TO_PATH
#    SRPM_PKG_NAMES
srpm_build_dictionary $SRPM_OUT

SRPMS_TO_COMPILE=()
SRPMS_LIST=""
RPMS_LIST=""

clean_list () {
   local SRPMS_LIST="$1"
   local RPMS_LIST="$2"
   local ALL=$3
   local TARGET
   local b
   local d
   local f
   local n
   local p
   local r
   local s
   local sn
   local t
   local SPEC_DIR

   echo "${FUNCNAME[0]}: '$SRPMS_LIST'  '$RPMS_LIST'  '$ALL'"
   if [ $ALL -eq 1 ]; then
       for r in $(find $RPM_DIR -name "*.rpm"); do
           \rm -f -v $r
       done

       if [ $CLEAN_FLAG -eq 1 ]; then
          for d in $(find $SPECS_DIR -type d); do
             echo "rm -rf $d"
              \rm -rf "$d" 2>> /dev/null
          done
       fi

       for d in $(find $RESULT_DIR/$USER-* -maxdepth 1 -type d 2>> /dev/null); do
           echo "rm -rf $d"
           \rm -rf "$d" 2>> /dev/null
       done
   else
       for s in $SRPMS_LIST; do
           (
           SPEC_DIR=$(spec_cache_dir_from_srpm $s)
           sn=$(rpm_get_name $s)
           update_spec_cache $s

           TARGET=$(rpm -qp --qf '%{NAME}-%{VERSION}\n' "$s")
           for d in $(find $RESULT_DIR/$USER-* -maxdepth 1 -name "$TARGET*" 2>> /dev/null); do
               echo "rm -rf $d"
               \rm -rf "$d" 2>> /dev/null
           done

           for p in $(ls -1 $SPEC_DIR/BUILDS); do
               for r in $(find $RESULT_DIR/$USER-* $RPM_DIR -name "$p-*.rpm" 2>> /dev/null); do
                   if [ -f $r ]; then
                       n=$(rpm_get_name $r)
                       if [ "$n" == "$p" ]; then
                          if [[ "$r" == *.src.rpm ]]; then
                              if [ "$n" != "$sn" ]; then
                                 continue
                              fi

                              TARGET=$(rpm -qp --qf '%{NAME}-%{VERSION}\n' "$r")
                              for d in $(find $RESULT_DIR/$USER-* -maxdepth 1 -name "$TARGET*" 2>> /dev/null); do
                                  echo "rm -rf $d"
                                  \rm -rf "$d" 2>> /dev/null
                              done
                              
                          else
                              rs=$(rpm_get_srpm $r)
                              if [[ "$rs" != "$sn"-[0-9]* ]]; then
                                  continue
                              fi
                          fi

                          \rm -f -v $r
                       fi
                   fi
               done
           done
    
           TARGET=$(rpm -qp --qf '%{NAME}-%{VERSION}\n' "$s")

           if [ $CLEAN_FLAG -eq 1 ]; then
               for d in $(find $SPECS_DIR -type d -name "$TARGET*" 2>> /dev/null); do
                   echo "rm -rf $d"
                    \rm -rf "$d" 2>> /dev/null
               done
           fi

           for d in $(find $RESULT_DIR/$USER-* -maxdepth 1 -name "$TARGET*" 2>> /dev/null); do
               echo "rm -rf $d"
               \rm -rf "$d" 2>> /dev/null
           done
           ) &
       done
       echo "waiting on file deletion"
       wait
       echo "wait complete"
   fi

   echo ""
   echo "Cleaning repodata"
   for d in $(find -L  $MY_WORKSPACE/rpmbuild $MY_WORKSPACE/results   -type d -name repodata); do
      recreate_repodata $(dirname $d)
   done

   echo ""
   echo "Cleaning mock environment"
   echo ""

   if [ $ALL -eq 1 ]; then
       # Wipe everything
       if [ "x$RPM_DIR" != "x" ]; then
           \rm -rf -v $RPM_DIR/* 2>> /dev/null
       fi

       \rm -f -v $RESULT_DIR/mockchain.log 2>> /dev/null
       mock_clean
   else  
       # Wipe only traces of what we built
       mock_partial_clean "$SRPMS_LIST" "$RPMS_LIST"
   fi
}

echo "ALL=$ALL"
(
trap my_exit EXIT
trap my_sigint INT
trap my_sighup HUP
echo "$CMDLINE"
echo "ALL=$ALL"

if [ $CLEAN_FLAG -eq 0 ]; then
  if [ -d $RESULT_DIR ]; then
    # in case previous build recieved a ctrl-C and didn't get a change to copy it's successful work into RPM_DIR
    for d in $(find $RESULT_DIR -name '*.rpm' | grep -v '[.]src[.]rpm' | xargs --no-run-if-empty --max-args=1 dirname | sort -u); do
        rsync -u $d/*.rpm $RPM_DIR
    done
    for d in $(find -L $RESULT_DIR  -type d -name repodata); do
       update_repodata $(dirname $d)
    done
  fi
fi

spec_cache_dir_from_srpm () {
   local SRPM=${1}
   local SPEC_DIR=$(echo $SRPM | sed 's#/SRPMS/#/SPECS/#')
   echo "$SPEC_DIR"
}

update_spec_cache () {
   local SRPM=${1}
   local SPEC_DIR=$(spec_cache_dir_from_srpm $SRPM)
   local NEED_UPDATE=0

   if [ ! -d $SPEC_DIR ]; then
      mkdir -p  $SPEC_DIR
      NEED_UPDATE=1
   else
      find "$SPEC_DIR" -not -newer "$SRPM" -name '*.spec' | grep 'spec' >> /dev/null
      if [ $? -eq 0 ]; then
         NEED_UPDATE=1
      fi
   fi

   if [ $NEED_UPDATE -ne 0 ]; then
      (
      cd $SPEC_DIR
      \rm -rf BUILDS BUILDS_VR *.spec 2>> /dev/null
      mkdir -p BUILDS
      mkdir -p NAMES
      mkdir -p SERVICES
      mkdir -p BUILDS_VR
      rpm2cpio $SRPM | pax -r '*.spec'
      if [ $? -ne 0 ]; then
         echo "ERROR: no spec file found in '$SRPM'"
      fi
      for f in $(find . -name '*.spec' | sort -V); do
         touch $f
         for p in $(spec_list_ver_rel_packages $f); do
            touch "BUILDS_VR/$p"
         done
         for p in $(spec_list_packages $f); do
            touch "BUILDS/$p"
         done
         for p in $(spec_find_tag Name $f 2>> /dev/null); do
            touch "NAMES/$p"
         done
         for p in $(spec_find_global service $f 2>> /dev/null); do
            touch "SERVICES/$p"
         done
      done
      )
   fi
}

# Find the list of packages we must compile

echo "Find the list of packages we must compile"

NEED_BUILD_DIR=$(mktemp -d $MY_WORKSPACE/tmp/$USER-$ME-need-build-XXXXXX)
if [ $? -ne 0 ] || [ "x$NEED_BUILD_DIR" == "x" ]; then
    echo "Failed to create temp directory under $MY_WORKSPACE/tmp"
    exit 1
fi

UNRESOLVED_TARGETS_DIR=$(mktemp -d $MY_WORKSPACE/tmp/$USER-$ME-unresolved-XXXXXX)
if [ $? -ne 0 ] || [ "x$UNRESOLVED_TARGETS_DIR" == "x" ]; then
    echo "Failed to create temp directory under $MY_WORKSPACE/tmp"
    exit 1
fi

for n in ${UNRESOLVED_TARGETS}; do
    touch $UNRESOLVED_TARGETS_DIR/$n
done

PLATFORM_RELEASE_CHANGED=0
if [ -f $LAST_PLATFORM_RELEASE_FILE ]; then
    LAST_PLATFORM_RELEASE=$(cat $LAST_PLATFORM_RELEASE_FILE)
    if [ "$LAST_PLATFORM_RELEASE" != "$PLATFORM_RELEASE" ]; then
        PLATFORM_RELEASE_CHANGED=1
    fi
else
    PLATFORM_RELEASE_CHANGED=1
fi

for n in "${SRPM_PKG_NAMES[@]}"; do
    (
    s=${SRPM_PKG_NAME_TO_PATH[$n]}
    SPEC_DIR=$(spec_cache_dir_from_srpm $s)
    update_spec_cache $s
    # echo "$BASHPID: considering $n: $s, SPEC_DIR=$SPEC_DIR"
    NEED_BUILD=0

    if [ "x$TARGETS" == "x" ]; then
        # We weren't given a list of build targets.
        # Build anything missing or out of date.
        NEED_BUILD=0

        if [ -f $RESULT_DIR/$USER-$PROJECT-$SRC_BUILD_ENVIRONMENT-$BUILD_TYPE/$(basename ${s//.src.rpm/})/fail ]; then
            echo "Previous build of $(basename ${s//.src.rpm/}) failed"
            NEED_BUILD=1
        elif [ ! -f $RESULT_DIR/$USER-$PROJECT-$SRC_BUILD_ENVIRONMENT-$BUILD_TYPE/$(basename ${s//.src.rpm/})/success ]; then
            echo "No previous build of $(basename ${s//.src.rpm/})"
            NEED_BUILD=1
        else
            LOCAL_RPMS_VRA_LIST=$(ls -1 $SPEC_DIR/BUILDS_VR | tr '\n' ' ')

            for f in $LOCAL_RPMS_VRA_LIST; do
                m=$(find $RPM_DIR/$f*rpm 2>> /dev/null | wc -l)
                if [ $m -eq 0 ] && [ -f "$UNBUILT_PATTERN_FILE" ]; then
                    echo $f | grep -f "$UNBUILT_PATTERN_FILE" >> /dev/null && m=1
                    if [ $m -eq 1 ]; then
                       echo "Excluding '$f' due to match in UNBUILT_PATTERN_FILE '$UNBUILT_PATTERN_FILE'"
                       if [ -f "$IMAGE_INC_FILE" ] ; then
                          for t in $(grep -v '^#' "$IMAGE_INC_FILE"); do
                             ii=$(echo $f | grep "^$t-[0-9]" | wc -l)
                             if [ $ii -gt 0 ]; then
                                echo "Including '$f' due to match in IMAGE_INC_FILE '$IMAGE_INC_FILE' due to pattern '^$t-[0-9]'"
                                m=0
                                break
                             fi
                          done
                       fi
                    fi
                fi

                newer=$(find $RPM_DIR/$f*rpm -type f -not -cnewer $s 2>> /dev/null | wc -l)
                # echo "$m  $newer=find $RPM_DIR/$f*rpm -type f -not -cnewer $s 2>> /dev/null | wc -l"
                if [ $m -eq 0 ] || [ $newer -gt 0 ] || [ $CLEAN_FLAG -eq 1 ]; then
                    if [ $newer -gt 0 ]; then
                        echo "Including '$f' due to newer code"
                        find $RPM_DIR/$f*rpm -type f -not -cnewer $s
                    else
                        if [ $m -eq 0 ]; then
                            echo "Including '$f' due to m=0"
                        else
                           if [ $CLEAN_FLAG -eq 1 ]; then
                               echo "Including '$f' due to CLEAN_FLAG=1"
                           fi
                        fi
                    fi
                    NEED_BUILD=1
                    break
                fi
            done
        fi
    else
        # We were given a list of build targets,
        # try to find packages matching that list.
        NEED_BUILD=0
        for f in $(find $SPEC_DIR/NAMES $SPEC_DIR/SERVICES $SPEC_DIR/BUILDS -type f 2>> /dev/null); do
            b=$(basename $f)
            for t in $TARGETS; do
                if [[ ( "$b" == "$t" ) || ( ( "$BUILD_TYPE" == "rt" ) && ( "$b" == "$t-rt" ) ) ]]; then
                    echo "Including named target '$f'"
                    TARGET_FOUND=$t
                    NEED_BUILD=1
                    # UNRESOLVED_TARGETS=$(echo "$UNRESOLVED_TARGETS" | sed "s/\(^\|[[:space:]]\)$TARGET_FOUND\([[:space:]]\|$\)/ /g")
                    if [ -f $UNRESOLVED_TARGETS_DIR/$TARGET_FOUND ]; then
                        \rm -f $UNRESOLVED_TARGETS_DIR/$TARGET_FOUND
                    fi
                    break
                fi
            done
        done
    fi

    if [ $NO_BUILD_INFO -eq 0 ]; then
        if [ "$n" == "build-info" ]; then
            echo "Including '$n' by default"
            NEED_BUILD=1
        fi
    fi

    if [ $PLATFORM_RELEASE_CHANGED -eq 1 ]; then
        grep '%{platform_release}' $SPEC_DIR/*.spec >> /dev/null
        if [ $? -eq 0 ]; then
            echo "Including '$n' due to changed platform_release"
            NEED_BUILD=1
        fi
    fi

    if [ $NEED_BUILD -eq 1 ]; then
        echo "found $n: $s"
        touch "$NEED_BUILD_DIR/$n"
        # SRPMS_TO_COMPILE+=("$n")
    fi
    ) &
done
echo "waiting"
wait
for n in $(ls -1 $NEED_BUILD_DIR); do
    SRPMS_TO_COMPILE+=("$n")
done
UNRESOLVED_TARGETS=" "
for n in $(ls -1 $UNRESOLVED_TARGETS_DIR); do
    UNRESOLVED_TARGETS="$UNRESOLVED_TARGETS $n"
done
\rm -rf $NEED_BUILD_DIR
\rm -rf $UNRESOLVED_TARGETS_DIR

ORIG_SRPMS_TO_COMPILE=( ${SRPMS_TO_COMPILE[@]} )

echo "SRPMS_TO_COMPILE = ${SRPMS_TO_COMPILE[@]}"


# adding dependant packages
if [ $CLEAN_FLAG -eq 0 ] && [ $NO_DESCENDANTS -eq 0 ] && [ -f $SRPM_DIRECT_DESCENDANTS_FILE ]; then
   echo
   echo "adding dependant packages"

   # This array will accumulate a list of secondary build targets.
   TRANSITIVE_SRPMS_TO_COMPILE=()

   # Add packages that directly depend on the primary build targets in ORIG_SRPMS_TO_COMPILE
   for n in ${ORIG_SRPMS_TO_COMPILE[@]}; do
       needs=( $(grep "^$n;" "$SRPM_DIRECT_DESCENDANTS_FILE" | sed "s/$n;//" | sed 's/,/ /g'; alt_n=$(echo "$n" | sed 's#-rt$##'); if [ "$alt_n" != "$n" ]; then grep "^$alt_n;" "$SRPM_DIRECT_DESCENDANTS_FILE" | sed "s/$alt_n;//" | sed 's/,/ /g' | sed 's#\([^[:space:]]*\)#\1-rt#g'; fi ) )

       # intersection of 'needs' and 'SRPM_PKG_NAMES' ... i.e. what should be compiled that we have source for
       compilable_needs=( $(intersection needs SRPM_PKG_NAMES) )
       TRANSITIVE_SRPMS_TO_COMPILE=( $(union compilable_needs TRANSITIVE_SRPMS_TO_COMPILE) )
   done

   # For non-std build, and if non specific build targets are named, then search all
   # packages that we might build and check if they require a package that DID build 
   # in the std build.  If so build the package as a secondary target, even though the
   # primary target was from a different build_type.
   if [ "$BUILD_TYPE" != "std" ] && [ $ALL -eq 1 ] && [ -f $SRPM_TO_RPM_MAP_FILE ] && [ -f $SRPM_RPM_DIRECT_REQUIRES_FILE ]; then
       # Test all that we can build ...
       for n in ${SRPM_PKG_NAMES[@]}; do
           contains ORIG_SRPMS_TO_COMPILE $n
           if [ $? -eq 0 ]; then
               # Already on the primary build list, skip it.
               echo "skip $n"
               continue
           fi

           STD_NEEDS_BUILD=0

           # Iterate over all binary rpms names produce by the candidate package
           for b in $(grep "^$n;" "$SRPM_TO_RPM_MAP_FILE" | sed "s/$n;//" | sed 's/,/ /g'); do
               # find an rpm file with the rpm name we seek
               for bp in $(find $RPM_DIR -name "$b-[0-9]*.rpm" | grep -v '.src.rpm'); do
                   if [ "$b" != "$(rpm_get_name $bp)" ]; then
                       # rpm name doesn't match
                       continue
                   fi

                   # Iterate over binary rpms names required by the candidate package
                   for r in $(grep "^$n;" "$SRPM_RPM_DIRECT_REQUIRES_FILE" | sed "s/$n;//" | sed 's/,/ /g'); do
                       if [ $r == $n ]; then
                           # Ignore self dependency
                           continue
                       fi

                       # find a required rpm file with the rpm name we seek, AND is newer than the produced rpm file
                       for rp in $(find $(echo $RPM_DIR | sed "s#/$BUILD_TYPE/#/std/#") -name "$r-[0-9]*.rpm" -newermm $bp | grep -v '.src.rpm'); do
                           if [ "$r" != "$(rpm_get_name $rp)" ]; then
                               # rpm name doesn't match
                               continue
                           fi

                           # Ok, a required rpm is newer than a built rpm, we should rebuild!
                           echo "rebuild '$n' due to newer '$r'"
                           STD_NEEDS_BUILD=1
                           break
                       done
                   done
               done

               # Avoid pointless processing if we already have a positive result.
               if [ $STD_NEEDS_BUILD -eq 1 ]; then
                   break
               fi
           done

           if [ $STD_NEEDS_BUILD -eq 1 ]; then
               # Compile is requires due to an updated required package in the std build.
               # Add 'n' to array TRANSITIVE_SRPMS_TO_COMPILE.
               TRANSITIVE_SRPMS_TO_COMPILE=( $(put TRANSITIVE_SRPMS_TO_COMPILE $n) )
           fi
       done
   fi

   # If the kernel or kernel-rt packages where absent from the primary build targets, but 
   # added as a secondary target, then make sure all out-of-tree kernel modules are also
   # added.
   for n in kernel kernel-rt; do
       KERNEL_IN_ORIG=0
       KERNEL_IN_TRANSITIVE=0
       contains ORIG_SRPMS_TO_COMPILE "$n" && KERNEL_IN_ORIG=1
       contains TRANSITIVE_SRPMS_TO_COMPILE "$n" && KERNEL_IN_TRANSITIVE=1
       if [ $KERNEL_IN_TRANSITIVE -eq 1 ] && [ $KERNEL_IN_ORIG -eq 0 ]; then
           needs=( $(grep "^$n;" "$SRPM_DIRECT_DESCENDANTS_FILE" | sed "s/$n;//" | sed 's/,/ /g'; alt_n=$(echo "$n" | sed 's#-rt$##'); if [ "$alt_n" != "$n" ]; then grep "^$alt_n;" "$SRPM_DIRECT_DESCENDANTS_FILE" | sed "s/$alt_n;//" | sed 's/,/ /g' | sed 's#\([^[:space:]]*\)#\1-rt#g'; fi ) )

           # intersection of 'needs' and 'SRPM_PKG_NAMES' ... i.e. what should be compiled that we have source for
           compilable_needs=( $(intersection needs SRPM_PKG_NAMES) )
           TRANSITIVE_SRPMS_TO_COMPILE=( $(union compilable_needs TRANSITIVE_SRPMS_TO_COMPILE) )
       fi
   done

   # Append the secondary targetc list to the primary list
   SRPMS_TO_COMPILE=( $(union SRPMS_TO_COMPILE TRANSITIVE_SRPMS_TO_COMPILE) )
   echo "SRPMS_TO_COMPILE = ${SRPMS_TO_COMPILE[@]}"
fi


MUST_SRPMS_TO_COMPILE=( ${SRPMS_TO_COMPILE[@]} )

# adding required packages
if [ $CLEAN_FLAG -eq 0 ] && [ "x$TARGETS" != "x" ] && [ $NO_REQUIRED -eq 0 ] && [ -f $SRPM_TRANSITIVE_REQUIRES_FILE ]; then
   echo
   echo "adding required packages"
   TRANSITIVE_SRPMS_TO_COMPILE=()
   for n in ${MUST_SRPMS_TO_COMPILE[@]}; do
       needs=( $(grep "^$n;" "$SRPM_TRANSITIVE_REQUIRES_FILE" | sed "s/$n;//" | sed 's/,/ /g') )

       # intersection of 'needs' and 'SRPM_PKG_NAMES' ... i.e. what should be compiled that we have source for
       compilable_needs=( $(intersection needs SRPM_PKG_NAMES) )
       TRANSITIVE_SRPMS_TO_COMPILE=( $(union compilable_needs TRANSITIVE_SRPMS_TO_COMPILE) )

       for b in "${un[@]}"; do
          echo $b
       done
   done

   SRPMS_TO_COMPILE=( $(union TRANSITIVE_SRPMS_TO_COMPILE SRPMS_TO_COMPILE) )
   echo "SRPMS_TO_COMPILE = ${SRPMS_TO_COMPILE[@]}"
fi


# Determine build order ... now done in mockchain4
SRPMS_TO_COMPILE=( $(echo ${SRPMS_TO_COMPILE[@]} | sed 's/ /\n/g' | sort -u) )


# convert pkg names to paths, clean work dirs if needed
echo
echo "Mapping packages to src rpm paths"
for n in ${SRPMS_TO_COMPILE[@]}; do
    s=${SRPM_PKG_NAME_TO_PATH[$n]}
    SPEC_DIR=$(spec_cache_dir_from_srpm $s)
    update_spec_cache $s

    SRPMS_LIST="$SRPMS_LIST $s"
    # echo "SRPMS_LIST = $SRPMS_LIST"

    TMP_RPMS_LIST=$(ls -1 $SPEC_DIR/BUILDS | tr '\n' ' ')
    RPMS_LIST="$RPMS_LIST $TMP_RPMS_LIST"
done
echo

if [ $CLEAN_FLAG -eq 0 ]; then
   update_cgcs_repo cgcs-centos-repo
   if [ -d $MY_REPO/cgcs-3rd-party-repo ]; then
       update_cgcs_repo cgcs-3rd-party-repo
   fi
fi

mock_clean_mounts

# clean work dirs if needed
CLEAN_BEFORE_BUILD_SRPM_LIST=""
CLEAN_BEFORE_BUILD_RPM_LIST=""
if [ $CLEAN_FLAG -eq 0 ]; then
    echo
    echo "Calculating minimal clean list"
    for nm in ${SRPMS_TO_COMPILE[@]}; do
        MUST_CLEAN=0
        contains MUST_SRPMS_TO_COMPILE $nm && MUST_CLEAN=1

        s=${SRPM_PKG_NAME_TO_PATH[$nm]}
        SPEC_DIR=$(spec_cache_dir_from_srpm $s)
        update_spec_cache $s

        LOCAL_RPMS_LIST=$(ls -1 $SPEC_DIR/BUILDS | tr '\n' ' ')
        LOCAL_RPMS_VRA_LIST=$(ls -1 $SPEC_DIR/BUILDS_VR | tr '\n' ' ')

        for f in $LOCAL_RPMS_VRA_LIST; do
            m=$(find $RPM_DIR/$f*rpm 2>> /dev/null | wc -l)
            if [ -f "$UNBUILT_PATTERN_FILE" ]; then
                echo $f | grep -f "$UNBUILT_PATTERN_FILE" >> /dev/null && m=1
            fi

            n=$(find $RPM_DIR/$f*rpm -type f -not -cnewer $s 2>> /dev/null | wc -l)
            # echo "$n=find $RPM_DIR/$f*rpm -type f -not -cnewer $s 2>> /dev/null | wc -l"
            if [ $m -eq 0 ] || [ $n -gt 0 ] || [ $MUST_CLEAN -eq 1 ]; then
                CLEAN_BEFORE_BUILD_SRPM_LIST="$CLEAN_BEFORE_BUILD_SRPM_LIST $s"
                CLEAN_BEFORE_BUILD_RPM_LIST="$CLEAN_BEFORE_BUILD_RPM_LIST $LOCAL_RPMS_LIST"
                break
            fi
        done
    done
fi


if [ "$UNRESOLVED_TARGETS" != " " ]; then
    if [ $CLEAN_FLAG -eq 0 ]; then
        echo ""
        echo "ERROR: failed to resolve build targets: $UNRESOLVED_TARGETS"
        exit 1
    fi
fi

echo "SRPMS_LIST = $SRPMS_LIST"
echo "RPMS_LIST = $RPMS_LIST"


echo 
if [ $CLEAN_FLAG -eq 0 ]; then
   # pre-create these directories as $USER, 
   # else mock will create them as root and fails to clean them.
   # Note: keep these in sync with mockchain-parallel!
   for i in $(seq 0 $((MAX_WORKERS-1))); do
      mkdir -p $MY_WORKSPACE/mock/b$i
      mkdir -p $MY_WORKSPACE/cache/b$i/mock
   done

   mock_update_or_init
fi
set_mock_symlinks $MY_BUILD_CFG

echo
echo "Cleaning"
if [ $CLEAN_FLAG -eq 1 ]; then
   # Clean what the user asked for
   echo "========= clean_list '$SRPMS_LIST' '$RPMS_LIST' $ALL"
   \rm -r -f -v $MY_WORKSPACE/mock-$USER-*
   clean_list "$SRPMS_LIST" "$RPMS_LIST" "$ALL"

   exit 0
else
   # Clean what we intend to build
   if [ $NO_AUTOCLEAN -eq 1 ]; then
      echo "no-autoclean was requested"
   else
      if [ "$CLEAN_BEFORE_BUILD_SRPM_LIST" != "" ]; then
         echo "========= clean_list '$CLEAN_BEFORE_BUILD_SRPM_LIST' '$CLEAN_BEFORE_BUILD_RPM_LIST' 0"
         clean_list "$CLEAN_BEFORE_BUILD_SRPM_LIST" "$CLEAN_BEFORE_BUILD_RPM_LIST" 0
      fi
   fi
fi

echo
echo "Cleaning repodata"

BUILD_ENVIRONMENT_DIR=$(basename $BUILD_CFG)
BUILD_ENVIRONMENT_DIR=${BUILD_ENVIRONMENT_DIR%.*}
LOCAL_URL=http://127.0.0.1:8088$BUILD_BASE/results/$BUILD_ENVIRONMENT_DIR/
LOCAL_SRC_URL=http://127.0.0.1:8088$BUILD_BASE/rpmbuild/SRPMS/

for d in $(find -L $RESULT_DIR  -type d -name repodata); do
(cd $d/..
 if [ -f repodata/*comps*xml ]; then
    \mv repodata/*comps*xml comps.xml
 fi
 \rm -rf repodata
)
done

echo
echo "Cleaning Metadata"

MOCKCHAIN_LOG="$RESULT_DIR/mockchain.log"
mkdir -p $RESULT_DIR
touch $RESULT_DIR/build_start
\rm -rf $MOCKCHAIN_LOG

mock_clean_metadata

echo
echo "Building"

CMD_PREFIX=""
if [ -x /bin/ionice ]; then
    CMD_PREFIX="nice -n 20 ionice -c Idle /bin/ionice "
fi

CMD_OPTIONS="-m --no-clean -m --no-cleanup-after"
if [ $CAREFUL -eq 1 ]; then
   CMD_OPTIONS="-m --no-cleanup-after"
fi
echo "CAREFUL=$CAREFUL"

# Sets WORKERS and MOCKCHAIN_RESOURCE_ALLOCATION
compute_resources $SRPMS_LIST


if [ -f $SRPM_RPM_DIRECT_REQUIRES_FILE ]; then
    CMD_OPTIONS="$CMD_OPTIONS --srpm-dependency-file $SRPM_RPM_DIRECT_REQUIRES_FILE"
fi
if [ -f "$RPM_DIRECT_REQUIRES_FILE" ]; then
    CMD_OPTIONS="$CMD_OPTIONS --rpm-dependency-file $RPM_DIRECT_REQUIRES_FILE"
fi
if [ -f "$RPM_TO_SRPM_MAP_FILE" ]; then
    CMD_OPTIONS="$CMD_OPTIONS --rpm-to-srpm-map-file $RPM_TO_SRPM_MAP_FILE"
fi


for s in $SRPMS_LIST; do
    d=$(echo "$s" | sed 's#/SRPMS/#/SOURCES/#')
    if [ -f $d/BIG ]; then
        BUILD_SIZE=$(cat $d/BIG | { read first rest ; echo $first ; })
        CMD_OPTIONS="$CMD_OPTIONS --mark-big-path $BUILD_SIZE:$s"
    fi
    if [ -f $d/SLOW ]; then
        BUILD_SPEED=$(cat $d/SLOW | { read first rest ; echo $first ; })
        CMD_OPTIONS="$CMD_OPTIONS --mark-slow-path $BUILD_SPEED:$s"
    fi
done
echo "CMD_OPTIONS=$CMD_OPTIONS"

echo "MAX_WORKERS=$MAX_WORKERS"
echo "MOCKCHAIN_RESOURCE_ALLOCATION=$MOCKCHAIN_RESOURCE_ALLOCATION"

CMD="$CMD_PREFIX mockchain-parallel -r $BUILD_CFG -l $BUILD_BASE --recurse --workers=$MAX_WORKERS --worker-resources=$MOCKCHAIN_RESOURCE_ALLOCATION --basedir=$MY_WORKSPACE --log=$MOCKCHAIN_LOG --tmp_prefix=$USER --addrepo=$LOCAL_URL --addrepo=$LOCAL_SRC_URL $CMD_OPTIONS -m --rebuild $SRPMS_LIST"
echo ""
echo "$CMD -m --define='_tis_dist .tis' -m --define='platform_release $PLATFORM_RELEASE'" 
echo ""

trapwrap stdbuf -o0 $CMD -m --define="_tis_dist .tis" -m --define="platform_release $PLATFORM_RELEASE"
MOCKCHAIN_RC=$?

echo $PLATFORM_RELEASE > $LAST_PLATFORM_RELEASE_FILE

if [ $CLEAN_FLAG -eq 0 ]; then
    umount_mock_root_as_tmpfs_all
fi

for d in $(find $RESULT_DIR -name '*.rpm' | grep -v '[.]src[.]rpm' | xargs --max-args=1 dirname | sort -u); do
    rsync -u $d/*.rpm $RPM_DIR 
done

if [ $ALL -eq 1 ]; then
    echo
    echo "Auditing for obsolete srpms"
    for r in $(find $RESULT_DIR $RPM_DIR -name '*.src.rpm'); do
        (
        f=$(basename $r)
        if [ ! -f "$SRPM_OUT/$f" ]; then
            \rm -fv $r
        fi
        ) &
    done
    echo "waiting for srpm audit to complete"
    wait
    echo "Auditing for obsolete rpms"
    for r in $(find $RESULT_DIR $RPM_DIR -name '*.rpm' | grep -v 'src.rpm'); do
        (
        s=$(rpm_get_srpm $r)
        if [ ! -f "$SRPM_OUT/$s" ]; then
            echo "Failed to find '$SRPM_OUT/$s'"
            \rm -fv $r
        fi
        ) &
    done
    echo "waiting for rpm audit to complete"
    wait
    echo "Audit complete"
    echo ""
fi

if [ $MOCKCHAIN_RC -ne 0 ]; then
   echo "ERROR: Failed to build rpms using '$CMD'"
   exit 1
fi

echo "Recreate repodata"
for d in $(find -L $MY_WORKSPACE/rpmbuild $MY_WORKSPACE/results  -type d -name repodata); do
   update_repodata $(dirname "$d")
done


if [ -f $MOCKCHAIN_LOG ]; then
    grep 'following pkgs could not be successfully built' $MOCKCHAIN_LOG >> /dev/null
    if [ $? -eq 0 ]; then
        FAILED_PKGS=""
        for p in $(sed -n '/following pkgs could not be successfully built:/,/Results out to/p' $MOCKCHAIN_LOG | grep -v '*** Build Failed ***'  | sed 1d | sed '$ d' | cut -d ':' -f2-); do
            PKG=$(basename $p)
            FAILED_PKGS="$PKG  $FAILED_PKGS"
        done
        echo
        echo "Failed to build packages:  $FAILED_PKGS"
        exit 1
    fi
fi

# If we're doing a nightly or formal build (i.e. not a developer build) then we
# want to sign certain packages.  Note that only certain users (i.e. jenkins)
# have the authority to requiest that packages be signed.
#
# Signing is not actually done on this server (the keys are kept safe on a
# different server with very limited access) but we can invoke a script to
# make calls to the signing server.  Note that this will NOT work if you are
# not Jenkins and don't have access to the Jenkins cross server login keys.
#
# Note that both std and rt builds must be complete before invoking the signing
# script
if [ 0$FORMAL_BUILD -eq 1 ] && [ "$USER" == "jenkins" ]; then
	if [ -e $MY_WORKSPACE_TOP/std ] && [ -e $MY_WORKSPACE_TOP/rt ]; then
		# Create dir for log, if it doesn't exit
		mkdir -p $MY_WORKSPACE_TOP/export
		echo "We are jenkins, and we are trying to do a formal build -- calling signing server"
		echo "  to sign boot RPMs with secure boot keys"

		MY_WORKSPACE=$MY_WORKSPACE_TOP ${SIGN_SECURE_BOOT} > $MY_WORKSPACE_TOP/export/${SIGN_SECURE_BOOT_LOG} 2>&1
		if [ $? -ne 0 ]; then
			echo "Signing of packages failed -- see $MY_WORKSPACE_TOP/export/${SIGN_SECURE_BOOT_LOG}"
			exit 1
		fi
	fi
fi

exit 0 
) 2>&1 | stdbuf -o0 awk '{ print strftime("%H:%M:%S"), $0; fflush(); }' | tee $(date "+$MY_WORKSPACE/build-rpms-parallel_%Y-%m-%d_%H-%M-%S.log") ; exit ${PIPESTATUS[0]}