starlingx/root
Code Issues Proposed changes
5c1e7e7b75
root/build-tools/build-srpms-parallel
Don Penney 427193dd6c Exclude .tox folder when checking file ages 
The content of a .tox folder should not be considered when building
srpms. Additionally, as this folder can be fairly large and include
tens of thousands of files, a find listing can take some time. This
commit updates these find commands to exclude the .tox folder from the
search in order to save time if a user has run tox tests prior to the
build.

Change-Id: I035d730af8cdd443487f6ead3eec9e1e54f28709
Related-Bug: 1908940
Signed-off-by: Don Penney <don.penney@windriver.com>
2020-12-21 15:25:27 -05:00

1606 lines
50 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
# set -x
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
#
# Create src.rpm files from source, or from a downloaded tarball
# or src.rpm plus our additional patches.
#
# This version tries to compile many packages in parallel.
#
# The location of packages to be build are identified by
# <distro>_pkg_dirs[_<opt-build-type>] files located at the root of
# any git tree (e.g. stx/integ/centos_pkg_dirs).
#
# The build of an individul package is driven by it's build_srpm.data
# file plus a <pkg-name>.spec file or an srpm_path file.
#
export ME=$(basename "$0")
CMDLINE="$ME $@"
BUILD_SRPMS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
source $BUILD_SRPMS_PARALLEL_DIR/git-utils.sh
source $BUILD_SRPMS_PARALLEL_DIR/spec-utils
source $BUILD_SRPMS_PARALLEL_DIR/srpm-utils
source $BUILD_SRPMS_PARALLEL_DIR/classify
source $BUILD_SRPMS_PARALLEL_DIR/build-srpms-common.sh
source $BUILD_SRPMS_PARALLEL_DIR/image-utils.sh
INITIAL_DIR=$(pwd)
export DISTRO="centos"
SRPM_SCRIPT="build_srpm"
SRPM_DATA="build_srpm.data"
PKG_DIRS_FILE="${DISTRO}_pkg_dirs"
DEFAULT_SRPM_SCRIPT="$BUILD_SRPMS_PARALLEL_DIR/default_$SRPM_SCRIPT"
SCRIPT_PATH="$DISTRO"
DATA_PATH="$DISTRO"
FILES_PATH="$DISTRO/files"
PATCHES_PATH="$DISTRO/patches"
ORIG_SPECS_PATH="$DISTRO"
SRPM_LIST_PATH="$DISTRO/srpm_path"
MIRROR_ROOT="$MY_REPO/${DISTRO}-repo"
if [ ! -d ${MIRROR_ROOT} ]; then
# Old value... a temporary measure for backward compatibility
MIRROR_ROOT="$MY_REPO/cgcs-${DISTRO}-repo"
if [ ! -d ${MIRROR_ROOT} ]; then
MIRROR_ROOT="$MY_REPO/${DISTRO}-repo"
fi
fi
REPO_DOWNLOADS_ROOT="$MY_REPO"
SRPM_REBUILT_LIST=""
SRPM_FAILED_REBUILD_LIST=""
STOP_SCHEDULING=0
ABSOLUTE_MAX_WORKERS=8
MAX_WORKERS=$(grep -c ^processor /proc/cpuinfo)
if [ "$MAX_WORKERS" == "" ] || [ "$MAX_WORKERS" == "0" ]; then
MAX_WORKERS=1
fi
if [ $MAX_WORKERS -gt $ABSOLUTE_MAX_WORKERS ]; then
MAX_WORKERS=$ABSOLUTE_MAX_WORKERS
fi
echo "MAX_WORKERS=$MAX_WORKERS"
CREATEREPO=$(which createrepo_c)
if [ $? -ne 0 ]; then
CREATEREPO="createrepo"
fi
#
# Create a list of rpms in the directory
#
create_lst () {
local DIR=${1}
(cd $DIR
[ -f rpm.lst ] && \rm -rf rpm.lst
[ -f srpm.lst ] && \rm -rf srpm.lst
find . -name '*.rpm' -and -not -name '*.src.rpm' | sed 's#^[.][/]##' | sort > rpm.lst
find . -name '*.src.rpm' | sed 's#^[.][/]##' | sort > srpm.lst
)
}
usage () {
echo ""
echo "Usage: "
echo " Create source rpms:"
echo " $ME [--rt | --std | --installer | --containers] [--layer=<layer>] [--no-descendants] [--formal] [ list of package names ]"
echo ""
echo " Delete source rpms, and the directories associated with it's creation:"
echo " Note: does not clean an edit environment"
echo " $ME --clean [--rt | --std | --installer | --containers] [optional list of package names]"
echo ""
echo " Extract an src.rpm into a pair of git trees to aid in editing it's contents,"
echo " one for source code and one for metadata such as the spec file."
echo " If --no-meta-patch is specified, then WRS patches are omitted."
echo " $ME --edit [--rt | --std | --installer | --containers] [--no-meta-patch] [list of package names]"
echo ""
echo " Delete an edit environment"
echo " $ME --edit --clean [--rt | --std | --installer | --containers] [list of package names]"
echo ""
echo " This help page"
echo " $ME --help"
echo ""
}
spec_cache_dir_from_srpm () {
local SRPM=${1}
local SPEC_DIR=$(echo $SRPM | sed 's#/SRPMS/#/SPECS/#')
echo "$SPEC_DIR"
}
result_dir_from_srpm () {
local SRPM=$(basename ${1} | sed 's#.src.rpm$##')
local RESULT_DIR="$MY_WORKSPACE/results/$MY_BUILD_ENVIRONMENT/$SRPM"
echo "$RESULT_DIR"
}
# This function creates a bunch of subdirs in $MY_WORKSPACE and makes sure
# that a $MY_BUILD_CFG file exists.
#
# The goal of this is to have a script do as much of the annoying
# grunt-work so that the "how to build it" instructions aren't 200 lines
create_output_dirs () {
# make sure variables are sane before continuing
# Note that $BUILD_ROOT contains either $MY_WORKSPACE or $MY_PATCH_WORKSPACE
if [ "x$BUILD_ROOT" == "x" ]; then
return
fi
if [ "x$MY_BUILD_CFG" == "x" ]; then
return
fi
if [ "x$MY_BUILD_DIR" == "x" ]; then
return
fi
if [ "x$MY_SRC_RPM_BUILD_DIR" == "x" ]; then
return
fi
# create output dirs
mkdir -p $MY_BUILD_DIR
mkdir -p $MY_SRC_RPM_BUILD_DIR
mkdir -p $MY_SRC_RPM_BUILD_DIR/SOURCES
mkdir -p $MY_SRC_RPM_BUILD_DIR/SPECS
mkdir -p $MY_SRC_RPM_BUILD_DIR/BUILD
mkdir -p $MY_SRC_RPM_BUILD_DIR/RPMS
mkdir -p $MY_SRC_RPM_BUILD_DIR/SRPMS
# create $MY_BUILD_CFG, if required
if [ ! -f $MY_BUILD_CFG ]; then
echo "FORMAL_BUILD=$FORMAL_BUILD"
echo "modify-build-cfg $MY_BUILD_CFG"
${DIR}/modify-build-cfg $MY_BUILD_CFG
if [ $? -ne 0 ]; then
echo "Could not modifiy $MY_BUILD_CFG";
exit 1
fi
fi
}
NO_DESCENDANTS=0
NO_BUILD_INFO=0
HELP=0
CLEAN_FLAG=0
FORMAL_FLAG=0
BUILD_TYPE_FLAG=0
EDIT_FLAG=0
NO_META_PATCH_FLAG=0
# read the options
TEMP=$(getopt -o h --long parallel,std,rt,installer,containers,no-descendants,no-meta-patch,no-build-info,help,formal,clean,edit,layer: -n "$ME" -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
fi
eval set -- "$TEMP"
export BUILD_TYPE=std
# extract options and their arguments into variables.
while true ; do
case "$1" in
--no-descendants) NO_DESCENDANTS=1 ; shift ;;
--no-build-info) NO_BUILD_INFO=1 ; shift ;;
-h|--help) HELP=1 ; shift ;;
--clean) CLEAN_FLAG=1 ; shift ;;
--formal) FORMAL_FLAG=1 ; shift ;;
--std) BUILD_TYPE_FLAG=1; BUILD_TYPE=std; shift ;;
--rt) BUILD_TYPE_FLAG=1; BUILD_TYPE=rt; shift ;;
--installer) BUILD_TYPE=installer; shift ;;
--containers) BUILD_TYPE=containers; shift ;;
--edit) EDIT_FLAG=1 ; shift ;;
--no-meta-patch) NO_META_PATCH_FLAG=1 ; shift ;;
--parallel) shift ;;
--layer) export LAYER=$2 ; shift ; shift ;;
--) shift ; break ;;
*) echo "Internal error!" ; exit 1 ;;
esac
done
# Reset variables
if [ -n "$MY_WORKSPACE" ]; then
export MY_WORKSPACE_TOP=${MY_WORKSPACE_TOP:-$MY_WORKSPACE}
export MY_WORKSPACE=$MY_WORKSPACE_TOP/$BUILD_TYPE
else
export MY_PATCH_WORKSPACE_TOP=${MY_PATCH_WORKSPACE_TOP:-$MY_PATCH_WORKSPACE}
export MY_PATCH_WORKSPACE=$MY_PATCH_WORKSPACE_TOP/$BUILD_TYPE
fi
export MY_BUILD_DIR_TOP=${MY_BUILD_DIR_TOP:-$MY_BUILD_DIR}
export MY_BUILD_DIR=$MY_BUILD_DIR_TOP/$BUILD_TYPE
export MY_BUILD_ENVIRONMENT_TOP=${MY_BUILD_ENVIRONMENT_TOP:-$MY_BUILD_ENVIRONMENT}
export MY_BUILD_ENVIRONMENT=$MY_BUILD_ENVIRONMENT_TOP-$BUILD_TYPE
export MY_BUILD_ENVIRONMENT_FILE=$MY_BUILD_ENVIRONMENT.cfg
export MY_SRC_RPM_BUILD_DIR=$MY_BUILD_DIR/rpmbuild
export MY_BUILD_CFG=$MY_WORKSPACE/$MY_BUILD_ENVIRONMENT_FILE
export MY_MOCK_ROOT=$MY_WORKSPACE/mock/root
if [ "$BUILD_TYPE" != "std" ]; then
PKG_DIRS_FILE="${DISTRO}_pkg_dirs_${BUILD_TYPE}"
fi
echo "CLEAN_FLAG=$CLEAN_FLAG"
TARGETS=$@
if [ $HELP -eq 1 ]; then
usage
exit 0
fi
if [ $FORMAL_FLAG -eq 1 ]; then
export FORMAL_BUILD="yes"
fi
if [ "x$TARGETS" == "x" ] && [ $EDIT_FLAG -eq 1 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): a package name is required when --edit is specified"
usage
exit 0
fi
SRC_ROOT="$MY_REPO"
if [ "x$MY_REPO" == "x" ]; then
SRC_ROOT=$INITIAL_DIR
fi
BUILD_ROOT="$MY_WORKSPACE"
if [ "x$MY_WORKSPACE" == "x" ]; then
BUILD_ROOT="$MY_PATCH_WORKSPACE"
if [ "x$MY_PATCH_WORKSPACE" == "x" ]; then
echo "ERROR: $FUNCNAME (${LINENO}): require one of MY_WORKSPACE or MY_PATCH_WORKSPACE be defined"
exit 1
fi
fi
export CCACHE_DIR="$BUILD_ROOT/.ccache"
export SRC_BASE="$SRC_ROOT"
export STX_BASE="$SRC_BASE/stx"
export CGCS_BASE="$STX_BASE"
export DISTRO_REPO_BASE=$MIRROR_ROOT
export SPECS_BASE="$ORIG_SPECS_PATH"
export FILES_BASE="$FILES_PATH"
export PATCHES_BASE="$PATCHES_PATH"
export BUILD_BASE="$BUILD_ROOT"
BUILD_INPUTS="$BUILD_BASE/inputs"
SRPM_ASSEMBLE="$BUILD_BASE/srpm_assemble"
SRPM_WORK="$BUILD_BASE/srpm_work"
if [ "x$MY_SRC_RPM_BUILD_DIR" != "x" ]; then
RPM_BUILD_ROOT=$MY_SRC_RPM_BUILD_DIR
else
RPM_BUILD_ROOT=$BUILD_BASE/rpmbuild
fi
create_output_dirs
export RPM_BUILD_BASE="$RPM_BUILD_ROOT"
export SRPM_OUT="$RPM_BUILD_BASE/SRPMS"
export SOURCE_OUT="$RPM_BUILD_BASE/SOURCES"
export RPM_DIR="$RPM_BUILD_BASE/RPMS"
if [ ! -d $CGCS_BASE ]; then
echo "ERROR: $FUNCNAME (${LINENO}): expected to find directory at '$CGCS_BASE'"
exit 1
fi
if [ ! -d $BUILD_BASE ]; then
if [ $CLEAN_FLAG -eq 1 ]; then
exit 0
fi
echo "ERROR: $FUNCNAME (${LINENO}): expected to find directory at '$BUILD_BASE'"
exit 1
fi
RELEASE_INFO_FILE="$(get_release_info)"
if [ -f "$RELEASE_INFO_FILE" ]; then
source "$RELEASE_INFO_FILE"
else
echo "Warning: $FUNCNAME (${LINENO}): failed to find RELEASE_INFO_FILE=$RELEASE_INFO_FILE"
fi
if [ "x$PLATFORM_RELEASE" == "x" ]; then
echo "Warning: $FUNCNAME (${LINENO}): PLATFORM_RELEASE is not defined in $RELEASE_INFO_FILE"
PLATFORM_RELEASE=00.00
fi
export PLATFORM_RELEASE
mkdir -p $RPM_BUILD_BASE
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$RPM_BUILD_BASE'"
exit 1
fi
mkdir -p $SRPM_OUT
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$SRPM_OUT'"
exit 1
fi
mkdir -p $RPM_DIR
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$RPM_DIR'"
exit 1
fi
mkdir -p $SRPM_ASSEMBLE
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$SRPM_ASSEMBLE'"
exit 1
fi
mkdir -p $BUILD_INPUTS
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$BUILD_INPUTS'"
exit 1
fi
build_dir () {
local build_idx=$1
local d=$2
local w=$3
export PKG_BASE=$d
export WORK_BASE=$w
export SPECS_BASE="$PKG_BASE/$ORIG_SPECS_PATH"
local RC
local ORIG_DIR=$(pwd)
# echo "build_dir: PKG_BASE=$PKG_BASE"
cd "$PKG_BASE"
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): failed to cd into '$PKG_BASE'"
return 1
fi
if [ ! -d $ORIG_SPECS_PATH ]; then
# nothing to do
echo "WARNING: '$ORIG_SPECS_PATH' not found in '$PKG_BASE'"
cd "$ORIG_DIR"
return 0
fi
SRPM_COUNT=0
ORIG_SRPM_PATH=""
if [ -f $SRPM_LIST_PATH ]; then
# we've found a file (ex centos/srpm_path) which lists a path to a source
# RPM file
#
# The specified file can be of the form
#
# repo:path/to/file.src.rpm
# mirror:path/to/file.src.rpm
# /path/to/file.rpm
# path/to/file.rpm
#
# If "repo:" is specified, then we search for the file relative to
# $REPO_DOWNLOADS_ROOT (i.e. a path to the file in a "downloads subgit)
#
# If "mirror:" is specified, then we search for the file relateive to
# $MIRROR_ROOT
#
# An absolute path is parsed as an absolute path (mainly intended for
# developer/experimental use without checking in files or messing with
# your git repos)
#
# A lack of prefix (relative path name) is interpretted as "mirror:"
# (legacy support for existing packages)
#
# Other prefixes (file:, http:, whatever:)are unsupported at this time
for p in $(grep -v '^#' $SRPM_LIST_PATH | grep -v '^$'); do
# absolute path source rpms
echo "$p" | grep "^/" >/dev/null && ORIG_SRPM_PATH=$p
if [ "${ORIG_SRPM_PATH}x" == "x" ]; then
# handle repo: definitions
echo "$p" | grep "^repo:" >/dev/null && ORIG_SRPM_PATH=$(echo $p | sed "s%^repo:%$REPO_DOWNLOADS_ROOT/%")
fi
if [ "${ORIG_SRPM_PATH}x" == "x" ]; then
# handle mirror: definitions
echo "$p" | grep "^mirror:" >/dev/null && ORIG_SRPM_PATH=$(echo $p | sed "s%^mirror:%$MIRROR_ROOT/%" | sed "s#CentOS/tis-r3-CentOS/kilo/##" | sed "s#CentOS/tis-r3-CentOS/mitaka/##")
fi
if [ "${ORIG_SRPM_PATH}x" == "x" ]; then
# we haven't found a valid prefix yet, so assume it's a legacy
# file (mirror: interpretation)
ORIG_SRPM_PATH="$MIRROR_ROOT/$p"
fi
# echo "ORIG_SRPM_PATH=$ORIG_SRPM_PATH"
if [ -f $ORIG_SRPM_PATH ]; then
SRPM_COUNT=$((SRPM_COUNT + 1))
else
echo "ERROR: $FUNCNAME (${LINENO}): Invalid srpm path '$p', evaluated as '$ORIG_SRPM_PATH', found in '$PKG_BASE/$SRPM_LIST_PATH'"
ORIG_SRPM_PATH=""
return 3
fi
done
fi
# Clean up an tmp_spec_*.spec file left by a prior failed build
for f in $(find $ORIG_SPECS_PATH -name 'tmp_spec_*.spec'); do
\rm -f $f
done
SPEC_COUNT=$(find $ORIG_SPECS_PATH -name '*.spec' | wc -l)
if [ $SPEC_COUNT -eq 0 ]; then
if [ -f $ORIG_SPECS_PATH/spec_path ]; then
SPECS_BASE=$SRC_BASE/$(cat $SPECS_BASE/spec_path)
SPEC_COUNT=$(find $SPECS_BASE -maxdepth 1 -name '*.spec' | wc -l)
fi
fi
if [ $SPEC_COUNT -eq 0 ] && [ $SRPM_COUNT -eq 0 ]; then
# nothing to do
echo "ERROR: $FUNCNAME (${LINENO}): Neither srpm_path nor .spec file not found in '$PKG_BASE/$ORIG_SPECS_PATH'"
cd "$ORIG_DIR"
return 0
fi
if [ $SPEC_COUNT -gt 0 ] && [ $SRPM_COUNT -gt 0 ]; then
# nothing to do
echo "ERROR: $FUNCNAME (${LINENO}): Please provide only one of srpm_path or .spec files, not both, in '$PKG_BASE/$ORIG_SPECS_PATH'"
cd $ORIG_DIR
return 0
fi
if [ $SPEC_COUNT -gt 0 ]; then
build_dir_spec $build_idx
RC=$?
cd "$ORIG_DIR"
return $RC
else
build_dir_srpm $build_idx $ORIG_SRPM_PATH
RC=$?
cd "$ORIG_DIR"
return $RC
fi
cd "$ORIG_DIR"
return 0
}
clean_srpm_dir () {
local build_idx=$1
local DIR=$2
local EXCLUDE_MD5=$3
local SRPM_PATH
local SRPM_FILE
local SRPM_OUT_PATH
local SRPM_NAME
local SRPM_OUT_NAME
local INPUTS_TO_CLEAN=""
if [ "$EXCLUDE_MD5" == "" ]; then
EXCLUDE_MD5=0
fi
echo "clean_srpm_dir build_idx=$build_idx DIR=$DIR"
INPUTS_TO_CLEAN=$(dirname $(dirname $DIR))
echo "$INPUTS_TO_CLEAN" | grep -q "^$BUILD_INPUTS/"
if [ $? -ne 0 ] ; then
INPUTS_TO_CLEAN=""
fi
for SRPM_PATH in $(find "$DIR" -name '*.src.rpm'); do
SRPM_FILE=$(basename $SRPM_PATH)
SRPM_NAME=$(rpm -q --queryformat '%{NAME}\n' --nosignature -p $SRPM_PATH 2>> /dev/null)
if [ $CLEAN_FLAG -eq 1 ]; then
sed -i "/^$SRPM_NAME$/d" $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_${build_idx}
fi
\rm -fv $SRPM_PATH $SRPM_OUT/$SRPM_FILE
if [ -d $SRPM_ASSEMBLE/$SRPM_NAME ]; then
echo "rm -rf $SRPM_ASSEMBLE/$SRPM_NAME"
\rm -rf $SRPM_ASSEMBLE/$SRPM_NAME
fi
if [ -d $SOURCE_OUT/$SRPM_FILE ]; then
echo "rm -rf $SOURCE_OUT/$SRPM_FILE"
\rm -rf $SOURCE_OUT/$SRPM_FILE
fi
if [ $EXCLUDE_MD5 -eq 0 ] && [ -d $SOURCE_OUT/$SRPM_NAME ]; then
echo "rm -rf $SOURCE_OUT/$SRPM_NAME"
\rm -rf $SOURCE_OUT/$SRPM_NAME
fi
local d
local src_d
local spec
local spec_name
for d in $(find $BUILD_INPUTS -type d -name "${SRPM_NAME}*") ;do
src_d=$(echo $d | sed "s#^$BUILD_INPUTS/#$MY_REPO/#")
for spec in $(find $src_d/${DISTRO} -name '*.spec'); do
spec_name=$(spec_find_tag Name $spec)
if [ "$spec_name" == "$SRPM_NAME" ]; then
INPUTS_TO_CLEAN=$(if [ "x$INPUTS_TO_CLEAN" != "x" ]; then echo $INPUTS_TO_CLEAN; fi; echo "$d")
fi
done
done
# Look for older versions of the same src rpm that also need cleaning
for SRPM_OUT_PATH in $(ls -1 $SRPM_OUT/$SRPM_NAME*.src.rpm 2>> /dev/null); do
SRPM_OUT_FILE=$(basename $SRPM_OUT_PATH)
SRPM_OUT_NAME=$(rpm -q --queryformat '%{NAME}\n' -p $SRPM_OUT_PATH 2>> /dev/null)
if [ "$SRPM_NAME" == "$SRPM_OUT_NAME" ]; then
\rm -fv $SRPM_OUT_PATH
if [ -d $SOURCE_OUT/$SRPM_OUT_FILE ]; then
echo "rm -rf $SOURCE_OUT/$SRPM_OUT_FILE"
\rm -rf $SOURCE_OUT/$SRPM_OUT_FILE
fi
fi
done
done
if [ "x$INPUTS_TO_CLEAN" != "x" ]; then
for d in $INPUTS_TO_CLEAN; do
if [ -d $d/rpmbuild ]; then
echo "rm -rf $d"
\rm -rf $d
fi
done
fi
}
build_dir_srpm () {
local build_idx=$1
local ORIG_SRPM_PATH=$2
local ORIG_SRPM=$(basename $ORIG_SRPM_PATH)
local NAME=$(rpm -q --queryformat '%{NAME}\n' --nosignature -p $ORIG_SRPM_PATH)
local PKG_NAME_VER=$(rpm -q --queryformat '%{NAME}-%{VERSION}-%{RELEASE}\n' --nosignature -p $ORIG_SRPM_PATH)
local PKG_DIR="$NAME"
local TARGET_FOUND=""
local RC=0
export SRPM_EXPORT_NAME=$NAME
export SRPM_EXPORT_VER=$VER
local NEED_BUILD=0
if [ "x$TARGETS" == "x" ]; then
NEED_BUILD=1
TARGET_FOUND=$NAME
else
TARGET_LIST=( $TARGETS )
TARGET_FOUND=$(srpm_match_target_list TARGET_LIST "$ORIG_SRPM_PATH" 2>> /dev/null)
if [ $? -eq 0 ]; then
echo "found target '$TARGET_FOUND' in '$ORIG_SRPM'"
NEED_BUILD=1
sed -i "/^$TARGET_FOUND$/d" $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_${build_idx}
fi
fi
if [ $NEED_BUILD -eq 0 ]; then
return 0
fi
local ROOT_DIR="$SRPM_ASSEMBLE"
if [ $EDIT_FLAG -eq 1 ]; then
mkdir -p $SRPM_WORK
ROOT_DIR="$SRPM_WORK"
fi
local PKG_ROOT_DIR="$ROOT_DIR/$PKG_DIR"
local BUILD_DIR="$PKG_DIR/rpmbuild"
local FULL_BUILD_DIR="$ROOT_DIR/$BUILD_DIR"
local SRPM_DIR="$FULL_BUILD_DIR/SRPMS"
local SOURCES_DIR="$SOURCE_OUT"
if [ $CLEAN_FLAG -eq 1 ]; then
# clean
echo "===== Cleaning '$TARGET_FOUND' ====="
if [ -d $SRPM_DIR ] && [ $EDIT_FLAG -eq 0 ]; then
clean_srpm_dir $build_idx "$SRPM_DIR" 0
fi
if [ -d $PKG_ROOT_DIR ]; then
echo "rm -rf $PKG_ROOT_DIR"
\rm -rf "$PKG_ROOT_DIR"
fi
else
#build
echo "===== Build SRPM for '$TARGET_FOUND' ====="
echo "PKG_BASE=$PKG_BASE"
echo "BUILD_DIR=$BUILD_DIR"
echo "SRPM_DIR=$SRPM_DIR"
if [ ! -d $ROOT_DIR ]; then
mkdir -p "$ROOT_DIR"
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): mkdir '$ROOT_DIR' failed"
return 1
fi
fi
#
# Load data from build_srpm.data
#
export DATA="$DATA_PATH/$SRPM_DATA"
if [ -f "$DATA" ]; then
srpm_source_build_data "$DATA" "$SRC_BUILD_TYPE_SRPM" "$ORIG_SRPM_PATH"
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): failed to source $DATA"
return 1
fi
fi
#
# Capture md5 data for all input files
#
local TARGET_SOURCES_DIR="$SOURCES_DIR/$TARGET_FOUND"
local INPUT_FILES_MD5="$TARGET_SOURCES_DIR/srpm_input.md5"
local REFERENCE_MD5="$TARGET_SOURCES_DIR/srpm_reference.md5"
mkdir -p "$TARGET_SOURCES_DIR"
md5sums_from_input_vars "$SRC_BUILD_TYPE_SRPM" "$ORIG_SRPM_PATH" "$TARGET_SOURCES_DIR" > "$INPUT_FILES_MD5"
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): md5sums_from_input_vars '$SRC_BUILD_TYPE_SRPM' '$ORIG_SRPM_PATH' '$TARGET_SOURCES_DIR'"
return 1
fi
echo "Wrote: $INPUT_FILES_MD5"
#
# Is a rebuild required?
# Compare md5 of current inputs vs md5 of previous build?
#
local BUILD_NEEDED=0
local SRPM_OUT_PATH2
local DIFF_LINE
local DIFF_FILE
if [ -f $REFERENCE_MD5 ]; then
DIFF_LINE=$(diff "$INPUT_FILES_MD5" "$REFERENCE_MD5" | head -n 2 | tail -n 1; exit ${PIPESTATUS[0]})
if [ $? -ne 0 ]; then
DIFF_FILE=$(echo "$DIFF_LINE" | cut -d ' ' -f4-)
BUILD_NEEDED=1
case ${DIFF_LINE:0:1} in
'>') echo "Rebuild required due to deleted file: $DIFF_FILE" ;;
'<') echo "Rebuild required due to new or changed file: $DIFF_FILE" ;;
*) echo "Rebuild required due to diff: $DIFF_LINE" ;;
esac
fi
else
echo "Rebuild required due to missing reference md5: $REFERENCE_MD5"
BUILD_NEEDED=1
fi
if [ -d "$FULL_BUILD_DIR/SRPMS" ]; then
b=""
for SRPM_PATH in $(find "$FULL_BUILD_DIR/SRPMS" -name '*.src.rpm' | sort -V); do
b=$(basename $SRPM_PATH)
SRPM_OUT_PATH2=$(find $SRPM_OUT -name $b)
if [ "x$SRPM_OUT_PATH2" == "x" ]; then
echo "Rebuild required due to missing srpm: $b"
BUILD_NEEDED=1
fi
done
if [ "$b" == "" ]; then
echo "Rebuild required due no src.rpm in directory: '$FULL_BUILD_DIR/SRPMS'"
BUILD_NEEDED=1
fi
else
echo "Rebuild required due to missing directory: '$FULL_BUILD_DIR/SRPMS'"
BUILD_NEEDED=1
fi
if [ $BUILD_NEEDED -eq 0 ]; then
echo "SRPM build not required for '$PKG_BASE'"
echo "===== Build complete for '$TARGET_FOUND' ====="
echo
return 0
fi
if [ $EDIT_FLAG -eq 0 ]; then
clean_srpm_dir $build_idx "$FULL_BUILD_DIR/SRPMS" 1
if [ -d $PKG_ROOT_DIR ]; then
echo "arf rm -rf $PKG_ROOT_DIR"
\rm -rf $PKG_ROOT_DIR
fi
fi
if [ $EDIT_FLAG -eq 1 ]; then
PKG_CLASSIFICATION=$(classify $PKG_BASE)
echo "$PKG_CLASSIFICATION = classify $PKG_BASE"
if [ "$PKG_CLASSIFICATION" == "spec + tarball" ] || [ "$PKG_CLASSIFICATION" == "srpm + patches" ]; then
echo "OK to edit $PKG_BASE"
else
echo "Can't edit this package, it is of type '$PKG_CLASSIFICATION', it is not derived from SRPM or tarball and patches"
return 1
fi
echo "srpm_extract_to_git '$ORIG_SRPM_PATH' '$PKG_BASE' '$ROOT_DIR' '$BUILD_DIR' '$PKG_NAME_VER' '$NO_META_PATCH_FLAG' '$TIS_PATCH_VER' '$PBR_VERSION'"
srpm_extract_to_git $ORIG_SRPM_PATH $PKG_BASE $ROOT_DIR $BUILD_DIR $PKG_NAME_VER $NO_META_PATCH_FLAG $TIS_PATCH_VER $PBR_VERSION
RC=$?
if [ $RC -ne 0 ]; then
if [ $RC -eq 1 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): failed to extract srpm '$ORIG_SRPM_PATH'"
fi
return $RC
fi
local LOC
LOC=$(git_list_containing_tag "${PKG_ROOT_DIR}/gits" "pre_wrs_$PKG_NAME_VER" | head -n 1 )
echo "===== '$TARGET_FOUND' has been extracted for editing. ====="
echo "===== Metadata can be found at: $PKG_ROOT_DIR/rpmbuild"
echo "===== Source code can be found at: $LOC"
return 0
fi
#
# Find age of youngest input file.
# We will apply this as the creation/modification timestamp of the src.rpm we produce.
#
AGE=$(find $PKG_BASE $ORIG_SRPM_PATH ! -path '*/.git/*' ! -path '*/.tox/*' -type f -exec stat --format '%Y' "{}" \; | sort -nr | head -n 1)
if [ -f $PKG_BASE/$DATA ]; then
AGE2=$(
cd $PKG_BASE
srpm_source_build_data "$DATA" "$SRC_BUILD_TYPE_SRPM" "$ORIG_SRPM_PATH"
PATH_LIST=""
# NOTE: SRC_DIR is not honored in this build path
if [ "x$COPY_LIST" != "x" ]; then
PATH_LIST="$PATH_LIST $COPY_LIST"
fi
# NOTE: COPY_LIST_TO_TAR is not honored in this build path
if [ "x$PATH_LIST" == "x" ]; then
echo "0"
else
AGE2=$(find -L $PATH_LIST ! -path '*/.git/*' ! -path '*/.tox/*' -type f -exec stat --format '%Y' "{}" \; | sort -nr | head -n 1)
echo "$AGE2"
fi
)
if [ $AGE2 -gt $AGE ]; then
AGE=$AGE2
fi
fi
srpm_extract $ORIG_SRPM_PATH $PKG_BASE $ROOT_DIR $BUILD_DIR $PKG_NAME_VER
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): failed to extract srpm '$ORIG_SRPM_PATH'"
return 1
fi
if [ "x$COPY_LIST" != "x" ]; then
echo "COPY_LIST: $COPY_LIST"
for p in $COPY_LIST; do
# echo "COPY_LIST: $p"
\cp -L -r -f -v $p $FULL_BUILD_DIR/SOURCES
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): COPY_LIST: file not found: '$p'"
return 1
fi
done
fi
srpm_assemble $FULL_BUILD_DIR $TIS_PATCH_VER $PBR_VERSION
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): failed to assemble srpm for '$PKG_NAME_VER'"
echo "$TARGET_FOUND" >> $MY_WORKSPACE/tmp/SRPM_FAILED_REBUILD_LIST_${build_idx}
return 1
fi
TS=$(date -d @$AGE +%Y-%m-%dT%H:%M:%S)
for s in $(find $FULL_BUILD_DIR/SRPMS -name '*.src.rpm'); do
\cp -L -f -v $s $SRPM_OUT/
ss=$(basename $s)
touch $SRPM_OUT/$ss --date=$TS
mkdir -p $SOURCES_DIR/$ss
BIG_FLAG_FILE="$SOURCES_DIR/$ss/BIG"
SLOW_FLAG_FILE="$SOURCES_DIR/$ss/SLOW"
if [ $BUILD_IS_BIG -gt 0 ]; then
echo "$BUILD_IS_BIG" > $BIG_FLAG_FILE
else
if [ -f $BIG_FLAG_FILE ]; then
\rm -f $BIG_FLAG_FILE
fi
fi
if [ $BUILD_IS_SLOW -gt 0 ]; then
echo "$BUILD_IS_SLOW" > $SLOW_FLAG_FILE
else
if [ -f $SLOW_FLAG_FILE ]; then
\rm -f $SLOW_FLAG_FILE
fi
fi
\rm -f -v "$REFERENCE_MD5"
\mv -v "$INPUT_FILES_MD5" "$REFERENCE_MD5"
local SPEC_DIR=$(spec_cache_dir_from_srpm $SRPM_OUT/$ss)
if [ -d $SPEC_DIR/BUILDS_VR ]; then
for f in $(ls -1 $SPEC_DIR/BUILDS_VR); do
for r in $(find $RPM_DIR -name "$f*rpm" 2>> /dev/null); do
\rm -f -v $r
done
done
fi
local RESULT_DIR=$(result_dir_from_srpm $SRPM_OUT/$ss)
if [ -d $RESULT_DIR ]; then
echo "rm -rf $RESULT_DIR"
\rm -rf $RESULT_DIR
fi
done
echo "$TARGET_FOUND" >> $MY_WORKSPACE/tmp/SRPM_REBUILT_LIST_${build_idx}
echo "SRPM build successful for '$PKG_NAME_VER'"
echo "===== Build complete for '$TARGET_FOUND' ====="
echo
fi
return 0
}
build_dir_spec () {
local build_idx=$1
local NEED_BUILD=0
local TARGET_FOUND=""
if [ "x$TARGETS" == "x" ]; then
NEED_BUILD=1
for f in $(find $SPECS_BASE -maxdepth 1 -name '*.spec'); do
TARGET_FOUND=$(spec_find_tag Name "$f" 2>> /dev/null)
if [ $? -ne 0 ]; then
TARGET_FOUND=$(spec_find_global service "$f" 2>> /dev/null)
if [ $? -ne 0 ]; then
TARGET_FOUND=""
fi
fi
done
else
TARGET_LIST=( $TARGETS )
for f in $(find $SPECS_BASE -maxdepth 1 -name '*.spec' 2>> /dev/null); do
TARGET_FOUND=$(spec_match_target_list TARGET_LIST "$f" 2>> /dev/null)
if [ $? -eq 0 ]; then
echo "found target '$TARGET_FOUND' in '$f'"
NEED_BUILD=1
sed -i "/^$TARGET_FOUND$/d" $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_${build_idx}
break
fi
done
fi
if [ $NEED_BUILD -eq 1 ]; then
MAKE_SRPM="$SCRIPT_PATH/$SRPM_SCRIPT"
export DATA="$DATA_PATH/$SRPM_DATA"
export RPMBUILD_BASE="$WORK_BASE/rpmbuild"
SRPM_PATH="$RPMBUILD_BASE/SRPMS"
SPEC_PATH="$RPMBUILD_BASE/SPECS"
SOURCES_PATH="$RPMBUILD_BASE/SOURCES"
local ROOT_DIR="$RPMBUILD_BASE"
local PKG_ROOT_DIR="$RPMBUILD_BASE"
local SPEC=$(find $SPECS_BASE -maxdepth 1 -name '*.spec' | head -n 1)
local NAME=$(spec_find_tag Name $SPEC)
local PKG_NAME_VER=$(spec_name_ver_rel $SPEC)
local PKG_DIR="$NAME"
local BUILD_DIR="$PKG_DIR/rpmbuild"
local FULL_BUILD_DIR="$ROOT_DIR"
local SRPM_DIR="$FULL_BUILD_DIR/SRPMS"
local SOURCES_DIR="$SOURCE_OUT"
if [ $EDIT_FLAG -eq 1 ]; then
mkdir -p $SRPM_WORK
ROOT_DIR="$SRPM_WORK"
PKG_ROOT_DIR="$ROOT_DIR/$PKG_DIR"
fi
if [ $CLEAN_FLAG -eq 1 ]; then
# clean
echo "===== Cleaning '$TARGET_FOUND' ====="
if [ -d $SRPM_PATH ] && [ $EDIT_FLAG -eq 0 ]; then
clean_srpm_dir $build_idx $SRPM_PATH 0
fi
if [ -d $PKG_ROOT_DIR ]; then
echo "rm -rf $PKG_ROOT_DIR"
\rm -rf "$PKG_ROOT_DIR"
fi
else
# build
echo "===== Build SRPM for '$TARGET_FOUND' ====="
echo "PKG_BASE=$PKG_BASE"
echo "WORK_BASE=$WORK_BASE"
echo "RPMBUILD_BASE=$RPMBUILD_BASE"
if [ ! -x $MAKE_SRPM ]; then
if [ ! -f $DATA ]; then
echo "expected to find an executable script at '$MAKE_SRPM' or data for the default script at '$DATA'"
cd $INITIAL_DIR
exit 1
else
MAKE_SRPM="$DEFAULT_SRPM_SCRIPT"
fi
fi
#
# Load data from build_srpm.data
#
srpm_source_build_data "$DATA" "$SRC_BUILD_TYPE_SPEC" "$SPEC"
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): failed to source $DATA"
return 1
fi
#
# Capture md5 data for all input files
#
local TARGET_SOURCES_DIR="$SOURCES_DIR/$TARGET_FOUND"
local INPUT_FILES_MD5="$TARGET_SOURCES_DIR/srpm_input.md5"
local REFERENCE_MD5="$TARGET_SOURCES_DIR/srpm_reference.md5"
mkdir -p "$TARGET_SOURCES_DIR"
md5sums_from_input_vars "$SRC_BUILD_TYPE_SPEC" "$SPEC" "$TARGET_SOURCES_DIR" > "$INPUT_FILES_MD5"
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): md5sums_from_input_vars '$SRC_BUILD_TYPE_SPEC' '$SPEC' '$TARGET_SOURCES_DIR'"
return 1
fi
echo "Wrote: $INPUT_FILES_MD5"
#
# Is a rebuild required?
# Compare md5 of current inputs vs md5 of previous build?
#
local BUILD_NEEDED=0
local SRPM_OUT_PATH2
local DIFF_LINE
local DIFF_FILE
if [ -f $REFERENCE_MD5 ]; then
DIFF_LINE=$(diff "$INPUT_FILES_MD5" "$REFERENCE_MD5" | head -n 2 | tail -n 1; exit ${PIPESTATUS[0]})
if [ $? -ne 0 ]; then
DIFF_FILE=$(echo "$DIFF_LINE" | cut -d ' ' -f4-)
BUILD_NEEDED=1
case ${DIFF_LINE:0:1} in
'>') echo "Rebuild required due to deleted file: $DIFF_FILE" ;;
'<') echo "Rebuild required due to new or changed file: $DIFF_FILE" ;;
*) echo "Rebuild required due to diff: $DIFF_LINE" ;;
esac
fi
else
echo "Rebuild required due to missing reference md5: $REFERENCE_MD5"
BUILD_NEEDED=1
fi
if [ -d "$FULL_BUILD_DIR/SRPMS" ]; then
if [ -d "$RPMBUILD_BASE/SRPMS" ]; then
b=""
for SRPM_PATH2 in $(find "$RPMBUILD_BASE/SRPMS" -name '*.src.rpm' | sort -V); do
b=$(basename $SRPM_PATH2)
SRPM_OUT_PATH2=$(find $SRPM_OUT -name $b)
if [ "x$SRPM_OUT_PATH2" == "x" ]; then
echo "Rebuild required due to missing srpm: $b"
BUILD_NEEDED=1
fi
done
if [ "$b" == "" ]; then
echo "Rebuild required due no src.rpm found in directory: '$RPMBUILD_BASE/SRPMS'"
BUILD_NEEDED=1
fi
else
echo "Rebuild required due to missing directory: '$RPMBUILD_BASE/SRPMS'"
BUILD_NEEDED=1
fi
else
echo "Rebuild required due to missing directory: '$FULL_BUILD_DIR/SRPMS'"
BUILD_NEEDED=1
fi
if [ $EDIT_FLAG -eq 1 ]; then
PKG_CLASSIFICATION=$(classify $PKG_BASE)
echo "$PKG_CLASSIFICATION = classify $PKG_BASE"
if [ "$PKG_CLASSIFICATION" == "spec + tarball" ] || [ "$PKG_CLASSIFICATION" == "srpm + patches" ]; then
echo "OK to edit $PKG_BASE"
else
echo "Can't edit this package, it is of type '$PKG_CLASSIFICATION', it is not derived from SRPM or tarball and patches"
return 1
fi
echo "tar_and_spec_extract_to_git '$SPEC' '$PKG_BASE' '$ROOT_DIR' '$BUILD_DIR' '$PKG_NAME_VER' '$NO_META_PATCH_FLAG' '$TIS_PATCH_VER' '$PBR_VERSION'"
tar_and_spec_extract_to_git "$SPEC" "$PKG_BASE" "$ROOT_DIR" "$BUILD_DIR" "$PKG_NAME_VER" "$NO_META_PATCH_FLAG" "$TIS_PATCH_VER" "$PBR_VERSION"
RC=$?
if [ $RC -ne 0 ]; then
if [ $RC -eq 1 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): failed to extract srpm '$ORIG_SRPM_PATH'"
fi
return $RC
fi
local LOC
LOC=$(git_list_containing_branch "${PKG_ROOT_DIR}/gits" "${PKG_NAME_VER}" | head -n 1 )
echo "===== '$TARGET_FOUND' has been extracted for editing. ====="
echo "===== Metadata can be found at: $PKG_ROOT_DIR/rpmbuild"
echo "===== Source code can be found at: $LOC"
return 0
fi
if [ $BUILD_NEEDED -eq 0 ]; then
echo "SRPM build not required for '$PKG_BASE'"
echo "===== Build complete for '$TARGET_FOUND' ====="
echo
return 0
fi
export SRC_BUILD_TYPE="$SRC_BUILD_TYPE_SPEC"
export SRPM_OR_SPEC_PATH="$SPEC"
echo "MAKE_SRPM=$MAKE_SRPM"
echo "DATA=$DATA"
echo "SRC_BUILD_TYPE=$SRC_BUILD_TYPE"
echo "SRPM_OR_SPEC_PATH=$SRPM_OR_SPEC_PATH"
if [ -d "$RPMBUILD_BASE/SRPMS" ]; then
clean_srpm_dir $build_idx "$RPMBUILD_BASE/SRPMS" 1
fi
if [ -d $RPMBUILD_BASE ]; then
echo "rm -rf $RPMBUILD_BASE"
\rm -rf "$RPMBUILD_BASE"
fi
echo "mkdir -p $WORK_BASE $SRPM_PATH $SPEC_PATH $SOURCES_PATH"
mkdir -p "$WORK_BASE" && \
mkdir -p "$SRPM_PATH" && \
mkdir -p "$SPEC_PATH" && \
mkdir -p "$SOURCES_PATH"
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directories under: $WORK_BASE"
fi
\cp -L -f -v $SPECS_BASE/*.spec $SPEC_PATH/
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): Failed to copy spec files from '$SPECS_BASE' to '$SPEC_PATH'"
fi
#
# build
#
$MAKE_SRPM
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): script failed '$MAKE_SRPM'"
echo "$TARGET_FOUND" >> $MY_WORKSPACE/tmp/SRPM_FAILED_REBUILD_LIST_${build_idx}
exit 1
fi
#
# Find age of youngest input file.
# We will apply this as the creation/modification timestamp of the src.rpm we produce.
#
AGE=$(find $PKG_BASE ! -path '*/.git/*' ! -path '*/.tox/*' -type f -exec stat --format '%Y' "{}" \; | sort -nr | head -n 1)
if [ -f $PKG_BASE/$DATA ]; then
AGE2=$(
cd $PKG_BASE
srpm_source_build_data "$DATA" "$SRC_BUILD_TYPE_SPEC" "$SPEC"
PATH_LIST=""
if [ "x$SRC_DIR" != "x" ]; then
if [ -d "$SRC_DIR" ]; then
PATH_LIST="$PATH_LIST $SRC_DIR"
fi
fi
if [ "x$COPY_LIST" != "x" ]; then
PATH_LIST="$PATH_LIST $COPY_LIST"
fi
if [ "x$COPY_LIST_TO_TAR" != "x" ]; then
PATH_LIST="$PATH_LIST $COPY_LIST_TO_TAR"
fi
if [ "x$PATH_LIST" == "x" ]; then
echo "0"
else
AGE2=$(find -L $PATH_LIST ! -path '*/.git/*' ! -path '*/.tox/*' -type f -exec stat --format '%Y' "{}" \; | sort -nr | head -n 1)
echo "$AGE2"
fi
)
if [ $AGE2 -gt $AGE ]; then
AGE=$AGE2
fi
fi
TS=$(date -d @$AGE +%Y-%m-%dT%H:%M:%S)
for s in $(find $SRPM_PATH -name '*.src.rpm'); do
\cp -L -f $s $SRPM_OUT/
ss=$(basename $s)
touch $SRPM_OUT/$ss --date=$TS
mkdir -p $SOURCES_DIR/$ss
BIG_FLAG_FILE="$SOURCES_DIR/$ss/BIG"
SLOW_FLAG_FILE="$SOURCES_DIR/$ss/SLOW"
if [ $BUILD_IS_BIG -gt 0 ]; then
echo $BUILD_IS_BIG > $BIG_FLAG_FILE
else
if [ -f $BIG_FLAG_FILE ]; then
\rm -f $BIG_FLAG_FILE
fi
fi
if [ $BUILD_IS_SLOW -gt 0 ]; then
echo $BUILD_IS_SLOW > $SLOW_FLAG_FILE
else
if [ -f $SLOW_FLAG_FILE ]; then
\rm -f $SLOW_FLAG_FILE
fi
fi
\rm -f -v "$REFERENCE_MD5"
\mv -v "$INPUT_FILES_MD5" "$REFERENCE_MD5"
local SPEC_DIR=$(spec_cache_dir_from_srpm $SRPM_OUT/$ss)
if [ -d $SPEC_DIR/BUILDS_VR ]; then
for f in $(ls -1 $SPEC_DIR/BUILDS_VR); do
for r in $(find $RPM_DIR -name "$f*rpm" 2>> /dev/null); do
\rm -f -v $r
done
done
fi
local RESULT_DIR=$(result_dir_from_srpm $SRPM_OUT/$ss)
if [ -d $RESULT_DIR ]; then
echo "rm -rf $RESULT_DIR"
\rm -rf $RESULT_DIR
fi
done
echo "$TARGET_FOUND" >> $MY_WORKSPACE/tmp/SRPM_REBUILT_LIST_${build_idx}
echo "===== Build complete for '$TARGET_FOUND' ====="
echo
fi
fi
return 0
}
(
echo "$CMDLINE"
if [ -L $BUILD_ROOT/repo ]; then
REPO_DEST=$(readlink $BUILD_ROOT/repo)
if [ "$REPO_DEST" != "$SRC_ROOT" ]; then
echo "Error: MY_REPO changed since last build"
echo " old path: $REPO_DEST"
echo " new path: $SRC_ROOT"
echo "Please run '$ME --clean' if you want to compile from a new source tree"
exit 1
fi
fi
if [ ! -L $BUILD_ROOT/repo ]; then
ln -s $SRC_ROOT $BUILD_ROOT/repo
fi
ALL=0
UNRESOLVED_TARGETS=""
if [ "x$TARGETS" == "x" ]; then
echo "make: all"
ALL=1
else
echo "make: $TARGETS"
UNRESOLVED_TARGETS="$TARGETS"
fi
workers=0
max_workers=$MAX_WORKERS
declare -A build_env
init_build_env () {
local i=0
local stop=$((max_workers-1))
for i in $(seq 0 $stop); do
build_env[$i]='Idle'
done
}
init_build_env
get_idle_build_env () {
local i=0
local stop=$((max_workers-1))
if [ $stop -ge 255 ]; then
stop=254
fi
for i in $(seq 0 $stop); do
if [ ${build_env[$i]} == 'Idle' ]; then
build_env[$i]='Busy'
return $i
fi
done
return 255
}
set_build_env_pid () {
local idx=$1
local val=$2
build_env[$idx]=$val
}
release_build_env () {
local idx=$1
build_env[$idx]='Idle'
}
reaper () {
local reaped=0
local last_reaped=-1
local i=0
local stop=$((max_workers-1))
local p=0
local ret=0
if [ $stop -ge 255 ]; then
stop=254
fi
while [ $reaped -gt $last_reaped ]; do
last_reaped=$reaped
for i in $(seq 0 $stop); do
p=${build_env[$i]}
if [ "$p" == "Idle" ] || [ "$p" == "Busy" ]; then
continue
fi
# echo "test $i $p"
kill -0 $p &> /dev/null
if [ $? -ne 0 ]; then
wait $p
ret=$?
workers=$((workers-1))
reaped=$((reaped+1))
release_build_env $i
if [ $ret -ne 0 ]; then
VERB="build"
if [ $EDIT_FLAG -eq 1 ]; then
VERB="edit"
if [ $CLEAN_FLAG -eq 1 ]; then
VERB="edit clean"
fi
else
if [ $CLEAN_FLAG -eq 1 ]; then
VERB="clean"
fi
fi
sleep 1
echo "ERROR: $FUNCNAME (${LINENO}): Failed to $VERB src.rpm from source at 'b$i'"
cat "$LOG_DIR/$i" >> $LOG_DIR/errors
echo "ERROR: $FUNCNAME (${LINENO}): Failed to $VERB src.rpm from source at 'b$i'" >> $LOG_DIR/errors
echo "" >> $LOG_DIR/errors
STOP_SCHEDULING=1
fi
fi
done
done
return $reaped
}
# Set up files to collect parallel build results ...
mkdir -p $MY_WORKSPACE/tmp
fn="$MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_merge"
if [ -f $fn ]; then
\rm -f $fn
fi
for n in $UNRESOLVED_TARGETS; do
echo $n >> $fn;
done
if [ -f $fn ]; then
sort $fn > $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS
else
\rm -f -v $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS
touch $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS
fi
for i in $(seq 0 $((max_workers-1))); do
for fn in $MY_WORKSPACE/tmp/SRPM_REBUILT_LIST_$i $MY_WORKSPACE/tmp/SRPM_FAILED_REBUILD_LIST_$i $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_$i; do
if [ -f $fn ]; then
\rm -f -v $fn
fi
done
\cp $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_$i
done
# create a build info
if [ $CLEAN_FLAG -eq 0 ] && [ $EDIT_FLAG -eq 0 ] && [ $NO_BUILD_INFO -eq 0 ]; then
set_build_info
fi
# Build src.rpm's
LOG_DIR=$(mktemp -d $MY_WORKSPACE/tmp/$USER-$ME-log-XXXXXX)
if [ "x$LOG_DIR" == "x" ]; then
echo "failed to create temporary directory"
exit 1;
fi
for GIT_ROOT in $GIT_LIST; do
export GIT_BASE="$GIT_ROOT"
if [ $STOP_SCHEDULING -eq 1 ]; then
break;
fi
if echo "$GIT_ROOT" | grep do-not-build; then
continue
fi
for p in $(sed 's/#.*//' $GIT_ROOT/$PKG_DIRS_FILE 2>> /dev/null); do
if [ $STOP_SCHEDULING -eq 1 ]; then
break;
fi
src_dir="$GIT_ROOT/$p"
if [ -d $src_dir ]; then
if [ -d $src_dir/${DISTRO} ]; then
rel_dir=$(echo $src_dir | sed "s:^$SRC_BASE::")
work_dir="$BUILD_INPUTS$rel_dir"
# Free up a worker
while [ $workers -ge $max_workers ]; do
reaper
reaped=$?
if [ $reaped -eq 0 ]; then
sleep 0.1
fi
done
workers=$((workers+1))
get_idle_build_env
b=$?
if [ $b -ge 255 ]; then
echo "get_idle_build_env failed to find a free slot"
exit 1
fi
PREFIX="b$b"
( build_dir $b $src_dir $work_dir 2>&1 | sed "s#^#${PREFIX}: #" | tee $LOG_DIR/$b; exit ${PIPESTATUS[0]} ) &
pp=$!
set_build_env_pid $b $pp
else
echo "ERROR: $FUNCNAME (${LINENO}): Failed to find 'centos' in '$p', found in file '$GIT_ROOT/$PKG_DIRS_FILE'"
fi
else
echo "ERROR: $FUNCNAME (${LINENO}): Bad path '$p' in file '$GIT_ROOT/$PKG_DIRS_FILE'"
fi
done
done
# Wait for remaining workers to exit
while [ $workers -gt 0 ]; do
reaper
reaped=$?
if [ $reaped -eq 0 ]; then
sleep 0.1
fi
done
if [ $STOP_SCHEDULING -eq 1 ]; then
echo "============ Build failed ============="
if [ -f $LOG_DIR/errors ]; then
cat $LOG_DIR/errors
fi
\rm -rf $LOG_DIR
exit 1
fi
\rm -rf $LOG_DIR
# Transfer results from files back into variables
SRPM_REBUILT_LIST=$((for i in $(seq 0 $((max_workers-1))); do
fn=$MY_WORKSPACE/tmp/SRPM_REBUILT_LIST_$i
if [ -f $fn ]; then
cat $fn | tr '\n' ' '
fi
done) | sed 's/ $//')
SRPM_FAILED_REBUILD_LIST=$((for i in $(seq 0 $((max_workers-1))); do
fn=$MY_WORKSPACE/tmp/SRPM_FAILED_REBUILD_LIST_$i
if [ -f $fn ]; then
cat $fn | tr '\n' ' '
fi
done) | sed 's/ $//')
UNRESOLVED_TARGETS=$(for i in $(seq 0 $((max_workers-1))); do
if [ -f $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_$i ]; then
comm -1 -2 $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_$i > $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_merge
\mv $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_merge $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS
fi
done
cat $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS | tr '\n' ' ' | sed 's/ $//')
\rm -rf $MY_WORKSPACE/tmp/SRPM_REBUILT_LIST_* $MY_WORKSPACE/tmp/SRPM_FAILED_REBUILD_LIST_* $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS* 2>> /dev/null
# Try to find and clean orphaned and discontinued .src.rpm's
if [ $ALL -eq 1 ]; then
echo
echo "Auditing for obsolete srpms"
AUDIT_DIR=$(mktemp -d $MY_WORKSPACE/tmp/$USER-$ME-audit-XXXXXX)
if [ $? -eq 0 ] && [ "x$AUDIT_DIR" != "x" ]; then
for GIT_ROOT in $GIT_LIST; do
if echo "$GIT_ROOT" | grep -q do-not-build; then
continue
fi
for p in $(cat $GIT_ROOT/$PKG_DIRS_FILE 2>> /dev/null); do
(
src_dir="$GIT_ROOT/$p"
if [ -d $src_dir ]; then
if [ -d $src_dir/$DISTRO ]; then
for f in $(find $src_dir/${DISTRO} -name '*.spec' | sort -V); do
NAME=$(spec_find_tag Name "$f" 2>> /dev/null)
if [ $? -eq 0 ]; then
touch "$AUDIT_DIR/$NAME"
fi
done
if [ -f $src_dir/$SRPM_LIST_PATH ]; then
for p in $(grep -v '^#' $src_dir/$SRPM_LIST_PATH | grep -v '^$'); do
ORIG_SRPM_PATH=""
# absolute path source rpms
echo "$p" | grep "^/" >/dev/null && ORIG_SRPM_PATH=$p
if [ "${ORIG_SRPM_PATH}x" == "x" ]; then
# handle repo: definitions
echo "$p" | grep "^repo:" >/dev/null && ORIG_SRPM_PATH=$(echo $p | sed "s%^repo:%$REPO_DOWNLOADS_ROOT/%")
fi
if [ "${ORIG_SRPM_PATH}x" == "x" ]; then
# handle mirror: definitions
echo "$p" | grep "^mirror:" >/dev/null && ORIG_SRPM_PATH=$(echo $p | sed "s%^mirror:%$MIRROR_ROOT/%" | sed "s#CentOS/tis-r3-CentOS/kilo/##" | sed "s#CentOS/tis-r3-CentOS/mitaka/##")
fi
if [ "${ORIG_SRPM_PATH}x" == "x" ]; then
# we haven't found a valid prefix yet, so assume it's a legacy
# file (mirror: interpretation)
ORIG_SRPM_PATH="$MIRROR_ROOT/$p"
fi
if [ -f $ORIG_SRPM_PATH ]; then
NAME=$(rpm -q --queryformat '%{NAME}\n' -p $ORIG_SRPM_PATH 2>> /dev/null)
if [ $? -eq 0 ]; then
touch "$AUDIT_DIR/$NAME"
fi
fi
done
fi
fi
fi
) &
done
done
echo "waiting"
wait
echo "Auditing for obsolete srpms Phase 2"
for r in $(find $SRPM_OUT -name '*.src.rpm' | sort -V); do
(
NAME=$(rpm -q --queryformat '%{NAME}\n' -p $r 2>> /dev/null)
ALT_NAME=$(echo $NAME | sed "s#-$BUILD_TYPE\$##")
FOUND=0
if [[ -f "$AUDIT_DIR/$NAME" || ( "$BUILD_TYPE" != "std" && -f "$AUDIT_DIR/$ALT_NAME" ) ]]; then
FOUND=1
fi
if [ $FOUND -eq 0 ]; then
for INPUT_DIR in $(find $BUILD_INPUTS -name $NAME | sort -V); do
if [ -d "$INPUT_DIR/rpmbuild/SRPMS" ]; then
clean_srpm_dir $build_idx "$INPUT_DIR/rpmbuild/SRPMS" 0
fi
if [ -d $INPUT_DIR ]; then
echo "rm -rf $r"
\rm -rf $r
fi
done
if [ -f $r ]; then
\rm -f -v $r
fi
fi
) &
done
echo "waiting"
wait
\rm -rf "$AUDIT_DIR"
fi
echo "Auditing for obsolete srpms done"
fi
if [ $CLEAN_FLAG -eq 1 ]; then
if [ $ALL -eq 1 ]; then
\rm -rf $BUILD_INPUTS
\rm -rf $SOURCE_OUT/*.src.rpm
fi
fi
if [ $EDIT_FLAG -ne 1 ]; then
echo "==== Update repodata ====="
mkdir -p $SRPM_OUT/repodata
for d in $(find -L $SRPM_OUT -type d -name repodata); do
(cd $d/..
\rm -rf repodata
$CREATEREPO $(pwd)
create_lst $(pwd)
)
done
echo "==== Update repodata complete ====="
fi
FINAL_RC=0
if [ $CLEAN_FLAG -eq 0 ] && [ $EDIT_FLAG -eq 0 ]; then
echo ""
if [ "$SRPM_FAILED_REBUILD_LIST" != "" ]; then
N=$(echo "$SRPM_FAILED_REBUILD_LIST" | wc -w)
echo "Failed to build $N packages:"
echo " $SRPM_FAILED_REBUILD_LIST"
FINAL_RC=1
fi
if [ "$SRPM_REBUILT_LIST" != "" ]; then
N=$(echo "$SRPM_REBUILT_LIST" | wc -w)
echo "Successfully built $N packages:"
echo " $SRPM_REBUILT_LIST"
echo ""
echo "Compiled src.rpm's can be found here: $SRPM_OUT"
fi
if [ "$SRPM_FAILED_REBUILD_LIST" == "" ] && [ "$SRPM_REBUILT_LIST" == "" ]; then
echo "No packages required a rebuild"
fi
fi
if [ "$UNRESOLVED_TARGETS" != "" ]; then
echo ""
echo "ERROR: $FUNCNAME (${LINENO}): failed to resolve build targets: $UNRESOLVED_TARGETS"
FINAL_RC=1
fi
exit $FINAL_RC
) 2>&1 | stdbuf -o0 awk '{ print strftime("%H:%M:%S"), $0; fflush(); }' | tee $(date "+$MY_WORKSPACE/build-srpms-parallel_%Y-%m-%d_%H-%M-%S.log") ; exit ${PIPESTATUS[0]}
View Git Blame Copy Permalink