starlingx/root
Code Issues Proposed changes

Merge "Use md5sums of inputs to determine if a package needs a rebuild"

Browse Source
This commit is contained in:
Zuul 2018-09-04 15:54:42 +00:00 committed by Gerrit Code Review
commit 792aa91dd7
4 changed files with 1418 additions and 451 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
build-tools/build-rpms-parallel
View File

@ -124,7 +124,7 @@ create-no-clean-list () {
else
if [ -f MY_BUILD_CFG ]; then
if [ -f MY_BUILD_CFG ]; then
find "$MY_BUILD_CFG" -not -newer "$NO_CLEAN_LIST_FILE" | grep $(basename $MY_BUILD_CFG) >> /dev/null
find "$MY_BUILD_CFG" -not -newermm "$NO_CLEAN_LIST_FILE" | grep -q $(basename $MY_BUILD_CFG)
if [ $? -eq 0 ]; then
NEED_REBUILD=1
fi
@ -930,7 +930,6 @@ mock_clean () {
( mock_clean_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
done
wait
# mock_clean_cfg $BUILD_CFG
echo "=================================="
remove_mock_symlinks $MY_BUILD_CFG
echo "${FUNCNAME[0]}: out"
@ -1050,7 +1049,6 @@ mock_partial_clean () {
( mock_partial_clean_cfg $SUB_CFG "$SRPMS_LIST" "$RPMS_LIST" 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
done
wait
# mock_partial_clean_cfg $BUILD_CFG "$SRPMS_LIST" "$RPMS_LIST"
echo "=================================="
echo "${FUNCNAME[0]}: out"
}
@ -1113,7 +1111,6 @@ mock_clean_cache_all () {
( mock_clean_cache_all_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
done
wait
# mock_clean_cache_all_cfg $BUILD_CFG
echo "${FUNCNAME[0]}: out"
}
@ -1263,7 +1260,6 @@ mock_clean_mounts () {
( mock_clean_mounts_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
done
wait
# mock_clean_mounts_cfg $BUILD_CFG
echo "${FUNCNAME[0]}: out"
}
@ -1309,7 +1305,6 @@ clean_yum_cache () {
( clean_yum_cache_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
done
wait
# clean_yum_cache_cfg $BUILD_CFG
echo "${FUNCNAME[0]}: out"
}
@ -1372,7 +1367,6 @@ mock_update_or_init () {
( mock_update_or_init_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) &
done
wait
# mock_update_or_init_cfg $BUILD_CFG
echo "${FUNCNAME[0]}: out"
}
@ -1396,7 +1390,7 @@ FORMAL_FLAG=0
CAREFUL=0
# read the options
TEMP=$(getopt -o ha::bc: --long std,rt,installer,no-required,no-descendants,no-autoclean,no-build-info,clean,tmpfs-clean,formal,careful,help,arga::,argb,argc: -n "$ME" -- "$@")
TEMP=$(getopt -o ha::bc: --long parallel,std,rt,installer,no-required,no-descendants,no-autoclean,no-build-info,clean,tmpfs-clean,formal,careful,help,arga::,argb,argc: -n "$ME" -- "$@")
if [ $? -ne 0 ]; then
usage
@ -1434,6 +1428,7 @@ while true ; do
"") shift 2 ;;
*) ARG_C=$2 ; shift 2 ;;
esac ;;
--parallel) shift ;;
--) shift ; break ;;
*) echo "Internal error!" ; exit 1 ;;
esac
@ -1764,8 +1759,15 @@ update_spec_cache () {
mkdir -p $SPEC_DIR
NEED_UPDATE=1
else
find "$SPEC_DIR" -not -newer "$SRPM" -name '*.spec' | grep 'spec' >> /dev/null
find "$SPEC_DIR" -name '*.spec' | grep 'spec' >> /dev/null
if [ $? -ne 0 ]; then
# No spec file
NEED_UPDATE=1
fi
find "$SPEC_DIR" -not -newermm "$SRPM" -name '*.spec' | grep -q 'spec'
if [ $? -eq 0 ]; then
# spec is older than src.rpm
NEED_UPDATE=1
fi
fi
@ -1778,7 +1780,7 @@ update_spec_cache () {
mkdir -p NAMES
mkdir -p SERVICES
mkdir -p BUILDS_VR
rpm2cpio $SRPM | pax -r '*.spec'
rpm2cpio $SRPM | cpio -civ '*.spec'
if [ $? -ne 0 ]; then
echo "ERROR: no spec file found in '$SRPM'"
fi
@ -1843,12 +1845,15 @@ for n in "${SRPM_PKG_NAMES[@]}"; do
# We weren't given a list of build targets.
# Build anything missing or out of date.
NEED_BUILD=0
BN=$(basename ${s//.src.rpm/})
if [ -f $RESULT_DIR/$USER-$PROJECT-$SRC_BUILD_ENVIRONMENT-$BUILD_TYPE/$(basename ${s//.src.rpm/})/fail ]; then
echo "Previous build of $(basename ${s//.src.rpm/}) failed"
if [ -f $RESULT_DIR/$MY_BUILD_ENVIRONMENT/$BN/fail ]; then
echo "Found: $RESULT_DIR/$MY_BUILD_ENVIRONMENT/$BN/fail"
echo "Previous build of $BN failed"
NEED_BUILD=1
elif [ ! -f $RESULT_DIR/$USER-$PROJECT-$SRC_BUILD_ENVIRONMENT-$BUILD_TYPE/$(basename ${s//.src.rpm/})/success ]; then
echo "No previous build of $(basename ${s//.src.rpm/})"
elif [ ! -f $RESULT_DIR/$MY_BUILD_ENVIRONMENT/$BN/success ]; then
echo "Not Found: $RESULT_DIR/$MY_BUILD_ENVIRONMENT/$BN/success"
echo "No previous build of $BN"
NEED_BUILD=1
else
LOCAL_RPMS_VRA_LIST=$(ls -1 $SPEC_DIR/BUILDS_VR | tr '\n' ' ')
@ -1872,12 +1877,12 @@ for n in "${SRPM_PKG_NAMES[@]}"; do
fi
fi
newer=$(find $RPM_DIR/$f*rpm -type f -not -cnewer $s 2>> /dev/null | wc -l)
# echo "$m $newer=find $RPM_DIR/$f*rpm -type f -not -cnewer $s 2>> /dev/null | wc -l"
newer=$(find $RPM_DIR/$f*rpm -type f -not -newermm $s 2>> /dev/null | wc -l)
# echo "$m $newer=find $RPM_DIR/$f*rpm -type f -not -newermm $s 2>> /dev/null | wc -l"
if [ $m -eq 0 ] || [ $newer -gt 0 ] || [ $CLEAN_FLAG -eq 1 ]; then
if [ $newer -gt 0 ]; then
echo "Including '$f' due to newer code"
find $RPM_DIR/$f*rpm -type f -not -cnewer $s
find $RPM_DIR/$f*rpm -type f -not -newermm $s
else
if [ $m -eq 0 ]; then
echo "Including '$f' due to m=0"
@ -2129,8 +2134,8 @@ if [ $CLEAN_FLAG -eq 0 ]; then
echo $f | grep -f "$UNBUILT_PATTERN_FILE" >> /dev/null && m=1
fi
n=$(find $RPM_DIR/$f*rpm -type f -not -cnewer $s 2>> /dev/null | wc -l)
# echo "$n=find $RPM_DIR/$f*rpm -type f -not -cnewer $s 2>> /dev/null | wc -l"
n=$(find $RPM_DIR/$f*rpm -type f -not -newermm $s 2>> /dev/null | wc -l)
# echo "$n=find $RPM_DIR/$f*rpm -type f -not -newermm $s 2>> /dev/null | wc -l"
if [ $m -eq 0 ] || [ $n -gt 0 ] || [ $MUST_CLEAN -eq 1 ]; then
CLEAN_BEFORE_BUILD_SRPM_LIST="$CLEAN_BEFORE_BUILD_SRPM_LIST $s"
CLEAN_BEFORE_BUILD_RPM_LIST="$CLEAN_BEFORE_BUILD_RPM_LIST $LOCAL_RPMS_LIST"

1240
build-tools/build-rpms-serial
View File

File diff suppressed because it is too large Load Diff

289
build-tools/build-srpms-parallel
View File

@ -72,6 +72,20 @@ usage () {
echo ""
}
spec_cache_dir_from_srpm () {
local SRPM=${1}
local SPEC_DIR=$(echo $SRPM | sed 's#/SRPMS/#/SPECS/#')
echo "$SPEC_DIR"
}
result_dir_from_srpm () {
local SRPM=$(basename ${1} | sed 's#.src.rpm$##')
local RESULT_DIR="$MY_WORKSPACE/results/$MY_BUILD_ENVIRONMENT/$SRPM"
echo "$RESULT_DIR"
}
# This function creates a bunch of subdirs in $MY_WORKSPACE and makes sure
# that a $MY_BUILD_CFG file exists.
#
@ -432,6 +446,8 @@ build_dir () {
clean_srpm_dir () {
local build_idx=$1
local DIR=$2
local EXCLUDE_MD5=$3
local SRPM_PATH
local SRPM_FILE
local SRPM_OUT_PATH
@ -439,6 +455,10 @@ clean_srpm_dir () {
local SRPM_OUT_NAME
local INPUTS_TO_CLEAN=""
if [ $EXCLUDE_MD5 == "" ]; then
EXCLUDE_MD5=0
fi
echo "clean_srpm_dir build_idx=$build_idx DIR=$DIR"
INPUTS_TO_CLEAN=$(dirname $(dirname $DIR))
@ -450,19 +470,28 @@ clean_srpm_dir () {
for SRPM_PATH in $(find "$DIR" -name '*.src.rpm'); do
SRPM_FILE=$(basename $SRPM_PATH)
SRPM_NAME=$(rpm -q --queryformat '%{NAME}\n' --nosignature -p $SRPM_PATH 2>> /dev/null)
if [ $CLEAN_FLAG -eq 1 ]; then
sed -i "/^$SRPM_NAME$/d" $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_${build_idx}
fi
\rm -fv $SRPM_PATH $SRPM_OUT/$SRPM_FILE
if [ -d $SRPM_ASSEMBLE/$SRPM_NAME ]; then
echo "rm -rf $SRPM_ASSEMBLE/$SRPM_NAME"
\rm -rf $SRPM_ASSEMBLE/$SRPM_NAME
fi
if [ -d $SOURCE_OUT/$SRPM_FILE ]; then
echo "rm -rf $SOURCE_OUT/$SRPM_FILE"
\rm -rf $SOURCE_OUT/$SRPM_FILE
fi
if [ $EXCLUDE_MD5 -eq 0 ] && [ -d $SOURCE_OUT/$SRPM_NAME ]; then
echo "rm -rf $SOURCE_OUT/$SRPM_NAME"
\rm -rf $SOURCE_OUT/$SRPM_NAME
fi
local d
local src_d
local spec
@ -552,7 +581,7 @@ build_dir_srpm () {
echo "===== Cleaning '$TARGET_FOUND' ====="
if [ -d $SRPM_DIR ] && [ $EDIT_FLAG -eq 0 ]; then
clean_srpm_dir $build_idx "$SRPM_DIR"
clean_srpm_dir $build_idx "$SRPM_DIR" 0
fi
if [ -d $PKG_ROOT_DIR ]; then
@ -593,54 +622,68 @@ build_dir_srpm () {
fi
fi
local BUILD_NEEDED=2
#
# Capture md5 data for all input files
#
local TARGET_SOURCES_DIR="$SOURCES_DIR/$TARGET_FOUND"
local INPUT_FILES="$TARGET_SOURCES_DIR/srpm_input.files"
local INPUT_FILES_SORTED="$TARGET_SOURCES_DIR/srpm_sorted_input.files"
local INPUT_FILES_MD5="$TARGET_SOURCES_DIR/srpm_input.md5"
local REFERENCE_MD5="$TARGET_SOURCES_DIR/srpm_reference.md5"
mkdir -p "$TARGET_SOURCES_DIR"
find $PKG_BASE -type f > $INPUT_FILES
find $ORIG_SRPM_PATH -type f >> $INPUT_FILES
(
cd $PKG_BASE
if [ "x$COPY_LIST" != "x" ]; then
find $COPY_LIST -type f >> $INPUT_FILES
fi
)
cat $INPUT_FILES | xargs readlink -f | sort --unique > $INPUT_FILES_SORTED
\rm -f $INPUT_FILES
cat $INPUT_FILES_SORTED | xargs md5sum | sed "s# $(readlink -f $MY_REPO)/# #" > $INPUT_FILES_MD5
\rm -f $INPUT_FILES_SORTED
echo "Wrote: $INPUT_FILES_MD5"
#
# Is a rebuild required?
# Compare md5 of current inputs vs md5 of previous build?
#
local BUILD_NEEDED=0
local SRPM_OUT_PATH2
local DIFF_LINE
local DIFF_FILE
for SRPM_PATH in $(find "$FULL_BUILD_DIR/SRPMS" -name '*.src.rpm' | sort -V); do
if [ $BUILD_NEEDED -eq 2 ]; then
BUILD_NEEDED=0
fi
b=$(basename $SRPM_PATH)
SRPM_OUT_PATH2=$(find $SRPM_OUT -name $b)
if [ "x$SRPM_OUT_PATH2" == "x" ]; then
BUILD_NEEDED=1
fi
n=$(find $PKG_BASE -type f -cnewer $SRPM_PATH | wc -l)
if [ $n -gt 0 ]; then
if [ -f $REFERENCE_MD5 ]; then
DIFF_LINE=$(diff "$INPUT_FILES_MD5" "$REFERENCE_MD5" | head -n 2 | tail -n 1; exit ${PIPESTATUS[0]})
if [ $? -ne 0 ]; then
DIFF_FILE=$(echo "$DIFF_LINE" | cut -d ' ' -f4-)
BUILD_NEEDED=1
case ${DIFF_LINE:0:1} in
'>') echo "Rebuild required due to deleted file: $DIFF_FILE" ;;
'<') echo "Rebuild required due to new or changed file: $DIFF_FILE" ;;
*) echo "Rebuild required due to diff: $DIFF_LINE" ;;
esac
fi
else
echo "Rebuild required due to missing reference md5: $REFERENCE_MD5"
BUILD_NEEDED=1
fi
n=$(find $ORIG_SRPM_PATH -type f -cnewer $SRPM_PATH | wc -l)
if [ $n -gt 0 ]; then
BUILD_NEEDED=1
fi
if [ -f $PKG_BASE/$DATA ]; then
(
cd $PKG_BASE
BUILD_NEEDED=0
srpm_source_build_data $DATA
# NOTE: SRC_DIR is not honored in this build path
if [ "x$COPY_LIST" != "x" ]; then
n=$(find $COPY_LIST -type f -cnewer $SRPM_PATH | wc -l)
if [ $n -gt 0 ]; then
BUILD_NEEDED=1
fi
fi
# NOTE: COPY_LIST_TO_TAR is not honored in this build path
exit $BUILD_NEEDED
)
if [ $? -gt 0 ]; then
if [ -d "$FULL_BUILD_DIR/SRPMS" ]; then
for SRPM_PATH in $(find "$FULL_BUILD_DIR/SRPMS" -name '*.src.rpm' | sort -V); do
b=$(basename $SRPM_PATH)
SRPM_OUT_PATH2=$(find $SRPM_OUT -name $b)
if [ "x$SRPM_OUT_PATH2" == "x" ]; then
echo "Rebuild required due to missing srpm: $b"
BUILD_NEEDED=1
fi
fi
done
done
else
echo "Rebuild required due to missing directory: '$FULL_BUILD_DIR/SRPMS'"
BUILD_NEEDED=1
fi
if [ $BUILD_NEEDED -eq 0 ]; then
echo "SRPM build not required for '$PKG_BASE'"
@ -650,7 +693,7 @@ build_dir_srpm () {
fi
if [ $EDIT_FLAG -eq 0 ]; then
clean_srpm_dir $build_idx "$FULL_BUILD_DIR/SRPMS"
clean_srpm_dir $build_idx "$FULL_BUILD_DIR/SRPMS" 1
if [ -d $PKG_ROOT_DIR ]; then
echo "arf rm -rf $PKG_ROOT_DIR"
@ -731,7 +774,7 @@ build_dir_srpm () {
\cp -L -r -f -v $p $FULL_BUILD_DIR/SOURCES
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): COPY_LIST: file not found: '$p'"
exit 1
return 1
fi
done
fi
@ -770,6 +813,23 @@ build_dir_srpm () {
fi
fi
\rm -f -v "$REFERENCE_MD5"
\mv -v "$INPUT_FILES_MD5" "$REFERENCE_MD5"
local SPEC_DIR=$(spec_cache_dir_from_srpm $SRPM_OUT/$ss)
if [ -d $SPEC_DIR/BUILDS_VR ]; then
for f in $(ls -1 $SPEC_DIR/BUILDS_VR); do
for r in $(find $RPM_DIR -name "$f*rpm" 2>> /dev/null); do
\rm -f -v $r
done
done
fi
local RESULT_DIR=$(result_dir_from_srpm $SRPM_OUT/$ss)
if [ -d $RESULT_DIR ]; then
echo "rm -rf $RESULT_DIR"
\rm -rf $RESULT_DIR
fi
done
echo "$TARGET_FOUND" >> $MY_WORKSPACE/tmp/SRPM_REBUILT_LIST_${build_idx}
@ -842,7 +902,7 @@ build_dir_spec () {
# clean
echo "===== Cleaning '$TARGET_FOUND' ====="
if [ -d $SRPM_PATH ] && [ $EDIT_FLAG -eq 0 ]; then
clean_srpm_dir $build_idx $SRPM_PATH
clean_srpm_dir $build_idx $SRPM_PATH 0
fi
if [ -d $PKG_ROOT_DIR ]; then
@ -865,9 +925,6 @@ build_dir_spec () {
fi
fi
local BUILD_NEEDED=2
local SRPM_OUT_PATH2
#
# Load data from build_srpm.data
#
@ -885,57 +942,82 @@ build_dir_spec () {
return 1
fi
for SRPM_PATH2 in $(find "$RPMBUILD_BASE/SRPMS" -name '*.src.rpm' | sort -V); do
if [ $BUILD_NEEDED -eq 2 ]; then
BUILD_NEEDED=0
#
# Capture md5 data for all input files
#
local TARGET_SOURCES_DIR="$SOURCES_DIR/$TARGET_FOUND"
local INPUT_FILES="$TARGET_SOURCES_DIR/srpm_input.files"
local INPUT_FILES_SORTED="$TARGET_SOURCES_DIR/srpm_sorted_input.files"
local INPUT_FILES_MD5="$TARGET_SOURCES_DIR/srpm_input.md5"
local REFERENCE_MD5="$TARGET_SOURCES_DIR/srpm_reference.md5"
mkdir -p "$TARGET_SOURCES_DIR"
find $PKG_BASE -type f > $INPUT_FILES
find $ORIG_SRPM_PATH -type f >> $INPUT_FILES
(
cd $PKG_BASE
if [ "x$SRC_DIR" != "x" ]; then
if [ -d "$SRC_DIR" ]; then
find $SRC_DIR -type f | grep -v '[/][.]git$' | grep -v '[/][.]git[/]' >> $INPUT_FILES
fi
fi
b=$(basename $SRPM_PATH2)
SRPM_OUT_PATH2=$(find $SRPM_OUT -name $b)
if [ "x$SRPM_OUT_PATH2" == "x" ]; then
if [ "x$COPY_LIST" != "x" ]; then
find $COPY_LIST -type f >> $INPUT_FILES
fi
if [ "x$COPY_LIST_TO_TAR" != "x" ]; then
find $COPY_LIST_TO_TAR -type f >> $INPUT_FILES
fi
if [ "x$OPT_DEP_LIST" != "x" ]; then
find $OPT_DEP_LIST -type f >> $INPUT_FILES 2> /dev/null
fi
)
cat $INPUT_FILES | xargs readlink -f | sort --unique > $INPUT_FILES_SORTED
\rm -f $INPUT_FILES
cat $INPUT_FILES_SORTED | xargs md5sum | sed "s# $(readlink -f $MY_REPO)/# #" > $INPUT_FILES_MD5
\rm -f $INPUT_FILES_SORTED
echo "Wrote: $INPUT_FILES_MD5"
#
# Is a rebuild required?
# Compare md5 of current inputs vs md5 of previous build?
#
local BUILD_NEEDED=0
local SRPM_OUT_PATH2
local DIFF_LINE
local DIFF_FILE
if [ -f $REFERENCE_MD5 ]; then
DIFF_LINE=$(diff "$INPUT_FILES_MD5" "$REFERENCE_MD5" | head -n 2 | tail -n 1; exit ${PIPESTATUS[0]})
if [ $? -ne 0 ]; then
DIFF_FILE=$(echo "$DIFF_LINE" | cut -d ' ' -f4-)
BUILD_NEEDED=1
case ${DIFF_LINE:0:1} in
'>') echo "Rebuild required due to deleted file: $DIFF_FILE" ;;
'<') echo "Rebuild required due to new or changed file: $DIFF_FILE" ;;
*) echo "Rebuild required due to diff: $DIFF_LINE" ;;
esac
fi
else
echo "Rebuild required due to missing reference md5: $REFERENCE_MD5"
BUILD_NEEDED=1
fi
n=$(find $PKG_BASE -type f -cnewer $SRPM_PATH2 | wc -l)
if [ $n -gt 0 ]; then
BUILD_NEEDED=1
fi
if [ -f $PKG_BASE/$DATA ]; then
(
cd $PKG_BASE
BUILD_NEEDED=0
srpm_source_build_data $DATA
if [ "x$SRC_DIR" != "x" ]; then
if [ -d "$SRC_DIR" ]; then
n=$(find $SRC_DIR -type f -cnewer $SRPM_PATH2 | grep -v '[/][.]git$' | grep -v '[/][.]git[/]' | wc -l)
if [ $n -gt 0 ]; then
BUILD_NEEDED=1
fi
fi
fi
if [ "x$COPY_LIST" != "x" ]; then
n=$(find $COPY_LIST -type f -cnewer $SRPM_PATH2 | wc -l)
if [ $n -gt 0 ]; then
BUILD_NEEDED=1
fi
fi
if [ "x$COPY_LIST_TO_TAR" != "x" ]; then
n=$(find $COPY_LIST_TO_TAR -type f -cnewer $SRPM_PATH2 | wc -l)
if [ $n -gt 0 ]; then
BUILD_NEEDED=1
fi
fi
exit $BUILD_NEEDED
)
if [ $? -gt 0 ]; then
if [ -d "$FULL_BUILD_DIR/SRPMS" ]; then
for SRPM_PATH2 in $(find "$RPMBUILD_BASE/SRPMS" -name '*.src.rpm' | sort -V); do
b=$(basename $SRPM_PATH2)
SRPM_OUT_PATH2=$(find $SRPM_OUT -name $b)
if [ "x$SRPM_OUT_PATH2" == "x" ]; then
echo "Rebuild required due to missing srpm: $b"
BUILD_NEEDED=1
fi
fi
done
done
else
echo "Rebuild required due to missing directory: '$FULL_BUILD_DIR/SRPMS'"
BUILD_NEEDED=1
fi
if [ $EDIT_FLAG -eq 1 ]; then
@ -978,8 +1060,9 @@ build_dir_spec () {
echo "DATA=$DATA"
if [ -d "$RPMBUILD_BASE/SRPMS" ]; then
clean_srpm_dir $build_idx "$RPMBUILD_BASE/SRPMS"
clean_srpm_dir $build_idx "$RPMBUILD_BASE/SRPMS" 1
fi
if [ -d $RPMBUILD_BASE ]; then
echo "rm -rf $RPMBUILD_BASE"
\rm -rf "$RPMBUILD_BASE"
@ -1070,6 +1153,24 @@ build_dir_spec () {
\rm -f $SLOW_FLAG_FILE
fi
fi
\rm -f -v "$REFERENCE_MD5"
\mv -v "$INPUT_FILES_MD5" "$REFERENCE_MD5"
local SPEC_DIR=$(spec_cache_dir_from_srpm $SRPM_OUT/$ss)
if [ -d $SPEC_DIR/BUILDS_VR ]; then
for f in $(ls -1 $SPEC_DIR/BUILDS_VR); do
for r in $(find $RPM_DIR -name "$f*rpm" 2>> /dev/null); do
\rm -f -v $r
done
done
fi
local RESULT_DIR=$(result_dir_from_srpm $SRPM_OUT/$ss)
if [ -d $RESULT_DIR ]; then
echo "rm -rf $RESULT_DIR"
\rm -rf $RESULT_DIR
fi
done
echo "$TARGET_FOUND" >> $MY_WORKSPACE/tmp/SRPM_REBUILT_LIST_${build_idx}
@ -1404,7 +1505,7 @@ if [ $ALL -eq 1 ]; then
if [ $FOUND -eq 0 ]; then
for INPUT_DIR in $(find $BUILD_INPUTS -name $NAME | sort -V); do
if [ -d "$INPUT_DIR/rpmbuild/SRPMS" ]; then
clean_srpm_dir $build_idx "$INPUT_DIR/rpmbuild/SRPMS"
clean_srpm_dir $build_idx "$INPUT_DIR/rpmbuild/SRPMS" 0
fi
if [ -d $INPUT_DIR ]; then
echo "rm -rf $r"

297
build-tools/build-srpms-serial
View File

@ -59,6 +59,19 @@ usage () {
echo ""
}
spec_cache_dir_from_srpm () {
local SRPM=${1}
local SPEC_DIR=$(echo $SRPM | sed 's#/SRPMS/#/SPECS/#')
echo "$SPEC_DIR"
}
result_dir_from_srpm () {
local SRPM=$(basename ${1} | sed 's#.src.rpm$##')
local RESULT_DIR="$MY_WORKSPACE/results/$MY_BUILD_ENVIRONMENT/$SRPM"
echo "$RESULT_DIR"
}
# This function creates a bunch of subdirs in $MY_WORKSPACE and makes sure
# that a $MY_BUILD_CFG file exists.
#
@ -365,7 +378,7 @@ build_dir () {
else
echo "ERROR: $FUNCNAME (${LINENO}): Invalid srpm path '$p', evaluated as '$ORIG_SRPM_PATH', found in '$PKG_BASE/$SRPM_LIST_PATH'"
ORIG_SRPM_PATH=""
exit 1
return 3
fi
done
fi
@ -417,6 +430,7 @@ build_dir () {
clean_srpm_dir () {
local DIR=$1
local EXCLUDE_MD5=$2
local SRPM_PATH
local SRPM_FILE
local SRPM_OUT_PATH
@ -424,6 +438,10 @@ clean_srpm_dir () {
local SRPM_OUT_NAME
local INPUTS_TO_CLEAN=""
if [ $EXCLUDE_MD5 == "" ]; then
EXCLUDE_MD5=0
fi
echo "clean_srpm_dir DIR=$DIR"
INPUTS_TO_CLEAN=$(dirname $(dirname $DIR))
@ -441,6 +459,16 @@ clean_srpm_dir () {
\rm -rf $SRPM_ASSEMBLE/$SRPM_NAME
fi
if [ -d $SOURCE_OUT/$SRPM_FILE ]; then
echo "rm -rf $SOURCE_OUT/$SRPM_FILE"
\rm -rf $SOURCE_OUT/$SRPM_FILE
fi
if [ $EXCLUDE_MD5 -eq 0 ] && [ -d $SOURCE_OUT/$SRPM_NAME ]; then
echo "rm -rf $SOURCE_OUT/$SRPM_NAME"
\rm -rf $SOURCE_OUT/$SRPM_NAME
fi
local d
local src_d
local spec
@ -457,17 +485,24 @@ clean_srpm_dir () {
done
done
# Look for older versions of the same src rpm that also need cleaning
for SRPM_OUT_PATH in $(ls -1 $SRPM_OUT/$SRPM_NAME*.src.rpm 2>> /dev/null); do
SRPM_OUT_FILE=$(basename $SRPM_OUT_PATH)
SRPM_OUT_NAME=$(rpm -q --queryformat '%{NAME}\n' -p $SRPM_OUT_PATH 2>> /dev/null)
if [ "$SRPM_NAME" == "$SRPM_OUT_NAME" ]; then
\rm -fv $SRPM_OUT_PATH
if [ -d $SOURCE_OUT/$SRPM_OUT_FILE ]; then
echo "rm -rf $SOURCE_OUT/$SRPM_OUT_FILE"
\rm -rf $SOURCE_OUT/$SRPM_OUT_FILE
fi
fi
done
done
if [ "x$INPUTS_TO_CLEAN" != "x" ]; then
for d in $INPUTS_TO_CLEAN; do
if [ -d $d ]; then
if [ -d $d/rpmbuild ]; then
echo "rm -rf $d"
\rm -rf $d
fi
@ -523,7 +558,7 @@ build_dir_srpm () {
echo "===== Cleaning '$TARGET_FOUND' ====="
if [ -d $SRPM_DIR ] && [ $EDIT_FLAG -eq 0 ]; then
clean_srpm_dir "$SRPM_DIR"
clean_srpm_dir "$SRPM_DIR" 0
fi
if [ -d $PKG_ROOT_DIR ]; then
@ -564,53 +599,69 @@ build_dir_srpm () {
fi
fi
local BUILD_NEEDED=2
#
# Capture md5 data for all input files
#
local TARGET_SOURCES_DIR="$SOURCES_DIR/$TARGET_FOUND"
local INPUT_FILES="$TARGET_SOURCES_DIR/srpm_input.files"
local INPUT_FILES_SORTED="$TARGET_SOURCES_DIR/srpm_sorted_input.files"
local INPUT_FILES_MD5="$TARGET_SOURCES_DIR/srpm_input.md5"
local REFERENCE_MD5="$TARGET_SOURCES_DIR/srpm_reference.md5"
mkdir -p "$TARGET_SOURCES_DIR"
find $PKG_BASE -type f > $INPUT_FILES
find $ORIG_SRPM_PATH -type f >> $INPUT_FILES
(
cd $PKG_BASE
if [ "x$COPY_LIST" != "x" ]; then
find $COPY_LIST -type f >> $INPUT_FILES
fi
)
cat $INPUT_FILES | xargs readlink -f | sort --unique > $INPUT_FILES_SORTED
\rm -f $INPUT_FILES
cat $INPUT_FILES_SORTED | xargs md5sum | sed "s# $(readlink -f $MY_REPO)/# #" > $INPUT_FILES_MD5
\rm -f $INPUT_FILES_SORTED
echo "Wrote: $INPUT_FILES_MD5"
#
# Is a rebuild required?
# Compare md5 of current inputs vs md5 of previous build?
#
local BUILD_NEEDED=0
local SRPM_OUT_PATH2
local DIFF_LINE
local DIFF_FILE
for SRPM_PATH in $(find "$FULL_BUILD_DIR/SRPMS" -name '*.src.rpm' | sort -V); do
if [ $BUILD_NEEDED -eq 2 ]; then
BUILD_NEEDED=0
fi
b=$(basename $SRPM_PATH)
SRPM_OUT_PATH2=$(find $SRPM_OUT -name $b)
if [ "x$SRPM_OUT_PATH2" == "x" ]; then
BUILD_NEEDED=1
fi
n=$(find $PKG_BASE -type f -cnewer $SRPM_PATH | wc -l)
if [ $n -gt 0 ]; then
if [ -f $REFERENCE_MD5 ]; then
DIFF_LINE=$(diff "$INPUT_FILES_MD5" "$REFERENCE_MD5" | head -n 2 | tail -n 1; exit ${PIPESTATUS[0]})
if [ $? -ne 0 ]; then
DIFF_FILE=$(echo "$DIFF_LINE" | cut -d ' ' -f4-)
BUILD_NEEDED=1
case ${DIFF_LINE:0:1} in
'>') echo "Rebuild required due to deleted file: $DIFF_FILE" ;;
'<') echo "Rebuild required due to new or changed file: $DIFF_FILE" ;;
*) echo "Rebuild required due to diff: $DIFF_LINE" ;;
esac
fi
else
echo "Rebuild required due to missing reference md5: $REFERENCE_MD5"
BUILD_NEEDED=1
fi
n=$(find $ORIG_SRPM_PATH -type f -cnewer $SRPM_PATH | wc -l)
if [ $n -gt 0 ]; then
BUILD_NEEDED=1
fi
if [ -f $PKG_BASE/$DATA ]; then
(
cd $PKG_BASE
BUILD_NEEDED=0
# NOTE: SRC_DIR is not honored in this build path
if [ "x$COPY_LIST" != "x" ]; then
n=$(find $COPY_LIST -type f -cnewer $SRPM_PATH | wc -l)
if [ $n -gt 0 ]; then
BUILD_NEEDED=1
fi
fi
# NOTE: COPY_LIST_TO_TAR is not honored in this build path
exit $BUILD_NEEDED
)
if [ $? -gt 0 ]; then
if [ -d "$FULL_BUILD_DIR/SRPMS" ]; then
for SRPM_PATH in $(find "$FULL_BUILD_DIR/SRPMS" -name '*.src.rpm' | sort -V); do
b=$(basename $SRPM_PATH)
SRPM_OUT_PATH2=$(find $SRPM_OUT -name $b)
if [ "x$SRPM_OUT_PATH2" == "x" ]; then
echo "Rebuild required due to missing srpm: $b"
BUILD_NEEDED=1
fi
fi
done
done
else
echo "Rebuild required due to missing directory: '$FULL_BUILD_DIR/SRPMS'"
BUILD_NEEDED=1
fi
if [ $BUILD_NEEDED -eq 0 ]; then
echo "SRPM build not required for '$PKG_BASE'"
@ -620,7 +671,7 @@ build_dir_srpm () {
fi
if [ $EDIT_FLAG -eq 0 ]; then
clean_srpm_dir "$FULL_BUILD_DIR/SRPMS"
clean_srpm_dir "$FULL_BUILD_DIR/SRPMS" 1
if [ -d $PKG_ROOT_DIR ]; then
echo "rm -rf $PKG_ROOT_DIR"
@ -700,7 +751,7 @@ build_dir_srpm () {
\cp -L -r -f -v $p $FULL_BUILD_DIR/SOURCES
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): COPY_LIST: file not found: '$p'"
exit 1
return 1
fi
done
fi
@ -737,6 +788,24 @@ build_dir_srpm () {
\rm -f $SLOW_FLAG_FILE
fi
fi
\rm -f -v "$REFERENCE_MD5"
\mv -v "$INPUT_FILES_MD5" "$REFERENCE_MD5"
local SPEC_DIR=$(spec_cache_dir_from_srpm $SRPM_OUT/$ss)
if [ -d $SPEC_DIR/BUILDS_VR ]; then
for f in $(ls -1 $SPEC_DIR/BUILDS_VR); do
for r in $(find $RPM_DIR -name "$f*rpm" 2>> /dev/null); do
\rm -f -v $r
done
done
fi
local RESULT_DIR=$(result_dir_from_srpm $SRPM_OUT/$ss)
if [ -d $RESULT_DIR ]; then
echo "rm -rf $RESULT_DIR"
\rm -rf $RESULT_DIR
fi
done
SRPM_REBUILT_LIST="$SRPM_REBUILT_LIST $TARGET_FOUND"
@ -807,7 +876,7 @@ build_dir_spec () {
# clean
echo "===== Cleaning '$TARGET_FOUND' ====="
if [ -d $SRPM_PATH ] && [ $EDIT_FLAG -eq 0 ]; then
clean_srpm_dir $SRPM_PATH
clean_srpm_dir $SRPM_PATH 0
fi
if [ -d $PKG_ROOT_DIR ]; then
@ -830,9 +899,6 @@ build_dir_spec () {
fi
fi
local BUILD_NEEDED=2
local SRPM_OUT_PATH2
#
# Load data from build_srpm.data
#
@ -852,56 +918,82 @@ build_dir_spec () {
fi
fi
for SRPM_PATH2 in $(find "$RPMBUILD_BASE/SRPMS" -name '*.src.rpm' | sort -V); do
if [ $BUILD_NEEDED -eq 2 ]; then
BUILD_NEEDED=0
#
# Capture md5 data for all input files
#
local TARGET_SOURCES_DIR="$SOURCES_DIR/$TARGET_FOUND"
local INPUT_FILES="$TARGET_SOURCES_DIR/srpm_input.files"
local INPUT_FILES_SORTED="$TARGET_SOURCES_DIR/srpm_sorted_input.files"
local INPUT_FILES_MD5="$TARGET_SOURCES_DIR/srpm_input.md5"
local REFERENCE_MD5="$TARGET_SOURCES_DIR/srpm_reference.md5"
mkdir -p "$TARGET_SOURCES_DIR"
find $PKG_BASE -type f > $INPUT_FILES
find $ORIG_SRPM_PATH -type f >> $INPUT_FILES
(
cd $PKG_BASE
if [ "x$SRC_DIR" != "x" ]; then
if [ -d "$SRC_DIR" ]; then
find $SRC_DIR -type f | grep -v '[/][.]git$' | grep -v '[/][.]git[/]' >> $INPUT_FILES
fi
fi
b=$(basename $SRPM_PATH2)
SRPM_OUT_PATH2=$(find $SRPM_OUT -name $b)
if [ "x$SRPM_OUT_PATH2" == "x" ]; then
if [ "x$COPY_LIST" != "x" ]; then
find $COPY_LIST -type f >> $INPUT_FILES
fi
if [ "x$COPY_LIST_TO_TAR" != "x" ]; then
find $COPY_LIST_TO_TAR -type f >> $INPUT_FILES
fi
if [ "x$OPT_DEP_LIST" != "x" ]; then
find $OPT_DEP_LIST -type f >> $INPUT_FILES 2> /dev/null
fi
)
cat $INPUT_FILES | xargs readlink -f | sort --unique > $INPUT_FILES_SORTED
\rm -f $INPUT_FILES
cat $INPUT_FILES_SORTED | xargs md5sum | sed "s# $(readlink -f $MY_REPO)/# #" > $INPUT_FILES_MD5
\rm -f $INPUT_FILES_SORTED
echo "Wrote: $INPUT_FILES_MD5"
#
# Is a rebuild required?
# Compare md5 of current inputs vs md5 of previous build?
#
local BUILD_NEEDED=0
local SRPM_OUT_PATH2
local DIFF_LINE
local DIFF_FILE
if [ -f $REFERENCE_MD5 ]; then
DIFF_LINE=$(diff "$INPUT_FILES_MD5" "$REFERENCE_MD5" | head -n 2 | tail -n 1; exit ${PIPESTATUS[0]})
if [ $? -ne 0 ]; then
DIFF_FILE=$(echo "$DIFF_LINE" | cut -d ' ' -f4-)
BUILD_NEEDED=1
case ${DIFF_LINE:0:1} in
'>') echo "Rebuild required due to deleted file: $DIFF_FILE" ;;
'<') echo "Rebuild required due to new or changed file: $DIFF_FILE" ;;
*) echo "Rebuild required due to diff: $DIFF_LINE" ;;
esac
fi
else
echo "Rebuild required due to missing reference md5: $REFERENCE_MD5"
BUILD_NEEDED=1
fi
n=$(find $PKG_BASE -type f -cnewer $SRPM_PATH2 | wc -l)
if [ $n -gt 0 ]; then
BUILD_NEEDED=1
fi
if [ -f $PKG_BASE/$DATA ]; then
(
cd $PKG_BASE
BUILD_NEEDED=0
if [ "x$SRC_DIR" != "x" ]; then
if [ -d "$SRC_DIR" ]; then
n=$(find $SRC_DIR -type f -cnewer $SRPM_PATH2 | grep -v '[/][.]git$' | grep -v '[/][.]git[/]' | wc -l)
if [ $n -gt 0 ]; then
BUILD_NEEDED=1
fi
fi
fi
if [ "x$COPY_LIST" != "x" ]; then
n=$(find $COPY_LIST -type f -cnewer $SRPM_PATH2 | wc -l)
if [ $n -gt 0 ]; then
BUILD_NEEDED=1
fi
fi
if [ "x$COPY_LIST_TO_TAR" != "x" ]; then
n=$(find $COPY_LIST_TO_TAR -type f -cnewer $SRPM_PATH2 | wc -l)
if [ $n -gt 0 ]; then
BUILD_NEEDED=1
fi
fi
exit $BUILD_NEEDED
)
if [ $? -gt 0 ]; then
if [ -d "$FULL_BUILD_DIR/SRPMS" ]; then
for SRPM_PATH2 in $(find "$RPMBUILD_BASE/SRPMS" -name '*.src.rpm' | sort -V); do
b=$(basename $SRPM_PATH2)
SRPM_OUT_PATH2=$(find $SRPM_OUT -name $b)
if [ "x$SRPM_OUT_PATH2" == "x" ]; then
echo "Rebuild required due to missing srpm: $b"
BUILD_NEEDED=1
fi
fi
done
done
else
echo "Rebuild required due to missing directory: '$FULL_BUILD_DIR/SRPMS'"
BUILD_NEEDED=1
fi
if [ $EDIT_FLAG -eq 1 ]; then
@ -944,7 +1036,7 @@ build_dir_spec () {
echo "DATA=$DATA"
if [ -d "$RPMBUILD_BASE/SRPMS" ]; then
clean_srpm_dir "$RPMBUILD_BASE/SRPMS"
clean_srpm_dir "$RPMBUILD_BASE/SRPMS" 1
fi
if [ -d $RPMBUILD_BASE ]; then
echo "rm -rf $RPMBUILD_BASE"
@ -1036,6 +1128,23 @@ build_dir_spec () {
fi
fi
\rm -f -v "$REFERENCE_MD5"
\mv -v "$INPUT_FILES_MD5" "$REFERENCE_MD5"
local SPEC_DIR=$(spec_cache_dir_from_srpm $SRPM_OUT/$ss)
if [ -d $SPEC_DIR/BUILDS_VR ]; then
for f in $(ls -1 $SPEC_DIR/BUILDS_VR); do
for r in $(find $RPM_DIR -name "$f*rpm" 2>> /dev/null); do
\rm -f -v $r
done
done
fi
local RESULT_DIR=$(result_dir_from_srpm $SRPM_OUT/$ss)
if [ -d $RESULT_DIR ]; then
echo "rm -rf $RESULT_DIR"
\rm -rf $RESULT_DIR
fi
done
SRPM_REBUILT_LIST="$SRPM_REBUILT_LIST $TARGET_FOUND"
@ -1204,7 +1313,7 @@ if [ $ALL -eq 1 ]; then
if [ $FOUND -eq 0 ]; then
for INPUT_DIR in $(find $BUILD_INPUTS -name $NAME | sort -V); do
if [ -d "$INPUT_DIR/rpmbuild/SRPMS" ]; then
clean_srpm_dir "$INPUT_DIR/rpmbuild/SRPMS"
clean_srpm_dir "$INPUT_DIR/rpmbuild/SRPMS" 0
fi
if [ -d $INPUT_DIR ]; then
echo "rm -rf $r"
@ -1217,10 +1326,12 @@ if [ $ALL -eq 1 ]; then
fi
done
fi
echo "Auditing for obsolete srpms done"
if [ $CLEAN_FLAG -eq 1 ]; then
if [ $ALL -eq 1 ]; then
\rm -rf $BUILD_INPUTS
\rm -rf $SOURCE_OUT/*.src.rpm
fi
fi