starlingx/root
Code Issues Proposed changes

Build avoidance for all build-type values

Browse Source 
Two improvements.

1) Extend build avoidance to work with any build type,
and built in any order.

Previously build avoidance did not support build types other
than 'std' and 'rt', and only supported thos properly
if both 'std' and 'rt' are built at the same time.

2) Allow for a build avoidance config that does not specify
a host.  In this mode the reference build must be on the same
machine.  The envisioned use case is to allow a build server
to use build avoidance internally for faster builds, without
potentially exposing an rsync port on the network.

Story: 2006166
Task: 39207
Signed-off-by: Scott Little <scott.little@windriver.com>
Change-Id: Idbd87bcc7683746d7b2ff8ac7488919c4a406b50
This commit is contained in:
Scott Little 2020-03-30 09:55:26 -04:00
parent b31cd4b01e
commit 61ef1d4124
5 changed files with 88 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
build-tools/build-avoidance-utils.sh
View File

@ -64,7 +64,6 @@ BUILD_AVOIDANCE_DATA_DIR="$MY_WORKSPACE/build_avoidance_data"
BUILD_AVOIDANCE_SOURCE="$MY_REPO/build-data/build_avoidance_source"
BUILD_AVOIDANCE_LOCAL_SOURCE="$MY_REPO/local-build-data/build_avoidance_source"
BUILD_AVOIDANCE_TEST_CONTEXT="$BUILD_AVOIDANCE_DATA_DIR/test_context"
BUILD_AVOIDANCE_LAST_SYNC_FILE="$BUILD_AVOIDANCE_DATA_DIR/last_sync_context"
if [ ! -f $BUILD_AVOIDANCE_SOURCE ]; then
echo "Couldn't read $BUILD_AVOIDANCE_SOURCE"
@ -102,9 +101,29 @@ echo "BUILD_AVOIDANCE_DIR=$BUILD_AVOIDANCE_DIR"
echo "BUILD_AVOIDANCE_HOST=$BUILD_AVOIDANCE_HOST"
echo "BUILD_AVOIDANCE_USR=$BUILD_AVOIDANCE_USR"
build_avoidance_last_sync_file () {
local BUILD_TYPE=$1
if [ -z "$BUILD_TYPE" ]; then
echo "build_avoidance_last_sync_file: Build type not set"
exit 1
fi
echo "$BUILD_AVOIDANCE_DATA_DIR/$BUILD_TYPE/last_sync_context"
}
build_avoidance_clean () {
if [ -f $BUILD_AVOIDANCE_LAST_SYNC_FILE ]; then
\rm -f -v "$BUILD_AVOIDANCE_LAST_SYNC_FILE"
local BUILD_TYPE=$1
local lsf
if [ "$BUILD_TYPE" == "" ]; then
for lsf in $(find $BUILD_AVOIDANCE_DATA_DIR -name last_sync_context); do
\rm -f -v "$lsf"
done
else
lsf="$(build_avoidance_last_sync_file $BUILD_TYPE)"
if [ -f $lsf ]; then
\rm -f -v "$lsf"
fi
fi
}
@ -321,10 +340,14 @@ test_build_avoidance_context () {
#
get_build_avoidance_context () {
(
local BUILD_TYPE=$1
local context
local BA_CONTEXT=""
local BA_LAST_SYNC_CONTEXT=""
export BUILD_AVOIDANCE_LAST_SYNC_FILE="$(build_avoidance_last_sync_file $BUILD_TYPE)"
mkdir -p "$(dirname $BUILD_AVOIDANCE_LAST_SYNC_FILE)"
# Load last synced context
if [ -f $BUILD_AVOIDANCE_LAST_SYNC_FILE ]; then
BA_LAST_SYNC_CONTEXT=$(head -n 1 $BUILD_AVOIDANCE_LAST_SYNC_FILE)
@ -345,7 +368,12 @@ get_build_avoidance_context () {
# Must set this prior to build_avoidance_copy_dir.
# The setting is not exported outside of the subshell.
BUILD_AVOIDANCE_URL="$BUILD_AVOIDANCE_HOST:$BUILD_AVOIDANCE_DIR"
if [ -z "$BUILD_AVOIDANCE_HOST" ]; then
BUILD_AVOIDANCE_URL="$BUILD_AVOIDANCE_DIR"
else
BUILD_AVOIDANCE_URL="$BUILD_AVOIDANCE_HOST:$BUILD_AVOIDANCE_DIR"
fi
build_avoidance_copy_dir "$REMOTE_CTX_DIR" "$LOCAL_CTX_DIR"
if [ $? -ne 0 ]; then
@ -454,9 +482,17 @@ get_build_avoidance_context () {
# test that the reference build context hasn't been deleted
local BA_CONTEXT_DIR="$BUILD_AVOIDANCE_DIR/$BA_CONTEXT"
>&2 echo "ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]'"
if ! ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]' ; then
return 1
if [ -z "$BUILD_AVOIDANCE_HOST" ]; then
>&2 echo "[ -d $BA_CONTEXT_DIR ]"
if ! [ -d $BA_CONTEXT_DIR ] ; then
return 1
fi
else
>&2 echo "ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]'"
if ! ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]' ; then
return 1
fi
fi
# Save the latest context
@ -465,7 +501,11 @@ get_build_avoidance_context () {
echo $BA_CONTEXT > $BUILD_AVOIDANCE_LAST_SYNC_FILE
# The location of the load with the most compatable new context
URL=$BUILD_AVOIDANCE_HOST:$BA_CONTEXT_DIR
if [ -z "$BUILD_AVOIDANCE_HOST" ]; then
URL=$BA_CONTEXT_DIR
else
URL=$BUILD_AVOIDANCE_HOST:$BA_CONTEXT_DIR
fi
# return URL to caller.
echo $URL
@ -538,9 +578,9 @@ build_avoidance_copy_dir_rsync () {
fi
if [ "$VERBOSE" != "" ]; then
FLAGS="$FLAGS -v"
echo "rsync $FLAGS '$BUILD_AVOIDANCE_URL/$FROM/*' '$TO/'"
echo "rsync $FLAGS '$BUILD_AVOIDANCE_URL/$FROM/' '$TO/'"
fi
rsync $FLAGS "$BUILD_AVOIDANCE_URL/$FROM/*" "$TO/"
rsync $FLAGS "$BUILD_AVOIDANCE_URL/$FROM/" "$TO/"
return $?
}
@ -767,21 +807,14 @@ build_avoidance () {
echo "==== Build Avoidance Start ===="
export BUILD_AVOIDANCE_LAST_SYNC_FILE="$(build_avoidance_last_sync_file $BUILD_TYPE)"
mkdir -p "$(dirname $BUILD_AVOIDANCE_LAST_SYNC_FILE)"
if [ "$BUILD_TYPE" == "" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required"
return 1
fi
if [ "$BUILD_TYPE" == "installer" ]; then
>&2 echo "build_avoidance: BUILD_TYPE==installer not supported"
return 1
fi
if [ "$BUILD_TYPE" == "containers" ]; then
>&2 echo "build_avoidance: BUILD_TYPE==containers not supported"
return 1
fi
if [ ! -d $MY_WORKSPACE/$BUILD_TYPE ]; then
mkdir -p $MY_WORKSPACE/$BUILD_TYPE
if [ $? -ne 0 ]; then
@ -790,7 +823,7 @@ build_avoidance () {
fi
fi
if [ ! -L $MY_WORKSPACE/$BUILD_TYPE ]; then
if [ ! -L $MY_WORKSPACE/$BUILD_TYPE/repo ]; then
ln -s $MY_REPO $MY_WORKSPACE/$BUILD_TYPE/repo
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): Failed to create symlink $MY_WORKSPACE/$BUILD_TYPE/repo -> $MY_REPO"

10
build-tools/build-pkgs-parallel
View File

@ -252,11 +252,6 @@ echo "BUILD_AVOIDANCE_FLAG=$BUILD_AVOIDANCE_FLAG"
echo "CLEAN_FLAG=$CLEAN_FLAG"
echo "EDIT_FLAG=$EDIT_FLAG"
if [ $BUILD_AVOIDANCE_FLAG -eq 1 ]; then
# Build Avoidance requested. Get URL of a usable context, if any.
export BUILD_AVOIDANCE_URL=$(get_build_avoidance_context)
fi
if [ "$TARGETS" != "$EMPTY_TARGETS" ]; then
TARGETS_STD="$(find_targets centos_pkg_dirs)"
@ -408,6 +403,11 @@ function launch_build()
echo -e "\n######## $(date): Launching build-srpms-parallel --$build_type $EXTRA_ARGS $@\n" | tee --append $logfile
if [ $BUILD_AVOIDANCE_FLAG -eq 1 ]; then
# Build Avoidance requested. Get URL of a usable context, if any.
export BUILD_AVOIDANCE_URL=$(get_build_avoidance_context $build_type)
fi
echo "BUILD_AVOIDANCE_URL=$BUILD_AVOIDANCE_URL" | tee --append $logfile
if [ "x$BUILD_AVOIDANCE_URL" != "x" ]; then
echo "build_avoidance $build_type" | tee --append $logfile

10
build-tools/build-pkgs-serial
View File

@ -252,11 +252,6 @@ echo "BUILD_AVOIDANCE_FLAG=$BUILD_AVOIDANCE_FLAG"
echo "CLEAN_FLAG=$CLEAN_FLAG"
echo "EDIT_FLAG=$EDIT_FLAG"
if [ $BUILD_AVOIDANCE_FLAG -eq 1 ]; then
# Build Avoidance requested. Get URL of a usable context, if any.
export BUILD_AVOIDANCE_URL=$(get_build_avoidance_context)
fi
if [ "$TARGETS" != "$EMPTY_TARGETS" ]; then
TARGETS_STD="$(find_targets centos_pkg_dirs)"
@ -407,6 +402,11 @@ function launch_build()
echo -e "\n######## $(date): Launching build-srpms-serial --$build_type $EXTRA_ARGS $@\n" | tee --append $logfile
if [ $BUILD_AVOIDANCE_FLAG -eq 1 ]; then
# Build Avoidance requested. Get URL of a usable context, if any.
export BUILD_AVOIDANCE_URL=$(get_build_avoidance_context $build_type)
fi
echo "BUILD_AVOIDANCE_URL=$BUILD_AVOIDANCE_URL" | tee --append $logfile
if [ "x$BUILD_AVOIDANCE_URL" != "x" ]; then
echo "build_avoidance $build_type" | tee --append $logfile

12
build-tools/build-srpms-parallel
View File

@ -337,6 +337,18 @@ if [ $? -ne 0 ]; then
exit 1
fi
mkdir -p $SRPM_ASSEMBLE
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$SRPM_ASSEMBLE'"
exit 1
fi
mkdir -p $BUILD_INPUTS
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$BUILD_INPUTS'"
exit 1
fi
build_dir () {
local build_idx=$1
local d=$2

12
build-tools/build-srpms-serial
View File

@ -324,6 +324,18 @@ if [ $? -ne 0 ]; then
exit 1
fi
mkdir -p $SRPM_ASSEMBLE
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$SRPM_ASSEMBLE'"
exit 1
fi
mkdir -p $BUILD_INPUTS
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$BUILD_INPUTS'"
exit 1
fi
build_dir () {
local d=$1
local w=$2