From 61ef1d4124ac858e71591f9c93200bf59a38ff5d Mon Sep 17 00:00:00 2001 From: Scott Little Date: Mon, 30 Mar 2020 09:55:26 -0400 Subject: [PATCH] Build avoidance for all build-type values Two improvements. 1) Extend build avoidance to work with any build type, and built in any order. Previously build avoidance did not support build types other than 'std' and 'rt', and only supported thos properly if both 'std' and 'rt' are built at the same time. 2) Allow for a build avoidance config that does not specify a host. In this mode the reference build must be on the same machine. The envisioned use case is to allow a build server to use build avoidance internally for faster builds, without potentially exposing an rsync port on the network. Story: 2006166 Task: 39207 Signed-off-by: Scott Little Change-Id: Idbd87bcc7683746d7b2ff8ac7488919c4a406b50 --- build-tools/build-avoidance-utils.sh | 75 ++++++++++++++++++++-------- build-tools/build-pkgs-parallel | 10 ++-- build-tools/build-pkgs-serial | 10 ++-- build-tools/build-srpms-parallel | 12 +++++ build-tools/build-srpms-serial | 12 +++++ 5 files changed, 88 insertions(+), 31 deletions(-) diff --git a/build-tools/build-avoidance-utils.sh b/build-tools/build-avoidance-utils.sh index 7bff8a63..5c09c3a7 100644 --- a/build-tools/build-avoidance-utils.sh +++ b/build-tools/build-avoidance-utils.sh @@ -64,7 +64,6 @@ BUILD_AVOIDANCE_DATA_DIR="$MY_WORKSPACE/build_avoidance_data" BUILD_AVOIDANCE_SOURCE="$MY_REPO/build-data/build_avoidance_source" BUILD_AVOIDANCE_LOCAL_SOURCE="$MY_REPO/local-build-data/build_avoidance_source" BUILD_AVOIDANCE_TEST_CONTEXT="$BUILD_AVOIDANCE_DATA_DIR/test_context" -BUILD_AVOIDANCE_LAST_SYNC_FILE="$BUILD_AVOIDANCE_DATA_DIR/last_sync_context" if [ ! -f $BUILD_AVOIDANCE_SOURCE ]; then echo "Couldn't read $BUILD_AVOIDANCE_SOURCE" @@ -102,9 +101,29 @@ echo "BUILD_AVOIDANCE_DIR=$BUILD_AVOIDANCE_DIR" echo "BUILD_AVOIDANCE_HOST=$BUILD_AVOIDANCE_HOST" echo "BUILD_AVOIDANCE_USR=$BUILD_AVOIDANCE_USR" +build_avoidance_last_sync_file () { + local BUILD_TYPE=$1 + + if [ -z "$BUILD_TYPE" ]; then + echo "build_avoidance_last_sync_file: Build type not set" + exit 1 + fi + echo "$BUILD_AVOIDANCE_DATA_DIR/$BUILD_TYPE/last_sync_context" +} + build_avoidance_clean () { - if [ -f $BUILD_AVOIDANCE_LAST_SYNC_FILE ]; then - \rm -f -v "$BUILD_AVOIDANCE_LAST_SYNC_FILE" + local BUILD_TYPE=$1 + local lsf + + if [ "$BUILD_TYPE" == "" ]; then + for lsf in $(find $BUILD_AVOIDANCE_DATA_DIR -name last_sync_context); do + \rm -f -v "$lsf" + done + else + lsf="$(build_avoidance_last_sync_file $BUILD_TYPE)" + if [ -f $lsf ]; then + \rm -f -v "$lsf" + fi fi } @@ -321,10 +340,14 @@ test_build_avoidance_context () { # get_build_avoidance_context () { ( + local BUILD_TYPE=$1 local context local BA_CONTEXT="" local BA_LAST_SYNC_CONTEXT="" + export BUILD_AVOIDANCE_LAST_SYNC_FILE="$(build_avoidance_last_sync_file $BUILD_TYPE)" + mkdir -p "$(dirname $BUILD_AVOIDANCE_LAST_SYNC_FILE)" + # Load last synced context if [ -f $BUILD_AVOIDANCE_LAST_SYNC_FILE ]; then BA_LAST_SYNC_CONTEXT=$(head -n 1 $BUILD_AVOIDANCE_LAST_SYNC_FILE) @@ -345,7 +368,12 @@ get_build_avoidance_context () { # Must set this prior to build_avoidance_copy_dir. # The setting is not exported outside of the subshell. - BUILD_AVOIDANCE_URL="$BUILD_AVOIDANCE_HOST:$BUILD_AVOIDANCE_DIR" + if [ -z "$BUILD_AVOIDANCE_HOST" ]; then + BUILD_AVOIDANCE_URL="$BUILD_AVOIDANCE_DIR" + else + BUILD_AVOIDANCE_URL="$BUILD_AVOIDANCE_HOST:$BUILD_AVOIDANCE_DIR" + fi + build_avoidance_copy_dir "$REMOTE_CTX_DIR" "$LOCAL_CTX_DIR" if [ $? -ne 0 ]; then @@ -454,9 +482,17 @@ get_build_avoidance_context () { # test that the reference build context hasn't been deleted local BA_CONTEXT_DIR="$BUILD_AVOIDANCE_DIR/$BA_CONTEXT" - >&2 echo "ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]'" - if ! ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]' ; then - return 1 + + if [ -z "$BUILD_AVOIDANCE_HOST" ]; then + >&2 echo "[ -d $BA_CONTEXT_DIR ]" + if ! [ -d $BA_CONTEXT_DIR ] ; then + return 1 + fi + else + >&2 echo "ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]'" + if ! ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]' ; then + return 1 + fi fi # Save the latest context @@ -465,7 +501,11 @@ get_build_avoidance_context () { echo $BA_CONTEXT > $BUILD_AVOIDANCE_LAST_SYNC_FILE # The location of the load with the most compatable new context - URL=$BUILD_AVOIDANCE_HOST:$BA_CONTEXT_DIR + if [ -z "$BUILD_AVOIDANCE_HOST" ]; then + URL=$BA_CONTEXT_DIR + else + URL=$BUILD_AVOIDANCE_HOST:$BA_CONTEXT_DIR + fi # return URL to caller. echo $URL @@ -538,9 +578,9 @@ build_avoidance_copy_dir_rsync () { fi if [ "$VERBOSE" != "" ]; then FLAGS="$FLAGS -v" - echo "rsync $FLAGS '$BUILD_AVOIDANCE_URL/$FROM/*' '$TO/'" + echo "rsync $FLAGS '$BUILD_AVOIDANCE_URL/$FROM/' '$TO/'" fi - rsync $FLAGS "$BUILD_AVOIDANCE_URL/$FROM/*" "$TO/" + rsync $FLAGS "$BUILD_AVOIDANCE_URL/$FROM/" "$TO/" return $? } @@ -767,21 +807,14 @@ build_avoidance () { echo "==== Build Avoidance Start ====" + export BUILD_AVOIDANCE_LAST_SYNC_FILE="$(build_avoidance_last_sync_file $BUILD_TYPE)" + mkdir -p "$(dirname $BUILD_AVOIDANCE_LAST_SYNC_FILE)" + if [ "$BUILD_TYPE" == "" ]; then >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required" return 1 fi - if [ "$BUILD_TYPE" == "installer" ]; then - >&2 echo "build_avoidance: BUILD_TYPE==installer not supported" - return 1 - fi - - if [ "$BUILD_TYPE" == "containers" ]; then - >&2 echo "build_avoidance: BUILD_TYPE==containers not supported" - return 1 - fi - if [ ! -d $MY_WORKSPACE/$BUILD_TYPE ]; then mkdir -p $MY_WORKSPACE/$BUILD_TYPE if [ $? -ne 0 ]; then @@ -790,7 +823,7 @@ build_avoidance () { fi fi - if [ ! -L $MY_WORKSPACE/$BUILD_TYPE ]; then + if [ ! -L $MY_WORKSPACE/$BUILD_TYPE/repo ]; then ln -s $MY_REPO $MY_WORKSPACE/$BUILD_TYPE/repo if [ $? -ne 0 ]; then >&2 echo "Error: $FUNCNAME (${LINENO}): Failed to create symlink $MY_WORKSPACE/$BUILD_TYPE/repo -> $MY_REPO" diff --git a/build-tools/build-pkgs-parallel b/build-tools/build-pkgs-parallel index 00ad0c4d..528b3ad9 100755 --- a/build-tools/build-pkgs-parallel +++ b/build-tools/build-pkgs-parallel @@ -252,11 +252,6 @@ echo "BUILD_AVOIDANCE_FLAG=$BUILD_AVOIDANCE_FLAG" echo "CLEAN_FLAG=$CLEAN_FLAG" echo "EDIT_FLAG=$EDIT_FLAG" -if [ $BUILD_AVOIDANCE_FLAG -eq 1 ]; then - # Build Avoidance requested. Get URL of a usable context, if any. - export BUILD_AVOIDANCE_URL=$(get_build_avoidance_context) -fi - if [ "$TARGETS" != "$EMPTY_TARGETS" ]; then TARGETS_STD="$(find_targets centos_pkg_dirs)" @@ -408,6 +403,11 @@ function launch_build() echo -e "\n######## $(date): Launching build-srpms-parallel --$build_type $EXTRA_ARGS $@\n" | tee --append $logfile + if [ $BUILD_AVOIDANCE_FLAG -eq 1 ]; then + # Build Avoidance requested. Get URL of a usable context, if any. + export BUILD_AVOIDANCE_URL=$(get_build_avoidance_context $build_type) + fi + echo "BUILD_AVOIDANCE_URL=$BUILD_AVOIDANCE_URL" | tee --append $logfile if [ "x$BUILD_AVOIDANCE_URL" != "x" ]; then echo "build_avoidance $build_type" | tee --append $logfile diff --git a/build-tools/build-pkgs-serial b/build-tools/build-pkgs-serial index ccb47d29..be871355 100755 --- a/build-tools/build-pkgs-serial +++ b/build-tools/build-pkgs-serial @@ -252,11 +252,6 @@ echo "BUILD_AVOIDANCE_FLAG=$BUILD_AVOIDANCE_FLAG" echo "CLEAN_FLAG=$CLEAN_FLAG" echo "EDIT_FLAG=$EDIT_FLAG" -if [ $BUILD_AVOIDANCE_FLAG -eq 1 ]; then - # Build Avoidance requested. Get URL of a usable context, if any. - export BUILD_AVOIDANCE_URL=$(get_build_avoidance_context) -fi - if [ "$TARGETS" != "$EMPTY_TARGETS" ]; then TARGETS_STD="$(find_targets centos_pkg_dirs)" @@ -407,6 +402,11 @@ function launch_build() echo -e "\n######## $(date): Launching build-srpms-serial --$build_type $EXTRA_ARGS $@\n" | tee --append $logfile + if [ $BUILD_AVOIDANCE_FLAG -eq 1 ]; then + # Build Avoidance requested. Get URL of a usable context, if any. + export BUILD_AVOIDANCE_URL=$(get_build_avoidance_context $build_type) + fi + echo "BUILD_AVOIDANCE_URL=$BUILD_AVOIDANCE_URL" | tee --append $logfile if [ "x$BUILD_AVOIDANCE_URL" != "x" ]; then echo "build_avoidance $build_type" | tee --append $logfile diff --git a/build-tools/build-srpms-parallel b/build-tools/build-srpms-parallel index 26a23602..f2610597 100755 --- a/build-tools/build-srpms-parallel +++ b/build-tools/build-srpms-parallel @@ -337,6 +337,18 @@ if [ $? -ne 0 ]; then exit 1 fi +mkdir -p $SRPM_ASSEMBLE +if [ $? -ne 0 ]; then + echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$SRPM_ASSEMBLE'" + exit 1 +fi + +mkdir -p $BUILD_INPUTS +if [ $? -ne 0 ]; then + echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$BUILD_INPUTS'" + exit 1 +fi + build_dir () { local build_idx=$1 local d=$2 diff --git a/build-tools/build-srpms-serial b/build-tools/build-srpms-serial index 0b73e287..aa00d8c1 100755 --- a/build-tools/build-srpms-serial +++ b/build-tools/build-srpms-serial @@ -324,6 +324,18 @@ if [ $? -ne 0 ]; then exit 1 fi +mkdir -p $SRPM_ASSEMBLE +if [ $? -ne 0 ]; then + echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$SRPM_ASSEMBLE'" + exit 1 +fi + +mkdir -p $BUILD_INPUTS +if [ $? -ne 0 ]; then + echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$BUILD_INPUTS'" + exit 1 +fi + build_dir () { local d=$1 local w=$2