Build avoidance for all build-type values

Two improvements.

1) Extend build avoidance to work with any build type,
and built in any order.

Previously build avoidance did not support build types other
than 'std' and 'rt', and only supported thos properly
if both 'std' and 'rt' are built at the same time.

2) Allow for a build avoidance config that does not specify
a host.  In this mode the reference build must be on the same
machine.  The envisioned use case is to allow a build server
to use build avoidance internally for faster builds, without
potentially exposing an rsync port on the network.

Story: 2006166
Task: 39207
Signed-off-by: Scott Little <scott.little@windriver.com>
Change-Id: Idbd87bcc7683746d7b2ff8ac7488919c4a406b50

build-tools/build-avoidance-utils.sh

@@ -64,7 +64,6 @@ BUILD_AVOIDANCE_DATA_DIR="$MY_WORKSPACE/build_avoidance_data"
 BUILD_AVOIDANCE_SOURCE="$MY_REPO/build-data/build_avoidance_source"
 BUILD_AVOIDANCE_LOCAL_SOURCE="$MY_REPO/local-build-data/build_avoidance_source"
 BUILD_AVOIDANCE_TEST_CONTEXT="$BUILD_AVOIDANCE_DATA_DIR/test_context"
-BUILD_AVOIDANCE_LAST_SYNC_FILE="$BUILD_AVOIDANCE_DATA_DIR/last_sync_context"
 
 if [ ! -f $BUILD_AVOIDANCE_SOURCE ]; then
     echo "Couldn't read $BUILD_AVOIDANCE_SOURCE"
@@ -102,9 +101,29 @@ echo "BUILD_AVOIDANCE_DIR=$BUILD_AVOIDANCE_DIR"
 echo "BUILD_AVOIDANCE_HOST=$BUILD_AVOIDANCE_HOST"
 echo "BUILD_AVOIDANCE_USR=$BUILD_AVOIDANCE_USR"
 
+build_avoidance_last_sync_file () {
+    local BUILD_TYPE=$1
+
+    if [ -z "$BUILD_TYPE" ]; then
+        echo "build_avoidance_last_sync_file: Build type not set"
+        exit 1
+    fi
+    echo "$BUILD_AVOIDANCE_DATA_DIR/$BUILD_TYPE/last_sync_context"
+}
+
 build_avoidance_clean () {
-    if [ -f $BUILD_AVOIDANCE_LAST_SYNC_FILE ]; then
+    local BUILD_TYPE=$1
-        \rm -f -v "$BUILD_AVOIDANCE_LAST_SYNC_FILE"
+    local lsf
+
+    if [ "$BUILD_TYPE" == "" ]; then
+        for lsf in $(find $BUILD_AVOIDANCE_DATA_DIR -name last_sync_context); do
+            \rm -f -v "$lsf"
+        done
+    else
+        lsf="$(build_avoidance_last_sync_file $BUILD_TYPE)"
+        if [ -f $lsf ]; then
+            \rm -f -v "$lsf"
+        fi
     fi
 }
 
@@ -321,10 +340,14 @@ test_build_avoidance_context () {
 #
 get_build_avoidance_context () {
     (
+    local BUILD_TYPE=$1
     local context
     local BA_CONTEXT=""
     local BA_LAST_SYNC_CONTEXT=""
 
+    export BUILD_AVOIDANCE_LAST_SYNC_FILE="$(build_avoidance_last_sync_file $BUILD_TYPE)"
+    mkdir -p "$(dirname $BUILD_AVOIDANCE_LAST_SYNC_FILE)"
+
     # Load last synced context
     if [ -f $BUILD_AVOIDANCE_LAST_SYNC_FILE ]; then
         BA_LAST_SYNC_CONTEXT=$(head -n 1 $BUILD_AVOIDANCE_LAST_SYNC_FILE)
@@ -345,7 +368,12 @@ get_build_avoidance_context () {
 
     # Must set this prior to build_avoidance_copy_dir.
     # The setting is not exported outside of the subshell.
-    BUILD_AVOIDANCE_URL="$BUILD_AVOIDANCE_HOST:$BUILD_AVOIDANCE_DIR"
+    if [ -z "$BUILD_AVOIDANCE_HOST" ]; then
+        BUILD_AVOIDANCE_URL="$BUILD_AVOIDANCE_DIR"
+    else
+        BUILD_AVOIDANCE_URL="$BUILD_AVOIDANCE_HOST:$BUILD_AVOIDANCE_DIR"
+    fi
+
 
     build_avoidance_copy_dir "$REMOTE_CTX_DIR" "$LOCAL_CTX_DIR"
     if [ $? -ne 0 ]; then
@@ -454,9 +482,17 @@ get_build_avoidance_context () {
 
     # test that the reference build context hasn't been deleted
     local BA_CONTEXT_DIR="$BUILD_AVOIDANCE_DIR/$BA_CONTEXT"
-    >&2 echo "ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]'"
+
-    if ! ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]' ; then
+    if [ -z "$BUILD_AVOIDANCE_HOST" ]; then
-        return 1
+        >&2 echo "[ -d $BA_CONTEXT_DIR ]"
+        if ! [ -d $BA_CONTEXT_DIR ] ; then
+            return 1
+        fi
+    else
+        >&2 echo "ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]'"
+        if ! ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]' ; then
+            return 1
+        fi
     fi
 
     # Save the latest context
@@ -465,7 +501,11 @@ get_build_avoidance_context () {
     echo $BA_CONTEXT > $BUILD_AVOIDANCE_LAST_SYNC_FILE
 
     # The location of the load with the most compatable new context
-    URL=$BUILD_AVOIDANCE_HOST:$BA_CONTEXT_DIR
+    if [ -z "$BUILD_AVOIDANCE_HOST" ]; then
+        URL=$BA_CONTEXT_DIR
+    else
+        URL=$BUILD_AVOIDANCE_HOST:$BA_CONTEXT_DIR
+    fi
 
     # return URL to caller.
     echo $URL
@@ -538,9 +578,9 @@ build_avoidance_copy_dir_rsync () {
     fi
     if [ "$VERBOSE" != "" ]; then
         FLAGS="$FLAGS -v"
-        echo "rsync $FLAGS '$BUILD_AVOIDANCE_URL/$FROM/*' '$TO/'"
+        echo "rsync $FLAGS '$BUILD_AVOIDANCE_URL/$FROM/' '$TO/'"
     fi
-    rsync $FLAGS "$BUILD_AVOIDANCE_URL/$FROM/*" "$TO/"
+    rsync $FLAGS "$BUILD_AVOIDANCE_URL/$FROM/" "$TO/"
     return $?
 }
 
@@ -767,21 +807,14 @@ build_avoidance () {
 
     echo "==== Build Avoidance Start ===="
 
+    export BUILD_AVOIDANCE_LAST_SYNC_FILE="$(build_avoidance_last_sync_file $BUILD_TYPE)"
+    mkdir -p "$(dirname $BUILD_AVOIDANCE_LAST_SYNC_FILE)"
+
     if [ "$BUILD_TYPE" == "" ]; then
         >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required"
         return 1
     fi
 
-    if [ "$BUILD_TYPE" == "installer" ]; then
-        >&2 echo "build_avoidance: BUILD_TYPE==installer not supported"
-        return 1
-    fi
-
-    if [ "$BUILD_TYPE" == "containers" ]; then
-        >&2 echo "build_avoidance: BUILD_TYPE==containers not supported"
-        return 1
-    fi
-
     if [ ! -d $MY_WORKSPACE/$BUILD_TYPE ]; then
         mkdir -p $MY_WORKSPACE/$BUILD_TYPE
         if [ $? -ne 0 ]; then
@@ -790,7 +823,7 @@ build_avoidance () {
         fi
     fi
 
-    if [ ! -L $MY_WORKSPACE/$BUILD_TYPE ]; then
+    if [ ! -L $MY_WORKSPACE/$BUILD_TYPE/repo ]; then
         ln -s $MY_REPO $MY_WORKSPACE/$BUILD_TYPE/repo
         if [ $? -ne 0 ]; then
             >&2 echo "Error: $FUNCNAME (${LINENO}): Failed to create symlink $MY_WORKSPACE/$BUILD_TYPE/repo -> $MY_REPO"

build-tools/build-pkgs-parallel

@@ -252,11 +252,6 @@ echo "BUILD_AVOIDANCE_FLAG=$BUILD_AVOIDANCE_FLAG"
 echo "CLEAN_FLAG=$CLEAN_FLAG"
 echo "EDIT_FLAG=$EDIT_FLAG"
 
-if [ $BUILD_AVOIDANCE_FLAG -eq 1 ]; then
-    # Build Avoidance requested. Get URL of a usable context, if any.
-    export BUILD_AVOIDANCE_URL=$(get_build_avoidance_context)
-fi
-
 if [ "$TARGETS" != "$EMPTY_TARGETS" ]; then
    TARGETS_STD="$(find_targets centos_pkg_dirs)"
 
@@ -408,6 +403,11 @@ function launch_build()
 
    echo -e "\n######## $(date): Launching build-srpms-parallel --$build_type $EXTRA_ARGS $@\n" | tee --append $logfile
 
+   if [ $BUILD_AVOIDANCE_FLAG -eq 1 ]; then
+      # Build Avoidance requested. Get URL of a usable context, if any.
+      export BUILD_AVOIDANCE_URL=$(get_build_avoidance_context $build_type)
+   fi
+
    echo "BUILD_AVOIDANCE_URL=$BUILD_AVOIDANCE_URL" | tee --append $logfile
    if [ "x$BUILD_AVOIDANCE_URL" != "x" ]; then
       echo "build_avoidance $build_type" | tee --append $logfile

build-tools/build-pkgs-serial

@@ -252,11 +252,6 @@ echo "BUILD_AVOIDANCE_FLAG=$BUILD_AVOIDANCE_FLAG"
 echo "CLEAN_FLAG=$CLEAN_FLAG"
 echo "EDIT_FLAG=$EDIT_FLAG"
 
-if [ $BUILD_AVOIDANCE_FLAG -eq 1 ]; then
-    # Build Avoidance requested. Get URL of a usable context, if any.
-    export BUILD_AVOIDANCE_URL=$(get_build_avoidance_context)
-fi
-
 if [ "$TARGETS" != "$EMPTY_TARGETS" ]; then
    TARGETS_STD="$(find_targets centos_pkg_dirs)"
 
@@ -407,6 +402,11 @@ function launch_build()
 
    echo -e "\n######## $(date): Launching build-srpms-serial --$build_type $EXTRA_ARGS $@\n" | tee --append $logfile
 
+   if [ $BUILD_AVOIDANCE_FLAG -eq 1 ]; then
+      # Build Avoidance requested. Get URL of a usable context, if any.
+      export BUILD_AVOIDANCE_URL=$(get_build_avoidance_context $build_type)
+   fi
+
    echo "BUILD_AVOIDANCE_URL=$BUILD_AVOIDANCE_URL" | tee --append $logfile
    if [ "x$BUILD_AVOIDANCE_URL" != "x" ]; then
       echo "build_avoidance $build_type" | tee --append $logfile

build-tools/build-srpms-parallel

@@ -337,6 +337,18 @@ if [ $? -ne 0 ]; then
    exit 1
 fi
 
+mkdir -p $SRPM_ASSEMBLE
+if [ $? -ne 0 ]; then
+   echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$SRPM_ASSEMBLE'"
+   exit 1
+fi
+
+mkdir -p $BUILD_INPUTS
+if [ $? -ne 0 ]; then
+   echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$BUILD_INPUTS'"
+   exit 1
+fi
+
 build_dir () {
    local build_idx=$1
    local d=$2

build-tools/build-srpms-serial

@@ -324,6 +324,18 @@ if [ $? -ne 0 ]; then
    exit 1
 fi
 
+mkdir -p $SRPM_ASSEMBLE
+if [ $? -ne 0 ]; then
+   echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$SRPM_ASSEMBLE'"
+   exit 1
+fi
+
+mkdir -p $BUILD_INPUTS
+if [ $? -ne 0 ]; then
+   echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$BUILD_INPUTS'"
+   exit 1
+fi
+
 build_dir () {
    local d=$1
    local w=$2