starlingx/root
Code Issues Proposed changes

Capture the git context of the build

Browse Source 
First step in build avoidance is the creation of a reference build.
We need to capture the state of all the git trees that contribute
to the reference build.

This update will save the git context to $MY_WORKSPACE/CONTEXT

The context file format is actually a shell script that can be run to
recreate the build context.

   repo init ...
   repo sync
   cd $MY_REPO/..
   source .../CONTEXT

Sample CONTEXT file ....
(cd ./cgcs-root && git checkout -f 12a1595941)
(cd ./cgcs-root/stx/git/calico && git checkout -f 12a1595941)
(cd ./cgcs-root/stx/git/ceilometer && git checkout -f 12a1595941)
...
(cd ./stx-tools && git checkout -f 12a1595941)
(cd ./.repo/manifests && git checkout -f 12a1595941)
(cd ./.repo/repo && git checkout -f 12a1595941)

This update also the git_list function, which populates the previously
introduced GIT_LIST variable, and can be used in several other code
paths where we want to find all gits in a directory tree.

Tested build/clean/edit  parallel/serial

Story: 2002835
Task: 22754
Change-Id: I5009b8a360bdb4a03e5a4e83430b7f2ef135115e
Signed-off-by: Scott Little <scott.little@windriver.com>
This commit is contained in:
Scott Little 2018-08-30 17:10:07 -04:00
parent 673c6f811c
commit 58b942baf1
5 changed files with 124 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
build-tools/build-pkgs-parallel
View File

@ -4,6 +4,7 @@
BUILD_PKGS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
source "${BUILD_PKGS_PARALLEL_DIR}/git-utils.sh"
source "${BUILD_PKGS_PARALLEL_DIR}/spec-utils"
@ -141,8 +142,7 @@ find_targets () {
local RESULT=" "
local FOUND=0
for g in $(find $MY_REPO -type d -name .git); do
d=$(dirname $g)
for d in $GIT_LIST; do
if [ -f $d/$centos_pkg_dirs ]; then
for d2 in $(grep -v '^#' $d/$centos_pkg_dirs); do
name=""
@ -364,7 +364,15 @@ function progbar()
}
# Create $MY_WORKSPACE if it doesn't exist already
mkdir -p $MY_WORKSPACE
mkdir -p "${MY_WORKSPACE}"
if [ $? -ne 0 ]; then
echo "Failed to create directory '${MY_WORKSPACE}'"
exit 1
fi
echo "Capture build context"
git_context > "${MY_WORKSPACE}/CONTEXT"
if [ $STD_BUILD -eq 1 ]; then
if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then

13
build-tools/build-pkgs-serial
View File

@ -4,6 +4,7 @@
BUILD_PKGS_SERIAL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
source "${BUILD_PKGS_SERIAL_DIR}/git-utils.sh"
source "${BUILD_PKGS_SERIAL_DIR}/spec-utils"
@ -97,8 +98,7 @@ find_targets () {
local RESULT=" "
local FOUND=0
for g in $(find $MY_REPO -type d -name .git); do
d=$(dirname $g)
for d in $GIT_LIST; do
if [ -f $d/$centos_pkg_dirs ]; then
for d2 in $(grep -v '^#' $d/$centos_pkg_dirs); do
name=""
@ -316,7 +316,14 @@ function progbar()
}
# Create $MY_WORKSPACE if it doesn't exist already
mkdir -p $MY_WORKSPACE
mkdir -p "${MY_WORKSPACE}"
if [ $? -ne 0 ]; then
echo "Failed to create directory '${MY_WORKSPACE}'"
exit 1
fi
echo "Capture build context"
git_context > "${MY_WORKSPACE}/CONTEXT"
if [ $STD_BUILD -eq 1 ]; then
if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then

14
build-tools/build-srpms-parallel
View File

@ -5,6 +5,7 @@ export ME=$(basename "$0")
CMDLINE="$ME $@"
BUILD_SRPMS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
source $BUILD_SRPMS_PARALLEL_DIR/git-utils.sh
source $BUILD_SRPMS_PARALLEL_DIR/spec-utils
source $BUILD_SRPMS_PARALLEL_DIR/srpm-utils
source $BUILD_SRPMS_PARALLEL_DIR/classify
@ -677,7 +678,8 @@ build_dir_srpm () {
return $RC
fi
local LOC=$(for g in $(find $PKG_ROOT_DIR/gits -type d -name .git); do d=$(dirname $g); (cd $d; git tag | grep "pre_wrs_$PKG_NAME_VER" >> /dev/null; if [ $? -eq 0 ]; then echo $d; fi); done | head -n 1 )
local LOC
LOC=$(git_list_containing_tag "${PKG_ROOT_DIR}/gits" "pre_wrs_$PKG_NAME_VER" | head -n 1 )
echo "===== '$TARGET_FOUND' has been extracted for editing. ====="
echo "===== Metadata can be found at: $PKG_ROOT_DIR/rpmbuild"
echo "===== Source code can be found at: $LOC"
@ -956,7 +958,8 @@ build_dir_spec () {
return $RC
fi
local LOC=$(for g in $(find $PKG_ROOT_DIR/gits -type d -name .git); do d=$(dirname $g); (cd $d; git branch --all | grep "$PKG_NAME_VER" >> /dev/null; if [ $? -eq 0 ]; then echo $d; fi); done | head -n 1 )
local LOC
LOC=$(git_list_containing_branch "${PKG_ROOT_DIR}/gits" "${PKG_NAME_VER}" | head -n 1 )
echo "===== '$TARGET_FOUND' has been extracted for editing. ====="
echo "===== Metadata can be found at: $PKG_ROOT_DIR/rpmbuild"
echo "===== Source code can be found at: $LOC"
@ -1235,8 +1238,8 @@ if [ "x$LOG_DIR" == "x" ]; then
echo "failed to create temporary directory"
exit 1;
fi
for g in $(find "$SRC_BASE" -type d -name .git | sort -V); do
GIT_ROOT=$(dirname $g)
for GIT_ROOT in $GIT_LIST; do
export GIT_BASE="$GIT_ROOT"
if [ $STOP_SCHEDULING -eq 1 ]; then
break;
@ -1330,8 +1333,7 @@ if [ $ALL -eq 1 ]; then
echo "Auditing for obsolete srpms"
AUDIT_DIR=$(mktemp -d $MY_WORKSPACE/tmp/$USER-$ME-audit-XXXXXX)
if [ $? -eq 0 ] && [ "x$AUDIT_DIR" != "x" ]; then
for g in $(find $SRC_BASE -type d -name .git | sort -V); do
GIT_ROOT=$(dirname $g)
for GIT_ROOT in $GIT_LIST; do
for p in $(cat $GIT_ROOT/$PKG_DIRS_FILE 2>> /dev/null); do
(
src_dir="$GIT_ROOT/$p"

13
build-tools/build-srpms-serial
View File

@ -6,6 +6,7 @@ CMDLINE="$ME $@"
BUILD_SRPMS_SERIAL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
source $BUILD_SRPMS_SERIAL_DIR/git-utils.sh
source $BUILD_SRPMS_SERIAL_DIR/spec-utils
source $BUILD_SRPMS_SERIAL_DIR/srpm-utils
source $BUILD_SRPMS_SERIAL_DIR/classify
@ -647,7 +648,8 @@ build_dir_srpm () {
return $RC
fi
local LOC=$(for g in $(find $PKG_ROOT_DIR/gits -type d -name .git); do d=$(dirname $g); (cd $d; git tag | grep "pre_wrs_$PKG_NAME_VER" >> /dev/null; if [ $? -eq 0 ]; then echo $d; fi); done | head -n 1 )
local LOC
LOC=$(git_list_containing_tag "${PKG_ROOT_DIR}/gits" "pre_wrs_$PKG_NAME_VER" | head -n 1 )
echo "===== '$TARGET_FOUND' has been extracted for editing. ====="
echo "===== Metadata can be found at: $PKG_ROOT_DIR/rpmbuild"
echo "===== Source code can be found at: $LOC"
@ -922,7 +924,8 @@ build_dir_spec () {
return $RC
fi
local LOC=$(for g in $(find $PKG_ROOT_DIR/gits -type d -name .git); do d=$(dirname $g); (cd $d; git branch --all | grep "$PKG_NAME_VER" >> /dev/null; if [ $? -eq 0 ]; then echo $d; fi); done | head -n 1 )
local LOC
LOC=$(git_list_containing_branch "${PKG_ROOT_DIR}/gits" "${PKG_NAME_VER}" | head -n 1 )
echo "===== '$TARGET_FOUND' has been extracted for editing. ====="
echo "===== Metadata can be found at: $PKG_ROOT_DIR/rpmbuild"
echo "===== Source code can be found at: $LOC"
@ -1093,8 +1096,7 @@ if [ $EDIT_FLAG -eq 0 ]; then
fi
fi
for g in $(find "$SRC_BASE" -type d -name .git | sort -V); do
GIT_ROOT=$(dirname $g)
for GIT_ROOT in $GIT_LIST; do
export GIT_BASE="$GIT_ROOT"
for p in $(cat $GIT_ROOT/$PKG_DIRS_FILE 2>> /dev/null); do
src_dir="$GIT_ROOT/$p"
@ -1136,8 +1138,7 @@ if [ $ALL -eq 1 ]; then
echo
echo "Auditing for obsolete srpms"
PACKAGES_CONSIDERED=""
for g in $(find $SRC_BASE -type d -name .git | sort -V); do
GIT_ROOT=$(dirname $g)
for GIT_ROOT in $GIT_LIST; do
for p in $(cat $GIT_ROOT/$PKG_DIRS_FILE 2>> /dev/null); do
src_dir="$GIT_ROOT/$p"
if [ -d $src_dir ]; then

89
build-tools/git-utils.sh
View File

@ -9,6 +9,91 @@
# by repo manifests.
#
# GIT_LIST: A list of root directories for all the gits under $MY_REPO
export GIT_LIST=$(find $MY_REPO -type d -name '.git' -exec dirname {} \;)
#
# git_list <dir>:
# Return a list of git root directories found under <dir>
#
git_list () {
local DIR=${1}
find "${DIR}" -type d -name '.git' -exec dirname {} \; | sort -V
}
# GIT_LIST: A list of root directories for all the gits under $MY_REPO/..
# as absolute paths.
export GIT_LIST=$(git_list "$(dirname "${MY_REPO}")")
# GIT_LIST_REL: A list of root directories for all the gits under $MY_REPO/..
# as relative paths.
export GIT_LIST_REL=$(for p in $GIT_LIST; do
echo .${p#$(dirname ${MY_REPO})};
done)
#
# git_list_containing_branch <dir> <branch>:
# Return a list of git root directories found under <dir> and
# having branch <branch>. The branch need not be current branch.
#
git_list_containing_branch () {
local DIR="${1}"
local BRANCH="${2}"
local d
for d in $(git_list "${DIR}"); do
(
cd "$d"
git branch --all | grep -q "$BRANCH"
if [ $? -eq 0 ]; then
echo "$d"
fi
)
done
}
#
# git_list_containing_tag <dir> <tag>:
# Return a list of git root directories found under <dir> and
# having tag <tag>.
#
git_list_containing_tag () {
local DIR="${1}"
local TAG="${2}"
local d
for d in $(git_list "${DIR}"); do
(
cd "$d"
git tag | grep -q "$TAG"
if [ $? -eq 0 ]; then
echo "$d"
fi
)
done
}
#
# git_context:
# Returns a bash script that can be used to recreate the current git context,
#
# Note: all paths are relative to $MY_REPO/..
#
git_context () {
(
cd $MY_REPO
for d in $GIT_LIST_REL; do
(
cd ${d}
echo -n "(cd ${d} && git checkout -f "
echo "$(git rev-list HEAD -1))"
)
done
)
}