diff --git a/.gitignore b/.gitignore index 56f0088c..72860aec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,6 @@ *.swp .tox __pycache__/ -/centos-repo -/cgcs-centos-repo -/cgcs-tis-repo /local-build-data /local-repo /public-keys/ diff --git a/build-data/build_avoidance_source b/build-data/build_avoidance_source deleted file mode 100644 index 1e371f24..00000000 --- a/build-data/build_avoidance_source +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2018 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# - - -# -# What files and directories need to be copied -# -BUILD_AVOIDANCE_SRPM_DIRECTORIES="inputs srpm_assemble rpmbuild/SRPMS rpmbuild/SOURCES" -BUILD_AVOIDANCE_SRPM_FILES="" -BUILD_AVOIDANCE_RPM_DIRECTORIES="results rpmbuild/RPMS rpmbuild/SPECS repo/local-repo/dependancy-cache" -BUILD_AVOIDANCE_RPM_FILES=".platform_release" - - -# -# Copy the lines below to $MY_REPO/local-build-data/build_avoidance_source, -# then uncomment and fill in the values giving the location of your local reference build. -# -# BUILD_AVOIDANCE_USR="jenkins" -# BUILD_AVOIDANCE_HOST="machine.corp.com" -# BUILD_AVOIDANCE_DIR="/localdisk/loadbuild/jenkins/StarlingX_Build" diff --git a/build-data/unbuilt_rpm_patterns b/build-data/unbuilt_rpm_patterns deleted file mode 100644 index 35df2a4f..00000000 --- a/build-data/unbuilt_rpm_patterns +++ /dev/null @@ -1,22 +0,0 @@ -[-]locale[-] -[-]doc[-] -[-]dbg[-] -vswitch-staticdev -vim-spell -openssh-server-sysvinit -openstack-neutron-linuxbridge -^libcacard- -^kernel-bootwrapper -^kernel-doc- -^kernel-abi-whitelists -^kernel-debug- -^kernel-kdump -^kernel-rt-bootwrapper -^kernel-rt-doc- -^kernel-rt-abi-whitelists -^kernel-rt-debug- -^kernel-rt-debuginfo -^kernel-rt-kdump -^kernel-rt-cross-headers -^kernel-rt-kvm-debuginfo -^kernel-rt-tools-debuginfo diff --git a/build-tools/Cached_Data.txt b/build-tools/Cached_Data.txt deleted file mode 100644 index e3a7d1fe..00000000 --- a/build-tools/Cached_Data.txt +++ /dev/null @@ -1,78 +0,0 @@ -Data on an source rpm: - - location: - ${MY_WORKSPACE}/${BUILD_TYPE}/rpmbuild/SPECS/${SRPM_FILE_NAME}/ - - files: - *.spec # spec file found in the source rpm - - subdirectories: - NAMES/ # Directory contains an emtpy file, where the file name - # is the name of the source rpm. - - SERVICES/ # Directory contains zero or more emtpy files, where the - # file name is the name of the service provided by one - # or more of the rpms. - - BUILDS/ # Directory contains emtpy files, where the file name is - # the name of a binary rpm built from the source rpm. - - BUILDS_VR/ # Directory contains emtpy files, where the file name is - # the name-verion-release of a binary rpm built from the - # source rpm. - - location: - ${MY_WORKSPACE}/${BUILD_TYPE}/rpmbuild/SOURCES/${SRPM_FILE_NAME}/ - - files: - BIG # if it exists, it contains one line, the numeric value - # extracted from build_srpms.data if the line - # BUILD_IS_BIG=### if present. - # This is the estimated filesystem size (GB) required to - # host a mock build of the package. - # Note: not all parallel build environments are the same - # size. The smallest build environmnet is 3 GB and this - # is sufficient for most packages. Don't bother adding a - # BUILD_IS_BIG=### directive unless 3 gb is proven to be - # insufficient. - - SLOW # if it exists, it contains one line, the numeric value i - # extracted from build_srpms.data if the line - # BUILD_IS_SLOW=### if present. - # This is the estimated build time (minutes) required to - # host perform a mock build of the package. - # Note: Currently we only use this value as a boolean. - # Non-zero and we try to start the build of this package - # earlier rather than later. Build times >= 3 minutes are - # worth anotating. Else don't bother adding a - # BUILD_IS_SLOW=### directive -e.g. - -cd $MY_WORKSPACE/std/rpmbuild/SPECS/openstack-cinder-9.1.1-0.tis.40.src.rpm -find . -./BUILDS -./BUILDS/openstack-cinder -./BUILDS/python-cinder -./BUILDS/python-cinder-tests -./NAMES -./NAMES/openstack-cinder -./SERVICES -./SERVICES/cinder -./BUILDS_VR -./BUILDS_VR/openstack-cinder-9.1.1-0.tis.40 -./BUILDS_VR/python-cinder-9.1.1-0.tis.40 -./BUILDS_VR/python-cinder-tests-9.1.1-0.tis.40 -./openstack-cinder.spec - - -e.g. -cd $MY_WORKSPACE/std/rpmbuild/SOURCES/kernel-3.10.0-514.16.1.el7.29.tis.src.rpm -find . -./BIG -./SLOW - -cat ./BIG -8 - -cat ./SLOW -12 diff --git a/build-tools/audit-pkgs b/build-tools/audit-pkgs deleted file mode 100755 index 370c9b0c..00000000 --- a/build-tools/audit-pkgs +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash - -rpm_compare () { - local r="$1" - local r2="$2" - local line - local f=$(basename $r) - local f2=$(basename $r2) - - rpm -q --dump --nosignature -p $r | awk ' { print $1 "\n" $1 " " $5 " " $6 " " $7 " " $8 " " $9 " " $10 " " $11 } ' > /tmp/dump.new - rpm -q --dump --nosignature -p $r2 | awk ' { print $1 "\n" $1 " " $5 " " $6 " " $7 " " $8 " " $9 " " $10 " " $11 } ' > /tmp/dump.old - first_line=1 - diff -y -W 200 --suppress-common-lines /tmp/dump.new /tmp/dump.old | grep '|' | - while read -r line; do - left=$(echo "$line" | awk -F '|' '{ print $1 }') - right=$(echo "$line" | awk -F '|' '{ print $2 }') - left_f=$(echo "$left" | awk '{ print $1 }') - right_f=$(echo "$right" | awk '{ print $1 }') - if [ "$left_f" != "$right_f" ];then - continue - fi - if [ $first_line -eq 1 ]; then - echo "" - echo "$f vs $f2" - first_line=0 - fi - echo "$line" - done -} - -# For backward compatibility. Old repo location or new? -CENTOS_REPO=${MY_REPO}/centos-repo -if [ ! -d ${CENTOS_REPO} ]; then - CENTOS_REPO=${MY_REPO}/cgcs-centos-repo - if [ ! -d ${CENTOS_REPO} ]; then - echo "ERROR: directory ${MY_REPO}/centos-repo not found." - exit 1 - fi -fi - -echo "" -echo "======================================================" -echo "Auditing built packages vs unpatched upstream packages" -echo "======================================================" -for r in $(find $MY_WORKSPACE/*/rpmbuild/RPMS -name '*.rpm' | grep -v '.src.rpm' | grep -v debuginfo); do - f=$(basename $r) - f2=$(echo $f | sed 's#[.]tis[.][0-9]*[.]#.#' | sed 's#[.]tis[.]#.#') - r2=$(find ${CENTOS_REPO}/Binary/ -name $f2) - if [ "$r2" == "" ]; then - # Probably one of our own - # echo "Couldn't find '$f2'" - continue - fi - rpm_compare "$r" "$r2" -done - -echo "" -echo "============================" -echo "Auditing built for conflicts" -echo "============================" -grep 'conflicts with file from package' -r --binary-files=without-match $MY_WORKSPACE/*/results/ | - -while read -r line; do - w=$(echo "$line" | awk '{ print $8 }')".rpm" - w2=$(echo "$line" | awk '{ print $14 }')".rpm" - echo "$w $w2" -done | sort --unique | sed 's#bash-completion-1:#bash-completion-#' | - -while read -r line2; do - f=$(echo "$line2" | awk '{ print $1 }') - f2=$(echo "$line2" | awk '{ print $2 }') - r=$(find ${CENTOS_REPO}/Binary/ $MY_WORKSPACE/*/rpmbuild/RPMS -name $f) - r2=$(find ${CENTOS_REPO}/Binary/ $MY_WORKSPACE/*/rpmbuild/RPMS -name $f2) - # echo "" - # echo "$f vs $f2" - # echo "$r vs $r2" - if [ "$r" != "" ] && [ "$r2" != "" ]; then - rpm_compare "$r" "$r2" - fi -done diff --git a/build-tools/build-avoidance-utils.sh b/build-tools/build-avoidance-utils.sh deleted file mode 100644 index 5ac5858d..00000000 --- a/build-tools/build-avoidance-utils.sh +++ /dev/null @@ -1,923 +0,0 @@ -# -# Copyright (c) 2018 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# - -# -# Functions related to build avoidance. -# -# Do not call directly. Used by build-pkgs. -# -# Build avoidance downloads rpm, src.rpm and other artifacts of -# build-pkgs for a local reference build. The reference would -# typically be an automated build run atleast daily. -# The MY_WORKSPACE directory for the reference build shall have -# a common root directory, and a leaf directory that is a time stamp -# in a sortable parsable format. Default YYYYMMDDThhmmssZ. -# e.g. /localdisk/loadbuild/jenkins/StarlingX/20180719T113021Z -# -# Other formats can be used by setting the following variables -# in $MY_REPO/local-build-data/build_avoidance_source. -# e.g. to allow format YYYY-MM-DD_hh-mm-ss -# BUILD_AVOIDANCE_DATE_FORMAT="%Y-%m-%d" -# BUILD_AVOIDANCE_TIME_FORMAT="%H-%M-%S" -# BUILD_AVOIDANCE_DATE_TIME_DELIM="_" -# BUILD_AVOIDANCE_DATE_TIME_POSTFIX="" -# -# Note: Must be able to rsync and ssh to the machine that holds the -# reference builds. -# -# In future alternative transfer protocols may be supported. -# Select the alternate protocol by setting the following variables -# in $MY_REPO/local-build-data/build_avoidance_source. -# e.g. -# BUILD_AVOIDANCE_FILE_TRANSFER="my-supported-prototcol" -# - -BUILD_AVOIDANCE_UTILS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" - -source "${BUILD_AVOIDANCE_UTILS_DIR}/git-utils.sh" - -BUILD_AVOIDANCE_USR="" -BUILD_AVOIDANCE_HOST="" -BUILD_AVOIDANCE_DIR="" -BUILD_AVOIDANCE_URL="" - -# Default date/time format, iso-8601 compact, 20180912T143913Z -# Syntax is a subset of that use by the unix 'date' command. -BUILD_AVOIDANCE_DATE_FORMAT="%Y%m%d" -BUILD_AVOIDANCE_TIME_FORMAT="%H%M%S" -BUILD_AVOIDANCE_DATE_TIME_DELIM="T" -BUILD_AVOIDANCE_DATE_TIME_POSTFIX="Z" - -# Default file transfer method -BUILD_AVOIDANCE_FILE_TRANSFER="rsync" - -# Default is to use timestamps and days in UTC -# -# If you prefer local time, then set 'BUILD_AVOIDANCE_DATE_UTC=0' -# in '$MY_REPO/local-build-data/build_avoidance_source' -BUILD_AVOIDANCE_DATE_UTC=1 - -BUILD_AVOIDANCE_DATA_DIR="$MY_WORKSPACE/build_avoidance_data" -BUILD_AVOIDANCE_SOURCE="$MY_REPO/build-data/build_avoidance_source" -BUILD_AVOIDANCE_LOCAL_SOURCE="$MY_REPO/local-build-data/build_avoidance_source" -BUILD_AVOIDANCE_TEST_CONTEXT="$BUILD_AVOIDANCE_DATA_DIR/test_context" - -if [ ! -f $BUILD_AVOIDANCE_SOURCE ]; then - echo "Couldn't read $BUILD_AVOIDANCE_SOURCE" - exit 1 -fi - -echo "Reading: $BUILD_AVOIDANCE_SOURCE" -source $BUILD_AVOIDANCE_SOURCE - -if [ -f $BUILD_AVOIDANCE_LOCAL_SOURCE ]; then - echo "Reading: $BUILD_AVOIDANCE_LOCAL_SOURCE" - source $BUILD_AVOIDANCE_LOCAL_SOURCE -fi - -UTC="" - -if [ $BUILD_AVOIDANCE_DATE_UTC -eq 1 ]; then - UTC="--utc" -fi - - -if [ "x$BUILD_AVOIDANCE_OVERRIDE_DIR" != "x" ]; then - BUILD_AVOIDANCE_DIR="$BUILD_AVOIDANCE_OVERRIDE_DIR" -fi - -if [ "x$BUILD_AVOIDANCE_OVERRIDE_HOST" != "x" ]; then - BUILD_AVOIDANCE_HOST="$BUILD_AVOIDANCE_OVERRIDE_HOST" -fi - -if [ "x$BUILD_AVOIDANCE_OVERRIDE_USR" != "x" ]; then - BUILD_AVOIDANCE_USR="$BUILD_AVOIDANCE_OVERRIDE_USR" -fi - -echo "BUILD_AVOIDANCE_DIR=$BUILD_AVOIDANCE_DIR" -echo "BUILD_AVOIDANCE_HOST=$BUILD_AVOIDANCE_HOST" -echo "BUILD_AVOIDANCE_USR=$BUILD_AVOIDANCE_USR" - -build_avoidance_last_sync_file () { - local BUILD_TYPE=$1 - - if [ -z "$BUILD_TYPE" ]; then - echo "build_avoidance_last_sync_file: Build type not set" - exit 1 - fi - echo "$BUILD_AVOIDANCE_DATA_DIR/$BUILD_TYPE/last_sync_context" -} - -build_avoidance_clean () { - local BUILD_TYPE=$1 - local lsf - - if [ "$BUILD_TYPE" == "" ]; then - for lsf in $(find $BUILD_AVOIDANCE_DATA_DIR -name last_sync_context); do - \rm -f -v "$lsf" - done - else - lsf="$(build_avoidance_last_sync_file $BUILD_TYPE)" - if [ -f $lsf ]; then - \rm -f -v "$lsf" - fi - fi -} - - -date_to_iso_8601 () { - local DATE="$1" - local CENTURY="" - local YEAR_IN_CENTURY="00" - local MONTH="01" - local DAY="01" - local DAY_OF_YEAR="" - - CENTURY="$(date '+%C')" - - for x in $(echo "${BUILD_AVOIDANCE_DATE_FORMAT}" | tr ' ' '#' | sed 's/%%/#/g' | tr '%' ' ' ); do - # Consume format case options - case ${x:0:1} in - ^) x=${x:1};; - \#) x=${x:1};; - *) ;; - esac - - # Process format - case $x in - Y*) CENTURY=${DATE:0:2}; YEAR_IN_CENTURY=${DATE:2:2}; DATE=${DATE:4}; x=${x:1};; - 0Y*) CENTURY=${DATE:0:2}; YEAR_IN_CENTURY=${DATE:2:2}; DATE=${DATE:4}; x=${x:2};; - _Y*) CENTURY=$(echo "${DATE:0:2}" | tr ' ' '0'); YEAR_IN_CENTURY=${DATE:2:2}; DATE=${DATE:4}; x=${x:2};; - - y*) YEAR_IN_CENTURY=${DATE:0:2}; DATE=${DATE:2}; x=${x:1};; - 0y*) YEAR_IN_CENTURY=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};; - _y*) YEAR_IN_CENTURY=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};; - - C*) CENTURY=${DATE:0:2}; DATE=${DATE:2}; x=${x:1};; - 0C*) CENTURY=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};; - _C*) CENTURY=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};; - - m*) MONTH=${DATE:0:2}; DATE=${DATE:2}; x=${x:1};; - 0m*) MONTH=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};; - _m*) MONTH=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};; - e*) MONTH=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:1};; - 0e*) MONTH=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};; - _e*) MONTH=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};; - b*) MONTH="$(date -d "${DATE:0:3} 1 2000" '+%m')"; DATE=${DATE:3}; x=${x:1};; - h*) MONTH="$(date -d "${DATE:0:3} 1 2000" '+%m')"; DATE=${DATE:3}; x=${x:1};; - - d*) DAY=${DATE:0:2}; DATE=${DATE:2}; x=${x:1};; - 0d*) DAY=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};; - _d*) DAY=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};; - - j*) DAY_OF_YEAR=${DATE:0:3}; DATE=${DATE:3}; x=${x:1};; - 0j*) DAY_OF_YEAR=${DATE:0:3}; DATE=${DATE:3}; x=${x:2};; - _j*) DAY_OF_YEAR=$(echo "${DATE:0:3}" | tr ' ' '0'); DATE=${DATE:3}; x=${x:2};; - - D*) MONTH=${DATE:0:2}; DAY=${DATE:3:2}; YEAR_IN_CENTURY=${DATE:6:2}; DATE=${DATE:8}; x=${x:1};; - F*) CENTURY=${DATE:0:2}; YEAR_IN_CENTURY=${DATE:2:2}; MONTH=${DATE:5:2}; DAY=${DATE:8:2}; DATE=${DATE:10}; x=${x:1};; - *) >&2 echo "$FUNCNAME (${LINENO}): Unsupported date format: ${BUILD_AVOIDANCE_DATE_FORMAT}"; return 1;; - esac - - # consume remaing non-interpreted content - if [ "$(echo "${DATE:0:${#x}}" | tr ' ' '#')" != "${x}" ]; then - >&2 echo "$FUNCNAME (${LINENO}): Unexpected content '${DATE:0:${#x}}' does not match expected '${x}': '$1' being parsed vs '${BUILD_AVOIDANCE_DATE_FORMAT}'" - return 1 - fi - DATE=${DATE:${#x}} - done - - if [ "${DAY_OF_YEAR}" != "" ]; then - local YEAR_SEC - local DOY_SEC - YEAR_SEC="$(date -d "${CENTURY}${YEAR_IN_CENTURY}-01-01" '+%s')" - DOY_SEC=$((YEAR_SEC+(DAY_OF_YEAR-1)*24*60*60)) - MONTH="$(date "@$DOY_SEC" "+%m")" - DAY="$(date "@$DOY_SEC" "+%d")" - fi - - echo "${CENTURY}${YEAR_IN_CENTURY}-${MONTH}-${DAY}" - return 0 -} - -time_to_iso_8601 () { - TIME="$1" - local HOUR="00" - local H12="" - local AMPM="" - local MINUTE="00" - local SECOND="00" - - CENTURY="$(date '+%C')" - - for x in $(echo "${BUILD_AVOIDANCE_TIME_FORMAT}" | tr ' ' '#' | sed 's/%%/#/g' | tr '%' ' ' ); do - # Consume format case options - case ${x:0:1} in - ^) x=${x:1};; - \#) x=${x:1};; - *) ;; - esac - - # Process format - case $x in - H*) HOUR=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};; - 0H*) HOUR=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};; - _H*) HOUR="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};; - k*) HOUR="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:1};; - 0k*) HOUR=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};; - _k*) HOUR="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};; - - I*) H12=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};; - 0I*) H12=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};; - _I*) H12="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};; - l*) H12="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:1};; - 0l*) H12=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};; - _l*) H12="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};; - p*) AMPM=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};; - - M*) MINUTE=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};; - 0M*) MINUTE=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};; - _M*) MINUTE="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};; - - S*) SECOND=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};; - 0S*) SECOND=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};; - _S*) SECOND="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};; - - R*) HOUR=${TIME:0:2}; MINUTE=${TIME:3:2} TIME=${TIME:5}; x=${x:1};; - r*) H12=${TIME:0:2}; MINUTE=${TIME:3:2}; SECOND=${TIME:6:2}; AMPM=${TIME:9:2}; TIME=${TIME:11}; x=${x:1};; - T*) HOUR=${TIME:0:2}; MINUTE=${TIME:3:2}; SECOND=${TIME:6:2}; TIME=${TIME:8}; x=${x:1};; - - *) >&2 echo "$FUNCNAME (${LINENO}): Unsupported time format: ${BUILD_AVOIDANCE_TIME_FORMAT}"; return 1;; - esac - - # consume remaing non-interpreted content - if [ "$(echo "${TIME:0:${#x}}" | tr ' ' '#')" != "${x}" ]; then - >&2 echo "$FUNCNAME (${LINENO}): Unexpected content '${TIME:0:${#x}}' does not match expected '${x}': '$1' being parsed vs '${BUILD_AVOIDANCE_TIME_FORMAT}'" - return 1 - fi - TIME=${TIME:${#x}} - done - - if [ "$H12" != "" ] && [ "$AMPM" != "" ]; then - HOUR="$(date "$H12:01:01 $AMPM" '+%H')" - else - if [ "$H12" != "" ] && [ "$AMPM" != "" ]; then - >&2 echo "$FUNCNAME (${LINENO}): Unsupported time format: ${BUILD_AVOIDANCE_TIME_FORMAT}" - return 1 - fi - fi - - echo "${HOUR}:${MINUTE}:${SECOND}" - return 0 -} - -date_time_to_iso_8601 () { - local DATE_TIME="$1" - local DATE - local TIME - local DECODED_DATE - local DECODED_TIME - DATE=$(echo "${DATE_TIME}" | cut -d ${BUILD_AVOIDANCE_DATE_TIME_DELIM} -f 1) - TIME=$(echo "${DATE_TIME}" | cut -d ${BUILD_AVOIDANCE_DATE_TIME_DELIM} -f 2 | sed "s#${BUILD_AVOIDANCE_DATE_TIME_POSTFIX}\$##") - DECODED_DATE=$(date_to_iso_8601 "${DATE}") - DECODED_TIME=$(time_to_iso_8601 "${TIME}") - echo "${DECODED_DATE}T${DECODED_TIME}$(date $UTC '+%:z')" -} - -# -# test_build_avoidance_context -# -# Is the provided context file compatible with the current -# state of all of our gits? A compatible context is one -# where every commit in the context file is visible in our -# current git history. -# -# Returns: Timestamp of context tested. -# Exit code: 0 = Compatible -# 1 = This context is older than the last applied -# build avoidance context. If you are searching -# newest to oldest, you might as well stop. -# 2 = Not compatible -# -test_build_avoidance_context () { - local context="$1" - local BA_LAST_SYNC_CONTEXT="$2" - local BA_CONTEXT="" - - BA_CONTEXT=$(basename $context | cut -d '.' -f 1) - >&2 echo "test: $BA_CONTEXT" - - if [ "$BA_CONTEXT" == "$BA_LAST_SYNC_CONTEXT" ]; then - # Stop the search. We've reached the last sync point - BA_CONTEXT="" - echo "$BA_CONTEXT" - return 1 - fi - - git_test_context "$context" - result=$? - if [ $result -eq 0 ]; then - # found a new context !!! - echo "$BA_CONTEXT" - return 0 - fi - - # Continue the search - BA_CONTEXT="" - echo "$BA_CONTEXT" - return 2 -} - - -# -# get_build_avoidance_context -# -# Return URL of the most recent jenkins build that is compatable with -# the current software context under $MY_REPO. -# -get_build_avoidance_context () { - ( - local BUILD_TYPE=$1 - local context - local BA_CONTEXT="" - local BA_LAST_SYNC_CONTEXT="" - - export BUILD_AVOIDANCE_LAST_SYNC_FILE="$(build_avoidance_last_sync_file $BUILD_TYPE)" - mkdir -p "$(dirname $BUILD_AVOIDANCE_LAST_SYNC_FILE)" - - # Load last synced context - if [ -f $BUILD_AVOIDANCE_LAST_SYNC_FILE ]; then - BA_LAST_SYNC_CONTEXT=$(head -n 1 $BUILD_AVOIDANCE_LAST_SYNC_FILE) - fi - - mkdir -p $BUILD_AVOIDANCE_DATA_DIR - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): mkdir -p $BUILD_AVOIDANCE_DATA_DIR" - return 1 - fi - - local REMOTE_CTX_DIR="context" - local LOCAL_CTX_DIR="$BUILD_AVOIDANCE_DATA_DIR/context" - - # First copy the directory containing all the context files for - # the reference builds. - >&2 echo "Download latest reference build contexts" - - # Must set this prior to build_avoidance_copy_dir. - # The setting is not exported outside of the subshell. - if [ -z "$BUILD_AVOIDANCE_HOST" ]; then - BUILD_AVOIDANCE_URL="$BUILD_AVOIDANCE_DIR" - else - BUILD_AVOIDANCE_URL="$BUILD_AVOIDANCE_HOST:$BUILD_AVOIDANCE_DIR" - fi - - - build_avoidance_copy_dir "$REMOTE_CTX_DIR" "$LOCAL_CTX_DIR" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_copy_dir '$REMOTE_CTX_DIR' '$LOCAL_CTX_DIR'" - return 1 - fi - - # Search for a new context to sync - cd $MY_REPO - - if [ "$BUILD_AVOIDANCE_DAY" == "" ]; then - # Normal case: - # Search all contexts, newest to oldest, for a good context. - for context in $(ls -1rd $LOCAL_CTX_DIR/*context); do - >&2 echo "context=$context" - BA_CONTEXT=$(test_build_avoidance_context $context $BA_LAST_SYNC_CONTEXT) - if [ $? -le 1 ]; then - # Stop search. Might or might not have found a good context. - break; - fi - done - else - # Special case when a target day is specified. Why would we do this? - # Reason is we might want the reference build to itself use build - # avoidance referencing prior builds of itself, except for one build - # a week when we use a full build rather than a build avoidance build. - # e.g. Sunday - full build - # Mon-Sat - avoidance builds that refernce Sunday build. - # - # Starting from last (e.g. "Sunday"), search newest to - # oldest for a good context. If none found, increment the target - # day (e.g. Monday) and search again. Keep incrementing until a - # good build is found, or target day + offset days would be a date - # in the furure. - # - local TARG_DAY=$BUILD_AVOIDANCE_DAY - local TODAY_DATE - local TODAY_DAY - local TARG_DATE="" - local TARG_TS - local TODAY_TS - - TODAY_DATE=$(date $UTC +%Y-%m-%d) - TODAY_DAY=$(date $UTC "+%A") - - for OFFSET_DAYS in 0 1 2 3 4 5 6; do - if [ "$TARG_DAY" != "" ]; then - # Convert TARG_DAY+OFFSET_DAYS to TARG_DATE - - if [ "$TODAY_DAY" == "$TARG_DAY" ]; then - TARG_DATE=$(date $UTC -d"$TARG_DAY+$OFFSET_DAYS days" +%Y-%m-%d) - else - TARG_DATE=$(date $UTC -d"last-$TARG_DAY+$OFFSET_DAYS days" +%Y-%m-%d) - fi - >&2 echo "TARG_DATE=$TARG_DATE" - - TARG_TS=$(date $UTC -d "$TARG_DATE" +%s) - TODAY_TS=$(date $UTC -d "$TODAY_DATE" +%s) - if [ $TARG_TS -gt $TODAY_TS ]; then - # Skip if offset has pushed us into future dates - continue; - fi - - if [ "$TARG_DATE" == "$TODAY_DATE" ]; then - TARG_DATE="" - fi - fi - - # Search build, newest to oldest, satisfying TARG_DATE - for f in $(ls -1rd $LOCAL_CTX_DIR/*context); do - DATE=$(date_to_iso_8601 $(basename "$f")) - if [ $? -ne 0 ]; then - >&2 echo "Failed to extract date from filename '$(basename "$f")', ignoring file" - continue - fi - - >&2 echo " DATE=$DATE, TARG_DATE=$TARG_DATE" - - if [ "$DATE" == "$TARG_DATE" ] || [ "$TARG_DATE" == "" ] ; then - context=$f; - else - continue - fi - - >&2 echo "context=$context" - - BA_CONTEXT=$(test_build_avoidance_context $context $BA_LAST_SYNC_CONTEXT) - - if [ $? -le 1 ]; then - # Stop search. Might or might not have found a good context. - break; - fi - done - - if [ "$BA_CONTEXT" != "" ]; then - # Found a good context. - break - fi - done - fi - - if [ "$BA_CONTEXT" == "" ]; then - # No new context found - return 1 - fi - - # test that the reference build context hasn't been deleted - local BA_CONTEXT_DIR="$BUILD_AVOIDANCE_DIR/$BA_CONTEXT" - - if [ -z "$BUILD_AVOIDANCE_HOST" ]; then - >&2 echo "[ -d $BA_CONTEXT_DIR ]" - if ! [ -d $BA_CONTEXT_DIR ] ; then - return 1 - fi - else - >&2 echo "ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]'" - if ! ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]' ; then - return 1 - fi - fi - - # Save the latest context - >&2 echo "BA_CONTEXT=$BA_CONTEXT" - >&2 echo "BUILD_AVOIDANCE_LAST_SYNC_FILE=$BUILD_AVOIDANCE_LAST_SYNC_FILE" - echo $BA_CONTEXT > $BUILD_AVOIDANCE_LAST_SYNC_FILE - - # The location of the load with the most compatable new context - if [ -z "$BUILD_AVOIDANCE_HOST" ]; then - URL=$BA_CONTEXT_DIR - else - URL=$BUILD_AVOIDANCE_HOST:$BA_CONTEXT_DIR - fi - - # return URL to caller. - echo $URL - return 0 - ) -} - - -# -# build_avoidance_pre_clean -# -# A place for any cleanup actions that must preceed a build avoidance build. -# -build_avoidance_pre_clean () { - local BUILD_TYPE="$1" - - if [ "$BUILD_TYPE" == "" ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required" - return 1 - fi - - # clean prior builds - if [ -d $MY_WORKSPACE/$BUILD_TYPE ]; then - build-pkgs --clean --$BUILD_TYPE --no-build-avoidance - if [ $? -ne 0 ]; then - return 1 - fi - fi - - for f in $BUILD_AVOIDANCE_SRPM_FILES $BUILD_AVOIDANCE_RPM_FILES; do - if [ -f $MY_WORKSPACE/$BUILD_TYPE/$f ]; then - \rm -f $MY_WORKSPACE/$BUILD_TYPE/$f - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): rm -f $MY_WORKSPACE/$BUILD_TYPE/$f" - return 1 - fi - fi - done - - for d in $BUILD_AVOIDANCE_SRPM_DIRECTORIES $BUILD_AVOIDANCE_RPM_DIRECTORIES; do - - if [ -d $MY_WORKSPACE/$BUILD_TYPE/$d ]; then - \rm -rf $MY_WORKSPACE/$BUILD_TYPE/$d - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): rm -rf $MY_WORKSPACE/$BUILD_TYPE/$d" - return 1 - fi - fi - done - - return 0 -} - - -# -# build_avoidance_copy_dir_rsync ['verbose'] -# -# Copy a file from $BUILD_AVOIDANCE_URL/ -# to using rsync. -# -build_avoidance_copy_dir_rsync () { - local FROM="$1" - local TO="$2" - local VERBOSE="$3" - local FLAGS="-a -u" - - if [ "$BUILD_AVOIDANCE_URL" == "" ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_AVOIDANCE_URL no set" - return 1 - fi - - if [ "$VERBOSE" != "" ]; then - FLAGS="$FLAGS -v" - echo "rsync $FLAGS '$BUILD_AVOIDANCE_URL/$FROM/' '$TO/'" - fi - - rsync $FLAGS "$BUILD_AVOIDANCE_URL/$FROM/" "$TO/" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): command failed: rsync $FLAGS '$BUILD_AVOIDANCE_URL/$FROM/' '$TO/'" - return 1 - fi - - chmod -R 'ug+w' "$TO/" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): command failed: chmod -R 'ug+w' '$TO/'" - return 1 - fi - return 0 -} - -# -# build_avoidance_copy_file_rsync ['verbose'] -# -# Copy a file from $BUILD_AVOIDANCE_URL/ -# to using rsync. -# -build_avoidance_copy_file_rsync () { - local FROM="$1" - local TO="$2" - local VERBOSE="$3" - local FLAGS="-a -u" - - if [ "$BUILD_AVOIDANCE_URL" == "" ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_AVOIDANCE_URL no set" - return 1 - fi - if [ "$VERBOSE" != "" ]; then - FLAGS="$FLAGS -v" - echo "rsync $FLAGS '$BUILD_AVOIDANCE_URL/$FROM' '$TO'" - fi - - rsync $FLAGS "$BUILD_AVOIDANCE_URL/$FROM" "$TO" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): command failed: rsync $FLAGS '$BUILD_AVOIDANCE_URL/$FROM' '$TO'" - return 1 - fi - - chmod -R 'ug+w' "$TO" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): command failed: chmod -R 'ug+w' '$TO'" - return 1 - fi - return $? -} - -# -# build_avoidance_copy_dir ['verbose'] -# -# Copy a file from $BUILD_AVOIDANCE_URL/ -# to . The copy method will be determined by -# BUILD_AVOIDANCE_FILE_TRANSFER. Only 'rsync' is supported at present. -# -# should be a directory, -# mkdir -p will be called on . -# -build_avoidance_copy_dir () { - local FROM="$1" - local TO="$2" - local VERBOSE="$3" - - if [ "$VERBOSE" != "" ]; then - echo "mkdir -p '$TO'" - fi - mkdir -p "$TO" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): mkdir -p $TO" - return 1 - fi - - case ${BUILD_AVOIDANCE_FILE_TRANSFER} in - rsync) - build_avoidance_copy_dir_rsync "$FROM" "$TO" "$VERBOSE" - return $? - ;; - *) - >&2 echo "Error: $FUNCNAME (${LINENO}): Unknown BUILD_AVOIDANCE_FILE_TRANSFER '${BUILD_AVOIDANCE_FILE_TRANSFER}'" - return 1 - ;; - esac - return 1 -} - -# -# build_avoidance_copy_file ['verbose'] -# -# Copy a file from $BUILD_AVOIDANCE_URL/ -# to . The copy method will be determined by -# BUILD_AVOIDANCE_FILE_TRANSFER. Only 'rsync' is supported at present. -# -# should be a file, not a directory, -# mkdir -p will be called on $(dirname ) -# -build_avoidance_copy_file () { - local FROM="$1" - local TO="$2" - local VERBOSE="$3" - - if [ "$VERBOSE" != "" ]; then - echo "mkdir -p $(dirname '$TO')" - fi - mkdir -p "$(dirname "$TO")" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): mkdir -p $(dirname "$TO")" - return 1 - fi - - case ${BUILD_AVOIDANCE_FILE_TRANSFER} in - rsync) - build_avoidance_copy_file_rsync "$FROM" "$TO" "$VERBOSE" - return $? - ;; - *) - >&2 echo "Error: $FUNCNAME (${LINENO}): Unknown BUILD_AVOIDANCE_FILE_TRANSFER '${BUILD_AVOIDANCE_FILE_TRANSFER}'" - return 1 - ;; - esac - return 1 -} - -# -# build_avoidance_copy ['verbose'] -# -# Copy the needed build artifacts for from $BUILD_AVOIDANCE_URL. -# -build_avoidance_copy () { - local BUILD_TYPE="$1" - local VERBOSE="$2" - - if [ "$BUILD_TYPE" == "" ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required" - return 1 - fi - - # Iterate through list of directories to copy - for d in $BUILD_AVOIDANCE_SRPM_DIRECTORIES $BUILD_AVOIDANCE_RPM_DIRECTORIES; do - build_avoidance_copy_dir "$BUILD_TYPE/$d" "$MY_WORKSPACE/$BUILD_TYPE/$d" "$VERBOSE" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_copy_dir '$BUILD_TYPE/$d' '$MY_WORKSPACE/$BUILD_TYPE/$d'" - return 1 - fi - done - - # Iterate through list of files to copy - for f in $BUILD_AVOIDANCE_SRPM_FILES $BUILD_AVOIDANCE_RPM_FILES; do - build_avoidance_copy_file "$BUILD_TYPE/$f" "$MY_WORKSPACE/$BUILD_TYPE/$f" "$VERBOSE" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_copy_file '$BUILD_TYPE/$f' '$MY_WORKSPACE/$BUILD_TYPE/$f'" - return 1 - fi - done - - return 0 -} - -# -# build_avoidance_fixups -# -# Fix paths in the build artifacts that we coppied that contain -# the user name. -# -# Also, our credentials may differ from the reference build, -# so substitute unsigned packages in place of signed packages. -# -build_avoidance_fixups () { - local BUILD_TYPE="$1" - - local BA_SOURCE_BUILD_ENVIRONMENT - BA_SOURCE_BUILD_ENVIRONMENT="${BUILD_AVOIDANCE_USR}-$(basename $(dirname $BUILD_AVOIDANCE_URL))-$(basename $BUILD_AVOIDANCE_URL)-${SRC_BUILD_ENVIRONMENT}" - local RESULT_DIR="" - local FROM_DIR="" - local TO_DIR="" - local rpm_path_post_signing - local rpm_path_pre_signing - local rpm_name - local md5sum_post_signing - local md5sum_pre_signing - - if [ "$BUILD_TYPE" == "" ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required" - return 1 - fi - - RESULT_DIR="$MY_WORKSPACE/$BUILD_TYPE/results" - FROM_DIR="${RESULT_DIR}/${BA_SOURCE_BUILD_ENVIRONMENT}-${BUILD_TYPE}" - TO_DIR="${RESULT_DIR}/${MY_BUILD_ENVIRONMENT}-${BUILD_TYPE}" - echo "$FUNCNAME: FROM_DIR=$FROM_DIR" - echo "$FUNCNAME: TO_DIR=$TO_DIR" - echo "$FUNCNAME: MY_BUILD_ENVIRONMENT=$MY_BUILD_ENVIRONMENT" - - # Fix patchs the use MY_BUILD_ENVIRONMENT - if [ ! -d "$FROM_DIR" ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): Expected directory '$FROM_DIR' is missing." - return 1 - fi - - echo "$FUNCNAME: mv '$FROM_DIR' '$TO_DIR'" - \mv "$FROM_DIR" "$TO_DIR" - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): mv '$FROM_DIR' '$TO_DIR'" - return 1 - fi - - local MY_WS_BT="$MY_WORKSPACE/$BUILD_TYPE" - - # Replace signed rpms with non-signed copies .... we aren't a formal build - for rpm_path_post_signing in $(find $MY_WS_BT/rpmbuild/RPMS -type f -name '*.rpm' | grep -v src.rpm); do - - rpm_name=$(basename $rpm_path_post_signing) - rpm_path_pre_signing=$(find $MY_WS_BT/results -name $rpm_name | head -n1) - if [ "$rpm_path_pre_signing" != "" ]; then - md5sum_post_signing=$(md5sum ${rpm_path_post_signing} | cut -d ' ' -f 1) - md5sum_pre_signing=$(md5sum ${rpm_path_pre_signing} | cut -d ' ' -f 1) - if [ "${md5sum_post_signing}" != "${md5sum_pre_signing}" ]; then - echo "$FUNCNAME: fixing $rpm_name" - \rm -f ${rpm_path_post_signing} - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): rm -f ${rpm_path_post_signing}" - return 1 - fi - - \cp ${rpm_path_pre_signing} ${rpm_path_post_signing} - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): cp ${rpm_path_pre_signing} ${rpm_path_post_signing}" - return 1 - fi - fi - fi; - done - - return 0 -} - - -# -# build_avoidance -# -# Look for a reference build that is applicable to our current git context. -# and copy it to our local workspace, if we haven't already done so. -# -build_avoidance () { - local BUILD_TYPE="$1" - - echo "==== Build Avoidance Start ====" - - export BUILD_AVOIDANCE_LAST_SYNC_FILE="$(build_avoidance_last_sync_file $BUILD_TYPE)" - mkdir -p "$(dirname $BUILD_AVOIDANCE_LAST_SYNC_FILE)" - - if [ "$BUILD_TYPE" == "" ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required" - return 1 - fi - - if [ ! -d $MY_WORKSPACE/$BUILD_TYPE ]; then - mkdir -p $MY_WORKSPACE/$BUILD_TYPE - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): Failed to create directory $MY_WORKSPACE/$BUILD_TYPE" - return 1 - fi - fi - - if [ ! -L $MY_WORKSPACE/$BUILD_TYPE/repo ]; then - ln -s $MY_REPO $MY_WORKSPACE/$BUILD_TYPE/repo - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): Failed to create symlink $MY_WORKSPACE/$BUILD_TYPE/repo -> $MY_REPO" - return 1 - fi - fi - - build_avoidance_pre_clean $BUILD_TYPE - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_pre_clean $BUILD_TYPE" - return 1 - fi - - build_avoidance_copy $BUILD_TYPE 'verbose' - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_copy $BUILD_TYPE" - return 1 - fi - - build_avoidance_fixups $BUILD_TYPE - if [ $? -ne 0 ]; then - >&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_fixups $BUILD_TYPE" - return 1 - fi - - echo "==== Build Avoidance Complete ====" - return 0 -} - -# -# build_avoidance_save_reference_context -# -# For use by a reference build. Copy the 'CONTEXT' file -# from the build into a central directory where we save -# the context of old builds. -# -# Individual reference builds use: -# MY_WORKSPACE=/ -# and context files are collected in dir: -# DEST_CTX_DIR=/context -# using name: -# DEST_CTX=.context - -build_avoidance_save_reference_context () { - local DIR - DIR=$(dirname "${MY_WORKSPACE}") - - # Note: SUB_DIR should be a timestamp - local SUB_DIR - SUB_DIR=$(basename "${MY_WORKSPACE}") - - local SRC_CTX="${MY_WORKSPACE}/CONTEXT" - local DEST_CTX_DIR="${DIR}/context" - local DEST_CTX="${DEST_CTX_DIR}/${SUB_DIR}.context" - - if [ ! -f "${SRC_CTX}" ]; then - echo "Context file not found at '${SRC_CTX}'" - return 1 - fi - - mkdir -p "${DEST_CTX_DIR}" - if [ $? -ne 0 ]; then - echo "Error: $FUNCNAME (${LINENO}): Failed to create directory '${DEST_CTX_DIR}'" - return 1 - fi - - cp "${SRC_CTX}" "${DEST_CTX}" - if [ $? -ne 0 ]; then - echo "Error: $FUNCNAME (${LINENO}): Failed to copy ${SRC_CTX} -> ${DEST_CTX}" - return 1 - fi - - return 0 -} diff --git a/build-tools/build-docker-images/README b/build-tools/build-docker-images/README index e961232e..c5dbf6ed 100644 --- a/build-tools/build-docker-images/README +++ b/build-tools/build-docker-images/README @@ -3,21 +3,22 @@ PRIVATE_REGISTRY_USERID=myuser PRIVATE_REGISTRY=xxx.xxx.xxx.xxx:9001 VERSION=2018.11.13 -OS=centos +OS=debian OS_VERSION=7.5.1804 BUILD_STREAM=stable HOST_PORT=8088 +PUBLISH_URL=https://mirror.starlingx.windriver.com/mirror/starlingx/master/${OS}/monolithic/latest_build/ -## Step 1: Build stx-centos +## Step 1: Build stx-debian time $MY_REPO/build-tools/build-docker-images/build-stx-base.sh \ --os ${OS} \ --os-version ${OS_VERSION} \ --version ${VERSION} \ --user ${PRIVATE_REGISTRY_USERID} \ --registry ${PRIVATE_REGISTRY} \ + --repo 'deb [trusted=yes check-valid-until=0] $PUBLISH_URL/inputs/packages ./' + --repo 'deb [trusted=yes check-valid-until=0] $PUBLISH_URL/outputs/std/packages ./' --push \ - --repo stx-local-build,http://${HOSTNAME}:${HOST_PORT}/${MY_WORKSPACE}/std/rpmbuild/RPMS \ - --repo stx-mirror-distro,http://${HOSTNAME}:${HOST_PORT}/${MY_REPO}/cgcs-root/cgcs-${OS}-repo/Binary \ --clean diff --git a/build-tools/build-docker-images/base-image-build-centos-dev.cfg b/build-tools/build-docker-images/base-image-build-centos-dev.cfg deleted file mode 100644 index e4ea3cd6..00000000 --- a/build-tools/build-docker-images/base-image-build-centos-dev.cfg +++ /dev/null @@ -1,2 +0,0 @@ -# one option per line, option=value -repo=ussuri-wsgi,https://mirror.starlingx.windriver.com/mirror/centos/centos/mirror.centos.org/centos/7/sclo/x86_64/rh/ diff --git a/build-tools/build-docker-images/base-image-build-centos-stable.cfg b/build-tools/build-docker-images/base-image-build-centos-stable.cfg deleted file mode 100644 index e4ea3cd6..00000000 --- a/build-tools/build-docker-images/base-image-build-centos-stable.cfg +++ /dev/null @@ -1,2 +0,0 @@ -# one option per line, option=value -repo=ussuri-wsgi,https://mirror.starlingx.windriver.com/mirror/centos/centos/mirror.centos.org/centos/7/sclo/x86_64/rh/ diff --git a/build-tools/build-docker-images/build-stx-base.sh b/build-tools/build-docker-images/build-stx-base.sh index acb60a00..7acd62f5 100755 --- a/build-tools/build-docker-images/build-stx-base.sh +++ b/build-tools/build-docker-images/build-stx-base.sh @@ -18,7 +18,7 @@ if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then exit 1 fi -SUPPORTED_OS_ARGS=('centos' 'debian') +SUPPORTED_OS_ARGS=( 'debian' ) OS= # default: autodetect OS_VERSION= # default: lookup "ARG RELEASE" in Dockerfile BUILD_STREAM=stable @@ -52,7 +52,6 @@ Options: --version: Specify version for output image --stream: Build stream, stable or dev (default: stable) --repo: Software repository, can be specified multiple times - * CentOS format: "NAME,BASEURL" * Debian format: "TYPE [OPTION=VALUE...] URL DISTRO COMPONENTS..." This will be added to /etc/apt/sources.list as is, see also sources.list(5) manpage. @@ -281,13 +280,7 @@ fi if [ ${#REPO_LIST[@]} -eq 0 ]; then # Either --repo or --local must be specified - if [ "${LOCAL}" = "yes" ]; then - if [[ "$OS" == "centos" ]] ; then - REPO_LIST+=("local-std,http://${HOST}:8088${MY_WORKSPACE}/std/rpmbuild/RPMS") - REPO_LIST+=("stx-distro,http://${HOST}:8089${MY_REPO}/cgcs-${OS}-repo/Binary") - fi - # debian is handled down below - elif [ "${BUILD_STREAM}" != "dev" -a "${BUILD_STREAM}" != "master" ]; then + if [ "${LOCAL}" != "yes" -a "${BUILD_STREAM}" != "dev" -a "${BUILD_STREAM}" != "master" ]; then echo "Either --local or --repo must be specified" >&2 exit 1 fi @@ -314,33 +307,7 @@ fi cp ${SRC_DOCKERFILE} ${BUILDDIR}/Dockerfile # Generate the stx.repo file -if [[ "$OS" == "centos" ]] ; then - STX_REPO_FILE=${BUILDDIR}/stx.repo - for repo in ${REPO_LIST[@]}; do - repo_name=$(echo $repo | awk -F, '{print $1}') - repo_baseurl=$(echo $repo | awk -F, '{print $2}') - - if [ -z "${repo_name}" -o -z "${repo_baseurl}" ]; then - echo "Invalid repo specified: ${repo}" >&2 - echo "Expected format: name,baseurl" >&2 - exit 1 - fi - - cat >>${STX_REPO_FILE} <&2 + exit 1 diff --git a/build-tools/build-docker-images/stx-centos/Dockerfile.dev b/build-tools/build-docker-images/stx-centos/Dockerfile.dev deleted file mode 100644 index af30a6b6..00000000 --- a/build-tools/build-docker-images/stx-centos/Dockerfile.dev +++ /dev/null @@ -1,16 +0,0 @@ -# Expected build arguments: -# RELEASE: centos release -# -ARG RELEASE=7.5.1804 -FROM centos:${RELEASE} - -RUN set -ex ;\ - sed -i '/\[main\]/ atimeout=120' /etc/yum.conf ;\ - yum install -y centos-release-openstack-stein ;\ - rm -rf \ - /var/log/* \ - /tmp/* \ - /var/tmp/* - -# root CA cert expired on October 1st, 2021 -RUN yum update -y ca-certificates diff --git a/build-tools/build-docker-images/stx-centos/Dockerfile.stable b/build-tools/build-docker-images/stx-centos/Dockerfile.stable deleted file mode 100644 index b30f615a..00000000 --- a/build-tools/build-docker-images/stx-centos/Dockerfile.stable +++ /dev/null @@ -1,31 +0,0 @@ -# Expected build arguments: -# RELEASE: centos release -# REPO_OPTS: yum options to enable StarlingX repo -# -ARG RELEASE=7.5.1804 -FROM centos:${RELEASE} - -ARG REPO_OPTS - -# The stx.repo file must be generated by the build tool first -COPY stx.repo / - -RUN set -ex ;\ - sed -i '/\[main\]/ atimeout=120' /etc/yum.conf ;\ - mv /stx.repo /etc/yum.repos.d/ ;\ - yum upgrade --disablerepo=* ${REPO_OPTS} -y ;\ - yum install --disablerepo=* ${REPO_OPTS} -y \ - qemu-img \ - openssh-clients \ - python3 \ - python3-pip \ - python3-wheel \ - rh-python36-mod_wsgi \ - ;\ - rm -rf \ - /var/log/* \ - /tmp/* \ - /var/tmp/* - -# root CA cert expired on October 1st, 2021 -RUN yum update -y ca-certificates diff --git a/build-tools/build-docker-images/update-stx-image.sh b/build-tools/build-docker-images/update-stx-image.sh index da6cd3d2..80fa8d8a 100755 --- a/build-tools/build-docker-images/update-stx-image.sh +++ b/build-tools/build-docker-images/update-stx-image.sh @@ -58,7 +58,7 @@ Options: --module-src: Specify path to module source to install/update (dir or git repo) Formats: dir[|version] url[|branch][|version] - --pkg: Specify path to distro package to install/update (ie. rpm) + --pkg: Specify path to distro package to install/update (ie. deb) --customize: Customization script --extra: Extra file (to be accessible to customization script) --push: Push to docker repo diff --git a/build-tools/build-guest b/build-tools/build-guest deleted file mode 100755 index ab1d9d69..00000000 --- a/build-tools/build-guest +++ /dev/null @@ -1,412 +0,0 @@ -#!/bin/env bash - -# -# Copyright (c) 2018 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# - -# -# Build the tis-centos-image.img or tis-centos-image-rt.img file -# - -BUILD_GUEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" -source "${BUILD_GUEST_DIR}/image-utils.sh" - -PROGNAME=$(basename "$0") - -# NOTE: TMP_DIR must end in '/' -# NOTE: /tmp/ is now tmpfs like. Can't be trusted across multiple mock commands -# TMP_DIR=/tmp/ -TMP_DIR=/ - -# Use RPMs from the std build only, for now -export BUILD_TYPE=std -export MY_BUILD_DIR_TOP=$MY_BUILD_DIR - -function init_vars { - # Output path (current dir unless MY_WORKSPACE defined) - OUTPUT_DIR="$PWD/export" - if [ ! -z "$MY_WORKSPACE" ] && [ -d "$MY_WORKSPACE" ] ; then - OUTPUT_DIR="$MY_WORKSPACE/export" - CGCS_REPO_DIR="$MY_WORKSPACE/rpmbuild/RPMS" - fi - - if [ -n "$MY_GUEST_DIR" ]; then - GUEST_DIR=$MY_GUEST_DIR - else - GUEST_DIR=$MY_WORKSPACE/guest - fi - - MOCK=/usr/bin/mock - if [ $VERBOSE -eq 0 ]; then - MOCK="$MOCK -q" - fi - - # Path to guest configuration - GUEST_BUILD_DIR="${BUILD_GUEST_DIR}/build_guest" - GUEST_BUILD_CMD=$GUEST_BUILD_DIR/build-guest-image.py - if [ $VERBOSE -eq 1 ]; then - GUEST_BUILD_CMD="$GUEST_BUILD_CMD -x" - fi - - if [ $BUILD_MODE == 'std' ]; then - OUTPUT_FILE=$OUTPUT_DIR/tis-centos-guest.img - elif [ $BUILD_MODE == 'rt' ]; then - OUTPUT_FILE=$OUTPUT_DIR/tis-centos-guest-rt.img - else - printf " Error -- unknown BUILD_MODE '$BUILD_MODE'\n"; - exit 1 - fi -} - - -function check_vars { - # Where to store data - printf "Finding cgcs-root\n" - printf " Checking \$MY_REPO (value \"$MY_REPO\")\n" - - if [ ! -z "$MY_REPO" ] && [ -d "$MY_REPO" ] ; then - INTERNAL_REPO_ROOT=$MY_REPO - printf " Found!\n" - fi - - if [ -z "$INTERNAL_REPO_ROOT" ] ; then - printf " No joy -- checking \$MY_REPO_ROOT_DIR (value \"$MY_REPO_ROOT_DIR\")\n" - if [ ! -z "$MY_REPO_ROOT_DIR" ] && [ -d "$MY_REPO_ROOT_DIR/cgcs-root" ] ; then - INTERNAL_REPO_ROOT=$MY_REPO_ROOT_DIR/cgcs-root - printf " Found!\n" - fi - fi - - if [ -z "$INTERNAL_REPO_ROOT" ] ; then - printf " No joy -- checking for \$MY_WORKSPACE/cgcs-root\n" - if [ -d "$MY_WORKSPACE/cgcs-root" ] ; then - INTERNAL_REPO_ROOT=$MY_WORKSPACE/cgcs-root - printf " Found!\n" - fi - fi - - if [ -z "$INTERNAL_REPO_ROOT" ] ; then - printf " Error -- could not locate cgcs-root repo.\n" - exit 1 - fi - - STX_DIR=$INTERNAL_REPO_ROOT/stx - - if [ "x$MY_BUILD_CFG" == "x" ];then - printf " Error -- reqiure MY_BUILD_CFG to be defined.\n" - exit 1 - fi - - RELEASE_INFO="$(get_release_info)" - if [ $? -ne 0 ]; then - echo "WARNING: failed to find a release info file." - else - export PLATFORM_RELEASE=$(source "$RELEASE_INFO" && echo $PLATFORM_RELEASE) - fi - -} - - -function create_rootfs { - printf "\nCreating guest file system\n" - - mkdir -p $GUEST_DIR - if [ $? -ne 0 ]; then - printf " Error -- Could not create $GUEST_DIR\n"; - exit 1 - fi - - # Place build-time environment variables in mock configuration - GUEST_ENV="${MY_BUILD_ENVIRONMENT}-guest" - GUEST_CFG=$GUEST_DIR/$MY_BUILD_ENVIRONMENT_FILE - - MY_BUILD_ENVIRONMENT=$GUEST_ENV "${BUILD_GUEST_DIR}/modify-build-cfg" $GUEST_CFG - if [ $? -ne 0 ]; then - printf " Error -- Could not update $GUEST_CFG\n"; - exit 1 - fi - - # Setup mock directories for the guest - if [ -d /localdisk/loadbuild/mock ]; then - LNK=/localdisk/loadbuild/mock/$GUEST_ENV - if [ ! -L $LNK ]; then - ln -s $GUEST_DIR $LNK - fi - fi - - if [ -d /localdisk/loadbuild/mock-cache ]; then - mkdir -p $GUEST_DIR/cache - LNK=/localdisk/loadbuild/mock-cache/$GUEST_ENV - if [ ! -L $LNK ]; then - ln -s $GUEST_DIR/cache $LNK - fi - fi - - # Setup mock chroot environment - $MOCK -r $GUEST_CFG --clean && $MOCK -r $GUEST_CFG --init - if [ $? -ne 0 ]; then - printf " Error -- Failed to setup guest mock chroot\n"; - exit 1 - fi - - # Install the RPMs to the root filesystem - - # Note that the "rt" build needs access to both local-std and local-rt repos - local EXTRA_REPOS="" - - if [ $BUILD_MODE == 'std' ]; then - INC_RPM_LIST=$(grep -v '^#' ${GUEST_BUILD_DIR}/rpm-install-list.txt) - TIS_RPM_LIST=$(image_inc_list guest std centos) - elif [ $BUILD_MODE == 'rt' ]; then - INC_RPM_LIST=$(grep -v '^#' ${GUEST_BUILD_DIR}/rpm-install-list-rt.txt) - TIS_RPM_LIST=$(image_inc_list guest rt centos) - EXTRA_REPOS="--enablerepo local-rt" - else - printf " Error -- unknown BUILD_MODE '$BUILD_MODE'\n"; - exit 1 - fi - - $MOCK -r $GUEST_CFG ${EXTRA_REPOS} --install ${INC_RPM_LIST} ${TIS_RPM_LIST} "$@" - if [ $? -ne 0 ]; then - printf "=====\n" - cat $GUEST_DIR/mock/result/root.log | sed -n '/Error:/,$p' | sed '/Child return code was:/q' - printf "=====\n" - printf " Error -- Failed to install RPM packages\n"; - exit 1 - fi - - # Make sure all requested packages are installed - MISSING=$( - extra_rpm_names="$( - for p in "$@" ; do - # skip URLs - if [[ "$p" =~ :// ]] ; then - continue - fi - # if it contains a slash or ends with .rpm, assume its a local file - # and read its embedded package name - if [[ "$p" =~ / || "$p" =~ [.]rpm$ ]] ; then - rpm -q --qf '%{name}\n' -p "$p" - # otherwise assume its a package name already - else - echo "$p" - fi - done - )" - $MOCK -r $GUEST_CFG --chroot -- rpm -q --whatprovides ${INC_RPM_LIST} ${TIS_RPM_LIST} $extra_rpm_names \ - | sed -n 's/^no package provides //p' \ - | sort -u - ) - if [ -n "$MISSING" ]; then - printf "=====\n" - printf "WARNING: The following RPMs are missing or could not be installed:\n" - local p - for p in $MISSING ; do - echo " [$p]" - done - printf "=====\n" - fi - - # Remove RPMs that are not required in image (pruned package list) - # NOTE: these are automatically installed from the mock init not - # through dependencies. - EXC_RPM_LIST=$(grep -v '^#' ${GUEST_BUILD_DIR}/rpm-remove-list.txt) - - $MOCK -r $GUEST_CFG --remove ${EXC_RPM_LIST} - if [ $? -ne 0 ]; then - printf " Error -- Failed to remove RPM packages\n"; - exit 1 - fi - - printf " Done\n" -} - - -function update_rootfs { - printf "\nCustomizing guest file system\n" - - # Copy over skeleton configuration files - for GUEST_ROOTFS in $GUEST_BUILD_DIR/rootfs $GUEST_BUILD_DIR/rootfs-$BUILD_MODE; - do - for f in $(cd $GUEST_ROOTFS && find . -type f | cut -c3-); - do - echo "$MOCK -r $GUEST_CFG --copyin $GUEST_ROOTFS/$f $f" - $MOCK -r $GUEST_CFG --copyin $GUEST_ROOTFS/$f $f - if [ $? -ne 0 ]; then - printf " Error -- Failed to copyin file $f\n"; - exit 1 - fi - done - done - - # Run the root file system setup script inside the chroot - ROOTFS_SETUP=rootfs-setup.sh - $MOCK -r $GUEST_CFG --copyin $GUEST_BUILD_DIR/$ROOTFS_SETUP $TMP_DIR && \ - if [ $BUILD_MODE == 'rt' ]; then - ROOTFS_SETUP_CMD="$TMP_DIR$ROOTFS_SETUP --rt" - elif [ $BUILD_MODE == 'std' ]; then - ROOTFS_SETUP_CMD="$TMP_DIR$ROOTFS_SETUP --std" - else - ROOTFS_SETUP_CMD="$TMP_DIR$ROOTFS_SETUP" - fi - $MOCK -r $GUEST_CFG --chroot "$ROOTFS_SETUP_CMD" - if [ $? -ne 0 ]; then - printf " Error -- Failed to run guest $ROOTFS_SETUP\n"; - exit 1 - fi - $MOCK -r $GUEST_CFG --chroot "rm -f $TMP_DIR$ROOTFS_SETUP" - if [ $? -ne 0 ]; then - printf " Error -- Failed to delete $ROOTFS_SETUP from guest\n"; - exit 1 - fi - - printf " Done\n" -} - - -function build_image { - # Build the image - printf "\nBuilding guest image $OUTPUT_FILE\n" - - mkdir -p $OUTPUT_DIR - if [ $? -ne 0 ]; then - printf " Error -- Could not create $OUTPUT_DIR\n"; - exit 1 - fi - - # Build guest rootfs archive - ROOTFS_SPACE=$((500*1024*1024)) - ROOTFS_TAR=rootfs.tar - ROOTFS_EXCLUDE=rootfs-exclude.txt - - $MOCK -r $GUEST_CFG --copyin $GUEST_BUILD_DIR/$ROOTFS_EXCLUDE $TMP_DIR - $MOCK -r $GUEST_CFG --chroot -- tar -cf $TMP_DIR$ROOTFS_TAR -X $TMP_DIR$ROOTFS_EXCLUDE --exclude=$TMP_DIR$ROOTFS_TAR --numeric-owner / - $MOCK -r $GUEST_CFG --copyout $TMP_DIR$ROOTFS_TAR $GUEST_DIR - $MOCK -r $GUEST_CFG --chroot -- rm -f $TMP_DIR$ROOTFS_TAR - - $GUEST_BUILD_CMD -i $GUEST_DIR/$ROOTFS_TAR -o $OUTPUT_FILE -s $ROOTFS_SPACE - if [ $? -ne 0 ]; then - printf " Error -- Failed to build guest image\n"; - exit 1 - fi - - printf " Done\n" -} - - -function clean_guest { - printf "\nCleaning the guest $GUEST_DIR\n" - - if [ ! -e $GUEST_DIR ]; then - printf " Done...nothing to do\n"; - exit 0 - fi - - # Place build-time environment variables in mock configuration - GUEST_ENV="${MY_BUILD_ENVIRONMENT}-guest" - GUEST_CFG=$GUEST_DIR/$MY_BUILD_ENVIRONMENT_FILE - - if [ ! -e $GUEST_CFG ]; then - MY_BUILD_ENVIRONMENT=$GUEST_ENV "${BUILD_GUEST_DIR}/modify-build-cfg" $GUEST_CFG - if [ $? -ne 0 ]; then - printf " Error -- Could not update $GUEST_CFG\n"; - exit 1 - fi - fi - - $MOCK -r $GUEST_CFG --clean - $MOCK -r $GUEST_CFG --scrub=cache - - rm -rf $GUEST_DIR - if [ $? -ne 0 ]; then - printf " Error -- Failed to remove guest $GUEST_DIR\n"; - exit 1 - fi - - printf " Done\n" -} - -############################################# -# Main code -############################################# - -usage () { - echo "" - echo "Usage: " - echo " build-guest [--rt | --std] [--verbose] [EXTRA_RPMS...]" - echo " build-guest [--help]" - echo " build-guest [--clean]" - echo "" - echo "EXTRA_RPMS are either package names or full RPM file paths" -} - -# Default argument values -HELP=0 -CLEAN=0 -VERBOSE=0 -BUILD_MODE='std' - -# read the options -TEMP=`getopt -o h --long clean,rt,std,verbose,help -n "$PROGNAME" -- "$@"` || exit 1 -eval set -- "$TEMP" - -# extract options and their arguments into variables. -while true ; do - case "$1" in - -h|--help) HELP=1 ; shift ;; - --clean) CLEAN=1 ; shift ;; - --verbose) VERBOSE=1 ; shift ;; - --rt) BUILD_MODE='rt' ; shift ;; - --std) BUILD_MODE='std' ; shift ;; - --) shift ; break ;; - *) echo "Internal error!" ; exit 1 ;; - esac -done - -if [ $HELP -eq 1 ]; then - usage - exit 0 -fi - -if [[ $CLEAN -eq 1 && "$#" -gt 0 ]] ; then - echo "Too many arguments!" >&2 ; exit 1 -else - # make sure extra RPM files exist - for p in "$@" ; do - # skip URLs - if [[ "$p" =~ :// ]] ; then - continue - fi - # if it contains a slash or ends with .rpm assume its a local file name - if [[ "$p" =~ / || "$p" =~ [.]rpm$ ]] ; then - # make sure it exists and is an RPM file - true <"$p" || exit 1 - if ! file --brief --mime-type "$p" | grep -q "^application/x-rpm$" ; then - echo "$p: not an RPM file" >&2 - exit 1 - fi - fi - done - unset p -fi - -( -printf "\n*****************************\n" -printf "Create Titanium Cloud/CentOS Guest Image\n" -printf "*****************************\n\n" - -init_vars -check_vars - -if [ $CLEAN -eq 1 ]; then - clean_guest - exit 0 -fi - -create_rootfs "$@" -update_rootfs -build_image - -) 2>&1 | stdbuf -o0 awk '{ print strftime("%H:%M:%S"), $0; fflush(); }' ; exit ${PIPESTATUS[0]} diff --git a/build-tools/build-helm-charts.sh b/build-tools/build-helm-charts.sh index e17cc95d..7e38ba76 100755 --- a/build-tools/build-helm-charts.sh +++ b/build-tools/build-helm-charts.sh @@ -12,7 +12,7 @@ BUILD_HELM_CHARTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source $BUILD_HELM_CHARTS_DIR/utils.sh || exit 1 -SUPPORTED_OS_ARGS=('centos' 'debian') +SUPPORTED_OS_ARGS=('debian') OS= LABEL="" APP_NAME="stx-openstack" @@ -35,7 +35,7 @@ Usage: $(basename $0) [--os ] [-a, --app ] [-A, --app-version-file /path/to/$APP_VERSION_BASE] [-B, --app-version ] - [-r, --rpm ] [-i, --image-record ] [--label