From e3460d7d11d0fda5420d71b22c3aad2e02901de7 Mon Sep 17 00:00:00 2001 From: Scott Little Date: Wed, 4 Mar 2020 12:25:05 -0500 Subject: [PATCH] Enable build of docker images, charts and wheels from a layered build 1) create merged wheels.inc file(s) as part of build-pkgs. These files can be published by CENGN layer builds, and be referenced by the 'container' layer build. 2) get-stx-wheels.sh needs to find wheels.inc files from lower layer builds. The will be fould under ${MY_REPO}/cgcs-centos-repo/layer_wheels_inc. 3) build-helm-charts.sh must now also search for helm rpms under ${MY_REPO}/cgcs-centos-repo/Binary/noarch for chart packages built by lower layers. Story: 2006166 Task: 38979 Depends-On: https://review.opendev.org/711773 Depends-On: https://review.opendev.org/711774 Change-Id: I91650d3e1746a45c8ac118cca58e9b64f30c5b72 Signed-off-by: Scott Little --- build-tools/build-helm-charts.sh | 18 ++++++++--- build-tools/build-rpms-parallel | 6 ++++ build-tools/build-rpms-serial | 6 ++++ build-tools/build-wheels/get-stx-wheels.sh | 22 +++++++++++-- build-tools/wheel-utils.sh | 36 ++++++++++++++++++++++ 5 files changed, 81 insertions(+), 7 deletions(-) create mode 100755 build-tools/wheel-utils.sh diff --git a/build-tools/build-helm-charts.sh b/build-tools/build-helm-charts.sh index 1bcdf298..50e6788d 100755 --- a/build-tools/build-helm-charts.sh +++ b/build-tools/build-helm-charts.sh @@ -149,6 +149,16 @@ function build_image_versions_to_manifest { done } +function find_chartfile { + local helm_rpm=$1 + + for RPMS_DIR in ${RPMS_DIRS}; do + if [ -d ${RPMS_DIR} ]; then + find ${RPMS_DIR} -name "${helm_rpm}${FIND_GLOB}" + fi + done +} + # Extract the helm charts from a rpm function extract_chartfile { local helm_rpm=$1 @@ -157,8 +167,7 @@ function extract_chartfile { centos) # Bash globbing does not handle [^-] like regex # so grep needed to be used - rpm_file=$(ls ${RPMS_DIR} | grep "^${helm_rpm}${GREP_GLOB}") - chartfile=${RPMS_DIR}/${rpm_file} + chartfile=$(find_chartfile ${helm_rpm}) if [ ! -f ${chartfile} ]; then echo "Failed to find helm package: ${helm_rpm}" >&2 exit 1 @@ -422,8 +431,9 @@ if [ ${#IMAGE_RECORDS[@]} -ne 0 ]; then fi # Extract helm charts and app version from the application rpm -RPMS_DIR=${MY_WORKSPACE}/std/rpmbuild/RPMS -GREP_GLOB="-[^-]*-[^-]*.tis.noarch.rpm" +RPMS_DIRS="${MY_WORKSPACE}/std/rpmbuild/RPMS ${MY_REPO}/cgcs-centos-repo/Binary/noarch" +FIND_GLOB="*.tis.noarch.rpm" + extract_application_rpms # Extract helm charts from the application dependent rpms if [ ${#APP_HELM_FILES[@]} -gt 0 ]; then diff --git a/build-tools/build-rpms-parallel b/build-tools/build-rpms-parallel index 5025a1ad..bdb04ed5 100755 --- a/build-tools/build-rpms-parallel +++ b/build-tools/build-rpms-parallel @@ -88,6 +88,7 @@ export MOCK=/usr/bin/mock BUILD_RPMS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" source "${BUILD_RPMS_PARALLEL_DIR}/image-utils.sh" +source "${BUILD_RPMS_PARALLEL_DIR}/wheel-utils.sh" source "${BUILD_RPMS_PARALLEL_DIR}/spec-utils" source "${BUILD_RPMS_PARALLEL_DIR}/srpm-utils" @@ -1485,6 +1486,11 @@ image_inc_list iso std ${DISTRO} > "${IMAGE_INC_FILE}" DEV_IMAGE_INC_FILE="${MY_WORKSPACE}/image-dev.inc" image_inc_list iso dev ${DISTRO} > "${DEV_IMAGE_INC_FILE}" +for STREAM in stable dev; do + WHEELS_INC_FILE="${MY_WORKSPACE}/${DISTRO}_${STREAM}_wheels.inc" + wheels_inc_list ${STREAM} ${DISTRO} > "${WHEELS_INC_FILE}" +done + LAST_PLATFORM_RELEASE_FILE="$MY_BUILD_DIR/.platform_release" TARGETS=$@ diff --git a/build-tools/build-rpms-serial b/build-tools/build-rpms-serial index c543fb4b..7eb87fc1 100755 --- a/build-tools/build-rpms-serial +++ b/build-tools/build-rpms-serial @@ -51,6 +51,7 @@ export MOCK=/usr/bin/mock BUILD_RPMS_SERIAL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" source "${BUILD_RPMS_SERIAL_DIR}/image-utils.sh" +source "${BUILD_RPMS_SERIAL_DIR}/wheel-utils.sh" source "${BUILD_RPMS_SERIAL_DIR}/spec-utils" source "${BUILD_RPMS_SERIAL_DIR}/srpm-utils" @@ -1253,6 +1254,11 @@ image_inc_list iso std ${DISTRO} > "${IMAGE_INC_FILE}" DEV_IMAGE_INC_FILE="${MY_WORKSPACE}/image-dev.inc" image_inc_list iso dev ${DISTRO} > "${DEV_IMAGE_INC_FILE}" +for STREAM in stable dev; do + WHEELS_INC_FILE="${MY_WORKSPACE}/${DISTRO}_${STREAM}_wheels.inc" + wheels_inc_list ${STREAM} ${DISTRO} > "${WHEELS_INC_FILE}" +done + LAST_PLATFORM_RELEASE_FILE="$MY_BUILD_DIR/.platform_release" TARGETS=$@ diff --git a/build-tools/build-wheels/get-stx-wheels.sh b/build-tools/build-wheels/get-stx-wheels.sh index c0dbe1fd..2c7123ad 100755 --- a/build-tools/build-wheels/get-stx-wheels.sh +++ b/build-tools/build-wheels/get-stx-wheels.sh @@ -88,7 +88,23 @@ function get_wheels_files { find ${GIT_LIST} -maxdepth 1 -name "${OS}_${BUILD_STREAM}_wheels.inc" } -declare -a WHEELS_FILES=($(get_wheels_files)) +function get_lower_layer_wheels_files { + find ${MY_REPO}/cgcs-centos-repo/layer_wheels_inc -maxdepth 1 -name "*_${OS}_${BUILD_STREAM}_wheels.inc" +} + +function find_wheel_rpm { + local wheel="$1" + local repo= + + for repo in ${MY_WORKSPACE}/std/rpmbuild/RPMS \ + ${MY_REPO}/cgcs-centos-repo/Binary; do + if [ -d $repo ]; then + find $repo -name "${wheel}-[^-]*-[^-]*[.][^.]*[.]rpm" + fi + done | head -n 1 +} + +declare -a WHEELS_FILES=($(get_wheels_files) $(get_lower_layer_wheels_files)) if [ ${#WHEELS_FILES[@]} -eq 0 ]; then echo "Could not find ${OS} wheels.inc files" >&2 exit 1 @@ -109,9 +125,9 @@ for wheel in $(sed -e 's/#.*//' ${WHEELS_FILES[@]} | sort -u); do centos) # Bash globbing does not handle [^\-] well, # so use grep instead - wheelfile=$(ls ${MY_WORKSPACE}/std/rpmbuild/RPMS/${wheel}-* | grep -- '[^\-]*-[^\-]*.rpm') + wheelfile="$(find_wheel_rpm ${wheel})" - if [ ! -f "${wheelfile}" ]; then + if [ ! -e "${wheelfile}" ]; then echo "Could not find ${wheel}" >&2 FAILED+=($wheel) continue diff --git a/build-tools/wheel-utils.sh b/build-tools/wheel-utils.sh new file mode 100755 index 00000000..044d5d2f --- /dev/null +++ b/build-tools/wheel-utils.sh @@ -0,0 +1,36 @@ +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# A place for any functions related to wheels.inc files +# + +WHEEL_UTILS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +source "${WHEEL_UTILS_DIR}/git-utils.sh" + +# +# wheels_inc_list [] +# +# Parameters: +# stream: One of 'stable', 'dev' +# distro: One of 'centos', ... +# +# Returns: A list of unique rpm packages that contain needed wheel +# files. This is the union per git wheels.inc files. + +wheels_inc_list () { + local stream=$1 + local distro=$2 + + local search_target=${distro}_${stream}_wheels.inc + + ( + for d in $GIT_LIST; do + find $d/ -maxdepth 1 -name "${search_target}" -exec grep '^[^#]' {} + + done + ) | sort --unique +}