helm charts: better build configuration
* Remove obsolete build parameter HELM_CHART_APPS * New build config option HELM_CHART_PACKAGES: list of DEB package name patterns for the "primary" helm chart(s), such as stx-openstack-helm * New build config option EXTRA_HELM_CHART_PACKAGES: list of DEB package name patterns for the additional helm chart(s), such as stx-monitor-helm Story: 2010226 Task: 46422 Signed-off-by: Davlet Panech <davlet.panech@windriver.com> Change-Id: I7b4fedfd95b294f35021ec876cf1b948bb1adf78
This commit is contained in:
Davlet Panech
parent
03f68d264f
commit
135e323b4d
|
|
@ -14,9 +14,92 @@ require_job_env DRY_RUN
|
|||
|
||||
load_build_env
|
||||
|
||||
unset GREP_COLOR GREP_COLORS GREP_OPTIONS
|
||||
|
||||
BUILD_STREAMS="stable dev"
|
||||
BUILD_TAGS="latest versioned"
|
||||
HELM_CHART_APPS=$(echo $HELM_CHART_APPS | sed -r 's/,+//g')
|
||||
|
||||
normalize_pattern_list() {
|
||||
echo "$1" | sed -r 's/[[:space:],]+/\n/g' | grep -v -E '^\s*$' || true
|
||||
}
|
||||
|
||||
# Convert helm chart package name patterns to arrays
|
||||
HELM_CHART_PACKAGES="$(normalize_pattern_list "$HELM_CHART_PACKAGES")"
|
||||
declare -a HELM_CHART_PKG_PATTENRS
|
||||
if [[ -n "$HELM_CHART_PACKAGES" ]] ; then
|
||||
readarray -t HELM_CHART_PKG_PATTERNS <<<"$HELM_CHART_PACKAGES"
|
||||
fi
|
||||
|
||||
EXTRA_HELM_CHART_PACKAGES="$(normalize_pattern_list "$EXTRA_HELM_CHART_PACKAGES")"
|
||||
declare -a EXTRA_HELM_CHART_PKG_PATTENRS
|
||||
if [[ -n "$EXTRA_HELM_CHART_PACKAGES" ]] ; then
|
||||
readarray -t EXTRA_HELM_CHART_PKG_PATTERNS <<<"$EXTRA_HELM_CHART_PACKAGES"
|
||||
fi
|
||||
|
||||
# Usage: fnmatch PATTERN STRING...
|
||||
fnmatch() {
|
||||
local ptn="$1" ; shift
|
||||
while [[ "$#" -gt 0 ]] ; do
|
||||
case "$1" in
|
||||
$ptn) return 0 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Usage: find_helm_apps PACKAGE_NAME_PATTERNS...
|
||||
# Prints "helm_tar_name:package_name" one per line
|
||||
find_helm_apps() {
|
||||
local inc_files_str
|
||||
inc_files_str=$(find "$BUILD_HOME/repo/cgcs-root" -name "${DOCKER_BASE_OS}_helm.inc") || return 1
|
||||
[[ -n "$inc_files_str" ]] || return 0
|
||||
local -a inc_files
|
||||
readarray -t inc_files <<<"$inc_files_str" || return 1
|
||||
[[ "${#inc_files[@]}" -gt 0 ]] || return 0
|
||||
local helm_packages
|
||||
helm_packages=$(cat "${inc_files[@]}" | \grep -v -E '^\s*(#.*)?$' ; [[ ${PIPESTATUS[0]} -eq 0 ]]) || return 1
|
||||
local pkg
|
||||
(
|
||||
for pkg in $helm_packages ; do
|
||||
local ptn
|
||||
for ptn in "$@" ; do
|
||||
if fnmatch "$ptn" "$pkg" ; then
|
||||
local app="${pkg%-helm}"
|
||||
echo "$app:$pkg"
|
||||
fi
|
||||
done
|
||||
done
|
||||
) | sort -u
|
||||
[[ "${PIPESTATUS[0]}" -eq 0 ]]
|
||||
}
|
||||
|
||||
# find main helm charts
|
||||
MAIN_HELM_CHARTS=$(find_helm_apps "${HELM_CHART_PKG_PATTERNS[@]}")
|
||||
info "found primary helm charts: [$(echo ${MAIN_HELM_CHARTS})] pattern=[${HELM_CHART_PKG_PATTERNS[*]}]"
|
||||
|
||||
# find extra helm charts
|
||||
EXTRA_HELM_CHARTS=$(
|
||||
set -e
|
||||
|
||||
# temp file - all charts matching HELM_CHART_PACKAGES
|
||||
file1="$(mktemp -t "build-helm-charts.1.XXXXXX")"
|
||||
exec 3<>"$file1"
|
||||
rm "$file1"
|
||||
file1="/proc/self/fd/3"
|
||||
echo "$MAIN_HELM_CHARTS" >"$file1"
|
||||
|
||||
# temp file all charts matching EXTRA_HELM_CHART_PACKAGES
|
||||
file2="$(mktemp -t "build-helm-charts.1.XXXXXX")"
|
||||
exec 4<>"$file2"
|
||||
rm "$file2"
|
||||
file2="/proc/self/fd/4"
|
||||
find_helm_apps "${EXTRA_HELM_CHART_PKG_PATTERNS[@]}" >"$file2"
|
||||
|
||||
# lines unique to file2
|
||||
comm -13 "$file1" "$file2"
|
||||
)
|
||||
info "found extra helm charts: [$(echo ${EXTRA_HELM_CHARTS})] pattern=[${EXTRA_HELM_CHART_PKG_PATTERNS[*]}]"
|
||||
|
||||
# find image dirs relative to WORKSPACE_ROOT
|
||||
declare -a image_dirs
|
||||
|
|
@ -28,13 +111,15 @@ if [[ -d "$WORKSPACE_ROOT/rt/build-images" ]] ; then
|
|||
fi
|
||||
# copy any extra image-*.lst files to workspace so that
|
||||
# build containers can see them
|
||||
if [[ -d "$EXTRA_IMAGE_RECORD_DIR" ]] ; then
|
||||
if [[ -n "$EXTRA_IMAGE_RECORD_DIR" ]] ; then
|
||||
info "looking for extra image records in $EXTRA_IMAGE_RECORD_DIR"
|
||||
( cd "$EXTRA_IMAGE_RECORD_DIR" ; ) || exit 1
|
||||
if ! $DRY_RUN ; then
|
||||
rm -rf --one-file-system "$WORKSPACE_ROOT/extra-image-records"/*
|
||||
mkdir -p "$WORKSPACE_ROOT/extra-image-records"
|
||||
find "$EXTRA_IMAGE_RECORD_DIR" \
|
||||
-mindepth 1 -maxdepth 1 -name 'images-*.lst' \
|
||||
-exec \cp --force --preserve=links --no-dereference -t "$WORKSPACE_ROOT/extra-image-records" '{}' '+' \
|
||||
-exec \cp -v --force --preserve=links --no-dereference -t "$WORKSPACE_ROOT/extra-image-records" '{}' '+' \
|
||||
|| exit 1
|
||||
image_dirs+=('extra-image-records')
|
||||
fi
|
||||
|
|
@ -49,12 +134,18 @@ copy_dir() {
|
|||
find "$1" -mindepth 1 -maxdepth 1 -exec cp -f -alr -t "$2" '{}' '+'
|
||||
}
|
||||
|
||||
# call build-helm-charts.sh in container for each stream/tag
|
||||
if [[ "${#image_dirs[@]}" -gt 0 ]] ; then
|
||||
output_dir="$BUILD_HOME/workspace/helm-charts"
|
||||
if [[ -d "$output_dir" ]] ; then
|
||||
rm -rf --one-file-system "$output_dir" || exit 1
|
||||
fi
|
||||
output_dir="$BUILD_HOME/workspace/helm-charts"
|
||||
tmp_output_dir=$BUILD_HOME/workspace/std/build-helm
|
||||
|
||||
if [[ -d "$output_dir" ]] ; then
|
||||
rm -rf --one-file-system "$output_dir" || exit 1
|
||||
fi
|
||||
mkdir -p "$output_dir"
|
||||
mkdir -p "$tmp_output_dir"
|
||||
|
||||
# replace image tags in the main helm chart with the images generated
|
||||
# by this build
|
||||
if [[ "${#image_dirs[@]}" -gt 0 && -n "${MAIN_HELM_CHARTS}" ]] ; then
|
||||
for build_stream in $BUILD_STREAMS ; do
|
||||
for build_tag in $BUILD_TAGS ; do
|
||||
for os in $DOCKER_BASE_OS ; do
|
||||
|
|
@ -77,29 +168,46 @@ if [[ "${#image_dirs[@]}" -gt 0 ]] ; then
|
|||
continue
|
||||
fi
|
||||
|
||||
tmp_output_dir=$BUILD_HOME/workspace/std/build-helm
|
||||
for spec in ${MAIN_HELM_CHARTS} ; do
|
||||
app="${spec%%:*}"
|
||||
pkg="${spec#*:}"
|
||||
|
||||
for app in ${HELM_CHART_APPS:-NONE} ; do
|
||||
cmd="build-helm-charts.sh"
|
||||
cmd+=" --verbose"
|
||||
cmd+=" --os ${os}"
|
||||
cmd+=" --label '${label}'"
|
||||
cmd+=" --image-record ${image_arg}"
|
||||
if [[ "$app" != "NONE" ]] ; then
|
||||
cmd+=" --app $app"
|
||||
fi
|
||||
cmd+=" | tee \"\$MY_WORKSPACE/helm-${label}.log\""
|
||||
cmd+=" --app $app"
|
||||
cmd+=" -r $pkg"
|
||||
cmd+=" 2>&1 | tee \"\$MY_WORKSPACE/std/build-helm/main-${label}.log\""
|
||||
cmd+=" ; [[ \${PIPESTATUS[0]} -eq 0 ]]"
|
||||
notice "building primary helm chart $app ($label)"
|
||||
build_helm_charts "$cmd" || exit 1
|
||||
if [[ -d "$tmp_output_dir" ]] ; then
|
||||
mkdir -p "$output_dir" || exit 1
|
||||
copy_dir "$tmp_output_dir" "$output_dir" || exit 1
|
||||
fi
|
||||
copy_dir "$tmp_output_dir" "$output_dir" || exit 1
|
||||
done
|
||||
done
|
||||
done
|
||||
done
|
||||
if [[ -d "$output_dir" ]] ; then
|
||||
notice "helm charts created in $output_dir"
|
||||
fi
|
||||
fi
|
||||
|
||||
# all other helm charts: extract them from DEBs w/o replacing image tags
|
||||
for spec in $EXTRA_HELM_CHARTS ; do
|
||||
app="${spec%%:*}"
|
||||
pkg="${spec#*:}"
|
||||
cmd="build-helm-charts.sh"
|
||||
cmd+=" --verbose"
|
||||
cmd+=" --os $DOCKER_BASE_OS"
|
||||
cmd+=" --app $app"
|
||||
cmd+=" -r $pkg"
|
||||
cmd+=" 2>&1 | tee \"\$MY_WORKSPACE/std/build-helm/$app.log\""
|
||||
notice "building extra helm chart $app"
|
||||
build_helm_charts "$cmd" || exit 1
|
||||
copy_dir "$tmp_output_dir" "$output_dir" || exit 1
|
||||
done
|
||||
|
||||
if [[ -d "$output_dir/stx" && $(find "$output_dir/stx" -maxdepth 1 -type f -name '*.tgz' -print -quit | wc -l) -gt 0 ]] ; then
|
||||
notice "helm charts created in $output_dir/stx"
|
||||
ls -al "$output_dir/stx"
|
||||
else
|
||||
notice "no helm charts found"
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -111,9 +111,42 @@ USE_POD_URLS_IN_DOCKER_IMAGES="false"
|
|||
# Base image for all docker images. If unset, we will build our own.
|
||||
DOCKER_IMAGE_BASE=
|
||||
|
||||
# Helm chart apps, comma or space-separated.
|
||||
# This will be passed to build-helm-charts.sh
|
||||
HELM_CHART_APPS=
|
||||
#
|
||||
# Space or comma-separated helm chart package names or glob patterns
|
||||
#
|
||||
# For each matching package we will extract the charts tarball & replace
|
||||
# image names & tags with images generated by the local build, once
|
||||
# for -$TIMSTAMP tags and once for -latest tags.
|
||||
#
|
||||
# These patterns will be matched against the names in debian_helm.inc files
|
||||
#
|
||||
# Example: HELM_CHART_PACKAGES="stx-openstack-helm" will create 2 tarballs
|
||||
# similar to:
|
||||
# stx-openstack-1.0-1.stx.4-debian-stable-latest.tgz
|
||||
# stx-openstack-1.0-1.stx.4-debian-stable-versioned.tgz
|
||||
#
|
||||
HELM_CHART_PACKAGES=
|
||||
|
||||
#
|
||||
# Space or comma-separated helm chart package names or glob patterns
|
||||
#
|
||||
# For each matching package we will extract the charts tarball and
|
||||
# save it among build artifacts, w/o replacing the image names.
|
||||
#
|
||||
# These patterns will be matched against the names in debian_helm.inc files
|
||||
#
|
||||
# Packages that also match HELM_CHART_PACKAGES will be automatically
|
||||
# excluded, even if they match this option.
|
||||
#
|
||||
# Example: EXTRA_HELM_CHART_PACKAGES="stx-*-helm" might create something
|
||||
# like:
|
||||
# stx-monitor-1.0-1.stx.1.tgz
|
||||
#
|
||||
EXTRA_HELM_CHART_PACKAGES=
|
||||
|
||||
# Directory containing additional image names & tags to be replaced in
|
||||
# helm charts
|
||||
EXTRA_IMAGE_RECORD_DIR=
|
||||
|
||||
##################################################
|
||||
# Jenkins & K8S parameters
|
||||
|
|
|
|||
Loading…
Reference in New Issue