starlingx/root
Code Issues Proposed changes
root/build-tools/build-helm-charts.sh
jbarboza b9e3fc0964 Fixed third-party packages bug 
With the introduction of the custom manifests
update [1], a new bug was found when third-party
packages were added to the project scope. This
new feature was ignoring all custom manifests
that were not part of the official build,
causing errors to happen when such packages
were used

[1] https://review.opendev.org/c/starlingx/root/+/938481

Test Plan:
- PASS: Build package with specific custom manifest
directory (/custom-manifests)
- PASS: Execute build-helm-charts.sh
- PASS: Compare diffs between official and merged manifests
- PASS: Apply generated tarball
- PASS: Execute any commands related to the third-party
package

Closes-Bug: 2095498

Change-Id: Id212e7dd63d7a6013f572a8c217fde6f1b663f9c
Signed-off-by: jbarboza <joaopedro.barbozalioneza@windriver.com>
(cherry picked from commit 3618d3046a5f1f2d897570754b465413b1920b5e)
2025-01-27 20:02:57 +00:00

951 lines
30 KiB
Bash
Executable File
Raw Permalink Blame History

#!/bin/bash
#
# Copyright (c) 2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# This utility retrieves StarlingX helm-charts
# from the build output and re-packages them
# in a single openstack-helm.tgz tarball
#
BUILD_HELM_CHARTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source $BUILD_HELM_CHARTS_DIR/utils.sh || exit 1
SUPPORTED_OS_ARGS=('debian')
OS=
LABEL=""
APP_NAME="stx-openstack"
APP_VERSION_BASE="helm-charts-release-info.inc"
APP_VERSION_FILE=""
APP_VERSION=""
declare -a IMAGE_RECORDS
declare -a PATCH_DEPENDENCIES
declare -a APP_PACKAGES
declare -a CHART_PACKAGE_FILES
declare -a IGNORE_CHARTS
# PYTHON_2_OR_3: initialized below
VERBOSE=false
CPIO_FLAGS=
TAR_FLAGS=
function usage {
cat >&2 <<EOF
Usage:
$(basename $0) [--os <os>] [-a, --app <app-name>]
[-A, --app-version-file /path/to/$APP_VERSION_BASE]
[-B, --app-version <version>]
[--package <package-name>] [-i, --image-record <image-record>] [--label <label>]
[--ignore <chart-name>]
[-p, --patch-dependency <patch-dependency>] [ --verbose ]
Options:
--os:
Specify base OS (valid options: ${SUPPORTED_OS_ARGS[@]})
-a, --app NAME:
Specify the application name.
When not specified will fall-back to its default value.
Default: stx-openstack
-A, --app-version-file FILENAME:
Specify the file containing version information for the helm
charts. By default we will search for a file named
$APP_VERSION_BASE in all git repos.
-B, --app-version VERSION:
Specify application (tarball) version, this overrides any other
version information.
--package PACKAGE_NAME,... :
Top-level package(s) containing the helm chart(s), comma-separated.
Default: ${APP_NAME}-helm
-i, --image-record FILENAME :
Specify the path to image record file(s) or url(s).
Multiple files/urls can be specified with a comma-separated
list, or with multiple --image-record arguments.
Note: Files are in order of priority. Images may appear
in multiple files, the last image reference has higher
priority.
-l, --label LABEL:
Specify the label of the application tarball. The label
will be appended to the version string in tarball name.
--ignore IGNORE_CHARTS,... :
Specify the Helm chart file name pattern(s) to be excluded
from the final application tarball.
-p, --patch-dependency DEPENDENCY,... :
Specify the patch dependency of the application tarball,
comma-separated
Multiple patches can be specified with a comma-separated
list, or with multiple --patch-dependency arguments.
--verbose:
Verbose output
--help:
Give this help list
EOF
}
function is_in {
local search=$1
shift
for v in $*; do
if [ "${search}" = "${v}" ]; then
return 0
fi
done
return 1
}
function get_image_record_file {
local image_record=$1
if [[ ${image_record} =~ ^https?://.*(.lst|.txt)$ ]]; then
wget --quiet --no-clobber ${image_record} \
--directory-prefix ${IMAGE_RECORD_PATH}
if [ $? -ne 0 ]; then
echo "Failed to download image record file from ${image_record}" >&2
exit 1
fi
elif [[ -f ${image_record} && ${image_record} =~ .lst|.txt ]]; then
cp ${image_record} ${IMAGE_RECORD_PATH}
if [ $? -ne 0 ]; then
echo "Failed to copy ${image_record} to ${IMAGE_RECORD_PATH}" >&2
exit 1
fi
else
echo "Cannot recognize the provided image record file:${image_record}" >&2
exit 1
fi
}
# Read the image versions from the passed image
# record files and build them into armada manifest
function build_image_versions_to_armada_manifest {
local manifest_file=$1
for image_record in ${IMAGE_RECORDS[@]}; do
get_image_record_file ${image_record}
# An image record file contains a list of images with the following format:
# <docker-registry>/<repository>/<repository>/.../<image-name>:<tag>
#
# An example of the content of an image record file:
# e.g. images-debian-stable-latest.lst
# docker.io/starlingx/stx-aodh:master-debian-stable-latest
# docker.io/starlingx/stx-ceilometer:master-debian-stable-latest
# docker.io/starlingx/stx-cinder:master-debian-stable-latest
# ...
#
# An example of the usage of an image reference in manifest file:
# e.g. manifest.yaml
# images:
# tags:
# aodh_api: docker.io/starlingx/stx-aodh:master-debian-stable-latest
# aodh_db_sync: docker.io/starlingx/stx-aodh:master-debian-stable-latest
# ...
#
# To replace the images in the manifest file with the images in image record file:
# For each image reference in the image record file,
# 1. extract image name
# e.g. image_name = stx-aodh
#
# 2. search the image reference in manifest yaml via image_name
# e.g. old_image_reference = docker.io/starlingx/stx-aodh:master-debian-stable-latest
#
# 3. update the manifest file to replace the old image references with the new one
# e.g. manifest.yaml
# images:
# tags:
# aodh_api: docker.io/starlingx/stx-aodh:master-debian-stable-latest
# aodh_db_sync: docker.io/starlingx/stx-aodh:master-debian-stable-latest
#
image_record=${IMAGE_RECORD_PATH}/$(basename ${image_record})
${PYTHON_2_OR_3} $BUILD_HELM_CHARTS_DIR/helm_chart_modify.py ${manifest_file} ${manifest_file}.tmp ${image_record}
if [ $? -ne 0 ]; then
echo "Failed to update manifest file" >&2
exit 1
fi
\mv -f ${manifest_file}.tmp ${manifest_file}
done
}
# Read the image versions from the passed image
# record files and build them into fluxcd manifests
function build_image_versions_to_fluxcd_manifests {
local manifest_folder=$1
for image_record in ${IMAGE_RECORDS[@]}; do
get_image_record_file ${image_record}
# An image record file contains a list of images with the following format:
# <docker-registry>/<repository>/<repository>/.../<image-name>:<tag>
#
# An example of the content of an image record file:
# e.g. images-debian-stable-latest.lst
# docker.io/starlingx/stx-aodh:master-debian-stable-latest
# docker.io/starlingx/stx-ceilometer:master-debian-stable-latest
# docker.io/starlingx/stx-cinder:master-debian-stable-latest
# ...
#
# An example of the usage of an image reference in manifest file:
# e.g. manifest.yaml
# images:
# tags:
# aodh_api: docker.io/starlingx/stx-aodh:master-debian-stable-latest
# aodh_db_sync: docker.io/starlingx/stx-aodh:master-debian-stable-latest
# ...
#
# To replace the images in the manifest file with the images in image record file:
# For each image reference in the image record file,
# 1. extract image name
# e.g. image_name = stx-aodh
#
# 2. search the image reference in manifest yaml via image_name
# e.g. old_image_reference = docker.io/starlingx/stx-aodh:master-debian-stable-latest
#
# 3. update the manifest file to replace the old image references with the new one
# e.g. manifest.yaml
# images:
# tags:
# aodh_api: docker.io/starlingx/stx-aodh:master-debian-stable-latest
# aodh_db_sync: docker.io/starlingx/stx-aodh:master-debian-stable-latest
#
image_record=${IMAGE_RECORD_PATH}/$(basename ${image_record})
find ${manifest_folder} -name "*.yaml" | while read manifest_file; do
${PYTHON_2_OR_3} $BUILD_HELM_CHARTS_DIR/helm_chart_modify.py ${manifest_file} ${manifest_file}.tmp ${image_record}
if [ $? -ne 0 ]; then
echo "Failed to update manifest file" >&2
exit 1
fi
\mv -f ${manifest_file}.tmp ${manifest_file}
done
done
}
function build_application_tarball {
if [ -n "$1" ] ; then
build_application_tarball_armada $1
else
build_application_tarball_fluxcd
fi
}
function build_application_tarball_armada {
local manifest=$1
manifest_file=$(basename ${manifest})
manifest_name=${manifest_file%.yaml}
deprecated_tarball_name="helm-charts-${manifest_name}"
build_image_versions_to_armada_manifest ${manifest}
cp ${manifest} staging/.
if [ $? -ne 0 ]; then
echo "Failed to copy the manifests to ${BUILD_OUTPUT_PATH}/staging" >&2
exit 1
fi
cd staging
# Add metadata file
touch metadata.yaml
if [ -n "${LABEL}" ]; then
APP_VERSION=${APP_VERSION}-${LABEL}
deprecated_tarball_name=${deprecated_tarball_name}-${LABEL}
fi
if ! grep -q "^app_name:" metadata.yaml ; then
echo "app_name: ${APP_NAME}" >> metadata.yaml
fi
echo "app_version: ${APP_VERSION}" >> metadata.yaml
if [ -n "${PATCH_DEPENDENCIES}" ]; then
echo "patch_dependencies:" >> metadata.yaml
for patch in ${PATCH_DEPENDENCIES[@]}; do
echo " - ${patch}" >> metadata.yaml
done
fi
# Add the tarball build date: For consistency with tooling that might use
# this metadata, match the date format used for BUILD_DATE in
# /etc/build.info
echo "build_date: $(date '+%Y-%m-%d %H:%M:%S %z')" >> metadata.yaml
# Add an md5
find . -type f ! -name '*.md5' -print0 | xargs -0 md5sum > checksum.md5
cd ..
tarball_name="${APP_NAME}-${APP_VERSION}.tgz"
tar ${TAR_FLAGS} ${tarball_name} -C staging/ .
if [ $? -ne 0 ]; then
echo "Failed to create the tarball" >&2
exit 1
fi
rm staging/${manifest_file}
rm staging/checksum.md5
echo " ${BUILD_OUTPUT_PATH}/${tarball_name}"
# Create a symbolic link to point to the generated tarball
# TODO: Remove the symboblic link once the community has an
# opportunity to adapt to the changes in filenames
if [ "${APP_NAME}" = "stx-openstack" ]; then
ln -s ${tarball_name} ${BUILD_OUTPUT_PATH}/${deprecated_tarball_name}.tgz
echo " ${BUILD_OUTPUT_PATH}/${deprecated_tarball_name}.tgz"
echo "Warning: The tarball ${deprecated_tarball_name}.tgz is a symbolic link for ${tarball_name}. It's deprecated and will be removed shortly."
fi
}
function compare_custom_manifests {
MANIFEST_LIST=$1
for manifest in ${MANIFEST_LIST}; do
if [ ! -s usr/lib/fluxcd/${manifest} ]; then
cp -R usr/lib/fluxcd/custom-manifests/${manifest} usr/lib/fluxcd/${manifest}
continue
fi
if [ -f usr/lib/fluxcd/custom-manifests/${manifest} ]; then
${PYTHON_2_OR_3} $BUILD_HELM_CHARTS_DIR/merge_manifests.py usr/lib/fluxcd/${manifest} usr/lib/fluxcd/custom-manifests/${manifest}
continue
fi
MAIN_MANIFESTS=$(ls usr/lib/fluxcd/${manifest})
CUSTOM_MANIFESTS=$(ls usr/lib/fluxcd/custom-manifests/${manifest})
for manifest_file in ${MAIN_MANIFESTS}; do
${PYTHON_2_OR_3} \
$BUILD_HELM_CHARTS_DIR/merge_manifests.py \
usr/lib/fluxcd/${manifest}/${manifest_file} \
usr/lib/fluxcd/custom-manifests/${manifest}/${manifest_file}
done
done
}
function build_application_tarball_fluxcd {
FLUXCD_MANIFEST_DIR='fluxcd-manifests'
CUSTOM_MANIFEST_LIST=$(ls usr/lib/fluxcd/custom-manifests)
if [ ${#CUSTOM_MANIFEST_LIST} -ne 0 ]; then
echo "Custom manifests detected. Merging custom manifests with FluxCD manifests"
compare_custom_manifests "${CUSTOM_MANIFEST_LIST}"
# Removing custom manifests directory to prevent unnecessary files from being included
rm -rd usr/lib/fluxcd/custom-manifests
fi
# Stage all the fluxcd manifests
cp -R usr/lib/fluxcd staging/${FLUXCD_MANIFEST_DIR}
if [ $? -ne 0 ]; then
echo "Failed to copy the FluxCD manifests from ${BUILD_OUTPUT_PATH}/usr/lib/fluxcd to ${BUILD_OUTPUT_PATH}/staging/fluxcd_manifests" >&2
exit 1
fi
cd staging
build_image_versions_to_fluxcd_manifests ${FLUXCD_MANIFEST_DIR}
# Add metadata file
touch metadata.yaml
if [ -n "${LABEL}" ]; then
APP_VERSION=${APP_VERSION}-${LABEL}
fi
if [ -z "${APP_NAME}" ] ; then
echo "No APP_NAME argument. Using metadata.yaml info instead..."
else
echo "Overriding metadata.yaml info... app_name: ${APP_NAME}"
sed -i "/^\([[:space:]]*app_name: \).*/s//\1${APP_NAME}/" metadata.yaml
fi
if [ -z "${APP_VERSION}" ] ; then
echo "No APP_VERSION argument. Using metadata.yaml info instead..."
else
echo "Overriding metadata.yaml info... app_version: ${APP_VERSION}"
sed -i "/^\([[:space:]]*app_version: \).*/s//\1${APP_VERSION}/" metadata.yaml
fi
if [ -n "${PATCH_DEPENDENCIES}" ]; then
echo "patch_dependencies:" >> metadata.yaml
for patch in ${PATCH_DEPENDENCIES[@]}; do
echo " - ${patch}" >> metadata.yaml
done
fi
# Add the tarball build date: For consistency with tooling that might use
# this metadata, match the date format used for BUILD_DATE in
# /etc/build.info
echo "build_date: $(date '+%Y-%m-%d %H:%M:%S %z')" >> metadata.yaml
# Add an md5
find . -type f ! -name '*.md5' -print0 | xargs -0 md5sum > checksum.md5
cd ..
tarball_name="${APP_NAME}-${APP_VERSION}.tgz"
tar ${TAR_FLAGS} ${tarball_name} -C staging/ .
if [ $? -ne 0 ]; then
echo "Failed to create the tarball" >&2
exit 1
fi
rm -fr staging/${FLUXCD_MANIFEST_DIR}
rm staging/checksum.md5
echo " ${BUILD_OUTPUT_PATH}/${tarball_name}"
}
function parse_yaml {
# Create a new yaml file based on sequentially merging a list of given yaml files
local yaml_script="
import sys
import collections
import ruamel.yaml as yaml
yaml_files = sys.argv[2:]
yaml_output = sys.argv[1]
def merge_yaml(yaml_merged, yaml_new):
for k in yaml_new.keys():
if not isinstance(yaml_new[k], dict):
yaml_merged[k] = yaml_new[k]
elif k not in yaml_merged:
yaml_merged[k] = yaml_new[k]
else:
merge_yaml(yaml_merged[k], yaml_new[k])
yaml_out = collections.OrderedDict()
for yaml_file in yaml_files:
print('Merging yaml from file: %s' % yaml_file)
for document in yaml.load_all(open(yaml_file), Loader=yaml.RoundTripLoader, preserve_quotes=True, version=(1, 1)):
document_name = (document['schema'], document['metadata']['schema'], document['metadata']['name'])
if document_name in yaml_out:
merge_yaml(yaml_out[document_name], document)
else:
yaml_out[document_name] = document
print('Writing merged yaml file: %s' % yaml_output)
yaml.dump_all(yaml_out.values(), open(yaml_output, 'w'), Dumper=yaml.RoundTripDumper, default_flow_style=False)
"
$PYTHON_2_OR_3 -c "${yaml_script}" ${@} || exit 1
}
# Find a file named $APP_VERSION_BASE at top-level of each git repo
function find_app_version_file {
echo "searching for $APP_VERSION_BASE" >&2
local dir file version_file root_dir
root_dir="$(cd "$MY_REPO"/.. && pwd)"
for dir in $(cd "$root_dir" && repo forall -c 'echo $REPO_PATH') ; do
file="$root_dir/$dir/$APP_VERSION_BASE"
[[ -f "$file" ]] || continue
if [[ -n "$version_file" ]] ; then
echo "Multiple $APP_VERSION_BASE files found:" >&2
echo " $version_file" >&2
echo " $file" >&2
return 1
fi
version_file="$file"
done
if [[ -z "$version_file" && -f "$MY_REPO/stx/utilities/utilities/build-info/$APP_VERSION_BASE" ]] ; then
version_file="$MY_REPO/stx/utilities/utilities/build-info/$APP_VERSION_BASE"
fi
if [[ -n "$version_file" ]] ; then
echo "$version_file"
fi
return 0
}
filter_existing_dirs() {
local d
for d in "$@" ; do
if [[ -d "$d" ]] ; then
echo "$d"
fi
done
}
#
# Usage:
# find_package_files
#
# Print noarch package files that might contain helm charts
#
function find_package_files {
local -a dirlist
local dir
if [[ "$OS" == "debian" ]] ; then
# FIXME: can't search 3rd-party binary debs because they are not accessible
# on the filesystem, but only as remote files in apt repos
readarray -t dirlist < <(filter_existing_dirs "${MY_WORKSPACE}/std")
if [[ "${#dirlist[@]}" -gt 0 ]] ; then
echo "looking for packages in ${dirlist[*]}" >&2
find "${dirlist[@]}" \
-mindepth 2 \
-maxdepth 2 \
"(" \
"(" \
-path "${MY_WORKSPACE}/build-wheels" \
-o -path "${MY_WORKSPACE}/build-images" \
-o -path "${MY_WORKSPACE}/build-helm" \
")" -prune \
")" \
-o \
"(" -xtype f -name "*.stx.*_all.deb" ")"
fi
fi
}
# Usage:
# find_helm_chart_packages PACKAGE_NAMES...
#
# Find helm chart packages and print their "NAME FILENAME" one per line
#
function find_helm_chart_package_files {
# hash: package files => package names
local -A package_files
# hash: package names => package files
local -A package_names
# load package files and names
echo "searching for package files" >&2
local package_file package_name
local failed=0
for package_file in $(find_package_files) ; do
package_name="$(
if [[ "$OS" == "debian" ]] ; then
deb_get_control "$package_file" | deb_get_field "Package"
check_pipe_status
fi
)" || exit 1
if [[ -n "${package_names[$package_name]}" && "${package_names[$package_name]}" != "$package_file" ]] ; then
echo "ERROR: found multiple packages named ${package_name}:" >&2
echo " $package_file" >&2
echo " ${package_names[$package_name]}" >&2
failed=1
continue
fi
package_names["$package_name"]="$package_file"
package_files["$package_file"]="$package_name"
done
[[ $failed -eq 0 ]] || exit 1
echo "looking for chart packages" >&2
# Make sure top-level chart packages requested by user exist
local failed=0
for package_name in "$@" ; do
if [[ -z "${package_names[$package_name]}" ]] ; then
echo "ERROR: required package ${package_name} not found" >&2
failed=1
fi
done
[[ $failed -eq 0 ]] || exit 1
# all chart package files
local -A chart_package_files
local -a ordered_chart_package_files
# Find immediate dependencies of each package as well
failed=0
for package_name in "$@" ; do
package_file="${package_names[$package_name]}"
# seen this file before, skip
if [[ -n "${chart_package_files[$package_file]}" ]] ; then
continue
fi
local -a dep_package_names=($(
if [[ "$OS" == "debian" ]] ; then
deb_get_control "$package_file" | deb_get_simple_depends
check_pipe_status || exit 1
fi
)) || exit 1
# save top-level package
chart_package_files["$package_file"]=1
ordered_chart_package_files+=("$package_file")
# make sure all dep_packages exist & save them as well
local dep_package_name dep_package_file
for dep_package_name in "${dep_package_names[@]}" ; do
dep_package_file="${package_names[$dep_package_name]}"
if [[ -z "$dep_package_file" ]] ; then
echo "ERROR: package ${package_file} requires package ${dep_package_name}, which does not exist" >&2
failed=1
continue
fi
# save dep_package_file, unless we've seen it before
if [[ -z "${chart_package_files[$dep_package_file]}" ]] ; then
chart_package_files["$dep_package_file"]=1
ordered_chart_package_files+=("$dep_package_file")
fi
done
done
[[ $failed -eq 0 ]] || exit 1
# make sure there's at least one
if [[ "${#chart_package_files[@]}" -eq 0 ]] ; then
echo "ERROR: could not find any chart packages" >&2
exit 1
fi
# print them
echo "found chart packages:" >&2
for package_file in "${ordered_chart_package_files[@]}" ; do
echo " $package_file" >&2
echo "$package_file"
done
}
#
# Usage:
# extract_chart_from_package PACKAGE_FILE
#
function extract_chart_from_package {
local package_file=$1
echo "extracting charts from package $package_file" >&2
case $OS in
debian)
deb_extract_content "$package_file" $([[ "$VERBOSE" == "true" ]] && echo --verbose || true)
if ! check_pipe_status ; then
echo "Failed to extract content of helm package: ${package_file}" >&2
exit 1
fi
;;
*)
echo "Unsupported OS ${OS}" >&2
;;
esac
}
# Usage: extract_charts CHART_PACKAGE_FILE...
function extract_charts {
local package_file
for package_file in "$@" ; do
extract_chart_from_package "$package_file"
done
}
#
# Usage:
# get_app_version CHART_PACKAGE_FILE...
#
# Print the app (tarball) version, based on command-line
# arguments, the version .inc file or the chart package files
#
function get_app_version {
# version provided on command line: use it
if [[ -n "$APP_VERSION" ]] ; then
echo "APP_VERSION=$APP_VERSION" >&2
echo "$APP_VERSION"
return 0
fi
# find app version file
local app_version_file="$APP_VERSION_FILE"
if [[ -z "$app_version_file" ]] ; then
app_version_file="$(find_app_version_file)" || exit 1
fi
if [[ -n "$app_version_file" ]] ; then
echo "reading $app_version_file" >&2
local app_version
app_version="$(
VERSION= RELEASE=
source "$app_version_file" || exit 1
if [[ -z "$VERSION" ]] ; then
echo "$app_version_file: missing VERSION" >&2
exit 1
fi
echo "${VERSION}-${RELEASE:-0}"
)" || exit 1
echo "APP_VERSION=$app_version" >&2
echo "$app_version"
return 0
fi
# this should never happen because we exit early if there are no chart
# packages
if [[ "$#" -eq 0 ]] ; then
echo "ERROR: unable to determine APP_VERSION" >&2
exit 1
fi
# app version file doesn't exist: use the version of
# the 1st chart package
echo "extracting version from $1" >&2
local app_version
app_version="$(
if [[ "$OS" == "debian" ]] ; then
control="$(deb_get_control "$1")" || exit 1
version="$(echo "$control" | deb_get_field "Version" | sed -r -e 's/^[^:]+:+//')"
if [[ -z "$version" ]] ; then
echo "ERROR: failed to determine the version of package $1" >&2
exit 1
fi
echo "${version}"
fi
)" || exit 1
echo "APP_VERSION=$app_version" >&2
echo "$app_version"
}
# TODO(awang): remove the deprecated image-file option
OPTS=$(getopt -o h,a:,A:,B:,i:,l:,p: -l help,os:,app:,app-version-file:,app-version:,package:,image-record:,image-file:,label:,ignore:,patch-dependency:,verbose -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
fi
eval set -- "${OPTS}"
while true; do
case $1 in
--)
# End of getopt arguments
shift
break
;;
--os)
OS=$2
shift 2
;;
-a | --app)
APP_NAME=$2
shift 2
;;
-A | --app-version-file)
APP_VERSION_FILE="$2"
shift 2
;;
-B | --app-version)
APP_VERSION="$2"
shift 2
;;
--package)
APP_PACKAGES+=(${2//,/ })
shift 2
;;
-i | --image-record | --image-file)
# Read comma-separated values into array
IMAGE_RECORDS+=(${2//,/ })
shift 2
;;
-l | --label)
LABEL=$2
shift 2
;;
--ignore)
# Read comma-separated values into array
IGNORE_CHARTS+=(${2//,/ })
shift 2
;;
-p | --patch-dependency)
# Read comma-separated values into array
PATCH_DEPENDENCIES+=(${2//,/ })
shift 2
;;
--verbose)
VERBOSE=true
shift
;;
-h | --help )
usage
exit 1
;;
*)
usage
exit 1
;;
esac
done
if [ "$VERBOSE" = true ] ; then
CPIO_FLAGS=-vidu
TAR_FLAGS=-zcvf
else
CPIO_FLAGS="-idu --quiet"
TAR_FLAGS=-zcf
fi
# Validate OS
if [ -z "$OS" ] ; then
OS="$(ID= && source /etc/os-release 2>/dev/null && echo $ID || true)"
if [[ -z "$OS" ]] ; then
echo "Unable to determine OS, please re-run with \`--os' option" >&2
exit 1
fi
fi
VALID_OS=1
for supported_os in ${SUPPORTED_OS_ARGS[@]}; do
if [ "$OS" = "${supported_os}" ]; then
VALID_OS=0
break
fi
done
if [ ${VALID_OS} -ne 0 ]; then
echo "Unsupported OS specified: ${OS}" >&2
echo "Supported OS options: ${SUPPORTED_OS_ARGS[@]}" >&2
exit 1
fi
# Required env vars
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
echo "Environment not setup for builds" >&2
exit 1
fi
# find a python interpreter
function find_python_2_or_3 {
local python python_found
for python in ${PYTHON2:-python2} ${PYTHON:-python} ${PYTHON3:-python3} ; do
if $python -c 'import ruamel.yaml' >/dev/null 2>&1 ; then
python_found=true
break
fi
done
if [[ -z "$python_found" ]] ; then
echo "ERROR: can't find python!" >&2
exit 1
fi
echo "$python"
}
PYTHON_2_OR_3="$(find_python_2_or_3)" || exit 1
# include packaging utils
if [[ "$OS" == "debian" ]] ; then
source $BUILD_HELM_CHARTS_DIR/deb-utils.sh || exit 1
fi
# Commenting out this code that attempts to validate the APP_NAME.
# It makes too many assumptions about the location and naming of apps.
#
# # Validate application
# APP_REPO=${MY_REPO}/stx/stx-config/kubernetes/applications/
# if [ ! -d ${APP_REPO} ];then
# echo "Unable to find the applications directory: ${APP_REPO}" >&2
# exit 1
# fi
# AVAILABLE_APPS=($(ls ${APP_REPO}))
# if [ ${#AVAILABLE_APPS[@]} -eq 0 ]; then
# echo "No application found" >&2
# exit 1
# fi
# if ! is_in ${APP_NAME} ${AVAILABLE_APPS[@]}; then
# echo "Invalid application: ${APP_NAME}" >&2
# exit 1
# fi
# Cleanup the previous chart build workspace
BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-helm/stx
echo "BUILD_OUTPUT_PATH=$BUILD_OUTPUT_PATH" >&2
if [ -d ${BUILD_OUTPUT_PATH} ]; then
# Wipe out the existing dir to ensure there are no stale files
rm -rf ${BUILD_OUTPUT_PATH}
if [ $? -ne 0 ]; then
echo "Failed to cleanup the workspace ${BUILD_OUTPUT_PATH}" >&2
exit 1
fi
fi
mkdir -p ${BUILD_OUTPUT_PATH}
if [ $? -ne 0 ]; then
echo "Failed to create the workspace ${BUILD_OUTPUT_PATH}" >&2
exit 1
fi
cd ${BUILD_OUTPUT_PATH}
# Create a directory to store image record files
IMAGE_RECORD_PATH=${BUILD_OUTPUT_PATH}/image_record
if [ ${#IMAGE_RECORDS[@]} -ne 0 ]; then
mkdir -p ${IMAGE_RECORD_PATH}
if [ $? -ne 0 ]; then
echo "Failed to create the ${IMAGE_RECORD_PATH}" >&2
exit 1
fi
fi
# Find chart packages
CHART_PACKAGE_FILES=($(
[[ "${#APP_PACKAGES[@]}" -gt 0 ]] || APP_PACKAGES+=("${APP_NAME}-helm")
find_helm_chart_package_files "${APP_PACKAGES[@]}"
)) || exit 1
# Initialize APP_VERSION
APP_VERSION="$(get_app_version "${CHART_PACKAGE_FILES[@]}")" || exit 1
# Extract chart files from packages
extract_charts "${CHART_PACKAGE_FILES[@]}" || exit 1
# Create a new tarball containing all the contents we extracted
# tgz files under helm are relocated to subdir charts.
# Files under armada are left at the top level
mkdir -p staging
if [ $? -ne 0 ]; then
echo "Failed to create ${BUILD_OUTPUT_PATH}/staging" >&2
exit 1
fi
if [ ! -d "usr/lib/fluxcd" ] || [ ! -d "usr/lib/helm" ]; then
# Armada Check: Remove with last supported Armada application
if [ ! -d "usr/lib/armada" ] || [ ! -d "usr/lib/helm" ]; then
echo "Failed to create the tarball. Mandatory files are missing." >&2
exit 1
fi
fi
# Transform array into the format expected by rsync
declare -a IGNORE_CHARTS_ARGS
if [ ${#IGNORE_CHARTS[@]} -ne 0 ]; then
for chart in "${IGNORE_CHARTS[@]}"; do
IGNORE_CHARTS_ARGS+=("--exclude=${chart}")
done
fi
# Stage the charts
rsync -a usr/lib/helm/ staging/charts/ "${IGNORE_CHARTS_ARGS[@]}"
if [ $? -ne 0 ]; then
echo "Failed to copy the charts from ${BUILD_OUTPUT_PATH}/usr/lib/helm to ${BUILD_OUTPUT_PATH}/staging/charts" >&2
exit 1
fi
# Stage all the plugin wheels, if present
if [ -d "plugins" ]; then
cp -R plugins staging/plugins
if [ $? -ne 0 ]; then
echo "Failed to copy the wheels from ${BUILD_OUTPUT_PATH}/wheels to ${BUILD_OUTPUT_PATH}/staging/plugins" >&2
exit 1
fi
fi
# Stage metadata file, if present
if [ -e usr/lib/application/metadata.yaml ]; then
cp usr/lib/application/metadata.yaml staging/.
fi
if [ ! -d "usr/lib/fluxcd" ] ; then
# Merge yaml files:
APP_YAML=${APP_NAME}.yaml
parse_yaml $APP_YAML `ls -rt usr/lib/armada/*.yaml`
echo "Results:"
# Build tarballs for merged yaml
build_application_tarball $APP_YAML
else
echo
echo "WARNING: Merging yaml manifests is currently not supported for FluxCD applications" >&2
echo
echo "Results:"
build_application_tarball
fi
export EXTRA_HELM_CHARTS="build-extra-helm-charts.sh"
echo "Running build extra helm charts script"
"${MY_REPO}/build-tools/$EXTRA_HELM_CHARTS" || exit 1
exit 0
View Git Blame Copy Permalink