From 0f3670fe64f298a2f94a86858ba7611bf923e550 Mon Sep 17 00:00:00 2001 From: Davlet Panech Date: Wed, 4 May 2022 13:57:52 -0400 Subject: [PATCH] debian: port helm chart script to debian * build-helm-charts.sh: - auto-detect $OS - process DEB files on Debian * deb-utils: new file with utilities for working with binary DEB files * tox.ini: run unit tests for deb-utils TESTS ======================================== Run script on CentOS and make sure the generated tarball's contents are the same as before the patch. Run script on Debian and make sure the generated tarball's contents look reasonable. Story: 2009897 Task: 45293 Depends-On: https://review.opendev.org/c/starlingx/openstack-armada-app/+/840561 Change-Id: Icbcb0bb7b47f623fac8d0851687423396edb5747 Signed-off-by: Davlet Panech --- build-tools/build-helm-charts.sh | 142 +++++++++++++++++----- build-tools/deb-utils.sh | 100 ++++++++++++++++ build-tools/deb-utils/deb_get_field.py | 56 +++++++++ build-tools/unit-tests/deb-utils.sh | 159 +++++++++++++++++++++++++ tox.ini | 13 +- 5 files changed, 439 insertions(+), 31 deletions(-) create mode 100644 build-tools/deb-utils.sh create mode 100755 build-tools/deb-utils/deb_get_field.py create mode 100755 build-tools/unit-tests/deb-utils.sh diff --git a/build-tools/build-helm-charts.sh b/build-tools/build-helm-charts.sh index 09f3a6fa..520bdd43 100755 --- a/build-tools/build-helm-charts.sh +++ b/build-tools/build-helm-charts.sh @@ -10,17 +10,10 @@ # BUILD_HELM_CHARTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source $BUILD_HELM_CHARTS_DIR/srpm-utils || exit 1 source $BUILD_HELM_CHARTS_DIR/utils.sh || exit 1 -# Required env vars -if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then - echo "Environment not setup for builds" >&2 - exit 1 -fi - -SUPPORTED_OS_ARGS=('centos') -OS=centos +SUPPORTED_OS_ARGS=('centos' 'debian') +OS= LABEL="" APP_NAME="stx-openstack" APP_VERSION_BASE="helm-charts-release-info.inc" @@ -31,6 +24,10 @@ declare -a PATCH_DEPENDENCIES declare -a APP_PACKAGES declare -a CHART_PACKAGE_FILES +VERBOSE=false +CPIO_FLAGS= +TAR_FLAGS= + function usage { cat >&2 </dev/null 2>&1 ; then + python_found=true + break + fi + done + if [[ -z "$python_found" ]] ; then + echo "ERROR: can't find python!" >&2 + exit 1 + fi + + $python -c "${yaml_script}" ${@} || exit 1 } # Find a file named $APP_VERSION_BASE at top-level of each git repo @@ -364,8 +378,20 @@ function find_package_files { "${centos_repo}/Binary/noarch" \ -type f -name "*.tis.noarch.rpm" else - echo "ERROR: unsupported OS $OS" >&2 - exit 1 + # FIXME: can't search 3rd-party binary debs because they are not accessible + # on the filesystem, but only as remote files in apt repos + find "${MY_WORKSPACE}/std" \ + -mindepth 2 \ + -maxdepth 2 \ + "(" \ + "(" \ + -path "${MY_WORKSPACE}/build-wheels" \ + -o -path "${MY_WORKSPACE}/build-images" \ + -o -path "${MY_WORKSPACE}/build-helm" \ + ")" -prune \ + ")" \ + -o \ + "(" -type f -name "*.stx.*_all.deb" ")" fi } @@ -385,7 +411,14 @@ function find_helm_chart_package_files { echo "searching for package files" >&2 local package_file package_name for package_file in $(find_package_files) ; do - package_name=$(rpm_get_name "$package_file") || exit 1 + package_name="$( + if [[ "$OS" == "centos" ]] ; then + rpm_get_name "$package_file" || exit 1 + else + deb_get_control "$package_file" | deb_get_field "Package" + check_pipe_status + fi + )" || exit 1 if [[ -n "${package_names[$package_name]}" && "${package_names[$package_name]}" != "$package_file" ]] ; then echo "ERROR: found multiple packages named ${package_name}:" >&2 echo " $package_file" >&2 @@ -423,8 +456,13 @@ function find_helm_chart_package_files { fi local -a dep_package_names=($( - rpm -qRp "$package_file" | sed 's/rpmlib([a-zA-Z0-9]*)[[:space:]]\?[><=!]\{0,2\}[[:space:]]\?[0-9.-]*//g' | grep -v '/' - check_pipe_status || exit 1 + if [[ "$OS" == "centos" ]] ; then + rpm -qRp "$package_file" | sed 's/rpmlib([a-zA-Z0-9]*)[[:space:]]\?[><=!]\{0,2\}[[:space:]]\?[0-9.-]*//g' | grep -v '/' + check_pipe_status || exit 1 + else + deb_get_control "$package_file" | deb_get_simple_depends + check_pipe_status || exit 1 + fi )) || exit 1 # save top-level package @@ -477,8 +515,16 @@ function extract_chart_from_package { echo "Failed to extract content of helm package: ${package_file}" >&2 exit 1 fi - ;; + + debian) + deb_extract_content "$package_file" $([[ "$VERBOSE" == "true" ]] && echo --verbose || true) + if ! check_pipe_status ; then + echo "Failed to extract content of helm package: ${package_file}" >&2 + exit 1 + fi + ;; + *) echo "Unsupported OS ${OS}" >&2 ;; @@ -543,8 +589,18 @@ function get_app_version { echo "extracting version from $1" >&2 local app_version app_version="$( - rpm -q --qf '%{VERSION}-%{RELEASE}' -p "$1" | sed 's![.]tis!!g' - check_pipe_status + if [[ "$OS" == "centos" ]] ; then + rpm -q --qf '%{VERSION}-%{RELEASE}' -p "$1" | sed 's![.]tis!!g' + check_pipe_status || exit 1 + else + control="$(deb_get_control "$1")" || exit 1 + version="$(echo "$control" | deb_get_field "Version" | sed -r -e 's/^[^:]+:+//')" + if [[ -z "$version" ]] ; then + echo "ERROR: failed to determine the version of package $1" >&2 + exit 1 + fi + echo "${version}" + fi )" || exit 1 echo "APP_VERSION=$app_version" >&2 echo "$app_version" @@ -582,7 +638,10 @@ while true; do APP_VERSION="$2" shift 2 ;; - -r | --rpm) + -r | --rpm | --package) + if [[ "$1" == "--rpm" ]] ; then + echo "WARNING: option $1 is deprecated, use --package instead" >&2 + fi APP_PACKAGES+=(${2//,/ }) shift 2 ;; @@ -623,6 +682,16 @@ else TAR_FLAGS=-zcf fi +# Validate OS +if [ -z "$OS" ] ; then + OS="$(ID= && source /etc/os-release 2>/dev/null && echo $ID || true)" + if [[ -z "$OS" ]] ; then + echo "Unable to determine OS, please re-run with \`--os' option" >&2 + exit 1 + elif [[ "$OS" != "debian" ]] ; then + OS="centos" + fi +fi VALID_OS=1 for supported_os in ${SUPPORTED_OS_ARGS[@]}; do if [ "$OS" = "${supported_os}" ]; then @@ -636,6 +705,19 @@ if [ ${VALID_OS} -ne 0 ]; then exit 1 fi +# Required env vars +if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then + echo "Environment not setup for builds" >&2 + exit 1 +fi + +# include SRPM utils +if [[ "$OS" == "centos" ]] ; then + source $BUILD_HELM_CHARTS_DIR/srpm-utils || exit 1 +else + source $BUILD_HELM_CHARTS_DIR/deb-utils.sh || exit 1 +fi + # Commenting out this code that attempts to validate the APP_NAME. # It makes too many assumptions about the location and naming of apps. # diff --git a/build-tools/deb-utils.sh b/build-tools/deb-utils.sh new file mode 100644 index 00000000..a8687b10 --- /dev/null +++ b/build-tools/deb-utils.sh @@ -0,0 +1,100 @@ +# bash +# vim: set syn=sh: + +__DEB_UTILS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")/deb-utils + +# +# Usage: __deb_get_section DEB_FILE {control|data} +# +# Uncompress and print the specified section to STDOUT in tar format. +# You should pipe it to "tar" to be useful. +# +function __deb_get_section { + local deb_file="$1" + local section="$2" + + # find $section.tar.{gz,bz2,xz} + local section_entry + section_entry="$( + ar t "$deb_file" | \grep "^$section[.]" || true + )" || return 1 + if [[ -z "$section_entry" ]] ; then + echo "$deb_file: couldn't find ${section}.*" >&2 + return 1 + fi + + # untar it to stdout + local uncompress + case "${section_entry#${section}.}" in + tar.gz | tgz) uncompress="gunzip" ;; + tar.bz2) uncompress="bunzip2" ;; + tar.xz) uncompress="unxz" ;; + *) + echo "$deb_file: unsupported archive format $section_entry" >&2 + return 1 + esac + ar p "$1" "$section_entry" | $uncompress + check_pipe_status +} + +# +# Usage: deb_get_control DEB_FILE +# +# Print the control file from the specified DEB package +# +function deb_get_control { + __deb_get_section "$1" control | tar -O -x ./control + check_pipe_status +} + +# +# Usage: deb_extract_content DEB_FILE [--verbose] [PATHS_IN_ARCHIVE...] +# +# Extract deb package content to current directory +# +function deb_extract_content { + __deb_get_section "$1" data | tar -x + check_pipe_status +} + +# +# Usage: deb_get_field KEY... +# +# Read a debian control file from STDIN, find the specified fields +# and print their values on STDOUT. With multiple fields, their values +# will be merged in the output w/no separators. +# +# See: https://www.debian.org/doc/debian-policy/ch-controlfields.html +# +function deb_get_field { + ${PYTHON3:-python3} "${__DEB_UTILS_DIR}/deb_get_field.py" "$@" +} + +# +# Usage: deb_get_simple_depends +# +# Read debian control file from STDIN, then print its immediate runtime +# dependencies to STDOUT, one per line, stripping any conditions and +# operators, e.g.: +# +# ... +# Depends: aaa, bbb [!amd64], ccc | ddd (>= 1.0) +# ... +# +# will be converted to +# +# aaa +# bbb +# ccc +# ddd +# +function deb_get_simple_depends { + local raw_depends + raw_depends=$(deb_get_field 'Pre-Depends' 'Depends') || return 1 + echo $raw_depends \ + | tr ',|' '\n' \ + | sed -r 's/^\s*([^[:space:](><=[]+).*$/\1/' \ + | grep -v -E '^\s*$' \ + | sort -u +} + diff --git a/build-tools/deb-utils/deb_get_field.py b/build-tools/deb-utils/deb_get_field.py new file mode 100755 index 00000000..230510ed --- /dev/null +++ b/build-tools/deb-utils/deb_get_field.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +import sys, re + +def usage(): + print ("""\ +Usage: %s KEY... +Read a debian control file from STDIN, print KEY values to STDOUT +""" % sys.argv[0]) + +if len (sys.argv) > 0 and sys.argv[1] == "--help": + usage() + sys.exit(0) + +# regex: "^(?:KEY1|KEY2|...)\s*:\s*(.*?)\s*$" +re_field = re.compile ( + "^(?:" + + "|".join ( + [ re.escape (key) for key in sys.argv[1:] ] + ) + + "):\s*(.*?)\s*$" +) + +re_ws = re.compile ("^\s*$") + +in_header = True +past_1st_paragraph = False +in_multiline_field = False + +for line in sys.stdin: + + # skip initial empty lines + if in_header and re_ws.fullmatch (line): + continue + in_header = False + + # skip everything past the 1st block + if past_1st_paragraph: + continue + if re_ws.fullmatch (line): + past_1st_paragraph = True + continue + + # Key: value + match = re_field.fullmatch (line) + if match: + print (match.group(1)) + in_multiline_field = True + continue + + # line starts with a space or tab: belongs to the previous field + if in_multiline_field and (line.startswith (" ") or line.startswith ("\t")): + print (line[1:].rstrip()) + continue + + in_multiline_field = False diff --git a/build-tools/unit-tests/deb-utils.sh b/build-tools/unit-tests/deb-utils.sh new file mode 100755 index 00000000..6be5c2ab --- /dev/null +++ b/build-tools/unit-tests/deb-utils.sh @@ -0,0 +1,159 @@ +#!/bin/bash + +PROGNAME="$(basename "$0")" + +REQUIRED_PROGS="${PYTHON3:-python3}" + +for prog in ${REQUIRED_PROGS} tar ar ; do + if ! $prog --version >/dev/null 2>&1 ; then + echo "$PROGNAME: WARNING: can't find \"$prog\", skipping tests" >&2 + exit 0 + fi +done + +source "$(dirname "$0")"/../deb-utils.sh || exit 1 +source "$(dirname "$0")"/../utils.sh || exit 1 + +declare -i FAIL_COUNT=0 + +# Usage: expect EXPECTED ACTUAL [DEPTH] +function expect { + local expected="$1" + local actual="$2" + if [[ "${actual}" != "${expected}" ]] ; then + let depth="${3:-0}" + echo >&2 + echo "${BASH_SOURCE[0]}:${BASH_LINENO[${depth}]}: expectation failed:" >&2 + echo " actual: [$actual]" >&2 + echo " expected: [$expected]" >&2 + echo >&2 + return 1 + fi + return 0 +} + +# Usage: echo ACTUAL | expect_stdin EXPECTED +function expect_stdin { + expect "$1" "$(cat)" 1 +} + +######################################################### +# deb_get_field +######################################################### + +##################### +echo "\ +Dummy1: dummy1 +Key1: value1 +Dummy2: dummy2 +" | deb_get_field "Key1" \ + | expect_stdin "value1" \ +|| let ++FAIL_COUNT + +##################### +echo " + +# 1st para +Dummy1: dummy1 +Key1: value1 +Dummy2: dummy2 + +# 2nd para +Dummy3: dummy3 +Key1: value1 +Dummy4: dummy4 + +" | deb_get_field "Key1" \ + | expect_stdin "value1" \ +|| let ++FAIL_COUNT + +##################### +echo " + +# 1st para +Dummy1: dummy1 +Dummy2: dummy2 + +# 2nd para +Dummy3: dummy3 +Key1: value1 +Dummy4: dummy4 + +" | deb_get_field "Key1" \ + | expect_stdin "" \ +|| let ++FAIL_COUNT + +##################### +echo " + +Dummy1: dummy1 +Key1: value1_line1 + value1_line2 + value1_line3 +Dummy2: dummy2_line1 + dummy2_line2 + dummy2_line3 + +" | deb_get_field "Key1" \ + | expect_stdin $'value1_line1\nvalue1_line2\nvalue1_line3' \ +|| let ++FAIL_COUNT + +##################### +echo " + +Dummy1: dummy1 +Key1: value1_line1 + value1_line2 + +" | deb_get_field "Key1" \ + | expect_stdin $'value1_line1\nvalue1_line2' \ +|| let ++FAIL_COUNT + +##################### +echo " + +Dummy1: dummy1 +Key1: value1_line1 + value1_line2 +Dummy2: dummy2 + +" | deb_get_field "Key1" \ + | expect_stdin $'value1_line1\nvalue1_line2' \ +|| let ++FAIL_COUNT + + +##################### +echo $' + +Dummy1: dummy1 +Key1: value1_line1 +\tvalue1_line2 +Dummy2: dummy2_line1 +\tdummy2_line2 + +' | deb_get_field "Key1" \ + | expect_stdin $'value1_line1\nvalue1_line2' \ +|| let ++FAIL_COUNT + + +######################################################### +# deb_get_simple_depends +######################################################### + +echo " +Depends: texinfo (>= 1.0), kernel-headers-2.2.10 [!hurd-i386], + hurd-dev [hurd-i386], gnumach-dev [hurd-i386], yy-foo (>= 1.0) | zz-bar +" | deb_get_simple_depends \ + | expect_stdin $'gnumach-dev\nhurd-dev\nkernel-headers-2.2.10\ntexinfo\nyy-foo\nzz-bar' \ +|| let ++FAIL_COUNT + + +if [[ $FAIL_COUNT -gt 0 ]] ; then + echo >&2 + echo "ERROR: ${FAIL_COUNT} test(s) failed" >&2 + echo >&2 + exit 1 +fi +echo "$PROGNAME: all tests passed" >&2 +exit 0 + diff --git a/tox.ini b/tox.ini index 18757c2b..0e42de60 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = linters +envlist = linters, unit-tests minversion = 2.3 skipsdist = True @@ -57,3 +57,14 @@ commands = [testenv:venv] basepython = python3 commands = {posargs} + +[testenv:unit-tests] +whitelist_externals = bash +basepython = python3 +setenv = PYTHON3=python +commands = + bash -c " \ + for f in {toxinidir}/build-tools/unit-tests/* ; do \ + $f || exit 1 ; \ + done ; \ + "