From cede6c0d482b22258c60b6f7ea9e24ac6cfcf68d Mon Sep 17 00:00:00 2001 From: Pete Birley Date: Sat, 22 Jun 2019 07:45:01 -0500 Subject: [PATCH] Gate: Simpler multiple distro/over-ride support This PS indroduces a simpler way to incorp over-rides into gate runs, and also ensures that they are scoped to a single chart, rather than all of the charts deployed within a gate run. Depends-On: https://review.opendev.org/666957 Change-Id: I49edf52cc1fc5ec60ee9754c28880c9c0c54492e Signed-off-by: Pete Birley --- .../values_overrides/ldap.yaml | 0 .../deployment/common/get-values-overrides.sh | 71 ++++++++++ tools/deployment/common/install-packages.sh | 3 +- .../component/ceph/ceph-ns-activate.sh | 12 +- tools/deployment/component/ceph/ceph.sh | 15 +- tools/deployment/component/cinder/cinder.sh | 7 +- tools/deployment/component/common/ingress.sh | 18 +-- tools/deployment/component/common/ldap.sh | 44 ++++++ tools/deployment/component/common/mariadb.sh | 11 +- .../deployment/component/common/memcached.sh | 11 +- tools/deployment/component/common/rabbitmq.sh | 12 +- .../component/compute-kit/compute-kit.sh | 18 ++- .../component/compute-kit/libvirt.sh | 12 +- .../component/compute-kit/openvswitch.sh | 13 +- tools/deployment/component/glance/glance.sh | 6 +- tools/deployment/component/heat/heat.sh | 6 +- tools/deployment/component/horizon/horizon.sh | 6 +- .../component/keystone/keystone-ldap.sh | 82 ----------- .../deployment/component/keystone/keystone.sh | 33 ++++- zuul.d/jobs-openstack-helm.yaml | 131 ++++++++++-------- 20 files changed, 309 insertions(+), 202 deletions(-) rename tools/overrides/keystone/ldap_domain_config.yaml => keystone/values_overrides/ldap.yaml (100%) create mode 100755 tools/deployment/common/get-values-overrides.sh create mode 100755 tools/deployment/component/common/ldap.sh delete mode 100755 tools/deployment/component/keystone/keystone-ldap.sh diff --git a/tools/overrides/keystone/ldap_domain_config.yaml b/keystone/values_overrides/ldap.yaml similarity index 100% rename from tools/overrides/keystone/ldap_domain_config.yaml rename to keystone/values_overrides/ldap.yaml diff --git a/tools/deployment/common/get-values-overrides.sh b/tools/deployment/common/get-values-overrides.sh new file mode 100755 index 0000000000..baf604e875 --- /dev/null +++ b/tools/deployment/common/get-values-overrides.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This script will generate a matrix of values-override file args to apply to +# charts, in the gate and development environments. It will 1st produce a +# consistenly ordered list of all permuations of filenames to try, and then +# if a file matching this name exists in the `values_overrides` directory within +# each chart, apply it upon install/upgrade. + +set -e +HELM_CHART="$1" +: "${HELM_CHART_ROOT_PATH:="../openstack-helm"}" +: "${OPENSTACK_RELEASE:="ocata"}" +: "${CONTAINER_DISTRO_NAME:="ubuntu"}" +: "${CONTAINER_DISTRO_VERSION:="xenial"}" +: "${FEATURE_GATES:=""}" +OSH_FEATURE_MIX="${FEATURE_GATES},${OPENSTACK_RELEASE},${CONTAINER_DISTRO_NAME}_${CONTAINER_DISTRO_VERSION},${CONTAINER_DISTRO_NAME}" + +function echoerr () { + echo "$@" 1>&2; +} + +function generate_awk_exp_from_mask () { + local POSITION=1 + for VALUE in $@; do + [ "${VALUE}" -eq 1 ] && echo -n "print \$${POSITION};" + POSITION=$((POSITION+1)) + done + echo -e "\n" +} + +function combination () { + POWER=$((2**$#)) + BITS="$(awk "BEGIN { while (c++ < $#) printf \"0\" }")" + while [ "${POWER}" -gt 1 ];do + POWER=$((POWER-1)) + BIN="$(bc <<< "obase=2; ${POWER}")" + MASK="$(echo "${BITS}" | sed -e "s/0\{${#BIN}\}$/$BIN/" | grep -o .)" + #NOTE: This line is odd, but written to support both BSD and GNU utils + awk -v ORS="-" "{$(generate_awk_exp_from_mask "$MASK")}" <<< "$@" | awk 1 | sed 's/-$//' + done +} + +function override_file_args () { + OVERRIDE_ARGS="" + echoerr "We will attempt to use values-override files with the following paths:" + for FILE in $(combination ${1//,/ } | uniq | tac); do + FILE_PATH="${HELM_CHART_ROOT_PATH}/${HELM_CHART}/values_overrides/${FILE}.yaml" + if [ -f "${FILE_PATH}" ]; then + OVERRIDE_ARGS+=" --values=${FILE_PATH} " + fi + echoerr "${FILE_PATH}" + done + echo "${OVERRIDE_ARGS}" +} + +echoerr "We are going to deploy the service ${HELM_CHART} for the OpenStack ${OPENSTACK_RELEASE} release, using ${CONTAINER_DISTRO_NAME} (${CONTAINER_DISTRO_VERSION}) distribution containers." +override_file_args "${OSH_FEATURE_MIX}" diff --git a/tools/deployment/common/install-packages.sh b/tools/deployment/common/install-packages.sh index fb83bc3ea8..6a126eb6e0 100755 --- a/tools/deployment/common/install-packages.sh +++ b/tools/deployment/common/install-packages.sh @@ -24,4 +24,5 @@ sudo apt-get install --no-install-recommends -y \ jq \ nmap \ curl \ - uuid-runtime + uuid-runtime \ + bc diff --git a/tools/deployment/component/ceph/ceph-ns-activate.sh b/tools/deployment/component/ceph/ceph-ns-activate.sh index 7f7f2a172a..110397837b 100755 --- a/tools/deployment/component/ceph/ceph-ns-activate.sh +++ b/tools/deployment/component/ceph/ceph-ns-activate.sh @@ -16,12 +16,14 @@ set -xe +#NOTE: Get the over-rides to use +export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${OSH_INFRA_PATH:="../openstack-helm-infra"}"}" +: ${OSH_EXTRA_HELM_ARGS_CEPH_NS_ACTIVATE:="$(./tools/deployment/common/get-values-overrides.sh ceph-provisioners)"} + #NOTE: Lint and package chart -: ${OSH_INFRA_PATH:="../openstack-helm-infra"} -make -C ${OSH_INFRA_PATH} ceph-provisioners +make -C ${HELM_CHART_ROOT_PATH} ceph-provisioners #NOTE: Deploy command -: ${OSH_EXTRA_HELM_ARGS:=""} tee /tmp/ceph-openstack-config.yaml < /tmp/ceph-fs-uuid.txt CEPH_FS_ID="$(cat /tmp/ceph-fs-uuid.txt)" #NOTE(portdirect): to use RBD devices with Ubuntu kernels < 4.5 this @@ -188,11 +186,14 @@ manifests: EOF for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do - helm upgrade --install ${CHART} ${OSH_INFRA_PATH}/${CHART} \ + #NOTE: Get the over-rides to use + : ${OSH_EXTRA_HELM_ARGS_CEPH:="$(./tools/deployment/common/get-values-overrides.sh ${CHART})"} + + helm upgrade --install ${CHART} ${HELM_CHART_ROOT_PATH}/${CHART} \ --namespace=ceph \ --values=/tmp/ceph.yaml \ - ${OSH_EXTRA_HELM_ARGS} \ - ${OSH_EXTRA_HELM_ARGS_CEPH_DEPLOY} + ${OSH_EXTRA_HELM_ARGS:=} \ + ${OSH_EXTRA_HELM_ARGS_CEPH} #NOTE: Wait for deploy ./tools/deployment/common/wait-for-pods.sh ceph diff --git a/tools/deployment/component/cinder/cinder.sh b/tools/deployment/component/cinder/cinder.sh index 6888e66a3e..789e8d8728 100755 --- a/tools/deployment/component/cinder/cinder.sh +++ b/tools/deployment/component/cinder/cinder.sh @@ -15,11 +15,13 @@ # under the License. set -xe +#NOTE: Get the over-rides to use +: ${OSH_EXTRA_HELM_ARGS_CINDER:="$(./tools/deployment/common/get-values-overrides.sh cinder)"} + #NOTE: Lint and package chart make cinder #NOTE: Deploy command -: ${OSH_EXTRA_HELM_ARGS:=""} tee /tmp/cinder.yaml <