diff --git a/README.md b/README.md index 6dc9ecd1e3..66e36b22a8 100644 --- a/README.md +++ b/README.md @@ -250,3 +250,42 @@ To setup a cells environment add the following to your `localrc`: enable_service n-cell Be aware that there are some features currently missing in cells, one notable one being security groups. The exercises have been patched to disable functionality not supported by cells. + + +# Local Configuration + +Historically DevStack has used ``localrc`` to contain all local configuration and customizations. More and more of the configuration variables available for DevStack are passed-through to the individual project configuration files. The old mechanism for this required specific code for each file and did not scale well. This is handled now by a master local configuration file. + +# local.conf + +The new config file ``local.conf`` is an extended-INI format that introduces a new meta-section header that provides some additional information such as a phase name and destination config filename: + + [[ | ]] + +where is one of a set of phase names defined by ``stack.sh`` and is the project config filename. The filename is eval'ed in the stack.sh context so all environment variables are available and may be used. Using the project config file variables in the header is strongly suggested (see example of NOVA_CONF below). If the path of the config file does not exist it is skipped. + +The defined phases are: + +* local - extracts ``localrc`` from ``local.conf`` before ``stackrc`` is sourced +* post-config - runs after the layer 2 services are configured and before they are started +* extra - runs after services are started and before any files in ``extra.d`` are executes + +The file is processed strictly in sequence; meta-sections may be specified more than once but if any settings are duplicated the last to appear in the file will be used. + + [[post-config|$NOVA_CONF]] + [DEFAULT] + use_syslog = True + + [osapi_v3] + enabled = False + +A specific meta-section ``local:localrc`` is used to provide a default localrc file. This allows all custom settings for DevStack to be contained in a single file. ``localrc`` is not overwritten if it exists to preserve compatability. + + [[local|localrc]] + FIXED_RANGE=10.254.1.0/24 + ADMIN_PASSWORD=speciale + LOGFILE=$DEST/logs/stack.sh.log + +Note that ``Q_PLUGIN_CONF_FILE`` is unique in that it is assumed to _NOT_ start with a ``/`` (slash) character. A slash will need to be added: + + [[post-config|/$Q_PLUGIN_CONF_FILE]] diff --git a/functions b/functions index 6f09630ebd..9e99cb22ce 100644 --- a/functions +++ b/functions @@ -155,6 +155,22 @@ function err_if_not_set() { } +# Prints line number and "message" in warning format +# warn $LINENO "message" +function warn() { + local exitcode=$? + errXTRACE=$(set +o | grep xtrace) + set +o xtrace + local msg="[WARNING] ${BASH_SOURCE[2]}:$1 $2" + echo $msg 1>&2; + if [[ -n ${SCREEN_LOGDIR} ]]; then + echo $msg >> "${SCREEN_LOGDIR}/error.log" + fi + $errXTRACE + return $exitcode +} + + # HTTP and HTTPS proxy servers are supported via the usual environment variables [1] # ``http_proxy``, ``https_proxy`` and ``no_proxy``. They can be set in # ``localrc`` or on the command line if necessary:: diff --git a/lib/config b/lib/config new file mode 100644 index 0000000000..6f686e9b5d --- /dev/null +++ b/lib/config @@ -0,0 +1,130 @@ +# lib/config - Configuration file manipulation functions + +# These functions have no external dependencies and the following side-effects: +# +# CONFIG_AWK_CMD is defined, default is ``awk`` + +# Meta-config files contain multiple INI-style configuration files +# using a specific new section header to delimit them: +# +# [[group-name|file-name]] +# +# group-name refers to the group of configuration file changes to be processed +# at a particular time. These are called phases in ``stack.sh`` but +# group here as these functions are not DevStack-specific. +# +# file-name is the destination of the config file + +# Save trace setting +C_XTRACE=$(set +o | grep xtrace) +set +o xtrace + + +# Allow the awk command to be overridden on legacy platforms +CONFIG_AWK_CMD=${CONFIG_AWK_CMD:-awk} + +# Get the section for the specific group and config file +# get_meta_section infile group configfile +function get_meta_section() { + local file=$1 + local matchgroup=$2 + local configfile=$3 + + [[ -r $file ]] || return 0 + [[ -z $configfile ]] && return 0 + + $CONFIG_AWK_CMD -v matchgroup=$matchgroup -v configfile=$configfile ' + BEGIN { group = "" } + /^\[\[.+|.*\]\]/ { + if (group == "") { + gsub("[][]", "", $1); + split($1, a, "|"); + if (a[1] == matchgroup && a[2] == configfile) { + group=a[1] + } + } else { + group="" + } + next + } + { + if (group != "") + print $0 + } + ' $file +} + + +# Get a list of config files for a specific group +# get_meta_section_files infile group +function get_meta_section_files() { + local file=$1 + local matchgroup=$2 + + [[ -r $file ]] || return 0 + + $CONFIG_AWK_CMD -v matchgroup=$matchgroup ' + /^\[\[.+\|.*\]\]/ { + gsub("[][]", "", $1); + split($1, a, "|"); + if (a[1] == matchgroup) + print a[2] + } + ' $file +} + + +# Merge the contents of a meta-config file into its destination config file +# If configfile does not exist it will be created. +# merge_config_file infile group configfile +function merge_config_file() { + local file=$1 + local matchgroup=$2 + local configfile=$3 + + [[ -r $configfile ]] || touch $configfile + + get_meta_section $file $matchgroup $configfile | \ + $CONFIG_AWK_CMD -v configfile=$configfile ' + BEGIN { section = "" } + /^\[.+\]/ { + gsub("[][]", "", $1); + section=$1 + next + } + /^ *\#/ { + next + } + /^.+/ { + split($0, d, " *= *") + print "iniset " configfile " " section " " d[1] " \"" d[2] "\"" + } + ' | while read a; do eval "$a"; done + +} + + +# Merge all of the files specified by group +# merge_config_group infile group [group ...] +function merge_config_group() { + local localfile=$1; shift + local matchgroups=$@ + + [[ -r $localfile ]] || return 0 + + for group in $matchgroups; do + for configfile in $(get_meta_section_files $localfile $group); do + if [[ -d $(dirname $configfile) ]]; then + merge_config_file $localfile $group $configfile + fi + done + done +} + + +# Restore xtrace +$C_XTRACE + +# Local variables: +# mode: shell-script +# End: diff --git a/stack.sh b/stack.sh index 7cd7e30d70..3d27f1a954 100755 --- a/stack.sh +++ b/stack.sh @@ -29,6 +29,9 @@ TOP_DIR=$(cd $(dirname "$0") && pwd) # Import common functions source $TOP_DIR/functions +# Import config functions +source $TOP_DIR/lib/config + # Determine what system we are running on. This provides ``os_VENDOR``, # ``os_RELEASE``, ``os_UPDATE``, ``os_PACKAGE``, ``os_CODENAME`` # and ``DISTRO`` @@ -38,6 +41,25 @@ GetDistro # Global Settings # =============== +# Check for a ``localrc`` section embedded in ``local.conf`` and extract if +# ``localrc`` does not already exist + +# Phase: local +rm -f $TOP_DIR/.localrc.auto +if [[ -r $TOP_DIR/local.conf ]]; then + LRC=$(get_meta_section_files $TOP_DIR/local.conf local) + for lfile in $LRC; do + if [[ "$lfile" == "localrc" ]]; then + if [[ -r $TOP_DIR/localrc ]]; then + warn $LINENO "localrc and local.conf:[[local]] both exist, using localrc" + else + echo "# Generated file, do not exit" >$TOP_DIR/.localrc.auto + get_meta_section $TOP_DIR/local.conf local $lfile >>$TOP_DIR/.localrc.auto + fi + fi + done +fi + # ``stack.sh`` is customizable by setting environment variables. Override a # default setting via export:: # @@ -812,6 +834,9 @@ if is_service_enabled sysstat;then fi +# Start Services +# ============== + # Keystone # -------- @@ -1068,6 +1093,14 @@ if is_service_enabled nova && is_baremetal; then fi +# Local Configuration +# =================== + +# Apply configuration from local.conf if it exists for layer 2 services +# Phase: post-config +merge_config_group $TOP_DIR/local.conf post-config + + # Launch Services # =============== @@ -1263,6 +1296,14 @@ for i in BASE_SQL_CONN ENABLED_SERVICES HOST_IP LOGFILE \ done +# Local Configuration +# =================== + +# Apply configuration from local.conf if it exists for layer 2 services +# Phase: extra +merge_config_group $TOP_DIR/local.conf extra + + # Run extras # ========== @@ -1335,5 +1376,66 @@ if [[ -n "$DEPRECATED_TEXT" ]]; then echo_summary "WARNING: $DEPRECATED_TEXT" fi +# Specific warning for deprecated configs +if [[ -n "$EXTRA_OPTS" ]]; then + echo "" + echo_summary "WARNING: EXTRA_OPTS is used" + echo "You are using EXTRA_OPTS to pass configuration into nova.conf." + echo "Please convert that configuration in localrc to a nova.conf section in local.conf:" + echo " +[[post-config|\$NOVA_CONF]] +[DEFAULT] +" + for I in "${EXTRA_OPTS[@]}"; do + # Replace the first '=' with ' ' for iniset syntax + echo ${I} + done +fi + +if [[ -n "$EXTRA_BAREMETAL_OPTS" ]]; then + echo "" + echo_summary "WARNING: EXTRA_OPTS is used" + echo "You are using EXTRA_OPTS to pass configuration into nova.conf." + echo "Please convert that configuration in localrc to a nova.conf section in local.conf:" + echo " +[[post-config|\$NOVA_CONF]] +[baremetal] +" + for I in "${EXTRA_BAREMETAL_OPTS[@]}"; do + # Replace the first '=' with ' ' for iniset syntax + echo ${I} + done +fi + +if [[ -n "$Q_DHCP_EXTRA_DEFAULT_OPTS" ]]; then + echo "" + echo_summary "WARNING: Q_DHCP_EXTRA_DEFAULT_OPTS is used" + echo "You are using Q_DHCP_EXTRA_DEFAULT_OPTS to pass configuration into $Q_DHCP_CONF_FILE." + echo "Please convert that configuration in localrc to a $Q_DHCP_CONF_FILE section in local.conf:" + echo " +[[post-config|\$Q_DHCP_CONF_FILE]] +[DEFAULT] +" + for I in "${Q_DHCP_EXTRA_DEFAULT_OPTS[@]}"; do + # Replace the first '=' with ' ' for iniset syntax + echo ${I} + done +fi + +if [[ -n "$Q_SRV_EXTRA_DEFAULT_OPTS" ]]; then + echo "" + echo_summary "WARNING: Q_SRV_EXTRA_DEFAULT_OPTS is used" + echo "You are using Q_SRV_EXTRA_DEFAULT_OPTS to pass configuration into $NEUTRON_CONF." + echo "Please convert that configuration in localrc to a $NEUTRON_CONF section in local.conf:" + echo " +[[post-config|\$NEUTRON_CONF]] +[DEFAULT] +" + for I in "${Q_SRV_EXTRA_DEFAULT_OPTS[@]}"; do + # Replace the first '=' with ' ' for iniset syntax + echo ${I} + done +fi + # Indicate how long this took to run (bash maintained variable ``SECONDS``) echo_summary "stack.sh completed in $SECONDS seconds." diff --git a/stackrc b/stackrc index 5c0baf7da9..3f740b5678 100644 --- a/stackrc +++ b/stackrc @@ -48,8 +48,12 @@ IDENTITY_API_VERSION=2.0 USE_SCREEN=True # allow local overrides of env variables, including repo config -if [ -f $RC_DIR/localrc ]; then +if [[ -f $RC_DIR/localrc ]]; then + # Old-style user-supplied config source $RC_DIR/localrc +elif [[ -f $RC_DIR/.localrc.auto ]]; then + # New-style user-supplied config extracted from local.conf + source $RC_DIR/.localrc.auto fi diff --git a/tests/test_config.sh b/tests/test_config.sh new file mode 100755 index 0000000000..fed2e7d477 --- /dev/null +++ b/tests/test_config.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash + +# Tests for DevStack meta-config functions + +TOP=$(cd $(dirname "$0")/.. && pwd) + +# Import common functions +source $TOP/functions + +# Import config functions +source $TOP/lib/config + +# check_result() tests and reports the result values +# check_result "actual" "expected" +function check_result() { + local actual=$1 + local expected=$2 + if [[ "$actual" == "$expected" ]]; then + echo "OK" + else + echo -e "failed: $actual != $expected\n" + fi +} + +TEST_1C_ADD="[eee] +type=new +multi = foo2" + +function create_test1c() { + cat >test1c.conf <test2a.conf <test.conf <