Add meta-config via local.conf

This defines a new local.conf file that is designed to take the place of all
of the 'pass-through'[1] configuration options that have been defined in DevStack.

* new local.conf file can contain multiple config file settings to be
  merged in to existing project config files
* localrc can be embedded into local.conf and will auto-extract if
  localrc does not exist
* Adds functions get_meta_section(), get_meta_section_files(),
  merge_config_file() and merge_config_group()
* Adds EXTRA_OPTS, EXTRA_BAREMETAL_OPTS, Q_DHCP_EXTRA_DEFAULT_OPTS and
  Q_SRV_EXTRA_DEFAULT_OPTS to the deprecated warning list at the end of stack.sh

[1] Pass-through options are those that do not configure or change DevStack's behaviour
but simply set a value in a project config file.  This includes most of the EXTRA_XXX_OPTS
configuration variables.

Change-Id: I367cadc86116621e9574ac203aafdab483d810d3

README.md

@@ -244,3 +244,42 @@ To setup a cells environment add the following to your `localrc`:
     enable_service n-cell
 
 Be aware that there are some features currently missing in cells, one notable one being security groups.  The exercises have been patched to disable functionality not supported by cells.
+
+
+# Local Configuration
+
+Historically DevStack has used ``localrc`` to contain all local configuration and customizations. More and more of the configuration variables available for DevStack are passed-through to the individual project configuration files.  The old mechanism for this required specific code for each file and did not scale well.  This is handled now by a master local configuration file.
+
+# local.conf
+
+The new config file ``local.conf`` is an extended-INI format that introduces a new meta-section header that provides some additional information such as a phase name and destination config filename:
+
+  [[ <phase> | <filename> ]]
+
+where <phase> is one of a set of phase names defined by ``stack.sh`` and <filename> is the project config filename.  The filename is eval'ed in the stack.sh context so all environment variables are available and may be used.  Using the project config file variables in the header is strongly suggested (see example of NOVA_CONF below).  If the path of the config file does not exist it is skipped.
+
+The defined phases are:
+
+* local - extracts ``localrc`` from ``local.conf`` before ``stackrc`` is sourced
+* post-config - runs after the layer 2 services are configured and before they are started
+* extra - runs after services are started and before any files in ``extra.d`` are executes
+
+The file is processed strictly in sequence; meta-sections may be specified more than once but if any settings are duplicated the last to appear in the file will be used.
+
+  [[post-config|$NOVA_CONF]]
+  [DEFAULT]
+  use_syslog = True
+
+  [osapi_v3]
+  enabled = False
+
+A specific meta-section ``local:localrc`` is used to provide a default localrc file.  This allows all custom settings for DevStack to be contained in a single file.  ``localrc`` is not overwritten if it exists to preserve compatability.
+
+  [[local|localrc]]
+  FIXED_RANGE=10.254.1.0/24
+  ADMIN_PASSWORD=speciale
+  LOGFILE=$DEST/logs/stack.sh.log
+
+Note that ``Q_PLUGIN_CONF_FILE`` is unique in that it is assumed to _NOT_ start with a ``/`` (slash) character.  A slash will need to be added:
+
+  [[post-config|/$Q_PLUGIN_CONF_FILE]]

functions

@@ -155,6 +155,22 @@ function err_if_not_set() {
 }
 
 
+# Prints line number and "message" in warning format
+# warn $LINENO "message"
+function warn() {
+    local exitcode=$?
+    errXTRACE=$(set +o | grep xtrace)
+    set +o xtrace
+    local msg="[WARNING] ${BASH_SOURCE[2]}:$1 $2"
+    echo $msg 1>&2;
+    if [[ -n ${SCREEN_LOGDIR} ]]; then
+        echo $msg >> "${SCREEN_LOGDIR}/error.log"
+    fi
+    $errXTRACE
+    return $exitcode
+}
+
+
 # HTTP and HTTPS proxy servers are supported via the usual environment variables [1]
 # ``http_proxy``, ``https_proxy`` and ``no_proxy``. They can be set in
 # ``localrc`` or on the command line if necessary::

lib/config

@@ -0,0 +1,130 @@
+# lib/config - Configuration file manipulation functions
+
+# These functions have no external dependencies and the following side-effects:
+#
+# CONFIG_AWK_CMD is defined, default is ``awk``
+
+# Meta-config files contain multiple INI-style configuration files
+# using a specific new section header to delimit them:
+#
+#   [[group-name|file-name]]
+#
+# group-name refers to the group of configuration file changes to be processed
+# at a particular time.  These are called phases in ``stack.sh`` but 
+# group here as these functions are not DevStack-specific.
+#
+# file-name is the destination of the config file
+
+# Save trace setting
+C_XTRACE=$(set +o | grep xtrace)
+set +o xtrace
+
+
+# Allow the awk command to be overridden on legacy platforms
+CONFIG_AWK_CMD=${CONFIG_AWK_CMD:-awk}
+
+# Get the section for the specific group and config file
+# get_meta_section infile group configfile
+function get_meta_section() {
+    local file=$1
+    local matchgroup=$2
+    local configfile=$3
+
+    [[ -r $file ]] || return 0
+    [[ -z $configfile ]] && return 0
+
+    $CONFIG_AWK_CMD -v matchgroup=$matchgroup -v configfile=$configfile '
+        BEGIN { group = "" }
+        /^\[\[.+|.*\]\]/ {
+            if (group == "") {
+                gsub("[][]", "", $1);
+                split($1, a, "|");
+                if (a[1] == matchgroup && a[2] == configfile) {
+                    group=a[1]
+                }
+            } else {
+                group=""
+            }
+            next
+        }
+        {
+            if (group != "")
+                print $0
+        }
+    ' $file
+}
+
+
+# Get a list of config files for a specific group
+# get_meta_section_files infile group
+function get_meta_section_files() {
+    local file=$1
+    local matchgroup=$2
+
+    [[ -r $file ]] || return 0
+
+    $CONFIG_AWK_CMD -v matchgroup=$matchgroup '
+      /^\[\[.+\|.*\]\]/ {
+          gsub("[][]", "", $1);
+          split($1, a, "|");
+          if (a[1] == matchgroup)
+              print a[2]
+      }
+    ' $file
+}
+
+
+# Merge the contents of a meta-config file into its destination config file
+# If configfile does not exist it will be created.
+# merge_config_file infile group configfile
+function merge_config_file() {
+    local file=$1
+    local matchgroup=$2
+    local configfile=$3
+
+    [[ -r $configfile ]] || touch $configfile
+
+    get_meta_section $file $matchgroup $configfile | \
+    $CONFIG_AWK_CMD -v configfile=$configfile '
+        BEGIN { section = "" }
+        /^\[.+\]/ {
+            gsub("[][]", "", $1);
+            section=$1
+            next
+        }
+        /^ *\#/ {
+            next
+        }
+        /^.+/ {
+            split($0, d, " *= *")
+            print "iniset " configfile " " section " " d[1] " \"" d[2] "\""
+        }
+    ' | while read a; do eval "$a"; done
+
+}
+
+
+# Merge all of the files specified by group
+# merge_config_group infile group [group ...]
+function merge_config_group() {
+    local localfile=$1; shift
+    local matchgroups=$@
+
+    [[ -r $localfile ]] || return 0
+
+    for group in $matchgroups; do
+        for configfile in $(get_meta_section_files $localfile $group); do
+            if [[ -d $(dirname $configfile) ]]; then
+                merge_config_file $localfile $group $configfile
+            fi
+        done
+    done
+}
+
+
+# Restore xtrace
+$C_XTRACE
+
+# Local variables:
+# mode: shell-script
+# End:

stack.sh

@@ -29,6 +29,9 @@ TOP_DIR=$(cd $(dirname "$0") && pwd)
 # Import common functions
 source $TOP_DIR/functions
 
+# Import config functions
+source $TOP_DIR/lib/config
+
 # Determine what system we are running on.  This provides ``os_VENDOR``,
 # ``os_RELEASE``, ``os_UPDATE``, ``os_PACKAGE``, ``os_CODENAME``
 # and ``DISTRO``
@@ -38,6 +41,25 @@ GetDistro
 # Global Settings
 # ===============
 
+# Check for a ``localrc`` section embedded in ``local.conf`` and extract if
+# ``localrc`` does not already exist
+
+# Phase: local
+rm -f $TOP_DIR/.localrc.auto
+if [[ -r $TOP_DIR/local.conf ]]; then
+    LRC=$(get_meta_section_files $TOP_DIR/local.conf local)
+    for lfile in $LRC; do
+        if [[ "$lfile" == "localrc" ]]; then
+            if [[ -r $TOP_DIR/localrc ]]; then
+                warn $LINENO "localrc and local.conf:[[local]] both exist, using localrc"
+            else
+                echo "# Generated file, do not exit" >$TOP_DIR/.localrc.auto
+                get_meta_section $TOP_DIR/local.conf local $lfile >>$TOP_DIR/.localrc.auto
+            fi
+        fi
+    done
+fi
+
 # ``stack.sh`` is customizable by setting environment variables.  Override a
 # default setting via export::
 #
@@ -842,6 +864,9 @@ if is_service_enabled sysstat;then
 fi
 
 
+# Start Services
+# ==============
+
 # Keystone
 # --------
 
@@ -1153,6 +1178,14 @@ if is_service_enabled nova && is_baremetal; then
 fi
 
 
+# Local Configuration
+# ===================
+
+# Apply configuration from local.conf if it exists for layer 2 services
+# Phase: post-config
+merge_config_group $TOP_DIR/local.conf post-config
+
+
 # Launch Services
 # ===============
 
@@ -1348,6 +1381,14 @@ for i in BASE_SQL_CONN ENABLED_SERVICES HOST_IP LOGFILE \
 done
 
 
+# Local Configuration
+# ===================
+
+# Apply configuration from local.conf if it exists for layer 2 services
+# Phase: extra
+merge_config_group $TOP_DIR/local.conf extra
+
+
 # Run extras
 # ==========
 
@@ -1420,5 +1461,66 @@ if [[ -n "$DEPRECATED_TEXT" ]]; then
     echo_summary "WARNING: $DEPRECATED_TEXT"
 fi
 
+# Specific warning for deprecated configs
+if [[ -n "$EXTRA_OPTS" ]]; then
+    echo ""
+    echo_summary "WARNING: EXTRA_OPTS is used"
+    echo "You are using EXTRA_OPTS to pass configuration into nova.conf."
+    echo "Please convert that configuration in localrc to a nova.conf section in local.conf:"
+    echo "
+[[post-config|\$NOVA_CONF]]
+[DEFAULT]
+"
+    for I in "${EXTRA_OPTS[@]}"; do
+        # Replace the first '=' with ' ' for iniset syntax
+        echo ${I}
+    done
+fi
+
+if [[ -n "$EXTRA_BAREMETAL_OPTS" ]]; then
+    echo ""
+    echo_summary "WARNING: EXTRA_OPTS is used"
+    echo "You are using EXTRA_OPTS to pass configuration into nova.conf."
+    echo "Please convert that configuration in localrc to a nova.conf section in local.conf:"
+    echo "
+[[post-config|\$NOVA_CONF]]
+[baremetal]
+"
+    for I in "${EXTRA_BAREMETAL_OPTS[@]}"; do
+        # Replace the first '=' with ' ' for iniset syntax
+        echo ${I}
+    done
+fi
+
+if [[ -n "$Q_DHCP_EXTRA_DEFAULT_OPTS" ]]; then
+    echo ""
+    echo_summary "WARNING: Q_DHCP_EXTRA_DEFAULT_OPTS is used"
+    echo "You are using Q_DHCP_EXTRA_DEFAULT_OPTS to pass configuration into $Q_DHCP_CONF_FILE."
+    echo "Please convert that configuration in localrc to a $Q_DHCP_CONF_FILE section in local.conf:"
+    echo "
+[[post-config|\$Q_DHCP_CONF_FILE]]
+[DEFAULT]
+"
+    for I in "${Q_DHCP_EXTRA_DEFAULT_OPTS[@]}"; do
+        # Replace the first '=' with ' ' for iniset syntax
+        echo ${I}
+    done
+fi
+
+if [[ -n "$Q_SRV_EXTRA_DEFAULT_OPTS" ]]; then
+    echo ""
+    echo_summary "WARNING: Q_SRV_EXTRA_DEFAULT_OPTS is used"
+    echo "You are using Q_SRV_EXTRA_DEFAULT_OPTS to pass configuration into $NEUTRON_CONF."
+    echo "Please convert that configuration in localrc to a $NEUTRON_CONF section in local.conf:"
+    echo "
+[[post-config|\$NEUTRON_CONF]]
+[DEFAULT]
+"
+    for I in "${Q_SRV_EXTRA_DEFAULT_OPTS[@]}"; do
+        # Replace the first '=' with ' ' for iniset syntax
+        echo ${I}
+    done
+fi
+
 # Indicate how long this took to run (bash maintained variable ``SECONDS``)
 echo_summary "stack.sh completed in $SECONDS seconds."

stackrc

@@ -48,8 +48,12 @@ IDENTITY_API_VERSION=2.0
 USE_SCREEN=True
 
 # allow local overrides of env variables, including repo config
-if [ -f $RC_DIR/localrc ]; then
+if [[ -f $RC_DIR/localrc ]]; then
+    # Old-style user-supplied config
     source $RC_DIR/localrc
+elif [[ -f $RC_DIR/.localrc.auto ]]; then
+    # New-style user-supplied config extracted from local.conf
+    source $RC_DIR/.localrc.auto
 fi
 
 

tests/test_config.sh

@@ -0,0 +1,179 @@
+#!/usr/bin/env bash
+
+# Tests for DevStack meta-config functions
+
+TOP=$(cd $(dirname "$0")/.. && pwd)
+
+# Import common functions
+source $TOP/functions
+
+# Import config functions
+source $TOP/lib/config
+
+# check_result() tests and reports the result values
+# check_result "actual" "expected"
+function check_result() {
+    local actual=$1
+    local expected=$2
+    if [[ "$actual" == "$expected" ]]; then
+        echo "OK"
+    else
+        echo -e "failed: $actual != $expected\n"
+    fi
+}
+
+TEST_1C_ADD="[eee]
+type=new
+multi = foo2"
+
+function create_test1c() {
+    cat >test1c.conf <<EOF
+[eee]
+# original comment
+type=original
+EOF
+}
+
+function create_test2a() {
+    cat >test2a.conf <<EOF
+[ddd]
+# original comment
+type=original
+EOF
+}
+
+cat >test.conf <<EOF
+[[test1|test1a.conf]]
+[default]
+# comment an option
+#log_file=./log.conf
+log_file=/etc/log.conf
+handlers=do not disturb
+
+[aaa]
+# the commented option should not change
+#handlers=cc,dd
+handlers = aa, bb
+
+[[test1|test1b.conf]]
+[bbb]
+handlers=ee,ff
+
+[ ccc ]
+spaces  =  yes
+
+[[test2|test2a.conf]]
+[ddd]
+# new comment
+type=new
+additional=true
+
+[[test1|test1c.conf]]
+$TEST_1C_ADD
+EOF
+
+
+echo -n "get_meta_section_files: test0 doesn't exist: "
+VAL=$(get_meta_section_files test.conf test0)
+check_result "$VAL" ""
+
+echo -n "get_meta_section_files: test1 3 files: "
+VAL=$(get_meta_section_files test.conf test1)
+EXPECT_VAL="test1a.conf
+test1b.conf
+test1c.conf"
+check_result "$VAL" "$EXPECT_VAL"
+
+echo -n "get_meta_section_files: test2 1 file: "
+VAL=$(get_meta_section_files test.conf test2)
+EXPECT_VAL="test2a.conf"
+check_result "$VAL" "$EXPECT_VAL"
+
+
+# Get a section from a group that doesn't exist
+echo -n "get_meta_section: test0 doesn't exist: "
+VAL=$(get_meta_section test.conf test0 test0.conf)
+check_result "$VAL" ""
+
+# Get a single section from a group with multiple files
+echo -n "get_meta_section: test1c single section: "
+VAL=$(get_meta_section test.conf test1 test1c.conf)
+check_result "$VAL" "$TEST_1C_ADD"
+
+# Get a single section from a group with a single file
+echo -n "get_meta_section: test2a single section: "
+VAL=$(get_meta_section test.conf test2 test2a.conf)
+EXPECT_VAL="[ddd]
+# new comment
+type=new
+additional=true"
+check_result "$VAL" "$EXPECT_VAL"
+
+# Get a single section that doesn't exist from a group
+echo -n "get_meta_section: test2z.conf not in test2: "
+VAL=$(get_meta_section test.conf test2 test2z.conf)
+check_result "$VAL" ""
+
+# Get a section from a conf file that doesn't exist
+echo -n "get_meta_section: nofile doesn't exist: "
+VAL=$(get_meta_section nofile.ini test1)
+check_result "$VAL" ""
+
+echo -n "get_meta_section: nofile doesn't exist: "
+VAL=$(get_meta_section nofile.ini test0 test0.conf)
+check_result "$VAL" ""
+
+echo -n "merge_config_file test1c exists: "
+create_test1c
+merge_config_file test.conf test1 test1c.conf
+VAL=$(cat test1c.conf)
+# iniset adds values immediately under the section header
+EXPECT_VAL="[eee]
+multi = foo2
+# original comment
+type=new"
+check_result "$VAL" "$EXPECT_VAL"
+
+echo -n "merge_config_file test2a exists: "
+create_test2a
+merge_config_file test.conf test2 test2a.conf
+VAL=$(cat test2a.conf)
+# iniset adds values immediately under the section header
+EXPECT_VAL="[ddd]
+additional = true
+# original comment
+type=new"
+check_result "$VAL" "$EXPECT_VAL"
+
+echo -n "merge_config_file test2a not exist: "
+rm test2a.conf
+merge_config_file test.conf test2 test2a.conf
+VAL=$(cat test2a.conf)
+# iniset adds a blank line if it creates the file...
+EXPECT_VAL="
+[ddd]
+additional = true
+type = new"
+check_result "$VAL" "$EXPECT_VAL"
+
+echo -n "merge_config_group test2: "
+rm test2a.conf
+merge_config_group test.conf test2
+VAL=$(cat test2a.conf)
+# iniset adds a blank line if it creates the file...
+EXPECT_VAL="
+[ddd]
+additional = true
+type = new"
+check_result "$VAL" "$EXPECT_VAL"
+
+echo -n "merge_config_group test2 no conf file: "
+rm test2a.conf
+merge_config_group x-test.conf test2
+if [[ ! -r test2a.conf ]]; then
+    echo "OK"
+else
+    echo "failed: $VAL != $EXPECT_VAL"
+fi
+
+rm -f test.conf test1c.conf test2a.conf