Dynamically create cinder.conf.sample

As it stands, the opts.py file that is passed into
oslo-config-generator isn't being generated dynamically
and the old way of generating the cinder.conf.sample is
dependent on oslo-incubator which Cinder is trying to
move away from. oslo-config-generator works differently
than oslo-incubator so a number of changes had to be made
in order to make this switch.

This patch adds the config directory to Cinder and in it
are two files:

    -generate_cinder_opts.py that will take the
     results of a grep command to create the opts.py
     file to be passed into oslo-config-generator.

    -cinder.conf which is the new configuration for
     oslo-config-generator. The file is inside the config
     directory to be consistent with other projects.

Some changes were made to the generate_sample.sh file in
order to give the base directories and target directories
to the generate_cinder_opts.py program.

tox.ini was edited to remove the checkonly option because
all that needs to happen in check_uptodate.sh is a check to
ensure that the cinder.conf.sample is actually being
generated with no issues.

All options were removed from the check_uptodate.sh
because they were unnecessary given the new, more simple
way of generating the cinder.conf.sample.

setup.cfg was also edited in order to add information
oslo-config-generator needs to run.

Co-Authored By: Jay Bryant <jsbryant@us.ibm.com>
Co-Authored By: Jacob Gregor <jgregor@us.ibm.com>

Change-Id: I643dbe5675ae9280e204f691781e617266f570d5
Closes-Bug: 1473768
Closes-Bug: 1437904
Closes-Bug: 1381563
This commit is contained in:
Kendall Nelson 2015-08-13 10:17:36 -05:00 committed by Walter A. Boring IV (hemna)
parent 351a432f59
commit 111a056c0f
7 changed files with 291 additions and 155 deletions

View File

@ -0,0 +1,21 @@
[DEFAULT]
output_file = etc/cinder/cinder.conf.sample
wrap_width = 79
namespace = cinder
namespace = keystonemiddleware.auth_token
namespace = oslo.config
namespace = oslo.concurrency
namespace = oslo.context
namespace = oslo.db
namesapce = oslo.i18n
namespace = oslo.log
namespace = oslo.messaging
namespace = oslo.middleware
namespace = oslo.policy
namespace = oslo.reports
namespace = oslo.rootwrap
namespace = oslo.serialization
namespace = oslo.service
namespace = oslo.utils
namespace = oslo.versionedobjects
namespace = oslo.vmware

View File

@ -0,0 +1,204 @@
#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import subprocess
if __name__ == "__main__":
opt_file = open("cinder/opts.py", 'a')
opt_dict = {}
dir_trees_list = []
opt_file.write("import copy\n")
opt_file.write("import itertools\n\n")
targetdir = os.environ['TARGETDIR']
basedir = os.environ['BASEDIRESC']
common_string = ('find ' + targetdir + ' -type f -name "*.py" ! '
'-path "*/tests/*" -exec grep -l "%s" {} '
'+ | sed -e "s/^' + basedir +
'\///g" | sort -u')
cmd_opts = common_string % "CONF.register_opts("
output_opts = subprocess.check_output('{}'.format(cmd_opts), shell = True)
dir_trees_list = output_opts.split()
cmd_opt = common_string % "CONF.register_opt("
output_opt = subprocess.check_output('{}'.format(cmd_opt), shell = True)
temp_list = output_opt.split()
for item in temp_list:
dir_trees_list.append(item)
dir_trees_list.sort()
flag = False
for atree in dir_trees_list:
if atree == "cinder/config/generate_cinder_opts.py":
continue
dirs_list = atree.split('/')
import_module = "from "
init_import_module = ""
import_name = ""
for dir in dirs_list:
if dir.find(".py") == -1:
import_module += dir + "."
init_import_module += dir + "."
import_name += dir + "_"
else:
if dir[:-3] != "__init__":
import_name += dir[:-3].replace("_", "")
import_module = (import_module[:-1] + " import " +
dir[:-3] + " as " + import_name)
opt_file.write(import_module + "\n")
else:
import_name = import_name[:-1].replace('/', '.')
init_import = atree[:-12].replace('/', '.')
opt_file.write("import " + init_import + "\n")
flag = True
if flag is False:
opt_dict[import_name] = atree
else:
opt_dict[init_import_module.strip(".")] = atree
flag = False
registered_opts_dict = {'fc-zone-manager': [],
'keymgr': [],
'BRCD_FABRIC_EXAMPLE': [],
'CISCO_FABRIC_EXAMPLE': [],
'profiler': [],
'DEFAULT': [], }
def _write_item(opts):
list_name = opts[-3:]
if list_name.lower() == "opts":
opt_file.write(" [" + opts.strip("\n") + "],\n")
else:
opt_file.write(" " + opts.strip("\n") + ",\n")
for key in opt_dict:
fd = os.open(opt_dict[key], os.O_RDONLY)
afile = os.fdopen(fd, "r")
for aline in afile:
exists = aline.find("CONF.register_opts(")
if exists != -1:
# TODO(kjnelson) FIX THIS LATER. These are instances where
# CONF.register_opts is happening without actually registering
# real lists of opts
exists = aline.find('base_san_opts')
if (exists != -1) or (key == 'cinder_volume_configuration'):
continue
if aline.find("fc-zone-manager") != -1:
fc_zm_list = aline.replace("CONF.register_opts(", '')
fc_zm_list = fc_zm_list.replace(", 'fc-zone-manager')", '')
fc_zm_list.strip()
line = key + "." + fc_zm_list
registered_opts_dict['fc-zone-manager'].append(line)
elif aline.find("keymgr") != -1:
keymgr_list = aline.replace("CONF.register_opts(", '')
keymgr_list = keymgr_list.replace(", group='keymgr')", '')
keymgr_list = keymgr_list.replace(", 'keymgr')", '')
keymgr_list.strip()
line = key + "." + keymgr_list
registered_opts_dict['keymgr'].append(line)
elif aline.find("BRCD_FABRIC_EXAMPLE") != -1:
brcd_list = aline.replace("CONF.register_opts(", '')
replace_string = ", 'BRCD_FABRIC_EXAMPLE')"
brcd_list = brcd_list.replace(replace_string, '')
brcd_list.strip()
line = key + "." + brcd_list
registered_opts_dict['BRCD_FABRIC_EXAMPLE'].append(line)
elif aline.find("CISCO_FABRIC_EXAMPLE") != -1:
cisco_list = aline.replace("CONF.register_opts(", '')
replace_string = ", 'CISCO_FABRIC_EXAMPLE')"
cisco_list = cisco_list.replace(replace_string, '')
cisco_list.strip()
line = key + "." + cisco_list
registered_opts_dict['CISCO_FABRIC_EXAMPLE'].append(line)
elif aline.find("profiler") != -1:
profiler_list = aline.replace("CONF.register_opts(", '')
replace_string = ', group="profiler")'
profiler_list = profiler_list.replace(replace_string, '')
profiler_list.strip()
line = key + "." + profiler_list
registered_opts_dict['profiler'].append(line)
else:
default_list = aline.replace("CONF.register_opts(", '')
default_list = default_list.replace(')', '').strip()
line = key + "." + default_list
registered_opts_dict['DEFAULT'].append(line)
opt_dict[key] = registered_opts_dict
list_str = ("def list_opts():\n"
" return [\n"
" ('DEFAULT',\n"
" itertools.chain(\n")
opt_file.write(list_str)
for item in registered_opts_dict["DEFAULT"]:
_write_item(item)
profiler_str = (" )),\n"
" ('profiler',\n"
" itertools.chain(\n")
opt_file.write(profiler_str)
for item in registered_opts_dict["profiler"]:
_write_item(item)
cisco_str = (" )),\n"
" ('CISCO_FABRIC_EXAMPLE',\n"
" itertools.chain(\n")
opt_file.write(cisco_str)
for item in registered_opts_dict["CISCO_FABRIC_EXAMPLE"]:
_write_item(item)
brcd_str = (" )),\n"
" ('BRCD_FABRIC_EXAMPLE',\n"
" itertools.chain(\n")
opt_file.write(brcd_str)
for item in registered_opts_dict["BRCD_FABRIC_EXAMPLE"]:
_write_item(item)
keymgr_str = (" )),\n"
" ('keymgr',\n"
" itertools.chain(\n")
opt_file.write(keymgr_str)
for item in registered_opts_dict["keymgr"]:
_write_item(item)
fczm_str = (" )),\n"
" ('fc-zone-manager',\n"
" itertools.chain(\n")
opt_file.write(fczm_str)
for item in registered_opts_dict["fc-zone-manager"]:
_write_item(item)
closing_str = (" )),\n"
"]\n\n\n")
opt_file.write(closing_str)
opt_file.close()

View File

@ -42,6 +42,12 @@ cinder.scheduler.weights =
ChanceWeigher = cinder.scheduler.weights.chance:ChanceWeigher
GoodnessWeigher = cinder.scheduler.weights.goodness:GoodnessWeigher
VolumeNumberWeigher = cinder.scheduler.weights.volume_number:VolumeNumberWeigher
oslo.config.opts =
cinder = cinder.opts:list_opts
keystonemiddleware = keystonemiddleware.auth_token:list_opts
oslo_concurrency = oslo_concurrency.opts:list_opts
oslo.messaging = oslo_messaging.opts:list_opts
oslo.db.concurrency = oslo.db.concurrency:list_opts
console_scripts =
cinder-all = cinder.cmd.all:main
cinder-api = cinder.cmd.api:main

View File

@ -12,33 +12,15 @@ CFGFILE_NAME=${PROJECT_NAME}.conf.sample
TEMPDIR=`mktemp -d /tmp/${PROJECT_NAME}.XXXXXX`
trap "rm -rf $TEMPDIR" EXIT
tools/config/generate_sample.sh -b ./ -p ${PROJECT_NAME} -o ${TEMPDIR}
tools/config/generate_sample.sh from_tox
# generate_sample.sh may return 0 even when it fails.
if [ $CHECKONLY -eq 1 ]; then
# Check whether something was generated.
if [ ! -s ${TEMPDIR}/${CFGFILE_NAME} ]; then
echo "Failed to generate ${CFGFILE_NAME}."
exit 1
fi
if [ -e etc/${PROJECT_NAME}/${CFGFILE_NAME} ]; then
CFGFILE=etc/${PROJECT_NAME}/${CFGFILE_NAME}
elif [ -e cinder/opts.py]; then
echo -en "\n\nWARNING: Found cinder/opts.py file. \n"
echo -en "Check for generate_cinder_opts.py failure.\n\n"
exit 1
else
if [ -e etc/${PROJECT_NAME}/${CFGFILE_NAME} ]; then
CFGFILE=etc/${PROJECT_NAME}/${CFGFILE_NAME}
elif [ -e etc/${CFGFILE_NAME} ]; then
CFGFILE=etc/${CFGFILE_NAME}
else
echo "${0##*/}: can not find config file"
exit 1
fi
if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE}
then
echo "${0##*/}: ${PROJECT_NAME}.conf.sample is not up to date."
echo "${0##*/}: Please run ${0%%${0##*/}}generate_sample.sh from within a VENV."
echo " \'source .venv/bin/activate; generate_sample.sh\'"
echo "OR simply run tox genconfig"
echo " \'tox -egenconfig\'"
exit 1
fi
echo "${0##*/}: Can't find config file."
exit 1
fi

View File

@ -10,66 +10,30 @@
# CINDER_CONFIG_GENERATOR_EXTRA_LIBRARIES: list of libraries to discover.
# CINDER_CONFIG_GENERATOR_EXCLUDED_FILES: list of files to remove from automatic listing.
print_hint() {
echo "Try \`${0##*/} --help' for more information." >&2
BASEDIR=${BASEDIR:-`pwd`}
print_error ()
{
echo -en "\n\n##########################################################"
echo -en "\nERROR: ${0} was not called from tox."
echo -en "\n Execute 'tox -e genconfig' for cinder.conf.sample"
echo -en "\n generation."
echo -en "\n##########################################################\n\n"
}
PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:m:l:o: \
--long help,base-dir:,package-name:,output-dir:,module:,library: -- "$@")
if [ -z ${1} ] ; then
print_error
exit 1
fi
if [ $? != 0 ] ; then print_hint ; exit 1 ; fi
if [ ${1} != "from_tox" ] ; then
print_error
exit 1
fi
eval set -- "$PARSED_OPTIONS"
while true; do
case "$1" in
-h|--help)
echo "${0##*/} [options]"
echo ""
echo "options:"
echo "-h, --help show brief help"
echo "-b, --base-dir=DIR project base directory"
echo "-p, --package-name=NAME project package name"
echo "-o, --output-dir=DIR file output directory"
echo "-m, --module=MOD extra python module to interrogate for options"
echo "-l, --library=LIB extra library that registers options for discovery"
exit 0
;;
-b|--base-dir)
shift
BASEDIR=`echo $1 | sed -e 's/\/*$//g'`
shift
;;
-p|--package-name)
shift
PACKAGENAME=`echo $1`
shift
;;
-o|--output-dir)
shift
OUTPUTDIR=`echo $1 | sed -e 's/\/*$//g'`
shift
;;
-m|--module)
shift
MODULES="$MODULES -m $1"
shift
;;
-l|--library)
shift
LIBRARIES="$LIBRARIES -l $1"
shift
;;
--)
break
;;
esac
done
BASEDIR=${BASEDIR:-`pwd`}
if ! [ -d $BASEDIR ]
then
echo "${0##*/}: missing project base directory" >&2 ; print_hint ; exit 1
echo "${0##*/}: missing project base directory" >&2 ; exit 1
elif [[ $BASEDIR != /* ]]
then
BASEDIR=$(cd "$BASEDIR" && pwd)
@ -77,84 +41,44 @@ fi
PACKAGENAME=${PACKAGENAME:-$(python setup.py --name)}
TARGETDIR=$BASEDIR/$PACKAGENAME
if ! [ -d $TARGETDIR ]
then
echo "${0##*/}: invalid project package name" >&2 ; print_hint ; exit 1
fi
OUTPUTDIR=${OUTPUTDIR:-$BASEDIR/etc}
# NOTE(bnemec): Some projects put their sample config in etc/,
# some in etc/$PACKAGENAME/
if [ -d $OUTPUTDIR/$PACKAGENAME ]
then
OUTPUTDIR=$OUTPUTDIR/$PACKAGENAME
elif ! [ -d $OUTPUTDIR ]
then
echo "${0##*/}: cannot access \`$OUTPUTDIR': No such file or directory" >&2
exit 1
if ! [ -d $TARGETDIR ] ; then
echo "${0##*/}: invalid project package name" >&2 ; exit 1
fi
BASEDIRESC=`echo $BASEDIR | sed -e 's/\//\\\\\//g'`
find $TARGETDIR -type f -name "*.pyc" -delete
FILES=$(find $TARGETDIR -type f -name "*.py" ! -path "*/tests/*" \
-exec grep -l "Opt(" {} + | sed -e "s/^$BASEDIRESC\///g" | sort -u)
RC_FILE="`dirname $0`/oslo.config.generator.rc"
if test -r "$RC_FILE"
export TARGETDIR=$TARGETDIR
export BASEDIRESC=$BASEDIRESC
python cinder/config/generate_cinder_opts.py
if [ $? -ne 0 ]
then
source "$RC_FILE"
fi
for filename in ${CINDER_CONFIG_GENERATOR_EXCLUDED_FILES}; do
FILES="${FILES[@]/$filename/}"
done
for mod in ${CINDER_CONFIG_GENERATOR_EXTRA_MODULES}; do
MODULES="$MODULES -m $mod"
done
for lib in ${CINDER_CONFIG_GENERATOR_EXTRA_LIBRARIES}; do
LIBRARIES="$LIBRARIES -l $lib"
done
export EVENTLET_NO_GREENDNS=yes
OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs)
[ "$OS_VARS" ] && eval "unset \$OS_VARS"
DEFAULT_MODULEPATH=cinder.openstack.common.config.generator
MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH}
OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample
python -m $MODULEPATH $MODULES $LIBRARIES $FILES > $OUTPUTFILE
if [ $? != 0 ]
then
echo "Can not generate $OUTPUTFILE"
echo -en "\n\n#################################################"
echo -en "\nERROR: Non-zero exit from generate_cinder_opts.py."
echo -en "\n See output above for details.\n"
echo -en "#################################################\n"
exit 1
fi
# Hook to allow projects to append custom config file snippets
CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null)
for CONCAT_FILE in $CONCAT_FILES; do
cat $CONCAT_FILE >> $OUTPUTFILE
done
oslo-config-generator --config-file=cinder/config/cinder-config-generator.conf
# NOTE(jsbryant): We collect the lib config options separately and
# append them. The generator requires that the name of the library be used
# as the entry point so we need to use oslo.*, not oslo_* .
oslo-config-generator \
--namespace oslo.concurrency \
--namespace oslo.config \
--namespace oslo.context \
--namespace oslo.log \
--namespace oslo.serialization \
--namespace oslo.utils \
--namespace oslo.db \
--namespace oslo.rootwrap \
--namespace oslo.messaging \
--namespace oslo.i18n \
--namespace oslo.middleware \
--namespace oslo.service.service \
--namespace oslo.service.periodic_task \
--namespace policy \
--namespace oslo.policy \
--namespace oslo.db.concurrency \
--namespace keystonemiddleware.auth_token | grep -v '^\[DEFAULT\]' >> $OUTPUTFILE
if [ $? -ne 0 ]
then
echo -en "\n\n#################################################"
echo -en "\nERROR: Non-zero exit from oslo-config-generator."
echo -en "\n See output above for details.\n"
echo -en "#################################################\n"
exit 1
fi
if [ ! -s ./etc/cinder/cinder.conf.sample ] ; then
echo -en "\n\n#########################################################"
echo -en "\nERROR: etc/cinder/cinder.sample.conf not created properly."
echo -en "\n See above output for details.\n"
echo -en "###########################################################\n"
exit 1
fi
rm -f cinder/opts.py
rm -f cinder/opts.pyc

View File

@ -1 +0,0 @@
export CINDER_CONFIG_GENERATOR_EXTRA_MODULES="keystonemiddleware.auth_token"

View File

@ -111,7 +111,7 @@ commands =
flake8 {posargs} . cinder/common
# Check that .po and .pot files are valid:
bash -c "find cinder -type f -regex '.*\.pot?' -print0|xargs -0 -n 1 msgfmt --check-format -o /dev/null"
{toxinidir}/tools/config/check_uptodate.sh --checkonly
{toxinidir}/tools/config/check_uptodate.sh
{toxinidir}/tools/check_exec.py {toxinidir}/cinder
[testenv:pylint]
@ -129,7 +129,7 @@ commands =
[testenv:genconfig]
sitepackages = False
envdir = {toxworkdir}/venv
commands = {toxinidir}/tools/config/generate_sample.sh -b . -p cinder -o etc/cinder
commands = {toxinidir}/tools/config/generate_sample.sh from_tox
[testenv:venv]
commands = {posargs}