Sync latest Oslo config code for i18n

This sync pulls the latest Oslo config code over
to Cinder.  This sync is needed to include a fix
to config generator which is needed as part of
the work to enabled i18n messages.  Config generator
was failing when lazy message translation was enabled.

The following changes are included with this patch. Each
patch includes the file(s) the patch touches:
-> c178e56 Add basic Python 3 tests
--> cinder/openstack/common/__init__.py
-> 12bcdb7 Remove vim header
--> cinder/openstack/common/__init__.py
-> 547ab34 Fix Copyright Headers - Rename LLC to Foundation
--> cinder/openstack/common/__init__.py
-> 44b6ea3 Import oslo.config configuration file generator
--> cinder/openstack/common/config/__init__.py
-> dd9aa2b Remove unused variables
--> cinder/openstack/common/config/generator.py
-> 5dce17b Use entry points to discover options in libraries
--> tools/config/generate_sample.sh
--> cinder/openstack/common/config/generator.py
-> e8e636c generator: add an EXTRA_LIBRARIES env variable
--> cinder/openstack/common/config/generator.py
-> e3dddd7 generator: use EXTRA_* env vars in the bash script
--> tools/config/generate_sample.sh
--> cinder/openstack/common/config/generator.py
-> 6da13e8 generator: rename EXTRA_MODULES_FILE to RC_FILE
--> tools/config/generate_sample.sh
-> 763eedf Fix DictOpt support in config sample generator
--> cinder/openstack/common/config/generator.py
-> e839886 Config generator fails with lazy messages
--> cinder/openstack/common/config/generator.py
-> 343686b Add check_uptodate to tools/config
--> tools/config/check_uptodate.sh

Oslo version:
-> 0f24d82 Fix migration.db_version when no tables
-> Date: Sat, 22 Feb 2014 00:32:18 +0000

Change-Id: I26a95fe96b08d6340b0fce1b9e2949c8e661a946
Closes-Bug: 1280826
Related-bp: i18n-messages
This commit is contained in:
Jay S. Bryant 2014-02-23 19:51:41 -06:00
parent f888e412b0
commit 9550b6b6b4
12 changed files with 283 additions and 188 deletions

View File

@ -1,15 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (c) 2011 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

68
cinder/openstack/common/config/generator.py Executable file → Normal file
View File

@ -1,4 +1,5 @@
# Copyright 2012 SINA Corporation
# Copyright 2014 Cisco Systems, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -18,6 +19,7 @@
from __future__ import print_function
import argparse
import imp
import os
import re
@ -27,6 +29,7 @@ import textwrap
from oslo.config import cfg
import six
import stevedore.named
from cinder.openstack.common import gettextutils
from cinder.openstack.common import importutils
@ -38,6 +41,7 @@ BOOLOPT = "BoolOpt"
INTOPT = "IntOpt"
FLOATOPT = "FloatOpt"
LISTOPT = "ListOpt"
DICTOPT = "DictOpt"
MULTISTROPT = "MultiStrOpt"
OPT_TYPES = {
@ -46,11 +50,12 @@ OPT_TYPES = {
INTOPT: 'integer value',
FLOATOPT: 'floating point value',
LISTOPT: 'list value',
DICTOPT: 'dict value',
MULTISTROPT: 'multi valued',
}
OPTION_REGEX = re.compile(r"(%s)" % "|".join([STROPT, BOOLOPT, INTOPT,
FLOATOPT, LISTOPT,
FLOATOPT, LISTOPT, DICTOPT,
MULTISTROPT]))
PY_EXT = ".py"
@ -59,34 +64,55 @@ BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
WORDWRAP_WIDTH = 60
def generate(srcfiles):
def generate(argv):
parser = argparse.ArgumentParser(
description='generate sample configuration file',
)
parser.add_argument('-m', dest='modules', action='append')
parser.add_argument('-l', dest='libraries', action='append')
parser.add_argument('srcfiles', nargs='*')
parsed_args = parser.parse_args(argv)
mods_by_pkg = dict()
for filepath in srcfiles:
for filepath in parsed_args.srcfiles:
pkg_name = filepath.split(os.sep)[1]
mod_str = '.'.join(['.'.join(filepath.split(os.sep)[:-1]),
os.path.basename(filepath).split('.')[0]])
mods_by_pkg.setdefault(pkg_name, list()).append(mod_str)
# NOTE(lzyeval): place top level modules before packages
pkg_names = filter(lambda x: x.endswith(PY_EXT), mods_by_pkg.keys())
pkg_names.sort()
ext_names = filter(lambda x: x not in pkg_names, mods_by_pkg.keys())
ext_names.sort()
pkg_names = sorted(pkg for pkg in mods_by_pkg if pkg.endswith(PY_EXT))
ext_names = sorted(pkg for pkg in mods_by_pkg if pkg not in pkg_names)
pkg_names.extend(ext_names)
# opts_by_group is a mapping of group name to an options list
# The options list is a list of (module, options) tuples
opts_by_group = {'DEFAULT': []}
extra_modules = os.getenv("OSLO_CONFIG_GENERATOR_EXTRA_MODULES", "")
if extra_modules:
for module_name in extra_modules.split(','):
module_name = module_name.strip()
if parsed_args.modules:
for module_name in parsed_args.modules:
module = _import_module(module_name)
if module:
for group, opts in _list_opts(module):
opts_by_group.setdefault(group, []).append((module_name,
opts))
# Look for entry points defined in libraries (or applications) for
# option discovery, and include their return values in the output.
#
# Each entry point should be a function returning an iterable
# of pairs with the group name (or None for the default group)
# and the list of Opt instances for that group.
if parsed_args.libraries:
loader = stevedore.named.NamedExtensionManager(
'oslo.config.opts',
names=list(set(parsed_args.libraries)),
invoke_on_load=False,
)
for ext in loader:
for group, opts in ext.plugin():
opt_list = opts_by_group.setdefault(group or 'DEFAULT', [])
opt_list.append((ext.name, opts))
for pkg_name in pkg_names:
mods = mods_by_pkg.get(pkg_name)
mods.sort()
@ -102,8 +128,8 @@ def generate(srcfiles):
opts_by_group.setdefault(group, []).append((mod_str, opts))
print_group_opts('DEFAULT', opts_by_group.pop('DEFAULT', []))
for group, opts in opts_by_group.items():
print_group_opts(group, opts)
for group in sorted(opts_by_group.keys()):
print_group_opts(group, opts_by_group[group])
def _import_module(mod_str):
@ -120,8 +146,10 @@ def _import_module(mod_str):
def _is_in_group(opt, group):
"Check if opt is in group."
for key, value in group._opts.items():
if value['opt'] == opt:
for value in group._opts.values():
# NOTE(llu): Temporary workaround for bug #1262148, wait until
# newly released oslo.config support '==' operator.
if not(value['opt'] != opt):
return True
return False
@ -132,7 +160,7 @@ def _guess_groups(opt, mod_obj):
return 'DEFAULT'
# what other groups is it in?
for key, value in cfg.CONF.items():
for value in cfg.CONF.values():
if isinstance(value, cfg.CONF.GroupAttr):
if _is_in_group(opt, value._group):
return value._group.name
@ -219,7 +247,8 @@ def _print_opt(opt):
except (ValueError, AttributeError) as err:
sys.stderr.write("%s\n" % str(err))
sys.exit(1)
opt_help += ' (' + OPT_TYPES[opt_type] + ')'
opt_help = u'%s (%s)' % (opt_help,
OPT_TYPES[opt_type])
print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH)))
if opt.deprecated_opts:
for deprecated_opt in opt.deprecated_opts:
@ -249,6 +278,11 @@ def _print_opt(opt):
elif opt_type == LISTOPT:
assert(isinstance(opt_default, list))
print('#%s=%s' % (opt_name, ','.join(opt_default)))
elif opt_type == DICTOPT:
assert(isinstance(opt_default, dict))
opt_default_strlist = [str(key) + ':' + str(value)
for (key, value) in opt_default.items()]
print('#%s=%s' % (opt_name, ','.join(opt_default_strlist)))
elif opt_type == MULTISTROPT:
assert(isinstance(opt_default, list))
if not opt_default:

View File

@ -1847,53 +1847,35 @@
#extra_capabilities={}
[fc-zone-manager]
[BRCD_FABRIC_EXAMPLE]
#
# Options defined in cinder.zonemanager.drivers.brocade.brcd_fc_zone_driver
# Options defined in cinder.zonemanager.drivers.brocade.brcd_fabric_opts
#
# Southbound connector for zoning operation (string value)
#brcd_sb_connector=cinder.zonemanager.drivers.brocade.brcd_fc_zone_client_cli.BrcdFCZoneClientCLI
# Management IP of fabric (string value)
#fc_fabric_address=
# Fabric user ID (string value)
#fc_fabric_user=
#
# Options defined in cinder.zonemanager.fc_zone_manager
#
# Password for user (string value)
#fc_fabric_password=
# FC Zone Driver responsible for zone management (string
# value)
#zone_driver=cinder.zonemanager.drivers.brocade.brcd_fc_zone_driver.BrcdFCZoneDriver
# Connecting port (integer value)
#fc_fabric_port=22
# Zoning policy configured by user (string value)
# overridden zoning policy (string value)
#zoning_policy=initiator-target
# Comma separated list of fibre channel fabric names. This
# list of names is used to retrieve other SAN credentials for
# connecting to each SAN fabric (string value)
#fc_fabric_names=<None>
# overridden zoning activation state (boolean value)
#zone_activate=true
# FC San Lookup Service (string value)
#fc_san_lookup_service=cinder.zonemanager.drivers.brocade.brcd_fc_san_lookup_service.BrcdFCSanLookupService
# overridden zone name prefix (string value)
#zone_name_prefix=<None>
[ssl]
#
# Options defined in cinder.openstack.common.sslutils
#
# CA certificate file to use to verify connecting clients
# (string value)
#ca_file=<None>
# Certificate file to use when starting the server securely
# (string value)
#cert_file=<None>
# Private key file to use when starting the server securely
# (string value)
#key_file=<None>
# Principal switch WWN of the fabric (string value)
#principal_switch_wwn=<None>
[database]
@ -1962,6 +1944,36 @@
#connection_trace=false
[fc-zone-manager]
#
# Options defined in cinder.zonemanager.drivers.brocade.brcd_fc_zone_driver
#
# Southbound connector for zoning operation (string value)
#brcd_sb_connector=cinder.zonemanager.drivers.brocade.brcd_fc_zone_client_cli.BrcdFCZoneClientCLI
#
# Options defined in cinder.zonemanager.fc_zone_manager
#
# FC Zone Driver responsible for zone management (string
# value)
#zone_driver=cinder.zonemanager.drivers.brocade.brcd_fc_zone_driver.BrcdFCZoneDriver
# Zoning policy configured by user (string value)
#zoning_policy=initiator-target
# Comma separated list of fibre channel fabric names. This
# list of names is used to retrieve other SAN credentials for
# connecting to each SAN fabric (string value)
#fc_fabric_names=<None>
# FC San Lookup Service (string value)
#fc_san_lookup_service=cinder.zonemanager.drivers.brocade.brcd_fc_san_lookup_service.BrcdFCSanLookupService
[keymgr]
#
@ -1982,32 +1994,6 @@
#fixed_key=<None>
[rpc_notifier2]
#
# Options defined in cinder.openstack.common.notifier.rpc_notifier2
#
# AMQP topic(s) used for OpenStack notifications (list value)
#topics=notifications
[matchmaker_redis]
#
# Options defined in cinder.openstack.common.rpc.matchmaker_redis
#
# Host to locate redis (string value)
#host=127.0.0.1
# Use this port to connect to redis host. (integer value)
#port=6379
# Password for Redis server. (optional) (string value)
#password=<None>
[keystone_authtoken]
#
@ -2139,35 +2125,20 @@
#enforce_token_bind=permissive
[BRCD_FABRIC_EXAMPLE]
[matchmaker_redis]
#
# Options defined in cinder.zonemanager.drivers.brocade.brcd_fabric_opts
# Options defined in cinder.openstack.common.rpc.matchmaker_redis
#
# Management IP of fabric (string value)
#fc_fabric_address=
# Host to locate redis (string value)
#host=127.0.0.1
# Fabric user ID (string value)
#fc_fabric_user=
# Use this port to connect to redis host. (integer value)
#port=6379
# Password for user (string value)
#fc_fabric_password=
# Connecting port (integer value)
#fc_fabric_port=22
# overridden zoning policy (string value)
#zoning_policy=initiator-target
# overridden zoning activation state (boolean value)
#zone_activate=true
# overridden zone name prefix (string value)
#zone_name_prefix=<None>
# Principal switch WWN of the fabric (string value)
#principal_switch_wwn=<None>
# Password for Redis server. (optional) (string value)
#password=<None>
[matchmaker_ring]
@ -2181,3 +2152,32 @@
#ringfile=/etc/oslo/matchmaker_ring.json
[rpc_notifier2]
#
# Options defined in cinder.openstack.common.notifier.rpc_notifier2
#
# AMQP topic(s) used for OpenStack notifications (list value)
#topics=notifications
[ssl]
#
# Options defined in cinder.openstack.common.sslutils
#
# CA certificate file to use to verify connecting clients
# (string value)
#ca_file=<None>
# Certificate file to use when starting the server securely
# (string value)
#cert_file=<None>
# Private key file to use when starting the server securely
# (string value)
#key_file=<None>

View File

@ -221,7 +221,7 @@ fi
if [ $just_pep8 -eq 1 ]; then
run_pep8
${wrapper} bash ./tools/conf/check_uptodate.sh
${wrapper} bash ./tools/config/check_uptodate.sh
exit
fi
@ -238,6 +238,6 @@ run_tests
if [ -z "$testrargs" ]; then
if [ $no_pep8 -eq 0 ]; then
run_pep8
${wrapper} bash ./tools/conf/check_uptodate.sh
${wrapper} bash ./tools/config/check_uptodate.sh
fi
fi

View File

@ -1,12 +0,0 @@
#!/bin/sh
TEMPDIR=`mktemp -d`
CFGFILE=cinder.conf.sample
tools/conf/generate_sample.sh -o $TEMPDIR
if ! diff $TEMPDIR/$CFGFILE etc/cinder/$CFGFILE
then
echo "E: cinder.conf.sample is not up to date, please run tools/conf/generate_sample.sh in venv"
echo "E: e.g. \$ source .venv/bin/activate; tools/conf/generate_sample.sh"
rm -rf $TEMPDIR
exit 42
fi
rm -rf $TEMPDIR

View File

@ -1,56 +0,0 @@
#!/usr/bin/env bash
print_hint() {
echo "Try \`${0##*/} --help' for more information." >&2
}
PARSED_OPTIONS=$(getopt -n "${0##*/}" -o ho: \
--long help,output-dir: -- "$@")
if [ $? != 0 ] ; then print_hint ; exit 1 ; fi
eval set -- "$PARSED_OPTIONS"
while true; do
case "$1" in
-h|--help)
echo "${0##*/} [options]"
echo ""
echo "options:"
echo "-h, --help show brief help"
echo "-o, --output-dir=DIR File output directory"
exit 0
;;
-o|--output-dir)
shift
OUTPUTDIR=`echo $1 | sed -e 's/\/*$//g'`
shift
;;
--)
break
;;
esac
done
OUTPUTDIR=${OUTPUTDIR:-etc/cinder}
if ! [ -d $OUTPUTDIR ]
then
echo "${0##*/}: cannot access \`$OUTPUTDIR': No such file or directory" >&2
exit 1
fi
OUTPUTFILE=$OUTPUTDIR/cinder.conf.sample
FILES=$(find cinder -type f -name "*.py" ! -path "cinder/tests/*" -exec \
grep -l "Opt(" {} \; | sort -u)
EXTRA_MODULES_FILE="`dirname $0`/oslo.config.generator.rc"
if test -r "$EXTRA_MODULES_FILE"
then
source "$EXTRA_MODULES_FILE"
fi
export EVENTLET_NO_GREENDNS=yes
PYTHONPATH=./:${PYTHONPATH} \
python $(dirname "$0")/../../cinder/openstack/common/config/generator.py ${FILES} > \
$OUTPUTFILE

View File

@ -1 +0,0 @@
export OSLO_CONFIG_GENERATOR_EXTRA_MODULES=keystoneclient.middleware.auth_token

25
tools/config/check_uptodate.sh Executable file
View File

@ -0,0 +1,25 @@
#!/usr/bin/env bash
PROJECT_NAME=${PROJECT_NAME:-cinder}
CFGFILE_NAME=${PROJECT_NAME}.conf.sample
if [ -e etc/${PROJECT_NAME}/${CFGFILE_NAME} ]; then
CFGFILE=etc/${PROJECT_NAME}/${CFGFILE_NAME}
elif [ -e etc/${CFGFILE_NAME} ]; then
CFGFILE=etc/${CFGFILE_NAME}
else
echo "${0##*/}: can not find config file"
exit 1
fi
TEMPDIR=`mktemp -d /tmp/${PROJECT_NAME}.XXXXXX`
trap "rm -rf $TEMPDIR" EXIT
tools/config/generate_sample.sh -b ./ -p ${PROJECT_NAME} -o ${TEMPDIR}
if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE}
then
echo "${0##*/}: ${PROJECT_NAME}.conf.sample is not up to date."
echo "${0##*/}: Please run ${0%%${0##*/}}generate_sample.sh."
exit 1
fi

119
tools/config/generate_sample.sh Executable file
View File

@ -0,0 +1,119 @@
#!/usr/bin/env bash
print_hint() {
echo "Try \`${0##*/} --help' for more information." >&2
}
PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:m:l:o: \
--long help,base-dir:,package-name:,output-dir:,module:,library: -- "$@")
if [ $? != 0 ] ; then print_hint ; exit 1 ; fi
eval set -- "$PARSED_OPTIONS"
while true; do
case "$1" in
-h|--help)
echo "${0##*/} [options]"
echo ""
echo "options:"
echo "-h, --help show brief help"
echo "-b, --base-dir=DIR project base directory"
echo "-p, --package-name=NAME project package name"
echo "-o, --output-dir=DIR file output directory"
echo "-m, --module=MOD extra python module to interrogate for options"
echo "-l, --library=LIB extra library that registers options for discovery"
exit 0
;;
-b|--base-dir)
shift
BASEDIR=`echo $1 | sed -e 's/\/*$//g'`
shift
;;
-p|--package-name)
shift
PACKAGENAME=`echo $1`
shift
;;
-o|--output-dir)
shift
OUTPUTDIR=`echo $1 | sed -e 's/\/*$//g'`
shift
;;
-m|--module)
shift
MODULES="$MODULES -m $1"
shift
;;
-l|--library)
shift
LIBRARIES="$LIBRARIES -l $1"
shift
;;
--)
break
;;
esac
done
BASEDIR=${BASEDIR:-`pwd`}
if ! [ -d $BASEDIR ]
then
echo "${0##*/}: missing project base directory" >&2 ; print_hint ; exit 1
elif [[ $BASEDIR != /* ]]
then
BASEDIR=$(cd "$BASEDIR" && pwd)
fi
PACKAGENAME=${PACKAGENAME:-${BASEDIR##*/}}
TARGETDIR=$BASEDIR/$PACKAGENAME
if ! [ -d $TARGETDIR ]
then
echo "${0##*/}: invalid project package name" >&2 ; print_hint ; exit 1
fi
OUTPUTDIR=${OUTPUTDIR:-$BASEDIR/etc}
# NOTE(bnemec): Some projects put their sample config in etc/,
# some in etc/$PACKAGENAME/
if [ -d $OUTPUTDIR/$PACKAGENAME ]
then
OUTPUTDIR=$OUTPUTDIR/$PACKAGENAME
elif ! [ -d $OUTPUTDIR ]
then
echo "${0##*/}: cannot access \`$OUTPUTDIR': No such file or directory" >&2
exit 1
fi
BASEDIRESC=`echo $BASEDIR | sed -e 's/\//\\\\\//g'`
find $TARGETDIR -type f -name "*.pyc" -delete
FILES=$(find $TARGETDIR -type f -name "*.py" ! -path "*/tests/*" \
-exec grep -l "Opt(" {} + | sed -e "s/^$BASEDIRESC\///g" | sort -u)
RC_FILE="`dirname $0`/oslo.config.generator.rc"
if test -r "$RC_FILE"
then
source "$RC_FILE"
fi
for mod in ${CINDER_CONFIG_GENERATOR_EXTRA_MODULES}; do
MODULES="$MODULES -m $mod"
done
for lib in ${CINDER_CONFIG_GENERATOR_EXTRA_LIBRARIES}; do
LIBRARIES="$LIBRARIES -l $lib"
done
export EVENTLET_NO_GREENDNS=yes
OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs)
[ "$OS_VARS" ] && eval "unset \$OS_VARS"
DEFAULT_MODULEPATH=cinder.openstack.common.config.generator
MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH}
OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample
python -m $MODULEPATH $MODULES $LIBRARIES $FILES > $OUTPUTFILE
# Hook to allow projects to append custom config file snippets
CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null)
for CONCAT_FILE in $CONCAT_FILES; do
cat $CONCAT_FILE >> $OUTPUTFILE
done

View File

@ -0,0 +1 @@
export CINDER_CONFIG_GENERATOR_EXTRA_MODULES=keystoneclient.middleware.auth_token

View File

@ -12,7 +12,7 @@ deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
commands =
python setup.py testr --slowest --testr-args='--concurrency 1 {posargs}'
{toxinidir}/tools/conf/check_uptodate.sh
{toxinidir}/tools/config/check_uptodate.sh
[tox:jenkins]
downloadcache = ~/cache/pip