Use cfg's new global CONF object

Implements blueprint cfg-global-object

Change-Id: Ic53b41dafa8666ce21f33697f7e8697f1e5cb0fd

bin/keystone-all

@@ -73,7 +73,7 @@ if __name__ == '__main__':
     if os.path.exists(dev_conf):
         config_files = [dev_conf]
 
-    CONF(config_files=config_files, args=sys.argv)
+    CONF(project='keystone', default_config_files=config_files)
 
     config.setup_logging(CONF)
 

keystone/cli.py

@@ -23,9 +23,7 @@ import textwrap
 from keystone import config
 from keystone.openstack.common import importutils
 
-
 CONF = config.CONF
-CONF.set_usage('%prog COMMAND')
 
 
 class BaseApp(object):
@@ -136,7 +134,10 @@ def run(cmd, args):
 
 def main(argv=None, config_files=None):
     CONF.reset()
-    args = CONF(config_files=config_files, args=argv)
+    args = CONF(args=argv,
+                project='keystone',
+                usage='%prog COMMAND',
+                default_config_files=config_files)
 
     if len(args) < 2:
         CONF.print_help()

keystone/config.py

@@ -25,24 +25,7 @@ from keystone.openstack.common import cfg
 gettext.install('keystone', unicode=1)
 
 
-class ConfigMixin(object):
+CONF = cfg.CONF
-    def __call__(self, config_files=None, *args, **kw):
-        if config_files is not None:
-            self._opts['config_file']['opt'].default = config_files
-        kw.setdefault('args', [])
-        return super(ConfigMixin, self).__call__(*args, **kw)
-
-    def set_usage(self, usage):
-        self.usage = usage
-        self._oparser.usage = usage
-
-
-class Config(ConfigMixin, cfg.ConfigOpts):
-    pass
-
-
-class CommonConfig(ConfigMixin, cfg.CommonConfigOpts):
-    pass
 
 
 def setup_logging(conf):
@@ -128,9 +111,6 @@ def register_cli_int(*args, **kw):
     return conf.register_cli_opt(cfg.IntOpt(*args, **kw), group=group)
 
 
-CONF = CommonConfig(project='keystone')
-
-
 register_str('admin_token', default='ADMIN')
 register_str('bind_host', default='0.0.0.0')
 register_str('compute_port', default=8774)

keystone/openstack/common/cfg.py

@@ -95,7 +95,7 @@ and --config-dir::
 
     class ConfigOpts(object):
 
-        def __init__(self, ...):
+        def __call__(self, ...):
 
             opts = [
                 MultiStrOpt('config-file',
@@ -233,12 +233,28 @@ log files:
         ...
      ]
 
+This module also contains a global instance of the CommonConfigOpts class
+in order to support a common usage pattern in OpenStack:
+
+  from openstack.common import cfg
+
+  opts = [
+    cfg.StrOpt('bind_host' default='0.0.0.0'),
+    cfg.IntOpt('bind_port', default=9292),
+  ]
+
+  CONF = cfg.CONF
+  CONF.register_opts(opts)
+
+  def start(server, app):
+      server.start(app, CONF.bind_port, CONF.bind_host)
+
 """
 
 import collections
 import copy
-import glob
 import functools
+import glob
 import optparse
 import os
 import string
@@ -768,6 +784,14 @@ class OptGroup(object):
 
         return True
 
+    def _unregister_opt(self, opt):
+        """Remove an opt from this group.
+
+        :param opt: an Opt object
+        """
+        if opt.dest in self._opts:
+            del self._opts[opt.dest]
+
     def _get_optparse_group(self, parser):
         """Build an optparse.OptionGroup for this group."""
         if self._optparse_group is None:
@@ -775,6 +799,10 @@ class OptGroup(object):
                                                         self.help)
         return self._optparse_group
 
+    def _clear(self):
+        """Clear this group's option parsing state."""
+        self._optparse_group = None
+
 
 class ParseError(iniparser.ParseError):
     def __init__(self, msg, lineno, line, filename):
@@ -849,57 +877,41 @@ class ConfigOpts(collections.Mapping):
     the values of options.
     """
 
-    def __init__(self,
+    def __init__(self):
-                 project=None,
+        """Construct a ConfigOpts object."""
-                 prog=None,
+        self._opts = {}  # dict of dicts of (opt:, override:, default:)
-                 version=None,
+        self._groups = {}
-                 usage=None,
-                 default_config_files=None):
-        """Construct a ConfigOpts object.
 
-        Automatically registers the --config-file option with either a supplied
+        self._args = None
-        list of default config files, or a list from find_config_files().
+        self._oparser = None
+        self._cparser = None
+        self._cli_values = {}
+        self.__cache = {}
+        self._config_opts = []
+        self._disable_interspersed_args = False
 
-        :param project: the toplevel project name, used to locate config files
+    def _setup(self, project, prog, version, usage, default_config_files):
-        :param prog: the name of the program (defaults to sys.argv[0] basename)
+        """Initialize a ConfigOpts object for option parsing."""
-        :param version: the program version (for --version)
-        :param usage: a usage string (%prog will be expanded)
-        :param default_config_files: config files to use by default
-        """
         if prog is None:
             prog = os.path.basename(sys.argv[0])
 
         if default_config_files is None:
             default_config_files = find_config_files(project, prog)
 
-        self.project = project
+        self._oparser = optparse.OptionParser(prog=prog,
-        self.prog = prog
+                                              version=version,
-        self.version = version
+                                              usage=usage)
-        self.usage = usage
+        if self._disable_interspersed_args:
-        self.default_config_files = default_config_files
+            self._oparser.disable_interspersed_args()
 
-        self._opts = {}  # dict of dicts of (opt:, override:, default:)
+        self._config_opts = [
-        self._groups = {}
-
-        self._args = None
-        self._cli_values = {}
-
-        self._oparser = optparse.OptionParser(prog=self.prog,
-                                              version=self.version,
-                                              usage=self.usage)
-        self._cparser = None
-
-        self.__cache = {}
-
-        opts = [
              MultiStrOpt('config-file',
-                         default=self.default_config_files,
+                         default=default_config_files,
                          metavar='PATH',
                          help='Path to a config file to use. Multiple config '
                               'files can be specified, with values in later '
                               'files taking precedence. The default files '
-                              ' used are: %s' %
+                              ' used are: %s' % (default_config_files, )),
-                              (self.default_config_files, )),
             StrOpt('config-dir',
                    metavar='DIR',
                    help='Path to a config directory to pull *.conf '
@@ -910,7 +922,13 @@ class ConfigOpts(collections.Mapping):
                         'hence over-ridden options in the directory take '
                         'precedence.'),
             ]
-        self.register_cli_opts(opts)
+        self.register_cli_opts(self._config_opts)
+
+        self.project = project
+        self.prog = prog
+        self.version = version
+        self.usage = usage
+        self.default_config_files = default_config_files
 
     def __clear_cache(f):
         @functools.wraps(f)
@@ -921,7 +939,13 @@ class ConfigOpts(collections.Mapping):
 
         return __inner
 
-    def __call__(self, args=None):
+    def __call__(self,
+                 args=None,
+                 project=None,
+                 prog=None,
+                 version=None,
+                 usage=None,
+                 default_config_files=None):
         """Parse command line arguments and config files.
 
         Calling a ConfigOpts object causes the supplied command line arguments
@@ -931,35 +955,34 @@ class ConfigOpts(collections.Mapping):
         The object may be called multiple times, each time causing the previous
         set of values to be overwritten.
 
+        Automatically registers the --config-file option with either a supplied
+        list of default config files, or a list from find_config_files().
+
         If the --config-dir option is set, any *.conf files from this
         directory are pulled in, after all the file(s) specified by the
         --config-file option.
 
-        :params args: command line arguments (defaults to sys.argv[1:])
+        :param args: command line arguments (defaults to sys.argv[1:])
+        :param project: the toplevel project name, used to locate config files
+        :param prog: the name of the program (defaults to sys.argv[0] basename)
+        :param version: the program version (for --version)
+        :param usage: a usage string (%prog will be expanded)
+        :param default_config_files: config files to use by default
         :returns: the list of arguments left over after parsing options
         :raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError,
-                 RequiredOptError
+                 RequiredOptError, DuplicateOptError
         """
         self.clear()
 
-        self._args = args
+        self._setup(project, prog, version, usage, default_config_files)
 
-        (values, args) = self._oparser.parse_args(self._args)
+        self._cli_values, leftovers = self._parse_cli_opts(args)
 
-        self._cli_values = vars(values)
+        self._parse_config_files()
-
-        def _list_config_dir():
-            return sorted(glob.glob(os.path.join(self.config_dir, '*.conf')))
-
-        from_file = list(self.config_file)
-
-        from_dir = _list_config_dir() if self.config_dir else []
-
-        self._parse_config_files(from_file + from_dir)
 
         self._check_required_opts()
 
-        return args
+        return leftovers
 
     def __getattr__(self, name):
         """Look up an option value and perform string substitution.
@@ -996,8 +1019,12 @@ class ConfigOpts(collections.Mapping):
     def clear(self):
         """Clear the state of the object to before it was called."""
         self._args = None
-        self._cli_values = {}
+        self._cli_values.clear()
+        self._oparser = None
         self._cparser = None
+        self.unregister_opts(self._config_opts)
+        for group in self._groups.values():
+            group._clear()
 
     @__clear_cache
     def register_opt(self, opt, group=None):
@@ -1044,15 +1071,7 @@ class ConfigOpts(collections.Mapping):
         if self._args is not None:
             raise ArgsAlreadyParsedError("cannot register CLI option")
 
-        if not self.register_opt(opt, group, clear_cache=False):
+        return self.register_opt(opt, group, clear_cache=False)
-            return False
-
-        if group is not None:
-            group = self._get_group(group, autocreate=True)
-
-        opt._add_to_cli(self._oparser, group)
-
-        return True
 
     @__clear_cache
     def register_cli_opts(self, opts, group=None):
@@ -1073,6 +1092,28 @@ class ConfigOpts(collections.Mapping):
 
         self._groups[group.name] = copy.copy(group)
 
+    @__clear_cache
+    def unregister_opt(self, opt, group=None):
+        """Unregister an option.
+
+        :param opt: an Opt object
+        :param group: an optional OptGroup object or group name
+        :raises: ArgsAlreadyParsedError, NoSuchGroupError
+        """
+        if self._args is not None:
+            raise ArgsAlreadyParsedError("reset before unregistering options")
+
+        if group is not None:
+            self._get_group(group)._unregister_opt(opt)
+        elif opt.dest in self._opts:
+            del self._opts[opt.dest]
+
+    @__clear_cache
+    def unregister_opts(self, opts, group=None):
+        """Unregister multiple CLI option schemas at once."""
+        for opt in opts:
+            self.unregister_opt(opt, group, clear_cache=False)
+
     @__clear_cache
     def set_override(self, name, override, group=None):
         """Override an opt value.
@@ -1103,16 +1144,24 @@ class ConfigOpts(collections.Mapping):
         opt_info = self._get_opt_info(name, group)
         opt_info['default'] = default
 
+    def _all_opt_infos(self):
+        """A generator function for iteration opt infos."""
+        for info in self._opts.values():
+            yield info, None
+        for group in self._groups.values():
+            for info in group._opts.values():
+                yield info, group
+
+    def _all_opts(self):
+        """A generator function for iteration opts."""
+        for info, group in self._all_opt_infos():
+            yield info['opt'], group
+
     def _unset_defaults_and_overrides(self):
         """Unset any default or override on all options."""
-        def unset(opts):
+        for info, group in self._all_opt_infos():
-            for info in opts.values():
+            info['default'] = None
-                info['default'] = None
+            info['override'] = None
-                info['override'] = None
-
-        unset(self._opts)
-        for group in self._groups.values():
-            unset(group._opts)
 
     def disable_interspersed_args(self):
         """Set parsing to stop on the first non-option.
@@ -1131,13 +1180,13 @@ class ConfigOpts(collections.Mapping):
 
         i.e. argument parsing is stopped at the first non-option argument.
         """
-        self._oparser.disable_interspersed_args()
+        self._disable_interspersed_args = True
 
     def enable_interspersed_args(self):
         """Set parsing to not stop on the first non-option.
 
         This it the default behaviour."""
-        self._oparser.enable_interspersed_args()
+        self._disable_interspersed_args = False
 
     def find_file(self, name):
         """Locate a file located alongside the config files.
@@ -1331,11 +1380,17 @@ class ConfigOpts(collections.Mapping):
 
         return opts[opt_name]
 
-    def _parse_config_files(self, config_files):
+    def _parse_config_files(self):
-        """Parse the supplied configuration files.
+        """Parse the config files from --config-file and --config-dir.
 
         :raises: ConfigFilesNotFoundError, ConfigFileParseError
         """
+        config_files = list(self.config_file)
+
+        if self.config_dir:
+            config_dir_glob = os.path.join(self.config_dir, '*.conf')
+            config_files += sorted(glob.glob(config_dir_glob))
+
         self._cparser = MultiConfigParser()
 
         try:
@@ -1347,8 +1402,12 @@ class ConfigOpts(collections.Mapping):
             not_read_ok = filter(lambda f: f not in read_ok, config_files)
             raise ConfigFilesNotFoundError(not_read_ok)
 
-    def _do_check_required_opts(self, opts, group=None):
+    def _check_required_opts(self):
-        for info in opts.values():
+        """Check that all opts marked as required have values specified.
+
+        :raises: RequiredOptError
+        """
+        for info, group in self._all_opt_infos():
             default, opt, override = [info[k] for k in sorted(info.keys())]
 
             if opt.required:
@@ -1359,15 +1418,25 @@ class ConfigOpts(collections.Mapping):
                 if self._get(opt.name, group) is None:
                     raise RequiredOptError(opt.name, group)
 
-    def _check_required_opts(self):
+    def _parse_cli_opts(self, args):
-        """Check that all opts marked as required have values specified.
+        """Parse command line options.
+
+        Initializes the command line option parser and parses the supplied
+        command line arguments.
+
+        :param args: the command line arguments
+        :returns: a dict of parsed option values
+        :raises: SystemExit, DuplicateOptError
 
-        :raises: RequiredOptError
         """
-        self._do_check_required_opts(self._opts)
+        self._args = args
 
-        for group in self._groups.values():
+        for opt, group in self._all_opts():
-            self._do_check_required_opts(group._opts, group)
+            opt._add_to_cli(self._oparser, group)
+
+        values, leftovers = self._oparser.parse_args(args)
+
+        return vars(values), leftovers
 
     class GroupAttr(collections.Mapping):
 
@@ -1483,7 +1552,10 @@ class CommonConfigOpts(ConfigOpts):
                help='syslog facility to receive log lines')
         ]
 
-    def __init__(self, **kwargs):
+    def __init__(self):
-        super(CommonConfigOpts, self).__init__(**kwargs)
+        super(CommonConfigOpts, self).__init__()
         self.register_cli_opts(self.common_cli_opts)
         self.register_cli_opts(self.logging_cli_opts)
+
+
+CONF = CommonConfigOpts()

keystone/test.py

@@ -165,13 +165,13 @@ class TestCase(NoModule, unittest.TestCase):
 
     def setUp(self):
         super(TestCase, self).setUp()
-        self.config()
+        self.config([etcdir('keystone.conf.sample'),
+                     testsdir('test_overrides.conf')])
         self.mox = mox.Mox()
         self.stubs = stubout.StubOutForTesting()
 
-    def config(self):
+    def config(self, config_files):
-        CONF(config_files=[etcdir('keystone.conf.sample'),
+        CONF(args=[], project='keystone', default_config_files=config_files)
-                           testsdir('test_overrides.conf')])
 
     def tearDown(self):
         try:

tests/_ldap_livetest.py

@@ -60,9 +60,9 @@ def clear_live_database():
 class LDAPIdentity(test.TestCase, test_backend.IdentityTests):
     def setUp(self):
         super(LDAPIdentity, self).setUp()
-        CONF(config_files=[test.etcdir('keystone.conf.sample'),
+        self.config([test.etcdir('keystone.conf.sample'),
-                           test.testsdir('test_overrides.conf'),
+                     test.testsdir('test_overrides.conf'),
-                           test.testsdir('backend_liveldap.conf')])
+                     test.testsdir('backend_liveldap.conf')])
         clear_live_database()
         self.identity_api = identity_ldap.Identity()
         self.load_fixtures(default_fixtures)

tests/test_backend_ldap.py

@@ -34,9 +34,9 @@ def clear_database():
 class LDAPIdentity(test.TestCase, test_backend.IdentityTests):
     def setUp(self):
         super(LDAPIdentity, self).setUp()
-        CONF(config_files=[test.etcdir('keystone.conf.sample'),
+        self.config([test.etcdir('keystone.conf.sample'),
-                           test.testsdir('test_overrides.conf'),
+                     test.testsdir('test_overrides.conf'),
-                           test.testsdir('backend_ldap.conf')])
+                     test.testsdir('backend_ldap.conf')])
         clear_database()
         self.identity_api = identity_ldap.Identity()
         self.load_fixtures(default_fixtures)

tests/test_backend_sql.py

@@ -33,9 +33,9 @@ CONF = config.CONF
 class SqlIdentity(test.TestCase, test_backend.IdentityTests):
     def setUp(self):
         super(SqlIdentity, self).setUp()
-        CONF(config_files=[test.etcdir('keystone.conf.sample'),
+        self.config([test.etcdir('keystone.conf.sample'),
-                           test.testsdir('test_overrides.conf'),
+                     test.testsdir('test_overrides.conf'),
-                           test.testsdir('backend_sql.conf')])
+                     test.testsdir('backend_sql.conf')])
         sql_util.setup_test_database()
         self.identity_api = identity_sql.Identity()
         self.load_fixtures(default_fixtures)
@@ -135,9 +135,9 @@ class SqlIdentity(test.TestCase, test_backend.IdentityTests):
 class SqlToken(test.TestCase, test_backend.TokenTests):
     def setUp(self):
         super(SqlToken, self).setUp()
-        CONF(config_files=[test.etcdir('keystone.conf.sample'),
+        self.config([test.etcdir('keystone.conf.sample'),
-                           test.testsdir('test_overrides.conf'),
+                     test.testsdir('test_overrides.conf'),
-                           test.testsdir('backend_sql.conf')])
+                     test.testsdir('backend_sql.conf')])
         sql_util.setup_test_database()
         self.token_api = token_sql.Token()
 

tests/test_import_legacy.py

@@ -33,9 +33,9 @@ CONF = config.CONF
 class ImportLegacy(test.TestCase):
     def setUp(self):
         super(ImportLegacy, self).setUp()
-        CONF(config_files=[test.etcdir('keystone.conf.sample'),
+        self.config([test.etcdir('keystone.conf.sample'),
-                           test.testsdir('test_overrides.conf'),
+                     test.testsdir('test_overrides.conf'),
-                           test.testsdir('backend_sql.conf')])
+                     test.testsdir('backend_sql.conf')])
         sql_util.setup_test_database()
         self.identity_api = identity_sql.Identity()
 

tests/test_keystoneclient_sql.py

@@ -27,10 +27,11 @@ CONF = config.CONF
 
 
 class KcMasterSqlTestCase(test_keystoneclient.KcMasterTestCase):
-    def config(self):
+    def config(self, config_files):
-        CONF(config_files=[test.etcdir('keystone.conf.sample'),
+        super(KcMasterSqlTestCase, self).config([
-                           test.testsdir('test_overrides.conf'),
+                test.etcdir('keystone.conf.sample'),
-                           test.testsdir('backend_sql.conf')])
+                test.testsdir('test_overrides.conf'),
+                test.testsdir('backend_sql.conf')])
         sql_util.setup_test_database()
 
     def test_endpoint_crud(self):

tests/test_migrate_nova_auth.py

@@ -68,9 +68,9 @@ FIXTURE = {
 class MigrateNovaAuth(test.TestCase):
     def setUp(self):
         super(MigrateNovaAuth, self).setUp()
-        CONF(config_files=[test.etcdir('keystone.conf.sample'),
+        self.config([test.etcdir('keystone.conf.sample'),
-                           test.testsdir('test_overrides.conf'),
+                     test.testsdir('test_overrides.conf'),
-                           test.testsdir('backend_sql.conf')])
+                     test.testsdir('backend_sql.conf')])
         sql_util.setup_test_database()
         self.identity_api = identity_sql.Identity()
         self.ec2_api = ec2_sql.Ec2()