Add config_source option

Define a config_source option that can be used to specify the
alternative sources that require drivers to load configuration
settings.

Co-Authored-By: Moises Guimaraes de Medeiros <moguimar@redhat.com>
Change-Id: Ibd5a6d306bb98d30d973dfe3604dcc0691d2e369
Blueprint: oslo-config-drivers
Signed-off-by: Doug Hellmann <doug@doughellmann.com>

lower-constraints.txt

@@ -36,7 +36,8 @@ python-subunit==1.0.0
 pytz==2013.6
 PyYAML==3.12
 reno==2.5.0
-requests==2.14.2
+requests==2.18.0
+requests_mock==1.5.0
 requestsexceptions==1.2.0
 rfc3986==0.3.1
 six==1.10.0

oslo_config/_list_opts.py

@@ -26,7 +26,8 @@ def list_opts():
         '/etc/project/project.conf.d/',
         '/etc/project.conf.d/',
     ]
-    return [
+    options = cfg.ConfigOpts._list_options_for_discovery(
-        (None, cfg.ConfigOpts._make_config_options(default_config_files,
+        default_config_files,
-                                                   default_config_dirs)),
+        default_config_dirs,
-    ]
+    )
+    return [(None, options)]

oslo_config/cfg.py

@@ -500,8 +500,11 @@ import enum
 import six
 
 from oslo_config import iniparser
+from oslo_config import sources
 from oslo_config import types
 
+from stevedore.extension import ExtensionManager
+
 LOG = logging.getLogger(__name__)
 
 
@@ -2351,6 +2354,16 @@ class ConfigOpts(collections.Mapping):
     disallow_names = ('project', 'prog', 'version',
                       'usage', 'default_config_files', 'default_config_dirs')
 
+    # NOTE(dhellmann): This instance is reused by list_opts().
+    _config_source_opt = ListOpt(
+        'config_source',
+        metavar='SOURCE',
+        default=[],
+        help=('Lists configuration groups that provide more '
+              'details for accessing configuration settings '
+              'from locations other than local files.'),
+    )
+
     def __init__(self):
         """Construct a ConfigOpts object."""
         self._opts = {}  # dict of dicts of (opt:, override:, default:)
@@ -2367,6 +2380,10 @@ class ConfigOpts(collections.Mapping):
         self._config_opts = []
         self._cli_opts = collections.deque()
         self._validate_default_values = False
+        self._sources = []
+        self._ext_mgr = None
+
+        self.register_opt(self._config_source_opt)
 
     def _pre_setup(self, project, prog, version, usage, description, epilog,
                    default_config_files, default_config_dirs):
@@ -2417,6 +2434,16 @@ class ConfigOpts(collections.Mapping):
                                'precedence.'),
         ]
 
+    @classmethod
+    def _list_options_for_discovery(cls,
+                                    default_config_files,
+                                    default_config_dirs):
+        "Return options to be used by list_opts() for the sample generator."
+        options = cls._make_config_options(default_config_files,
+                                           default_config_dirs)
+        options.append(cls._config_source_opt)
+        return options
+
     def _setup(self, project, prog, version, usage, default_config_files,
                default_config_dirs):
         """Initialize a ConfigOpts object for option parsing."""
@@ -2504,8 +2531,57 @@ class ConfigOpts(collections.Mapping):
             raise ConfigFilesPermissionDeniedError(
                 self._namespace._files_permission_denied)
 
+        self._load_alternative_sources()
+
         self._check_required_opts()
 
+    def _load_alternative_sources(self):
+        # Look for other sources of option data.
+        for source_group_name in self.config_source:
+            source = self._open_source_from_opt_group(source_group_name)
+            if source is not None:
+                self._sources.append(source)
+
+    def _open_source_from_opt_group(self, group_name):
+        if not self._ext_mgr:
+            self._ext_mgr = ExtensionManager(
+                "oslo.config.driver",
+                invoke_on_load=True)
+
+        self.register_opt(
+            StrOpt('driver',
+                   choices=self._ext_mgr.names(),
+                   help=('The name of the driver that can load this '
+                         'configuration source.')),
+            group=group_name)
+
+        try:
+            driver_name = self[group_name].driver
+        except ConfigFileValueError as err:
+            LOG.error(
+                "could not load configuration from %r. %s",
+                group_name, err.msg)
+            return None
+
+        if driver_name is None:
+            LOG.error(
+                "could not load configuration from %r, no 'driver' is set.",
+                group_name)
+            return None
+
+        LOG.info('loading configuration from %r using %r',
+                 group_name, driver_name)
+
+        driver = self._ext_mgr[driver_name].obj
+
+        try:
+            return driver.open_source_from_opt_group(self, group_name)
+        except Exception as err:
+            LOG.error(
+                "could not load configuration from %r using %s driver: %s",
+                group_name, driver_name, err)
+            return None
+
     def __getattr__(self, name):
         """Look up an option value and perform string substitution.
 
@@ -2982,12 +3058,13 @@ class ConfigOpts(collections.Mapping):
             return self._convert_value(
                 self._substitute(value, group, namespace), opt)
 
+        group_name = group.name if group else None
+
         if opt.mutable and namespace is None:
             namespace = self._mutable_ns
         if namespace is None:
             namespace = self._namespace
         if namespace is not None:
-            group_name = group.name if group else None
             try:
                 val, alt_loc = opt._get_from_namespace(namespace, group_name)
                 return (convert(val), alt_loc)
@@ -2998,6 +3075,11 @@ class ConfigOpts(collections.Mapping):
                     "Value for option %s is not valid: %s"
                     % (opt.name, str(ve)))
 
+        for source in self._sources:
+            val = source.get(group_name, name, opt)
+            if val[0] != sources._NoValue:
+                return (convert(val[0]), val[1])
+
         if 'default' in info:
             return (self._substitute(info['default']), loc)
 

oslo_config/sources/_uri.py

@@ -17,8 +17,8 @@ from oslo_config import cfg
 from oslo_config import sources
 
 
-class INIConfigurationSourceDriver(sources.ConfigurationSourceDriver):
+class URIConfigurationSourceDriver(sources.ConfigurationSourceDriver):
-    """A configuration source driver for INI files served through http[s].
+    """A configuration source driver for remote files served through http[s].
 
     Required options:
       - uri: URI containing the file location.
@@ -47,15 +47,15 @@ class INIConfigurationSourceDriver(sources.ConfigurationSourceDriver):
         conf.register_opt(cfg.StrOpt("client_cert"), group)
         conf.register_opt(cfg.StrOpt("client_key"), group)
 
-        return INIConfigurationSource(
+        return URIConfigurationSource(
             conf[group_name].uri,
             conf[group_name].ca_path,
             conf[group_name].client_cert,
             conf[group_name].client_key)
 
 
-class INIConfigurationSource(sources.ConfigurationSource):
+class URIConfigurationSource(sources.ConfigurationSource):
-    """A configuration source for INI files server through http[s].
+    """A configuration source for remote files served through http[s].
 
     :uri: The Uniform Resource Identifier of the configuration to be
           retrieved.
@@ -71,7 +71,7 @@ class INIConfigurationSource(sources.ConfigurationSource):
                  specified but does not includes the private key.
     """
 
-    def __init__(self, uri, ca_path, client_cert, client_key):
+    def __init__(self, uri, ca_path=None, client_cert=None, client_key=None):
         self._uri = uri
         self._namespace = cfg._Namespace(cfg.ConfigOpts())
 
@@ -102,7 +102,9 @@ class INIConfigurationSource(sources.ConfigurationSource):
         :type option_name: str
         :param opt: The option definition.
         :type opt: Opt
-        :return: Option value or NoValue.
+        :returns: A tuple (value, location) where value is the option value
+                  or oslo_config.sources._NoValue if the (group, option) is
+                  not present in the source, and location is a LocationInfo.
         """
         try:
             return self._namespace._get_value(

oslo_config/tests/test_cfg.py

@@ -2472,7 +2472,7 @@ class MappingInterfaceTestCase(BaseTestCase):
 
         self.assertIn('foo', self.conf)
         self.assertIn('config_file', self.conf)
-        self.assertEqual(len(self.conf), 3)
+        self.assertEqual(len(self.conf), 4)
         self.assertEqual('bar', self.conf['foo'])
         self.assertEqual('bar', self.conf.get('foo'))
         self.assertIn('bar', list(self.conf.values()))
@@ -3874,6 +3874,7 @@ class OptDumpingTestCase(BaseTestCase):
                          "=" * 80,
                          "config_dir                     = []",
                          "config_file                    = []",
+                         "config_source                  = []",
                          "foo                            = this",
                          "passwd                         = ****",
                          "blaa.bar                       = that",
@@ -3900,6 +3901,7 @@ class OptDumpingTestCase(BaseTestCase):
                          "command line args: None",
                          "config files: []",
                          "=" * 80,
+                         "config_source                  = []",
                          "*" * 80,
                          ], logger.logged)
 

oslo_config/tests/test_cfgfilter.py

@@ -38,11 +38,11 @@ class RegisterTestCase(BaseTestCase):
         self.assertEqual('bar', self.fconf.foo)
         self.assertEqual('bar', self.fconf['foo'])
         self.assertIn('foo', self.fconf)
-        self.assertEqual(['foo'], list(self.fconf))
+        self.assertEqual(set(['config_source', 'foo']), set(self.fconf))
-        self.assertEqual(1, len(self.fconf))
+        self.assertEqual(2, len(self.fconf))
 
         self.assertNotIn('foo', self.conf)
-        self.assertEqual(0, len(self.conf))
+        self.assertEqual(1, len(self.conf))
         self.assertRaises(cfg.NoSuchOptError, getattr, self.conf, 'foo')
 
     def test_register_opt_none_default(self):
@@ -51,11 +51,11 @@ class RegisterTestCase(BaseTestCase):
         self.assertIsNone(self.fconf.foo)
         self.assertIsNone(self.fconf['foo'])
         self.assertIn('foo', self.fconf)
-        self.assertEqual(['foo'], list(self.fconf))
+        self.assertEqual(set(['config_source', 'foo']), set(self.fconf))
-        self.assertEqual(1, len(self.fconf))
+        self.assertEqual(2, len(self.fconf))
 
         self.assertNotIn('foo', self.conf)
-        self.assertEqual(0, len(self.conf))
+        self.assertEqual(1, len(self.conf))
         self.assertRaises(cfg.NoSuchOptError, getattr, self.conf, 'foo')
 
     def test_register_grouped_opt_default(self):
@@ -66,13 +66,13 @@ class RegisterTestCase(BaseTestCase):
         self.assertEqual('bar', self.fconf['blaa']['foo'])
         self.assertIn('blaa', self.fconf)
         self.assertIn('foo', self.fconf.blaa)
-        self.assertEqual(['blaa'], list(self.fconf))
+        self.assertEqual(set(['config_source', 'blaa']), set(self.fconf))
         self.assertEqual(['foo'], list(self.fconf.blaa))
-        self.assertEqual(1, len(self.fconf))
+        self.assertEqual(2, len(self.fconf))
         self.assertEqual(1, len(self.fconf.blaa))
 
         self.assertNotIn('blaa', self.conf)
-        self.assertEqual(0, len(self.conf))
+        self.assertEqual(1, len(self.conf))
         self.assertRaises(cfg.NoSuchOptError, getattr, self.conf, 'blaa')
 
     def test_register_grouped_opt_none_default(self):
@@ -82,13 +82,13 @@ class RegisterTestCase(BaseTestCase):
         self.assertIsNone(self.fconf['blaa']['foo'])
         self.assertIn('blaa', self.fconf)
         self.assertIn('foo', self.fconf.blaa)
-        self.assertEqual(['blaa'], list(self.fconf))
+        self.assertEqual(set(['config_source', 'blaa']), set(self.fconf))
         self.assertEqual(['foo'], list(self.fconf.blaa))
-        self.assertEqual(1, len(self.fconf))
+        self.assertEqual(2, len(self.fconf))
         self.assertEqual(1, len(self.fconf.blaa))
 
         self.assertNotIn('blaa', self.conf)
-        self.assertEqual(0, len(self.conf))
+        self.assertEqual(1, len(self.conf))
         self.assertRaises(cfg.NoSuchOptError, getattr, self.conf, 'blaa')
 
     def test_register_group(self):
@@ -100,13 +100,13 @@ class RegisterTestCase(BaseTestCase):
         self.assertIsNone(self.fconf['blaa']['foo'])
         self.assertIn('blaa', self.fconf)
         self.assertIn('foo', self.fconf.blaa)
-        self.assertEqual(['blaa'], list(self.fconf))
+        self.assertEqual(set(['config_source', 'blaa']), set(self.fconf))
         self.assertEqual(['foo'], list(self.fconf.blaa))
-        self.assertEqual(1, len(self.fconf))
+        self.assertEqual(2, len(self.fconf))
         self.assertEqual(1, len(self.fconf.blaa))
 
         self.assertNotIn('blaa', self.conf)
-        self.assertEqual(0, len(self.conf))
+        self.assertEqual(1, len(self.conf))
         self.assertRaises(cfg.NoSuchOptError, getattr, self.conf, 'blaa')
 
     def test_register_opts(self):
@@ -181,7 +181,7 @@ class RegisterTestCase(BaseTestCase):
 
     def test_unknown_opt(self):
         self.assertNotIn('foo', self.fconf)
-        self.assertEqual(0, len(self.fconf))
+        self.assertEqual(1, len(self.fconf))
         self.assertRaises(cfg.NoSuchOptError, getattr, self.fconf, 'foo')
         self.assertNotIn('blaa', self.conf)
 
@@ -189,10 +189,10 @@ class RegisterTestCase(BaseTestCase):
         self.conf.register_opt(cfg.StrOpt('foo'))
 
         self.assertIn('foo', self.conf)
-        self.assertEqual(1, len(self.conf))
+        self.assertEqual(2, len(self.conf))
         self.assertIsNone(self.conf.foo)
         self.assertNotIn('foo', self.fconf)
-        self.assertEqual(0, len(self.fconf))
+        self.assertEqual(1, len(self.fconf))
         self.assertRaises(cfg.NoSuchOptError, getattr, self.fconf, 'foo')
 
     def test_already_registered_opt(self):
@@ -200,10 +200,10 @@ class RegisterTestCase(BaseTestCase):
         self.fconf.register_opt(cfg.StrOpt('foo'))
 
         self.assertIn('foo', self.conf)
-        self.assertEqual(1, len(self.conf))
+        self.assertEqual(2, len(self.conf))
         self.assertIsNone(self.conf.foo)
         self.assertIn('foo', self.fconf)
-        self.assertEqual(1, len(self.fconf))
+        self.assertEqual(2, len(self.fconf))
         self.assertIsNone(self.fconf.foo)
 
         self.conf.set_override('foo', 'bar')
@@ -220,13 +220,13 @@ class RegisterTestCase(BaseTestCase):
         self.assertIn('foo', self.conf)
         self.assertIn('fu', self.conf)
         self.assertNotIn('bu', self.conf)
-        self.assertEqual(2, len(self.conf))
+        self.assertEqual(3, len(self.conf))
         self.assertIsNone(self.conf.foo)
         self.assertIsNone(self.conf.fu)
         self.assertIn('foo', self.fconf)
         self.assertIn('bu', self.fconf)
         self.assertNotIn('fu', self.fconf)
-        self.assertEqual(2, len(self.fconf))
+        self.assertEqual(3, len(self.fconf))
         self.assertIsNone(self.fconf.foo)
         self.assertIsNone(self.fconf.bu)
 
@@ -240,10 +240,10 @@ class RegisterTestCase(BaseTestCase):
         self.fconf.register_cli_opt(cfg.StrOpt('foo'))
 
         self.assertIn('foo', self.conf)
-        self.assertEqual(1, len(self.conf))
+        self.assertEqual(2, len(self.conf))
         self.assertIsNone(self.conf.foo)
         self.assertIn('foo', self.fconf)
-        self.assertEqual(1, len(self.fconf))
+        self.assertEqual(2, len(self.fconf))
         self.assertIsNone(self.fconf.foo)
 
         self.conf.set_override('foo', 'bar')
@@ -259,12 +259,12 @@ class RegisterTestCase(BaseTestCase):
 
         self.assertIn('foo', self.conf)
         self.assertIn('fu', self.conf)
-        self.assertEqual(2, len(self.conf))
+        self.assertEqual(3, len(self.conf))
         self.assertIsNone(self.conf.foo)
         self.assertIsNone(self.conf.fu)
         self.assertIn('foo', self.fconf)
         self.assertIn('fu', self.fconf)
-        self.assertEqual(2, len(self.fconf))
+        self.assertEqual(3, len(self.fconf))
         self.assertIsNone(self.fconf.foo)
         self.assertIsNone(self.fconf.fu)
 

oslo_config/tests/test_generator.py

@@ -1085,7 +1085,8 @@ GENERATOR_OPTS = {'format_': 'yaml',
                   'namespace': ['test'],
                   'output_file': None,
                   'summarize': False,
-                  'wrap_width': 70}
+                  'wrap_width': 70,
+                  'config_source': []}
 
 
 class MachineReadableGeneratorTestCase(base.BaseTestCase):

oslo_config/tests/test_sources.py

@@ -10,59 +10,163 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import tempfile
+from requests import HTTPError
 
 from oslo_config import cfg
+from oslo_config import fixture
 from oslo_config import sources
+from oslo_config.sources import _uri
 
 from oslotest import base
+import requests_mock
 
-_GROUP = "group"
-_OPTIONS = "options"
-_DEFAULT = "DEFAULT"
 
-_extra_ini_opt_group = "extra_conf_from_ini"
+class TestProcessingSources(base.BaseTestCase):
-_extra_conf_url = "https://oslo.config/extra.conf"
 
-_conf_data = {
+    # NOTE(dhellmann): These tests use the config() method of the
-    _DEFAULT: {
+    # fixture because that invokes set_override() on the option. The
-        "config_sources": (cfg.StrOpt, _extra_ini_opt_group)
+    # load_raw_values() method injects data underneath the option, but
+    # only after invoking __call__ on the ConfigOpts instance, which
+    # is when the 'config_source' option is processed.
+
+    def setUp(self):
+        super(TestProcessingSources, self).setUp()
+        self.conf = cfg.ConfigOpts()
+        self.conf_fixture = self.useFixture(fixture.Config(self.conf))
+
+    def test_no_sources_default(self):
+        with base.mock.patch.object(
+                self.conf,
+                '_open_source_from_opt_group') as open_source:
+            open_source.side_effect = AssertionError('should not be called')
+            self.conf([])
+
+    def test_no_sources(self):
+        self.conf_fixture.config(
+            config_source=[],
+        )
+        with base.mock.patch.object(
+                self.conf,
+                '_open_source_from_opt_group') as open_source:
+            open_source.side_effect = AssertionError('should not be called')
+            self.conf([])
+
+    def test_source_named(self):
+        self.conf_fixture.config(
+            config_source=['missing_source'],
+        )
+        with base.mock.patch.object(
+                self.conf,
+                '_open_source_from_opt_group') as open_source:
+            self.conf([])
+            open_source.assert_called_once_with('missing_source')
+
+    def test_multiple_sources_named(self):
+        self.conf_fixture.config(
+            config_source=['source1', 'source2'],
+        )
+        with base.mock.patch.object(
+                self.conf,
+                '_open_source_from_opt_group') as open_source:
+            self.conf([])
+            open_source.assert_has_calls([
+                base.mock.call('source1'),
+                base.mock.call('source2'),
+            ])
+
+
+class TestLoading(base.BaseTestCase):
+
+    # NOTE(dhellmann): These tests can use load_raw_values() because
+    # they explicitly call _open_source_from_opt_group() after the
+    # ConfigOpts setup is done in __call__().
+
+    def setUp(self):
+        super(TestLoading, self).setUp()
+        self.conf = cfg.ConfigOpts()
+        self.conf_fixture = self.useFixture(fixture.Config(self.conf))
+
+    def test_source_missing(self):
+        # The group being loaded does not exist at all.
+        source = self.conf._open_source_from_opt_group('missing_source')
+        self.assertIsNone(source)
+
+    def test_driver_missing(self):
+        # The group exists and has other options, but does not specify
+        # a driver.
+        self.conf_fixture.load_raw_values(
+            group='missing_driver',
+            not_driver='foo',
+        )
+        source = self.conf._open_source_from_opt_group('missing_driver')
+        self.assertIsNone(source)
+
+    def test_unknown_driver(self):
+        # The group exists, but does not specify a valid driver.
+        self.conf_fixture.load_raw_values(
+            group='unknown_driver',
+            driver='very_unlikely_to_exist_driver_name',
+        )
+        source = self.conf._open_source_from_opt_group('unknown_driver')
+        self.assertIsNone(source)
+
+
+def make_uri(name):
+    return "https://oslo.config/{}.conf".format(name)
+
+
+_extra_configs = {
+    make_uri("types"): {
+        "name": "types",
+        "data": {
+            "DEFAULT": {
+                "foo": (cfg.StrOpt, "bar")
+            },
+            "test": {
+                "opt_str": (cfg.StrOpt, "a nice string"),
+                "opt_bool": (cfg.BoolOpt, True),
+                "opt_int": (cfg.IntOpt, 42),
+                "opt_float": (cfg.FloatOpt, 3.14),
+                "opt_ip": (cfg.IPOpt, "127.0.0.1"),
+                "opt_port": (cfg.PortOpt, 443),
+                "opt_host": (cfg.HostnameOpt, "www.openstack.org"),
+                "opt_uri": (cfg.URIOpt, "https://www.openstack.org"),
+                "opt_multi": (cfg.MultiStrOpt, ["abc", "def", "ghi"])
+            }
+        }
     },
-    _extra_ini_opt_group: {
+    make_uri("ini_1"): {
-        "driver": (cfg.StrOpt, "ini"),
+        "name": "ini_1",
-        "uri": (cfg.URIOpt, _extra_conf_url)
+        "data": {
-    }
+            "DEFAULT": {
-}
+                "abc": (cfg.StrOpt, "abc")
-
+            }
-_extra_conf_data = {
+        }
-    _DEFAULT: {
-        "foo": (cfg.StrOpt, "bar")
     },
-    "test": {
+    make_uri("ini_2"): {
-        "opt_str": (cfg.StrOpt, "a nice string"),
+        "name": "ini_2",
-        "opt_bool": (cfg.BoolOpt, True),
+        "data": {
-        "opt_int": (cfg.IntOpt, 42),
+            "DEFAULT": {
-        "opt_float": (cfg.FloatOpt, 3.14),
+                "abc": (cfg.StrOpt, "foo"),
-        "opt_ip": (cfg.IPOpt, "127.0.0.1"),
+                "def": (cfg.StrOpt, "def")
-        "opt_port": (cfg.PortOpt, 443),
+            }
-        "opt_host": (cfg.HostnameOpt, "www.openstack.org"),
+        }
-        "opt_uri": (cfg.URIOpt, "https://www.openstack.org"),
+    },
-        "opt_multi": (cfg.MultiStrOpt, ["abc", "def", "ghi"])
+    make_uri("ini_3"): {
+        "name": "ini_3",
+        "data": {
+            "DEFAULT": {
+                "abc": (cfg.StrOpt, "bar"),
+                "def": (cfg.StrOpt, "bar"),
+                "ghi": (cfg.StrOpt, "ghi")
+            }
+        }
     }
 }
 
 
-def register_opts(conf, opts):
+def opts_to_ini(uri, *args, **kwargs):
-    # 'g': group, 'o': option, and 't': type
+    opts = _extra_configs[uri]["data"]
-    for g in opts.keys():
-        for o, (t, _) in opts[g].items():
-            try:
-                conf.register_opt(t(o), g if g != "DEFAULT" else None)
-            except cfg.DuplicateOptError:
-                pass
-
-
-def opts_to_ini(opts):
     result = ""
 
     # 'g': group, 'o': option, 't': type, and 'v': value
@@ -78,53 +182,100 @@ def opts_to_ini(opts):
     return result
 
 
-def mocked_get(*args, **kwargs):
+class URISourceTestCase(base.BaseTestCase):
-    class MockResponse(object):
-        def __init__(self, text_data, status_code):
-            self.text = text_data
-            self.status_code = status_code
-
-        def __enter__(self, *args, **kwargs):
-            return self
-
-        def __exit__(self, *args, **kwargs):
-            pass
-
-        def raise_for_status(self):
-            if self.status_code != 200:
-                raise
-
-    if args[0] in _extra_conf_url:
-        return MockResponse(opts_to_ini(_extra_conf_data), 200)
-
-    return MockResponse(None, 404)
-
-
-class INISourceTestCase(base.BaseTestCase):
 
     def setUp(self):
-        super(INISourceTestCase, self).setUp()
+        super(URISourceTestCase, self).setUp()
-
         self.conf = cfg.ConfigOpts()
+        self.conf_fixture = self.useFixture(fixture.Config(self.conf))
 
-        with tempfile.NamedTemporaryFile() as tmp_file:
+    def _register_opts(self, opts):
-            tmp_file.write(opts_to_ini(_conf_data).encode("utf-8"))
+        # 'g': group, 'o': option, and 't': type
-            tmp_file.flush()
+        for g in opts.keys():
+            for o, (t, _) in opts[g].items():
+                self.conf.register_opt(t(o), g if g != "DEFAULT" else None)
 
-            self.conf(["--config-file", tmp_file.name])
+    def test_incomplete_driver(self):
+        # The group exists, but does not specify the
+        # required options for this driver.
+        self.conf_fixture.load_raw_values(
+            group='incomplete_ini_driver',
+            driver='remote_file',
+        )
+        source = self.conf._open_source_from_opt_group('incomplete_ini_driver')
+        self.assertIsNone(source)
+
+    @requests_mock.mock()
+    def test_fetch_uri(self, m):
+        m.get("https://bad.uri", status_code=404)
+
+        self.assertRaises(
+            HTTPError, _uri.URIConfigurationSource, "https://bad.uri")
+
+        m.get("https://good.uri", text="[DEFAULT]\nfoo=bar\n")
+        source = _uri.URIConfigurationSource("https://good.uri")
+
+        self.assertEqual(
+            "bar", source.get("DEFAULT", "foo", cfg.StrOpt("foo"))[0])
 
     @base.mock.patch(
-        "oslo_config.sources.ini.requests.get", side_effect=mocked_get)
+        "oslo_config.sources._uri.URIConfigurationSource._fetch_uri",
-    def test_configuration_source(self, mock_requests_get):
+        side_effect=opts_to_ini)
-        driver = sources.ini.INIConfigurationSourceDriver()
+    def test_configuration_source(self, mock_fetch_uri):
-        source = driver.open_source_from_opt_group(
+        group = "types"
-            self.conf, _extra_ini_opt_group)
+        uri = make_uri(group)
+
+        self.conf_fixture.load_raw_values(
+            group=group,
+            driver='remote_file',
+            uri=uri
+        )
+        self.conf_fixture.config(config_source=[group])
+
+        # testing driver loading
+        self.assertEqual(self.conf._sources, [])
+        self.conf._load_alternative_sources()
+        self.assertEqual(type(self.conf._sources[0]),
+                         _uri.URIConfigurationSource)
+
+        source = self.conf._open_source_from_opt_group(group)
+
+        self._register_opts(_extra_configs[uri]["data"])
 
         # non-existing option
         self.assertIs(sources._NoValue,
                       source.get("DEFAULT", "bar", cfg.StrOpt("bar"))[0])
 
         # 'g': group, 'o': option, 't': type, and 'v': value
-        for g in _extra_conf_data:
+        for g in _extra_configs[uri]["data"]:
-            for o, (t, v) in _extra_conf_data[g].items():
+            for o, (t, v) in _extra_configs[uri]["data"][g].items():
                 self.assertEqual(str(v), str(source.get(g, o, t(o))[0]))
+                self.assertEqual(v,
+                                 self.conf[g][o] if g != "DEFAULT" else
+                                 self.conf[o])
+
+    @base.mock.patch(
+        "oslo_config.sources._uri.URIConfigurationSource._fetch_uri",
+        side_effect=opts_to_ini)
+    def test_multiple_configuration_sources(self, mock_fetch_uri):
+        groups = ["ini_1", "ini_2", "ini_3"]
+        uri = make_uri("ini_3")
+
+        for group in groups:
+            self.conf_fixture.load_raw_values(
+                group=group,
+                driver='remote_file',
+                uri=make_uri(group)
+            )
+
+        self.conf_fixture.config(config_source=groups)
+        self.conf._load_alternative_sources()
+
+        # ini_3 contains all options to be tested
+        self._register_opts(_extra_configs[uri]["data"])
+
+        # where options are 'abc', 'def', and 'ghi'
+        # if the extra configs are loaded in the right order
+        # the option name and option value will match correctly
+        for option in _extra_configs[uri]["data"]["DEFAULT"]:
+            self.assertEqual(option, self.conf[option])

requirements.txt

@@ -10,3 +10,4 @@ oslo.i18n>=3.15.3 # Apache-2.0
 rfc3986>=0.3.1 # Apache-2.0
 PyYAML>=3.12 # MIT
 enum34>=1.0.4;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD
+requests>=2.18.0 # Apache-2.0

setup.cfg

@@ -32,6 +32,8 @@ console_scripts =
     oslo-config-generator = oslo_config.generator:main
 oslo.config.opts =
     oslo.config = oslo_config._list_opts:list_opts
+oslo.config.driver =
+    remote_file = oslo_config.sources._uri:URIConfigurationSourceDriver
 
 [wheel]
 universal = 1

test-requirements.txt

@@ -19,6 +19,7 @@ sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD
 
 # mocking framework
 mock>=2.0.0 # BSD
+requests_mock>=1.5.0 # Apache-2.0
 
 # Bandit security code scanner
 bandit>=1.1.0 # Apache-2.0