diff --git a/doc/source/reference/defining.rst b/doc/source/reference/defining.rst index 65a602fe..59d80d56 100644 --- a/doc/source/reference/defining.rst +++ b/doc/source/reference/defining.rst @@ -2,7 +2,9 @@ Defining Options ================== -Configuration options may be set on the command line or in config files. +Configuration options may be set on the command line, in the +:mod:`environment `, or in config files. +Options are processed in that order. The schema for each option is defined using the :class:`Opt` class or its sub-classes, for example: diff --git a/doc/source/reference/drivers.rst b/doc/source/reference/drivers.rst index 3790675c..0c06ab17 100644 --- a/doc/source/reference/drivers.rst +++ b/doc/source/reference/drivers.rst @@ -10,3 +10,4 @@ Known Backend Drivers --------------------- .. automodule:: oslo_config.sources._uri +.. automodule:: oslo_config.sources._environment diff --git a/doc/source/reference/locations.rst b/doc/source/reference/locations.rst index bfade1ab..40b2625c 100644 --- a/doc/source/reference/locations.rst +++ b/doc/source/reference/locations.rst @@ -47,6 +47,10 @@ describing the location. Its value depends on the ``location``. - ``True`` - A value set by the user on the command line. - Empty string. + * - ``environment`` + - ``True`` + - A value set by the user in the process environment. + - The name of the environment variable. Did a user set a configuration option? ====================================== diff --git a/oslo_config/cfg.py b/oslo_config/cfg.py index a914f650..5ba83858 100644 --- a/oslo_config/cfg.py +++ b/oslo_config/cfg.py @@ -40,6 +40,8 @@ except ImportError: from oslo_config import iniparser from oslo_config import sources +# Absolute import to avoid circular import in Python 2.7 +import oslo_config.sources._environment as _environment from oslo_config import types import stevedore @@ -58,6 +60,7 @@ class Locations(enum.Enum): set_override = (3, False) user = (4, True) command_line = (5, True) + environment = (6, True) def __init__(self, num, is_user_controlled): self.num = num @@ -189,7 +192,12 @@ class ConfigFileParseError(Error): return 'Failed to parse %s: %s' % (self.config_file, self.msg) -class ConfigFileValueError(Error, ValueError): +class ConfigSourceValueError(Error, ValueError): + """Raised if a config source value does not match its opt type.""" + pass + + +class ConfigFileValueError(ConfigSourceValueError): """Raised if a config file value does not match its opt type.""" pass @@ -1946,6 +1954,9 @@ class ConfigOpts(collections.Mapping): self._validate_default_values = False self._sources = [] self._ext_mgr = None + # Though the env_driver is a Source, we load it by default. + self._use_env = True + self._env_driver = _environment.EnvironmentConfigurationSource() self.register_opt(self._config_source_opt) @@ -2009,7 +2020,7 @@ class ConfigOpts(collections.Mapping): return options def _setup(self, project, prog, version, usage, default_config_files, - default_config_dirs): + default_config_dirs, use_env): """Initialize a ConfigOpts object for option parsing.""" self._config_opts = self._make_config_options(default_config_files, default_config_dirs) @@ -2021,6 +2032,7 @@ class ConfigOpts(collections.Mapping): self.usage = usage self.default_config_files = default_config_files self.default_config_dirs = default_config_dirs + self._use_env = use_env def __clear_cache(f): @functools.wraps(f) @@ -2056,7 +2068,8 @@ class ConfigOpts(collections.Mapping): default_config_dirs=None, validate_default_values=False, description=None, - epilog=None): + epilog=None, + use_env=True): """Parse command line arguments and config files. Calling a ConfigOpts object causes the supplied command line arguments @@ -2084,6 +2097,8 @@ class ConfigOpts(collections.Mapping): :param default_config_files: config files to use by default :param default_config_dirs: config dirs to use by default :param validate_default_values: whether to validate the default values + :param use_env: If True (the default) look in the environment as one + source of option values. :raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError, ConfigFilesPermissionDeniedError, RequiredOptError, DuplicateOptError @@ -2097,7 +2112,7 @@ class ConfigOpts(collections.Mapping): default_config_files, default_config_dirs) self._setup(project, prog, version, usage, default_config_files, - default_config_dirs) + default_config_dirs, use_env) self._namespace = self._parse_cli_opts(args if args is not None else sys.argv[1:]) @@ -2634,6 +2649,13 @@ class ConfigOpts(collections.Mapping): self._substitute(value, group, namespace), opt) group_name = group.name if group else None + key = (group_name, name) + + # If use_env is true, get a value from the environment but don't use + # it yet. We will look at the command line first, below. + env_val = (sources._NoValue, None) + if self._use_env: + env_val = self._env_driver.get(group_name, name, opt) if opt.mutable and namespace is None: namespace = self._mutable_ns @@ -2641,16 +2663,33 @@ class ConfigOpts(collections.Mapping): namespace = self._namespace if namespace is not None: try: - val, alt_loc = opt._get_from_namespace(namespace, group_name) - return (convert(val), alt_loc) - except KeyError: # nosec: Valid control flow instruction - pass + try: + val, alt_loc = opt._get_from_namespace(namespace, + group_name) + # Try command line first + if (val != sources._NoValue + and alt_loc.location == Locations.command_line): + return (convert(val), alt_loc) + # Environment source second + if env_val[0] != sources._NoValue: + return (convert(env_val[0]), env_val[1]) + # Default file source third + if val != sources._NoValue: + return (convert(val), alt_loc) + except KeyError: # nosec: Valid control flow instruction + # If there was a KeyError looking at config files or + # command line, retry the env_val. + if env_val[0] != sources._NoValue: + return (convert(env_val[0]), env_val[1]) except ValueError as ve: - raise ConfigFileValueError( - "Value for option %s is not valid: %s" - % (opt.name, str(ve))) + message = "Value for option %s from %s is not valid: %s" % ( + opt.name, alt_loc, str(ve)) + # Preserve backwards compatibility for file-based value + # errors. + if alt_loc.location == Locations.user: + raise ConfigFileValueError(message) + raise ConfigSourceValueError(message) - key = (group_name, name) try: return self.__drivers_cache[key] except KeyError: # nosec: Valid control flow instruction diff --git a/oslo_config/sources/_environment.py b/oslo_config/sources/_environment.py new file mode 100644 index 00000000..19dc093c --- /dev/null +++ b/oslo_config/sources/_environment.py @@ -0,0 +1,92 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +r""" +Environment +----------- + +The **environment** backend driver provides a method of accessing +configuration data in environment variables. It is enabled by default +and requires no additional configuration to use. The environment is +checked after command line options, but before configuration files. + + +Environment variables are checked for any configuration data. The variable +names take the form: + +* A prefix of ``OS_`` +* The group name, uppercased +* Separated from the option name by a `__` (double underscore) +* Followed by the name + +For an option that looks like this in the usual INI format:: + + [placement_database] + connection = sqlite:/// + +the corresponding environment variable would be +``OS_PLACEMENT_DATABASE__CONNECTION``. + +The Driver Class +================ + +.. autoclass:: EnvironmentConfigurationSourceDriver + +The Configuration Source Class +============================== + +.. autoclass:: EnvironmentConfigurationSource + +""" + +import os + +# Avoid circular import +import oslo_config.cfg +from oslo_config import sources + + +# In current practice this class is not used because the +# EnvironmentConfigurationSource is loaded by default, but we keep it +# here in case we choose to change that behavior in the future. +class EnvironmentConfigurationSourceDriver(sources.ConfigurationSourceDriver): + """A backend driver for environment variables. + + This configuration source is available by default and does not need special + configuration to use. The sample config is generated automatically but is + not necessary. + """ + + def list_options_for_discovery(self): + """There are no options for this driver.""" + return [] + + def open_source_from_opt_group(self, conf, group_name): + return EnvironmentConfigurationSource() + + +class EnvironmentConfigurationSource(sources.ConfigurationSource): + """A configuration source for options in the environment.""" + + @staticmethod + def _make_name(group_name, option_name): + group_name = group_name or 'DEFAULT' + return 'OS_{}__{}'.format(group_name.upper(), option_name.upper()) + + def get(self, group_name, option_name, opt): + env_name = self._make_name(group_name, option_name) + try: + value = os.environ[env_name] + loc = oslo_config.cfg.LocationInfo( + oslo_config.cfg.Locations.environment, env_name) + return (value, loc) + except KeyError: + return (sources._NoValue, None) diff --git a/oslo_config/tests/test_sources.py b/oslo_config/tests/test_sources.py index 8b8588fe..dca83b88 100644 --- a/oslo_config/tests/test_sources.py +++ b/oslo_config/tests/test_sources.py @@ -10,6 +10,8 @@ # License for the specific language governing permissions and limitations # under the License. +import os + from requests import HTTPError from oslo_config import _list_opts @@ -112,6 +114,65 @@ class TestLoading(base.BaseTestCase): self.assertIsNone(source) +class TestEnvironmentConfigurationSource(base.BaseTestCase): + + def setUp(self): + super(TestEnvironmentConfigurationSource, self).setUp() + self.conf = cfg.ConfigOpts() + self.conf_fixture = self.useFixture(fixture.Config(self.conf)) + self.conf.register_opt(cfg.StrOpt('bar'), 'foo') + + def cleanup(): + if 'OS_FOO__BAR' in os.environ: + del os.environ['OS_FOO__BAR'] + self.addCleanup(cleanup) + + def test_simple_environment_get(self): + self.conf(args=[]) + env_value = 'goodbye' + os.environ['OS_FOO__BAR'] = env_value + + self.assertEqual(env_value, self.conf['foo']['bar']) + + def test_env_beats_files(self): + file_value = 'hello' + env_value = 'goodbye' + self.conf(args=[]) + self.conf_fixture.load_raw_values( + group='foo', + bar=file_value, + ) + + self.assertEqual(file_value, self.conf['foo']['bar']) + self.conf.reload_config_files() + os.environ['OS_FOO__BAR'] = env_value + self.assertEqual(env_value, self.conf['foo']['bar']) + + def test_cli_beats_env(self): + env_value = 'goodbye' + cli_value = 'cli' + os.environ['OS_FOO__BAR'] = env_value + self.conf.register_cli_opt(cfg.StrOpt('bar'), 'foo') + self.conf(args=['--foo=%s' % cli_value]) + + self.assertEqual(cli_value, self.conf['foo']['bar']) + + def test_use_env_false_allows_files(self): + file_value = 'hello' + env_value = 'goodbye' + os.environ['OS_FOO__BAR'] = env_value + self.conf(args=[], use_env=False) + self.conf_fixture.load_raw_values( + group='foo', + bar=file_value, + ) + + self.assertEqual(file_value, self.conf['foo']['bar']) + self.conf.reset() + self.conf(args=[], use_env=True) + self.assertEqual(env_value, self.conf['foo']['bar']) + + def make_uri(name): return "https://oslo.config/{}.conf".format(name) diff --git a/releasenotes/notes/config-from-environment-3feba7b4cc747d2b.yaml b/releasenotes/notes/config-from-environment-3feba7b4cc747d2b.yaml new file mode 100644 index 00000000..d78337e6 --- /dev/null +++ b/releasenotes/notes/config-from-environment-3feba7b4cc747d2b.yaml @@ -0,0 +1,22 @@ +--- +features: + - | + Support for accessing configuration data in environment variables via the + environment backend driver, enabled by default. The environment is checked + after command line options, but before configuration files. + + Environment variables are checked for any configuration data. The variable + names take the form: + + * A prefix of ``OS_`` + * The group name, uppercased + * Separated from the option name by a `__` (double underscore) + * Followed by the name + + For an option that looks like this in the usual INI format:: + + [placement_database] + connection = sqlite:/// + + the corresponding environment variable would be + ``OS_PLACEMENT_DATABASE__CONNECTION``.