Add support for looking in environment for config

An _environment source is added that looks in os.environ for
values.

Using the environment is on by default, but can be shut down
by setting `use_env` to False when __call__ is called.

The enviroment is inspected before any other sources
of config data but the value is used after command line
arguments and before config file options.

This is done by checking both the command line and config
files and then inspecting the location of the result. If
it is command_line, we use it. If not, we use the environment
value (if any). If there's no environment value, the config
file value is used. If checking the command line and config
file results in a KeyError, the environment value is used,
if set.

The names of the environment variables follow the rules
described in oslo_config.sources._environment.

A new exception has been added: ConfigSourceValueError, this
is the superclass of the existing ConfigFileValueError. The
code in _do_get has been updated to only use
ConfigFileValueError when it is in fact a file from whence a
ValueError came.

Documentation has been updated and a rlease note created to
indicate the new functionality.

Change-Id: I3245c40ebdcc96f8e3b2dc0bab3b4aa71d07ad15
This commit is contained in:
Chris Dent 2018-07-25 20:01:37 +01:00
parent b12783ea0f
commit ea8a0f6a8b
7 changed files with 234 additions and 13 deletions

View File

@ -2,7 +2,9 @@
Defining Options
==================
Configuration options may be set on the command line or in config files.
Configuration options may be set on the command line, in the
:mod:`environment <oslo_config.sources._environment>`, or in config files.
Options are processed in that order.
The schema for each option is defined using the
:class:`Opt` class or its sub-classes, for example:

View File

@ -10,3 +10,4 @@ Known Backend Drivers
---------------------
.. automodule:: oslo_config.sources._uri
.. automodule:: oslo_config.sources._environment

View File

@ -47,6 +47,10 @@ describing the location. Its value depends on the ``location``.
- ``True``
- A value set by the user on the command line.
- Empty string.
* - ``environment``
- ``True``
- A value set by the user in the process environment.
- The name of the environment variable.
Did a user set a configuration option?
======================================

View File

@ -40,6 +40,8 @@ except ImportError:
from oslo_config import iniparser
from oslo_config import sources
# Absolute import to avoid circular import in Python 2.7
import oslo_config.sources._environment as _environment
from oslo_config import types
import stevedore
@ -58,6 +60,7 @@ class Locations(enum.Enum):
set_override = (3, False)
user = (4, True)
command_line = (5, True)
environment = (6, True)
def __init__(self, num, is_user_controlled):
self.num = num
@ -189,7 +192,12 @@ class ConfigFileParseError(Error):
return 'Failed to parse %s: %s' % (self.config_file, self.msg)
class ConfigFileValueError(Error, ValueError):
class ConfigSourceValueError(Error, ValueError):
"""Raised if a config source value does not match its opt type."""
pass
class ConfigFileValueError(ConfigSourceValueError):
"""Raised if a config file value does not match its opt type."""
pass
@ -1946,6 +1954,9 @@ class ConfigOpts(collections.Mapping):
self._validate_default_values = False
self._sources = []
self._ext_mgr = None
# Though the env_driver is a Source, we load it by default.
self._use_env = True
self._env_driver = _environment.EnvironmentConfigurationSource()
self.register_opt(self._config_source_opt)
@ -2009,7 +2020,7 @@ class ConfigOpts(collections.Mapping):
return options
def _setup(self, project, prog, version, usage, default_config_files,
default_config_dirs):
default_config_dirs, use_env):
"""Initialize a ConfigOpts object for option parsing."""
self._config_opts = self._make_config_options(default_config_files,
default_config_dirs)
@ -2021,6 +2032,7 @@ class ConfigOpts(collections.Mapping):
self.usage = usage
self.default_config_files = default_config_files
self.default_config_dirs = default_config_dirs
self._use_env = use_env
def __clear_cache(f):
@functools.wraps(f)
@ -2056,7 +2068,8 @@ class ConfigOpts(collections.Mapping):
default_config_dirs=None,
validate_default_values=False,
description=None,
epilog=None):
epilog=None,
use_env=True):
"""Parse command line arguments and config files.
Calling a ConfigOpts object causes the supplied command line arguments
@ -2084,6 +2097,8 @@ class ConfigOpts(collections.Mapping):
:param default_config_files: config files to use by default
:param default_config_dirs: config dirs to use by default
:param validate_default_values: whether to validate the default values
:param use_env: If True (the default) look in the environment as one
source of option values.
:raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError,
ConfigFilesPermissionDeniedError,
RequiredOptError, DuplicateOptError
@ -2097,7 +2112,7 @@ class ConfigOpts(collections.Mapping):
default_config_files, default_config_dirs)
self._setup(project, prog, version, usage, default_config_files,
default_config_dirs)
default_config_dirs, use_env)
self._namespace = self._parse_cli_opts(args if args is not None
else sys.argv[1:])
@ -2634,6 +2649,13 @@ class ConfigOpts(collections.Mapping):
self._substitute(value, group, namespace), opt)
group_name = group.name if group else None
key = (group_name, name)
# If use_env is true, get a value from the environment but don't use
# it yet. We will look at the command line first, below.
env_val = (sources._NoValue, None)
if self._use_env:
env_val = self._env_driver.get(group_name, name, opt)
if opt.mutable and namespace is None:
namespace = self._mutable_ns
@ -2641,16 +2663,33 @@ class ConfigOpts(collections.Mapping):
namespace = self._namespace
if namespace is not None:
try:
val, alt_loc = opt._get_from_namespace(namespace, group_name)
return (convert(val), alt_loc)
except KeyError: # nosec: Valid control flow instruction
pass
try:
val, alt_loc = opt._get_from_namespace(namespace,
group_name)
# Try command line first
if (val != sources._NoValue
and alt_loc.location == Locations.command_line):
return (convert(val), alt_loc)
# Environment source second
if env_val[0] != sources._NoValue:
return (convert(env_val[0]), env_val[1])
# Default file source third
if val != sources._NoValue:
return (convert(val), alt_loc)
except KeyError: # nosec: Valid control flow instruction
# If there was a KeyError looking at config files or
# command line, retry the env_val.
if env_val[0] != sources._NoValue:
return (convert(env_val[0]), env_val[1])
except ValueError as ve:
raise ConfigFileValueError(
"Value for option %s is not valid: %s"
% (opt.name, str(ve)))
message = "Value for option %s from %s is not valid: %s" % (
opt.name, alt_loc, str(ve))
# Preserve backwards compatibility for file-based value
# errors.
if alt_loc.location == Locations.user:
raise ConfigFileValueError(message)
raise ConfigSourceValueError(message)
key = (group_name, name)
try:
return self.__drivers_cache[key]
except KeyError: # nosec: Valid control flow instruction

View File

@ -0,0 +1,92 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
r"""
Environment
-----------
The **environment** backend driver provides a method of accessing
configuration data in environment variables. It is enabled by default
and requires no additional configuration to use. The environment is
checked after command line options, but before configuration files.
Environment variables are checked for any configuration data. The variable
names take the form:
* A prefix of ``OS_``
* The group name, uppercased
* Separated from the option name by a `__` (double underscore)
* Followed by the name
For an option that looks like this in the usual INI format::
[placement_database]
connection = sqlite:///
the corresponding environment variable would be
``OS_PLACEMENT_DATABASE__CONNECTION``.
The Driver Class
================
.. autoclass:: EnvironmentConfigurationSourceDriver
The Configuration Source Class
==============================
.. autoclass:: EnvironmentConfigurationSource
"""
import os
# Avoid circular import
import oslo_config.cfg
from oslo_config import sources
# In current practice this class is not used because the
# EnvironmentConfigurationSource is loaded by default, but we keep it
# here in case we choose to change that behavior in the future.
class EnvironmentConfigurationSourceDriver(sources.ConfigurationSourceDriver):
"""A backend driver for environment variables.
This configuration source is available by default and does not need special
configuration to use. The sample config is generated automatically but is
not necessary.
"""
def list_options_for_discovery(self):
"""There are no options for this driver."""
return []
def open_source_from_opt_group(self, conf, group_name):
return EnvironmentConfigurationSource()
class EnvironmentConfigurationSource(sources.ConfigurationSource):
"""A configuration source for options in the environment."""
@staticmethod
def _make_name(group_name, option_name):
group_name = group_name or 'DEFAULT'
return 'OS_{}__{}'.format(group_name.upper(), option_name.upper())
def get(self, group_name, option_name, opt):
env_name = self._make_name(group_name, option_name)
try:
value = os.environ[env_name]
loc = oslo_config.cfg.LocationInfo(
oslo_config.cfg.Locations.environment, env_name)
return (value, loc)
except KeyError:
return (sources._NoValue, None)

View File

@ -10,6 +10,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
from requests import HTTPError
from oslo_config import _list_opts
@ -112,6 +114,65 @@ class TestLoading(base.BaseTestCase):
self.assertIsNone(source)
class TestEnvironmentConfigurationSource(base.BaseTestCase):
def setUp(self):
super(TestEnvironmentConfigurationSource, self).setUp()
self.conf = cfg.ConfigOpts()
self.conf_fixture = self.useFixture(fixture.Config(self.conf))
self.conf.register_opt(cfg.StrOpt('bar'), 'foo')
def cleanup():
if 'OS_FOO__BAR' in os.environ:
del os.environ['OS_FOO__BAR']
self.addCleanup(cleanup)
def test_simple_environment_get(self):
self.conf(args=[])
env_value = 'goodbye'
os.environ['OS_FOO__BAR'] = env_value
self.assertEqual(env_value, self.conf['foo']['bar'])
def test_env_beats_files(self):
file_value = 'hello'
env_value = 'goodbye'
self.conf(args=[])
self.conf_fixture.load_raw_values(
group='foo',
bar=file_value,
)
self.assertEqual(file_value, self.conf['foo']['bar'])
self.conf.reload_config_files()
os.environ['OS_FOO__BAR'] = env_value
self.assertEqual(env_value, self.conf['foo']['bar'])
def test_cli_beats_env(self):
env_value = 'goodbye'
cli_value = 'cli'
os.environ['OS_FOO__BAR'] = env_value
self.conf.register_cli_opt(cfg.StrOpt('bar'), 'foo')
self.conf(args=['--foo=%s' % cli_value])
self.assertEqual(cli_value, self.conf['foo']['bar'])
def test_use_env_false_allows_files(self):
file_value = 'hello'
env_value = 'goodbye'
os.environ['OS_FOO__BAR'] = env_value
self.conf(args=[], use_env=False)
self.conf_fixture.load_raw_values(
group='foo',
bar=file_value,
)
self.assertEqual(file_value, self.conf['foo']['bar'])
self.conf.reset()
self.conf(args=[], use_env=True)
self.assertEqual(env_value, self.conf['foo']['bar'])
def make_uri(name):
return "https://oslo.config/{}.conf".format(name)

View File

@ -0,0 +1,22 @@
---
features:
- |
Support for accessing configuration data in environment variables via the
environment backend driver, enabled by default. The environment is checked
after command line options, but before configuration files.
Environment variables are checked for any configuration data. The variable
names take the form:
* A prefix of ``OS_``
* The group name, uppercased
* Separated from the option name by a `__` (double underscore)
* Followed by the name
For an option that looks like this in the usual INI format::
[placement_database]
connection = sqlite:///
the corresponding environment variable would be
``OS_PLACEMENT_DATABASE__CONNECTION``.