openstack/oslo.config
Code Issues Proposed changes

Merge "Add support for looking in environment for config"

Browse Source
This commit is contained in:
Zuul 2018-11-02 19:58:06 +00:00 committed by Gerrit Code Review
commit 4e0e72d480
7 changed files with 234 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/source/reference/defining.rst
View File

@ -2,7 +2,9 @@
Defining Options
==================
Configuration options may be set on the command line or in config files.
Configuration options may be set on the command line, in the
:mod:`environment <oslo_config.sources._environment>`, or in config files.
Options are processed in that order.
The schema for each option is defined using the
:class:`Opt` class or its sub-classes, for example:

1
doc/source/reference/drivers.rst
View File

@ -10,3 +10,4 @@ Known Backend Drivers
---------------------
.. automodule:: oslo_config.sources._uri
.. automodule:: oslo_config.sources._environment

4
doc/source/reference/locations.rst
View File

@ -47,6 +47,10 @@ describing the location. Its value depends on the ``location``.
- ``True``
- A value set by the user on the command line.
- Empty string.
* - ``environment``
- ``True``
- A value set by the user in the process environment.
- The name of the environment variable.
Did a user set a configuration option?
======================================

63
oslo_config/cfg.py
View File

@ -40,6 +40,8 @@ except ImportError:
from oslo_config import iniparser
from oslo_config import sources
# Absolute import to avoid circular import in Python 2.7
import oslo_config.sources._environment as _environment
from oslo_config import types
import stevedore
@ -58,6 +60,7 @@ class Locations(enum.Enum):
set_override = (3, False)
user = (4, True)
command_line = (5, True)
environment = (6, True)
def __init__(self, num, is_user_controlled):
self.num = num
@ -189,7 +192,12 @@ class ConfigFileParseError(Error):
return 'Failed to parse %s: %s' % (self.config_file, self.msg)
class ConfigFileValueError(Error, ValueError):
class ConfigSourceValueError(Error, ValueError):
"""Raised if a config source value does not match its opt type."""
pass
class ConfigFileValueError(ConfigSourceValueError):
"""Raised if a config file value does not match its opt type."""
pass
@ -1946,6 +1954,9 @@ class ConfigOpts(collections.Mapping):
self._validate_default_values = False
self._sources = []
self._ext_mgr = None
# Though the env_driver is a Source, we load it by default.
self._use_env = True
self._env_driver = _environment.EnvironmentConfigurationSource()
self.register_opt(self._config_source_opt)
@ -2009,7 +2020,7 @@ class ConfigOpts(collections.Mapping):
return options
def _setup(self, project, prog, version, usage, default_config_files,
default_config_dirs):
default_config_dirs, use_env):
"""Initialize a ConfigOpts object for option parsing."""
self._config_opts = self._make_config_options(default_config_files,
default_config_dirs)
@ -2021,6 +2032,7 @@ class ConfigOpts(collections.Mapping):
self.usage = usage
self.default_config_files = default_config_files
self.default_config_dirs = default_config_dirs
self._use_env = use_env
def __clear_cache(f):
@functools.wraps(f)
@ -2056,7 +2068,8 @@ class ConfigOpts(collections.Mapping):
default_config_dirs=None,
validate_default_values=False,
description=None,
epilog=None):
epilog=None,
use_env=True):
"""Parse command line arguments and config files.
Calling a ConfigOpts object causes the supplied command line arguments
@ -2084,6 +2097,8 @@ class ConfigOpts(collections.Mapping):
:param default_config_files: config files to use by default
:param default_config_dirs: config dirs to use by default
:param validate_default_values: whether to validate the default values
:param use_env: If True (the default) look in the environment as one
source of option values.
:raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError,
ConfigFilesPermissionDeniedError,
RequiredOptError, DuplicateOptError
@ -2097,7 +2112,7 @@ class ConfigOpts(collections.Mapping):
default_config_files, default_config_dirs)
self._setup(project, prog, version, usage, default_config_files,
default_config_dirs)
default_config_dirs, use_env)
self._namespace = self._parse_cli_opts(args if args is not None
else sys.argv[1:])
@ -2635,6 +2650,13 @@ class ConfigOpts(collections.Mapping):
self._substitute(value, group, namespace), opt)
group_name = group.name if group else None
key = (group_name, name)
# If use_env is true, get a value from the environment but don't use
# it yet. We will look at the command line first, below.
env_val = (sources._NoValue, None)
if self._use_env:
env_val = self._env_driver.get(group_name, name, opt)
if opt.mutable and namespace is None:
namespace = self._mutable_ns
@ -2642,16 +2664,33 @@ class ConfigOpts(collections.Mapping):
namespace = self._namespace
if namespace is not None:
try:
val, alt_loc = opt._get_from_namespace(namespace, group_name)
return (convert(val), alt_loc)
except KeyError: # nosec: Valid control flow instruction
pass
try:
val, alt_loc = opt._get_from_namespace(namespace,
group_name)
# Try command line first
if (val != sources._NoValue
and alt_loc.location == Locations.command_line):
return (convert(val), alt_loc)
# Environment source second
if env_val[0] != sources._NoValue:
return (convert(env_val[0]), env_val[1])
# Default file source third
if val != sources._NoValue:
return (convert(val), alt_loc)
except KeyError: # nosec: Valid control flow instruction
# If there was a KeyError looking at config files or
# command line, retry the env_val.
if env_val[0] != sources._NoValue:
return (convert(env_val[0]), env_val[1])
except ValueError as ve:
raise ConfigFileValueError(
"Value for option %s is not valid: %s"
% (opt.name, str(ve)))
message = "Value for option %s from %s is not valid: %s" % (
opt.name, alt_loc, str(ve))
# Preserve backwards compatibility for file-based value
# errors.
if alt_loc.location == Locations.user:
raise ConfigFileValueError(message)
raise ConfigSourceValueError(message)
key = (group_name, name)
try:
return self.__drivers_cache[key]
except KeyError: # nosec: Valid control flow instruction

92
oslo_config/sources/_environment.py Normal file
View File

@ -0,0 +1,92 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
r"""
Environment
-----------
The **environment** backend driver provides a method of accessing
configuration data in environment variables. It is enabled by default
and requires no additional configuration to use. The environment is
checked after command line options, but before configuration files.
Environment variables are checked for any configuration data. The variable
names take the form:
* A prefix of ``OS_``
* The group name, uppercased
* Separated from the option name by a `__` (double underscore)
* Followed by the name
For an option that looks like this in the usual INI format::
[placement_database]
connection = sqlite:///
the corresponding environment variable would be
``OS_PLACEMENT_DATABASE__CONNECTION``.
The Driver Class
================
.. autoclass:: EnvironmentConfigurationSourceDriver
The Configuration Source Class
==============================
.. autoclass:: EnvironmentConfigurationSource
"""
import os
# Avoid circular import
import oslo_config.cfg
from oslo_config import sources
# In current practice this class is not used because the
# EnvironmentConfigurationSource is loaded by default, but we keep it
# here in case we choose to change that behavior in the future.
class EnvironmentConfigurationSourceDriver(sources.ConfigurationSourceDriver):
"""A backend driver for environment variables.
This configuration source is available by default and does not need special
configuration to use. The sample config is generated automatically but is
not necessary.
"""
def list_options_for_discovery(self):
"""There are no options for this driver."""
return []
def open_source_from_opt_group(self, conf, group_name):
return EnvironmentConfigurationSource()
class EnvironmentConfigurationSource(sources.ConfigurationSource):
"""A configuration source for options in the environment."""
@staticmethod
def _make_name(group_name, option_name):
group_name = group_name or 'DEFAULT'
return 'OS_{}__{}'.format(group_name.upper(), option_name.upper())
def get(self, group_name, option_name, opt):
env_name = self._make_name(group_name, option_name)
try:
value = os.environ[env_name]
loc = oslo_config.cfg.LocationInfo(
oslo_config.cfg.Locations.environment, env_name)
return (value, loc)
except KeyError:
return (sources._NoValue, None)

61
oslo_config/tests/test_sources.py
View File

@ -10,6 +10,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
from requests import HTTPError
from oslo_config import _list_opts
@ -112,6 +114,65 @@ class TestLoading(base.BaseTestCase):
self.assertIsNone(source)
class TestEnvironmentConfigurationSource(base.BaseTestCase):
def setUp(self):
super(TestEnvironmentConfigurationSource, self).setUp()
self.conf = cfg.ConfigOpts()
self.conf_fixture = self.useFixture(fixture.Config(self.conf))
self.conf.register_opt(cfg.StrOpt('bar'), 'foo')
def cleanup():
if 'OS_FOO__BAR' in os.environ:
del os.environ['OS_FOO__BAR']
self.addCleanup(cleanup)
def test_simple_environment_get(self):
self.conf(args=[])
env_value = 'goodbye'
os.environ['OS_FOO__BAR'] = env_value
self.assertEqual(env_value, self.conf['foo']['bar'])
def test_env_beats_files(self):
file_value = 'hello'
env_value = 'goodbye'
self.conf(args=[])
self.conf_fixture.load_raw_values(
group='foo',
bar=file_value,
)
self.assertEqual(file_value, self.conf['foo']['bar'])
self.conf.reload_config_files()
os.environ['OS_FOO__BAR'] = env_value
self.assertEqual(env_value, self.conf['foo']['bar'])
def test_cli_beats_env(self):
env_value = 'goodbye'
cli_value = 'cli'
os.environ['OS_FOO__BAR'] = env_value
self.conf.register_cli_opt(cfg.StrOpt('bar'), 'foo')
self.conf(args=['--foo=%s' % cli_value])
self.assertEqual(cli_value, self.conf['foo']['bar'])
def test_use_env_false_allows_files(self):
file_value = 'hello'
env_value = 'goodbye'
os.environ['OS_FOO__BAR'] = env_value
self.conf(args=[], use_env=False)
self.conf_fixture.load_raw_values(
group='foo',
bar=file_value,
)
self.assertEqual(file_value, self.conf['foo']['bar'])
self.conf.reset()
self.conf(args=[], use_env=True)
self.assertEqual(env_value, self.conf['foo']['bar'])
def make_uri(name):
return "https://oslo.config/{}.conf".format(name)

22
releasenotes/notes/config-from-environment-3feba7b4cc747d2b.yaml Normal file
View File

@ -0,0 +1,22 @@
---
features:
- |
Support for accessing configuration data in environment variables via the
environment backend driver, enabled by default. The environment is checked
after command line options, but before configuration files.
Environment variables are checked for any configuration data. The variable
names take the form:
* A prefix of ``OS_``
* The group name, uppercased
* Separated from the option name by a `__` (double underscore)
* Followed by the name
For an option that looks like this in the usual INI format::
[placement_database]
connection = sqlite:///
the corresponding environment variable would be
``OS_PLACEMENT_DATABASE__CONNECTION``.