openstack/deb-python-dcos
Code Issues Proposed changes

make resolution of ENV_VAR precedence over config property clear (#644)

Browse Source
This commit is contained in:
tamarrow
2016-06-06 16:58:29 -07:00
parent 0739c974f9
commit 70146dbbb5
16 changed files with 102 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
cli/dcoscli/auth/main.py
View File

@@ -75,8 +75,7 @@ def _login():
# every call to login will generate a new token if applicable
_logout()
conf = config.get_config()
dcos_url = conf.get("core.dcos_url")
dcos_url = config.get_config_val("core.dcos_url")
if dcos_url is None:
msg = ("Please provide the url to your DCOS cluster: "
"`dcos config set core.dcos_url`")
@@ -103,6 +102,6 @@ def _logout():
:rtype: int
"""
if config.get_config().get("core.dcos_acs_token") is not None:
if config.get_config_val("core.dcos_acs_token") is not None:
config.unset("core.dcos_acs_token")
return 0

9
cli/dcoscli/data/help/config.txt
View File

@@ -32,3 +32,12 @@ Positional Arguments:
The name of the property.
<value>
The value of the property.
Environment Variables:
Configuration properties all have corresponding environment variables. If a
property is in the "core" section (ex. "core.foo"), it corresponds to
environment variable DCOS_FOO. All other properties (ex "foo.bar")
correspond to enviroment variable DCOS_FOO_BAR.
Environment variables take precendence over corresponding configuration
property.

5
cli/dcoscli/data/help/dcos.txt
View File

@@ -32,8 +32,3 @@ Environment Variables:
DCOS_LOG_LEVEL
Prints log messages to stderr at or above the level indicated. This is
equivalent to the --log-level command-line option.
DCOS_SSL_VERIFY
Indicates whether to verify SSL certificates for HTTPS (true) or set the
path to the SSL certificates (false). By default, this is variable is
set to true. This is equivalent to setting the `core.ssl_config` option
in the DCOS configuration file.

21
cli/dcoscli/main.py
View File

@@ -4,7 +4,6 @@ import sys
import dcoscli
import docopt
import six
from dcos import config, constants, emitting, errors, http, subcommand, util
from dcos.errors import DCOSException
from dcoscli.subcommand import SubcommandMain, default_doc
@@ -57,8 +56,6 @@ def _main():
signal.signal(signal.SIGINT, signal_handler)
http.silence_requests_warnings()
toml_config = config.get_config()
set_ssl_info_env_vars(toml_config)
args = docopt.docopt(default_doc("dcos"), options_first=True)
@@ -72,7 +69,7 @@ def _main():
util.configure_process_from_environ()
if args['--version']:
return _get_versions(toml_config.get("core.dcos_url"))
return _get_versions(config.get_config_val("core.dcos_url"))
command = args['<command>']
@@ -115,21 +112,5 @@ def signal_handler(signal, frame):
errors.DefaultError("User interrupted command with Ctrl-C"))
sys.exit(0)
def set_ssl_info_env_vars(toml_config):
"""Set SSL info from toml_config to environment variable if enviornment
variable doesn't exist
:param toml_config: config
:type toml_config: Toml
:rtype: None
"""
if 'core.ssl_verify' in toml_config and (
not os.environ.get(constants.DCOS_SSL_VERIFY_ENV)):
os.environ[constants.DCOS_SSL_VERIFY_ENV] = six.text_type(
toml_config['core.ssl_verify'])
if __name__ == "__main__":
main()

6
cli/dcoscli/package/main.py
View File

@@ -525,9 +525,11 @@ def _get_cosmos_url():
:rtype: str
"""
toml_config = config.get_config()
cosmos_url = toml_config.get("package.cosmos_url")
cosmos_url = config.get_config_val("package.cosmos_url", toml_config)
if cosmos_url is None:
cosmos_url = config.get_config_vals(['core.dcos_url'], toml_config)[0]
cosmos_url = config.get_config_val("core.dcos_url", toml_config)
if cosmos_url is None:
raise config.missing_config_exception(["core.dcos_url"])
return cosmos_url

9
cli/tests/data/help/config.txt
View File

@@ -32,3 +32,12 @@ Positional Arguments:
The name of the property.
<value>
The value of the property.
Environment Variables:
Configuration properties all have corresponding environment variables. If a
property is in the "core" section (ex. "core.foo"), it corresponds to
environment variable DCOS_FOO. All other properties (ex "foo.bar")
correspond to enviroment variable DCOS_FOO_BAR.
Environment variables take precendence over corresponding configuration
property.

5
cli/tests/data/help/dcos.txt
View File

@@ -32,8 +32,3 @@ Environment Variables:
DCOS_LOG_LEVEL
Prints log messages to stderr at or above the level indicated. This is
equivalent to the --log-level command-line option.
DCOS_SSL_VERIFY
Indicates whether to verify SSL certificates for HTTPS (true) or set the
path to the SSL certificates (false). By default, this is variable is
set to true. This is equivalent to setting the `core.ssl_config` option
in the DCOS configuration file.

2
cli/tests/integrations/common.py
View File

@@ -321,7 +321,7 @@ def delete_zk_node(znode):
:rtype: None
"""
dcos_url = config.get_config_vals(['core.dcos_url'])[0]
dcos_url = config.get_config_val('core.dcos_url')
znode_url = urllib.parse.urljoin(
dcos_url,
'/exhibitor/exhibitor/v1/explorer/znode/{}'.format(znode))

22
cli/tests/integrations/test_config.py
View File

@@ -266,6 +266,28 @@ def test_bad_port_fail_url_validation(env):
'http://localhost:bad_port/', env)
def test_dcos_acs_token_env_var(env):
env['DCOS_ACS_TOKEN'] = 'foobar'
msg = (b'Your core.dcos_acs_token is invalid. '
b'Please run: `dcos auth login`\n')
assert_command(['dcos', 'package', 'list'],
returncode=1,
stderr=msg,
env=env)
env.pop('DCOS_ACS_TOKEN')
def test_dcos_dcos_acs_token_env_var(env):
env['DCOS_DCOS_ACS_TOKEN'] = 'foobar'
msg = (b'Your core.dcos_acs_token is invalid. '
b'Please run: `dcos auth login`\n')
assert_command(['dcos', 'package', 'list'],
returncode=1,
stderr=msg,
env=env)
env.pop('DCOS_DCOS_ACS_TOKEN')
@pytest.mark.skipif(
True, reason='Network tests are unreliable')
def test_timeout(env):

16
cli/tests/integrations/test_ssl.py
View File

@@ -28,14 +28,14 @@ def setup_env(env):
def test_dont_verify_ssl_with_env_var(env):
env[constants.DCOS_SSL_VERIFY_ENV] = 'false'
env['DCOS_SSL_VERIFY'] = 'false'
returncode, stdout, stderr = exec_command(
['dcos', 'marathon', 'app', 'list'], env)
assert returncode == 0
assert stderr == b''
env.pop(constants.DCOS_SSL_VERIFY_ENV)
env.pop('DCOS_SSL_VERIFY')
def test_dont_verify_ssl_with_config(env):
@@ -47,14 +47,14 @@ def test_dont_verify_ssl_with_config(env):
def test_verify_ssl_without_cert_env_var(env):
env[constants.DCOS_SSL_VERIFY_ENV] = 'true'
env['DCOS_SSL_VERIFY'] = 'true'
with update_config('core.ssl_verify', None, env):
returncode, stdout, stderr = exec_command(
['dcos', 'marathon', 'app', 'list'], env)
assert returncode == 1
assert stderr.decode('utf-8') == _ssl_error_msg()
env.pop(constants.DCOS_SSL_VERIFY_ENV)
env.pop('DCOS_SSL_VERIFY')
def test_verify_ssl_without_cert_config(env):
@@ -66,7 +66,7 @@ def test_verify_ssl_without_cert_config(env):
def test_verify_ssl_with_bad_cert_env_var(env):
env[constants.DCOS_SSL_VERIFY_ENV] = 'tests/data/ssl/fake.pem'
env['DCOS_SSL_VERIFY'] = 'tests/data/ssl/fake.pem'
with update_config('core.ssl_verify', None, env):
returncode, stdout, stderr = exec_command(
@@ -74,7 +74,7 @@ def test_verify_ssl_with_bad_cert_env_var(env):
assert returncode == 1
assert stderr.decode('utf-8') == _ssl_error_msg()
env.pop(constants.DCOS_SSL_VERIFY_ENV)
env.pop('DCOS_SSL_VERIFY')
def test_verify_ssl_with_bad_cert_config(env):
@@ -86,7 +86,7 @@ def test_verify_ssl_with_bad_cert_config(env):
def test_verify_ssl_with_good_cert_env_var(env):
env[constants.DCOS_SSL_VERIFY_ENV] = '/dcos-cli/adminrouter/snakeoil.crt'
env['DCOS_SSL_VERIFY'] = '/dcos-cli/adminrouter/snakeoil.crt'
with update_config('core.ssl_verify', None, env):
returncode, stdout, stderr = exec_command(
@@ -94,7 +94,7 @@ def test_verify_ssl_with_good_cert_env_var(env):
assert returncode == 0
assert stderr == b''
env.pop(constants.DCOS_SSL_VERIFY_ENV)
env.pop('DCOS_SSL_VERIFY')
def test_verify_ssl_with_good_cert_config(env):

43
dcos/config.py
View File

@@ -35,8 +35,10 @@ def get_default_config_path():
def get_config(mutable=False):
"""Returns the DCOS configuration object and creates config file is none
found and `DCOS_CONFIG` set to default value
"""Returns the DCOS configuration object and creates config file is None
found and `DCOS_CONFIG` set to default value. Only use to get the config,
not to resolve a specific config parameter. This should be done with
`get_config_val`.
:param mutable: True if the returned Toml object should be mutable
:type mutable: boolean
@@ -52,24 +54,37 @@ def get_config(mutable=False):
return load_from_path(path, mutable)
def get_config_vals(keys, config=None):
"""Gets config values for each of the keys. Raises a DCOSException if
any of the keys don't exist.
def get_config_val(name, config=None):
"""Returns the config value for the specified key. Looks for corresponding
environment variable first, and if it doesn't exist, uses the config value.
- "core" properties get resolved to env variable DCOS_SUBKEY. With the
exception of subkeys that already start with DCOS, in which case we look
for SUBKEY first, and "DCOS_SUBKEY" second, and finally the config value.
- everything else gets resolved to DCOS_SECTION_SUBKEY
:param name: name of paramater
:type name: str
:param config: config
:type config: Toml
:param keys: keys in the config dict
:type keys: [str]
:returns: values for each of the keys
:rtype: [object]
:returns: value of 'name' parameter
:rtype: object | None
"""
config = config or get_config()
missing = [key for key in keys if key not in config]
if missing:
raise missing_config_exception(keys)
if config is None:
config = get_config()
return [config[key] for key in keys]
section, subkey = split_key(name.upper())
env_var = None
if section == "CORE":
if subkey.startswith("DCOS") and os.environ.get(subkey):
env_var = subkey
else:
env_var = "DCOS_{}".format(subkey)
else:
env_var = "DCOS_{}_{}".format(section, subkey)
return os.environ.get(env_var) or config.get(name)
def missing_config_exception(keys):

3
dcos/constants.py
View File

@@ -20,9 +20,6 @@ DCOS_DEBUG_ENV = 'DCOS_DEBUG'
DCOS_PAGER_COMMAND_ENV = 'PAGER'
"""Command to use to page long command output (e.g. 'less -R')"""
DCOS_SSL_VERIFY_ENV = 'DCOS_SSL_VERIFY'
"""Whether or not ot verify SSL certs for HTTPS or path to certificate(s)"""
PATH_ENV = 'PATH'
"""Name of the environment variable pointing to the executable directories."""

2
dcos/emitting.py
View File

@@ -172,7 +172,7 @@ def _page(output, pager_command=None):
num_lines = output.count('\n')
exceeds_tty_height = pager.getheight() - 1 < num_lines
paginate = config.get_config().get("core.pagination", True)
paginate = config.get_config_val("core.pagination") or True
if exceeds_tty_height and paginate:
pydoc.pipepager(output, cmd=pager_command)
else:

13
dcos/http.py
View File

@@ -1,10 +1,9 @@
import getpass
import os
import sys
import threading
import requests
from dcos import config, constants, util
from dcos import config, util
from dcos.errors import (DCOSAuthenticationException,
DCOSAuthorizationException, DCOSException,
DCOSHTTPException)
@@ -43,8 +42,8 @@ def _verify_ssl(verify=None):
:rtype: bool | str
"""
if verify is None and constants.DCOS_SSL_VERIFY_ENV in os.environ:
verify = os.environ[constants.DCOS_SSL_VERIFY_ENV]
if verify is None:
verify = config.get_config_val("core.ssl_verify")
if verify.lower() == "true":
verify = True
elif verify.lower() == "false":
@@ -172,7 +171,7 @@ def _request_with_auth(response,
elif response.status_code == 401 and \
auth_scheme in ["acsjwt", "oauthjwt"]:
if config.get_config().get("core.dcos_acs_token") is not None:
if config.get_config_val("core.dcos_acs_token") is not None:
msg = ("Your core.dcos_acs_token is invalid. "
"Please run: `dcos auth login`")
raise DCOSException(msg)
@@ -473,9 +472,9 @@ def _get_dcos_auth(auth_scheme, username, password, hostname):
"""
toml_config = config.get_config()
token = toml_config.get("core.dcos_acs_token")
token = config.get_config_val("core.dcos_acs_token", toml_config)
if token is None:
dcos_url = toml_config.get("core.dcos_url")
dcos_url = config.get_config_val("core.dcos_url", toml_config)
if auth_scheme == "acsjwt":
creds = _get_dcos_acs_auth_creds(username, password, hostname)
else:

8
dcos/marathon.py
View File

@@ -22,7 +22,7 @@ def create_client(toml_config=None):
toml_config = config.get_config()
marathon_url = _get_marathon_url(toml_config)
timeout = toml_config.get('core.timeout', http.DEFAULT_TIMEOUT)
timeout = config.get_config_val('core.timeout') or http.DEFAULT_TIMEOUT
logger.info('Creating marathon client with: %r', marathon_url)
return Client(marathon_url, timeout=timeout)
@@ -36,9 +36,11 @@ def _get_marathon_url(toml_config):
:rtype: str
"""
marathon_url = toml_config.get('marathon.url')
marathon_url = config.get_config_val('marathon.url', toml_config)
if marathon_url is None:
dcos_url = config.get_config_vals(['core.dcos_url'], toml_config)[0]
dcos_url = config.get_config_val('core.dcos_url', toml_config)
if dcos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
marathon_url = urllib.parse.urljoin(dcos_url, 'service/marathon/')
return marathon_url

9
dcos/mesos.py
View File

@@ -31,12 +31,13 @@ class DCOSClient(object):
def __init__(self):
toml_config = config.get_config()
self._dcos_url = toml_config.get("core.dcos_url")
self._dcos_url = config.get_config_val("core.dcos_url", toml_config)
if self._dcos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
self._mesos_master_url = toml_config.get('core.mesos_master_url')
self._mesos_master_url = config.get_config_val(
'core.mesos_master_url', toml_config)
self._timeout = toml_config.get('core.timeout')
self._timeout = config.get_config_val('core.timeout', toml_config)
def get_dcos_url(self, path):
""" Create a DCOS URL
@@ -269,7 +270,7 @@ class MesosDNSClient(object):
"""
def __init__(self, url=None):
self.url = url or urllib.parse.urljoin(
config.get_config_vals(['core.dcos_url'])[0], '/mesos_dns/')
config.get_config_val('core.dcos_url'), '/mesos_dns/')
def _path(self, path):
""" Construct a full path