diff --git a/cli/dcoscli/auth/main.py b/cli/dcoscli/auth/main.py index 988248e..274bf04 100644 --- a/cli/dcoscli/auth/main.py +++ b/cli/dcoscli/auth/main.py @@ -75,8 +75,7 @@ def _login(): # every call to login will generate a new token if applicable _logout() - conf = config.get_config() - dcos_url = conf.get("core.dcos_url") + dcos_url = config.get_config_val("core.dcos_url") if dcos_url is None: msg = ("Please provide the url to your DCOS cluster: " "`dcos config set core.dcos_url`") @@ -103,6 +102,6 @@ def _logout(): :rtype: int """ - if config.get_config().get("core.dcos_acs_token") is not None: + if config.get_config_val("core.dcos_acs_token") is not None: config.unset("core.dcos_acs_token") return 0 diff --git a/cli/dcoscli/data/help/config.txt b/cli/dcoscli/data/help/config.txt index 5e1568f..902836b 100644 --- a/cli/dcoscli/data/help/config.txt +++ b/cli/dcoscli/data/help/config.txt @@ -32,3 +32,12 @@ Positional Arguments: The name of the property. The value of the property. + +Environment Variables: + Configuration properties all have corresponding environment variables. If a + property is in the "core" section (ex. "core.foo"), it corresponds to + environment variable DCOS_FOO. All other properties (ex "foo.bar") + correspond to enviroment variable DCOS_FOO_BAR. + + Environment variables take precendence over corresponding configuration + property. diff --git a/cli/dcoscli/data/help/dcos.txt b/cli/dcoscli/data/help/dcos.txt index b3d9fd3..aa10136 100644 --- a/cli/dcoscli/data/help/dcos.txt +++ b/cli/dcoscli/data/help/dcos.txt @@ -32,8 +32,3 @@ Environment Variables: DCOS_LOG_LEVEL Prints log messages to stderr at or above the level indicated. This is equivalent to the --log-level command-line option. - DCOS_SSL_VERIFY - Indicates whether to verify SSL certificates for HTTPS (true) or set the - path to the SSL certificates (false). By default, this is variable is - set to true. This is equivalent to setting the `core.ssl_config` option - in the DCOS configuration file. diff --git a/cli/dcoscli/main.py b/cli/dcoscli/main.py index b112d59..b37e001 100644 --- a/cli/dcoscli/main.py +++ b/cli/dcoscli/main.py @@ -4,7 +4,6 @@ import sys import dcoscli import docopt -import six from dcos import config, constants, emitting, errors, http, subcommand, util from dcos.errors import DCOSException from dcoscli.subcommand import SubcommandMain, default_doc @@ -57,8 +56,6 @@ def _main(): signal.signal(signal.SIGINT, signal_handler) http.silence_requests_warnings() - toml_config = config.get_config() - set_ssl_info_env_vars(toml_config) args = docopt.docopt(default_doc("dcos"), options_first=True) @@ -72,7 +69,7 @@ def _main(): util.configure_process_from_environ() if args['--version']: - return _get_versions(toml_config.get("core.dcos_url")) + return _get_versions(config.get_config_val("core.dcos_url")) command = args[''] @@ -115,21 +112,5 @@ def signal_handler(signal, frame): errors.DefaultError("User interrupted command with Ctrl-C")) sys.exit(0) - -def set_ssl_info_env_vars(toml_config): - """Set SSL info from toml_config to environment variable if enviornment - variable doesn't exist - - :param toml_config: config - :type toml_config: Toml - :rtype: None - """ - - if 'core.ssl_verify' in toml_config and ( - not os.environ.get(constants.DCOS_SSL_VERIFY_ENV)): - - os.environ[constants.DCOS_SSL_VERIFY_ENV] = six.text_type( - toml_config['core.ssl_verify']) - if __name__ == "__main__": main() diff --git a/cli/dcoscli/package/main.py b/cli/dcoscli/package/main.py index 788a186..4ac0599 100644 --- a/cli/dcoscli/package/main.py +++ b/cli/dcoscli/package/main.py @@ -525,9 +525,11 @@ def _get_cosmos_url(): :rtype: str """ toml_config = config.get_config() - cosmos_url = toml_config.get("package.cosmos_url") + cosmos_url = config.get_config_val("package.cosmos_url", toml_config) if cosmos_url is None: - cosmos_url = config.get_config_vals(['core.dcos_url'], toml_config)[0] + cosmos_url = config.get_config_val("core.dcos_url", toml_config) + if cosmos_url is None: + raise config.missing_config_exception(["core.dcos_url"]) return cosmos_url diff --git a/cli/tests/data/help/config.txt b/cli/tests/data/help/config.txt index 5e1568f..902836b 100644 --- a/cli/tests/data/help/config.txt +++ b/cli/tests/data/help/config.txt @@ -32,3 +32,12 @@ Positional Arguments: The name of the property. The value of the property. + +Environment Variables: + Configuration properties all have corresponding environment variables. If a + property is in the "core" section (ex. "core.foo"), it corresponds to + environment variable DCOS_FOO. All other properties (ex "foo.bar") + correspond to enviroment variable DCOS_FOO_BAR. + + Environment variables take precendence over corresponding configuration + property. diff --git a/cli/tests/data/help/dcos.txt b/cli/tests/data/help/dcos.txt index b3d9fd3..aa10136 100644 --- a/cli/tests/data/help/dcos.txt +++ b/cli/tests/data/help/dcos.txt @@ -32,8 +32,3 @@ Environment Variables: DCOS_LOG_LEVEL Prints log messages to stderr at or above the level indicated. This is equivalent to the --log-level command-line option. - DCOS_SSL_VERIFY - Indicates whether to verify SSL certificates for HTTPS (true) or set the - path to the SSL certificates (false). By default, this is variable is - set to true. This is equivalent to setting the `core.ssl_config` option - in the DCOS configuration file. diff --git a/cli/tests/integrations/common.py b/cli/tests/integrations/common.py index 334d7af..cda523f 100644 --- a/cli/tests/integrations/common.py +++ b/cli/tests/integrations/common.py @@ -321,7 +321,7 @@ def delete_zk_node(znode): :rtype: None """ - dcos_url = config.get_config_vals(['core.dcos_url'])[0] + dcos_url = config.get_config_val('core.dcos_url') znode_url = urllib.parse.urljoin( dcos_url, '/exhibitor/exhibitor/v1/explorer/znode/{}'.format(znode)) diff --git a/cli/tests/integrations/test_config.py b/cli/tests/integrations/test_config.py index 86bc300..8e87dce 100644 --- a/cli/tests/integrations/test_config.py +++ b/cli/tests/integrations/test_config.py @@ -266,6 +266,28 @@ def test_bad_port_fail_url_validation(env): 'http://localhost:bad_port/', env) +def test_dcos_acs_token_env_var(env): + env['DCOS_ACS_TOKEN'] = 'foobar' + msg = (b'Your core.dcos_acs_token is invalid. ' + b'Please run: `dcos auth login`\n') + assert_command(['dcos', 'package', 'list'], + returncode=1, + stderr=msg, + env=env) + env.pop('DCOS_ACS_TOKEN') + + +def test_dcos_dcos_acs_token_env_var(env): + env['DCOS_DCOS_ACS_TOKEN'] = 'foobar' + msg = (b'Your core.dcos_acs_token is invalid. ' + b'Please run: `dcos auth login`\n') + assert_command(['dcos', 'package', 'list'], + returncode=1, + stderr=msg, + env=env) + env.pop('DCOS_DCOS_ACS_TOKEN') + + @pytest.mark.skipif( True, reason='Network tests are unreliable') def test_timeout(env): diff --git a/cli/tests/integrations/test_ssl.py b/cli/tests/integrations/test_ssl.py index 0319f79..f4c34ea 100644 --- a/cli/tests/integrations/test_ssl.py +++ b/cli/tests/integrations/test_ssl.py @@ -28,14 +28,14 @@ def setup_env(env): def test_dont_verify_ssl_with_env_var(env): - env[constants.DCOS_SSL_VERIFY_ENV] = 'false' + env['DCOS_SSL_VERIFY'] = 'false' returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'list'], env) assert returncode == 0 assert stderr == b'' - env.pop(constants.DCOS_SSL_VERIFY_ENV) + env.pop('DCOS_SSL_VERIFY') def test_dont_verify_ssl_with_config(env): @@ -47,14 +47,14 @@ def test_dont_verify_ssl_with_config(env): def test_verify_ssl_without_cert_env_var(env): - env[constants.DCOS_SSL_VERIFY_ENV] = 'true' + env['DCOS_SSL_VERIFY'] = 'true' with update_config('core.ssl_verify', None, env): returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'list'], env) assert returncode == 1 assert stderr.decode('utf-8') == _ssl_error_msg() - env.pop(constants.DCOS_SSL_VERIFY_ENV) + env.pop('DCOS_SSL_VERIFY') def test_verify_ssl_without_cert_config(env): @@ -66,7 +66,7 @@ def test_verify_ssl_without_cert_config(env): def test_verify_ssl_with_bad_cert_env_var(env): - env[constants.DCOS_SSL_VERIFY_ENV] = 'tests/data/ssl/fake.pem' + env['DCOS_SSL_VERIFY'] = 'tests/data/ssl/fake.pem' with update_config('core.ssl_verify', None, env): returncode, stdout, stderr = exec_command( @@ -74,7 +74,7 @@ def test_verify_ssl_with_bad_cert_env_var(env): assert returncode == 1 assert stderr.decode('utf-8') == _ssl_error_msg() - env.pop(constants.DCOS_SSL_VERIFY_ENV) + env.pop('DCOS_SSL_VERIFY') def test_verify_ssl_with_bad_cert_config(env): @@ -86,7 +86,7 @@ def test_verify_ssl_with_bad_cert_config(env): def test_verify_ssl_with_good_cert_env_var(env): - env[constants.DCOS_SSL_VERIFY_ENV] = '/dcos-cli/adminrouter/snakeoil.crt' + env['DCOS_SSL_VERIFY'] = '/dcos-cli/adminrouter/snakeoil.crt' with update_config('core.ssl_verify', None, env): returncode, stdout, stderr = exec_command( @@ -94,7 +94,7 @@ def test_verify_ssl_with_good_cert_env_var(env): assert returncode == 0 assert stderr == b'' - env.pop(constants.DCOS_SSL_VERIFY_ENV) + env.pop('DCOS_SSL_VERIFY') def test_verify_ssl_with_good_cert_config(env): diff --git a/dcos/config.py b/dcos/config.py index 0f5990e..2f6a854 100644 --- a/dcos/config.py +++ b/dcos/config.py @@ -35,8 +35,10 @@ def get_default_config_path(): def get_config(mutable=False): - """Returns the DCOS configuration object and creates config file is none - found and `DCOS_CONFIG` set to default value + """Returns the DCOS configuration object and creates config file is None + found and `DCOS_CONFIG` set to default value. Only use to get the config, + not to resolve a specific config parameter. This should be done with + `get_config_val`. :param mutable: True if the returned Toml object should be mutable :type mutable: boolean @@ -52,24 +54,37 @@ def get_config(mutable=False): return load_from_path(path, mutable) -def get_config_vals(keys, config=None): - """Gets config values for each of the keys. Raises a DCOSException if - any of the keys don't exist. +def get_config_val(name, config=None): + """Returns the config value for the specified key. Looks for corresponding + environment variable first, and if it doesn't exist, uses the config value. + - "core" properties get resolved to env variable DCOS_SUBKEY. With the + exception of subkeys that already start with DCOS, in which case we look + for SUBKEY first, and "DCOS_SUBKEY" second, and finally the config value. + - everything else gets resolved to DCOS_SECTION_SUBKEY + :param name: name of paramater + :type name: str :param config: config :type config: Toml - :param keys: keys in the config dict - :type keys: [str] - :returns: values for each of the keys - :rtype: [object] + :returns: value of 'name' parameter + :rtype: object | None """ - config = config or get_config() - missing = [key for key in keys if key not in config] - if missing: - raise missing_config_exception(keys) + if config is None: + config = get_config() - return [config[key] for key in keys] + section, subkey = split_key(name.upper()) + + env_var = None + if section == "CORE": + if subkey.startswith("DCOS") and os.environ.get(subkey): + env_var = subkey + else: + env_var = "DCOS_{}".format(subkey) + else: + env_var = "DCOS_{}_{}".format(section, subkey) + + return os.environ.get(env_var) or config.get(name) def missing_config_exception(keys): diff --git a/dcos/constants.py b/dcos/constants.py index 42ba248..f61c26a 100644 --- a/dcos/constants.py +++ b/dcos/constants.py @@ -20,9 +20,6 @@ DCOS_DEBUG_ENV = 'DCOS_DEBUG' DCOS_PAGER_COMMAND_ENV = 'PAGER' """Command to use to page long command output (e.g. 'less -R')""" -DCOS_SSL_VERIFY_ENV = 'DCOS_SSL_VERIFY' -"""Whether or not ot verify SSL certs for HTTPS or path to certificate(s)""" - PATH_ENV = 'PATH' """Name of the environment variable pointing to the executable directories.""" diff --git a/dcos/emitting.py b/dcos/emitting.py index a9f22bf..b98f606 100644 --- a/dcos/emitting.py +++ b/dcos/emitting.py @@ -172,7 +172,7 @@ def _page(output, pager_command=None): num_lines = output.count('\n') exceeds_tty_height = pager.getheight() - 1 < num_lines - paginate = config.get_config().get("core.pagination", True) + paginate = config.get_config_val("core.pagination") or True if exceeds_tty_height and paginate: pydoc.pipepager(output, cmd=pager_command) else: diff --git a/dcos/http.py b/dcos/http.py index f24693d..fe96a35 100644 --- a/dcos/http.py +++ b/dcos/http.py @@ -1,10 +1,9 @@ import getpass -import os import sys import threading import requests -from dcos import config, constants, util +from dcos import config, util from dcos.errors import (DCOSAuthenticationException, DCOSAuthorizationException, DCOSException, DCOSHTTPException) @@ -43,8 +42,8 @@ def _verify_ssl(verify=None): :rtype: bool | str """ - if verify is None and constants.DCOS_SSL_VERIFY_ENV in os.environ: - verify = os.environ[constants.DCOS_SSL_VERIFY_ENV] + if verify is None: + verify = config.get_config_val("core.ssl_verify") if verify.lower() == "true": verify = True elif verify.lower() == "false": @@ -172,7 +171,7 @@ def _request_with_auth(response, elif response.status_code == 401 and \ auth_scheme in ["acsjwt", "oauthjwt"]: - if config.get_config().get("core.dcos_acs_token") is not None: + if config.get_config_val("core.dcos_acs_token") is not None: msg = ("Your core.dcos_acs_token is invalid. " "Please run: `dcos auth login`") raise DCOSException(msg) @@ -473,9 +472,9 @@ def _get_dcos_auth(auth_scheme, username, password, hostname): """ toml_config = config.get_config() - token = toml_config.get("core.dcos_acs_token") + token = config.get_config_val("core.dcos_acs_token", toml_config) if token is None: - dcos_url = toml_config.get("core.dcos_url") + dcos_url = config.get_config_val("core.dcos_url", toml_config) if auth_scheme == "acsjwt": creds = _get_dcos_acs_auth_creds(username, password, hostname) else: diff --git a/dcos/marathon.py b/dcos/marathon.py index 40a769d..fc9b829 100644 --- a/dcos/marathon.py +++ b/dcos/marathon.py @@ -22,7 +22,7 @@ def create_client(toml_config=None): toml_config = config.get_config() marathon_url = _get_marathon_url(toml_config) - timeout = toml_config.get('core.timeout', http.DEFAULT_TIMEOUT) + timeout = config.get_config_val('core.timeout') or http.DEFAULT_TIMEOUT logger.info('Creating marathon client with: %r', marathon_url) return Client(marathon_url, timeout=timeout) @@ -36,9 +36,11 @@ def _get_marathon_url(toml_config): :rtype: str """ - marathon_url = toml_config.get('marathon.url') + marathon_url = config.get_config_val('marathon.url', toml_config) if marathon_url is None: - dcos_url = config.get_config_vals(['core.dcos_url'], toml_config)[0] + dcos_url = config.get_config_val('core.dcos_url', toml_config) + if dcos_url is None: + raise config.missing_config_exception(['core.dcos_url']) marathon_url = urllib.parse.urljoin(dcos_url, 'service/marathon/') return marathon_url diff --git a/dcos/mesos.py b/dcos/mesos.py index 23c3948..108a0ed 100644 --- a/dcos/mesos.py +++ b/dcos/mesos.py @@ -31,12 +31,13 @@ class DCOSClient(object): def __init__(self): toml_config = config.get_config() - self._dcos_url = toml_config.get("core.dcos_url") + self._dcos_url = config.get_config_val("core.dcos_url", toml_config) if self._dcos_url is None: raise config.missing_config_exception(['core.dcos_url']) - self._mesos_master_url = toml_config.get('core.mesos_master_url') + self._mesos_master_url = config.get_config_val( + 'core.mesos_master_url', toml_config) - self._timeout = toml_config.get('core.timeout') + self._timeout = config.get_config_val('core.timeout', toml_config) def get_dcos_url(self, path): """ Create a DCOS URL @@ -269,7 +270,7 @@ class MesosDNSClient(object): """ def __init__(self, url=None): self.url = url or urllib.parse.urljoin( - config.get_config_vals(['core.dcos_url'])[0], '/mesos_dns/') + config.get_config_val('core.dcos_url'), '/mesos_dns/') def _path(self, path): """ Construct a full path