openstack/deb-python-dcos
Code Issues Proposed changes

cluster: add subcommand for easy setup of multiple clusters (#983)

Browse Source 
We are introducing a new subcommand for managing your clusters. Configuring your CLI to talk to your cluster is a single command now `dcos cluster setup`. Moreover, the CLI can now be aware of multiple clusters with cluster specific configuration managed by the CLI. 

Subcommands will be installed for the current "attached" cluster only. To install a subcommand for all your configured clusters, use `--global`. 

Note that `DCOS_CONFIG` environment variable will not take effect in
"cluster" mode since we are now managing different clusters in the CLI.
This commit is contained in:
tamarrow
2017-05-03 10:41:45 -07:00
committed by GitHub
parent b9043397bd
commit 63bcddaa20
38 changed files with 1423 additions and 309 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
README.rst
View File

@@ -106,8 +106,6 @@ make sure you set owner only permissions on these files:
:code:`chmod 600 cli/tests/data/dcos.toml`
:code:`chmod 600 cli/tests/data/config/parse_error.toml`
The :code:`node` integration tests use :code:`CLI_TEST_SSH_KEY_PATH` for ssh
credentials to your cluster.
@@ -124,10 +122,6 @@ Running
Tox will run unit and integration tests in Python 3.5 using a temporarily
created virtualenv.
You can set :code:`DCOS_CONFIG` to a config file that points to a DC/OS
cluster you want to use for integration tests. This defaults to
:code:`~/.dcos/dcos.toml`
Note that in order for all the integration tests to pass, your DC/OS cluster
must have the experimental packaging features enabled. In order to enable
these features the :code:`staged_package_storage_uri` and :code:`package_storage_uri`

1
cli/bin/test.sh
View File

@@ -11,6 +11,5 @@ fi
echo "Virtualenv activated."
chmod 600 $BASEDIR/tests/data/dcos.toml
chmod 600 $BASEDIR/tests/data/config/parse_error.toml
tox

88
cli/dcoscli/auth/main.py
View File

@@ -144,37 +144,8 @@ def _login(password_str, password_env, password_file,
# every call to login will generate a new token if applicable
_logout()
password = _get_password(password_str, password_env, password_file)
if provider is None:
if username and password:
auth.dcos_uid_password_auth(dcos_url, username, password)
elif username and key_path:
auth.servicecred_auth(dcos_url, username, key_path)
else:
try:
providers = auth.get_providers()
# Let users know if they have non-default providers configured
# This is a weak check, we should check default versions per
# DC/OS version since defaults will change. jj
if len(providers) > 2:
msg = ("\nYour cluster has several authentication "
"providers enabled. Run `dcos auth "
"list-providers` to see all providers and `dcos "
"auth login --provider <provider-id>` to "
"authenticate with a specific provider\n")
emitter.publish(DefaultError(msg))
except DCOSException:
pass
finally:
auth.header_challenge_auth(dcos_url)
else:
providers = auth.get_providers()
if providers.get(provider):
_trigger_client_method(
provider, providers[provider], username, password, key_path)
else:
msg = "Provider [{}] not configured on your cluster"
raise DCOSException(msg.format(provider))
login(dcos_url, password_str, password_env, password_file,
provider, username, key_path)
emitter.publish("Login successful!")
return 0
@@ -232,3 +203,58 @@ def _logout():
if config.get_config_val("core.dcos_acs_token") is not None:
config.unset("core.dcos_acs_token")
return 0
def login(dcos_url, password_str, password_env, password_file,
provider, username, key_path):
"""
:param dcos_url: URL of DC/OS cluster
:type dcos_url: str
:param password_str: password
:type password_str: str
:param password_env: name of environment variable with password
:type password_env: str
:param password_file: path to file with password
:type password_file: bool
:param provider: name of provider to authentication with
:type provider: str
:param username: username
:type username: str
:param key_path: path to file with private key
:type param: str
:rtype: int
"""
password = _get_password(password_str, password_env, password_file)
if provider is None:
if username and password:
auth.dcos_uid_password_auth(dcos_url, username, password)
elif username and key_path:
auth.servicecred_auth(dcos_url, username, key_path)
else:
try:
providers = auth.get_providers()
# Let users know if they have non-default providers configured
# This is a weak check, we should check default versions per
# DC/OS version since defaults will change. jj
if len(providers) > 2:
msg = ("\nYour cluster has several authentication "
"providers enabled. Run `dcos auth "
"list-providers` to see all providers and `dcos "
"auth login --provider <provider-id>` to "
"authenticate with a specific provider\n")
emitter.publish(DefaultError(msg))
except DCOSException:
pass
finally:
auth.header_challenge_auth(dcos_url)
else:
providers = auth.get_providers()
if providers.get(provider):
_trigger_client_method(
provider, providers[provider], username, password, key_path)
else:
msg = "Provider [{}] not configured on your cluster"
raise DCOSException(msg.format(provider))
return 0

0
cli/dcoscli/cluster/__init__.py Normal file
View File

290
cli/dcoscli/cluster/main.py Normal file
View File

@@ -0,0 +1,290 @@
import os
import docopt
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
import dcoscli
from dcos import cluster, cmds, config, emitting, http, util
from dcos.errors import DCOSAuthenticationException, DCOSException
from dcoscli.auth.main import login
from dcoscli.subcommand import default_command_info, default_doc
from dcoscli.tables import clusters_table
from dcoscli.util import confirm, decorate_docopt_usage
emitter = emitting.FlatEmitter()
logger = util.get_logger(__name__)
def main(argv):
try:
return _main(argv)
except DCOSException as e:
emitter.publish(e)
return 1
@decorate_docopt_usage
def _main(argv):
args = docopt.docopt(
default_doc("cluster"),
argv=argv,
version='dcos-cluster version {}'.format(dcoscli.version))
http.silence_requests_warnings()
return cmds.execute(_cmds(), args)
def _cmds():
"""
:returns: all the supported commands
:rtype: list of dcos.cmds.Command
"""
return [
cmds.Command(
hierarchy=['cluster', 'setup'],
arg_keys=['<dcos_url>',
'--insecure', '--no-check', '--ca-certs',
'--password', '--password-env', '--password-file',
'--provider', '--username', '--private-key'],
function=setup),
cmds.Command(
hierarchy=['cluster', 'list'],
arg_keys=['--json', '--attached'],
function=_list),
cmds.Command(
hierarchy=['cluster', 'remove'],
arg_keys=['<name>'],
function=_remove),
cmds.Command(
hierarchy=['cluster', 'attach'],
arg_keys=['<name>'],
function=_attach),
cmds.Command(
hierarchy=['cluster', 'rename'],
arg_keys=['<name>', '<new_name>'],
function=_rename),
cmds.Command(
hierarchy=['cluster'],
arg_keys=['--info'],
function=_info),
]
def _info(info):
"""
:param info: Whether to output a description of this subcommand
:type info: boolean
:returns: process status
:rtype: int
"""
emitter.publish(default_command_info("cluster"))
return 0
def _list(json_, attached):
"""
List configured clusters.
:param json_: output json if True
:type json_: bool
:param attached: return only attached cluster
:type attached: True
:rtype: None
"""
clusters = [c.dict() for c in cluster.get_clusters()
if not attached or c.is_attached()]
if json_:
emitter.publish(clusters)
elif len(clusters) == 0:
if attached:
msg = ("No cluster is attached. "
"Please run `dcos cluster attach <cluster-name>")
else:
msg = ("No clusters are currently configured. "
"To configure one, run `dcos cluster setup <dcos_url>`")
raise DCOSException(msg)
else:
emitter.publish(clusters_table(clusters))
return
def _remove(name):
"""
:param name: name of cluster
:type name: str
:rtype: None
"""
return cluster.remove(name)
def _attach(name):
"""
:param name: name of cluster
:type name: str
:rtype: None
"""
c = cluster.get_cluster(name)
if c is not None:
return cluster.set_attached(c.get_cluster_path())
else:
raise DCOSException("Cluster [{}] does not exist".format(name))
def _rename(name, new_name):
"""
:param name: name of cluster
:type name: str
:param new_name: new_name of cluster
:type new_name: str
:rtype: None
"""
c = cluster.get_cluster(name)
other = cluster.get_cluster(new_name)
if c is None:
raise DCOSException("Cluster [{}] does not exist".format(name))
elif other and other != c:
msg = "A cluster with name [{}] already exists"
raise DCOSException(msg.format(new_name))
else:
config.set_val("cluster.name", new_name, c.get_config_path())
def setup(dcos_url,
insecure=False, no_check=False, ca_certs=None,
password_str=None, password_env=None, password_file=None,
provider=None, username=None, key_path=None):
"""
Setup the CLI to talk to your DC/OS cluster.
:param dcos_url: master ip of cluster
:type dcos_url: str
:param insecure: whether or not to verify ssl certs
:type insecure: bool
:param no_check: whether or not to verify downloaded ca cert
:type no_check: bool
:param ca_certs: path to root CA to verify requests
:type ca_certs: str
:param password_str: password
:type password_str: str
:param password_env: name of environment variable with password
:type password_env: str
:param password_file: path to file with password
:type password_file: bool
:param provider: name of provider to authentication with
:type provider: str
:param username: username
:type username: str
:param key_path: path to file with private key
:type param: str
:returns: process status
:rtype: int
"""
with cluster.setup_directory() as temp_path:
# set cluster as attached
cluster.set_attached(temp_path)
# authenticate
config.set_val("core.dcos_url", dcos_url)
# get validated dcos_url
dcos_url = config.get_config_val("core.dcos_url")
# configure ssl settings
stored_cert = False
if insecure:
config.set_val("core.ssl_verify", "false")
elif ca_certs:
config.set_val("core.ssl_verify", ca_certs)
else:
cert = cluster.get_cluster_cert(dcos_url)
# if we don't have a cert don't try to verify one
if cert is False:
config.set_val("core.ssl_verify", "false")
else:
stored_cert = _store_cluster_cert(cert, no_check)
try:
login(dcos_url,
password_str, password_env, password_file,
provider, username, key_path)
except DCOSAuthenticationException:
msg = ("Authentication failed. "
"Please run `dcos cluster setup <dcos_url>`")
raise DCOSException(msg)
# configure cluster directory
cluster.setup_cluster_config(dcos_url, temp_path, stored_cert)
return 0
def _user_cert_validation(cert_str):
"""Prompt user for validation of certification from cluster
:param cert_str: cluster certificate bundle
:type cert_str: str
:returns whether or not user validated cert
:rtype: bool
"""
cert = x509.load_pem_x509_certificate(
cert_str.encode('utf-8'), default_backend())
fingerprint = cert.fingerprint(hashes.SHA256())
pp_fingerprint = ":".join("{:02x}".format(c) for c in fingerprint).upper()
msg = "SHA256 fingerprint of cluster certificate bundle:\n{}".format(
pp_fingerprint)
return confirm(msg, False)
def _store_cluster_cert(cert, no_check):
"""Store cluster certificate bundle downloaded from cluster and store
settings in core.ssl_verify
:param cert: ca cert from cluster
:type cert: str
:param no_check: whether to verify downloaded cert
:type no_check: bool
:returns: whether or not we are storing the downloaded cert bundle
:rtype: bool
"""
if not no_check:
if not _user_cert_validation(cert):
# we don't have a cert, but we still want to validate SSL
config.set_val("core.ssl_verify", "true")
return False
with util.temptext() as temp_file:
_, temp_path = temp_file
with open(temp_path, 'w') as f:
f.write(cert)
cert_path = os.path.join(
config.get_attached_cluster_path(), "dcos_ca.crt")
util.sh_copy(temp_path, cert_path)
config.set_val("core.ssl_verify", cert_path)
return True

18
cli/dcoscli/config/main.py
View File

@@ -5,6 +5,7 @@ import docopt
import dcoscli
from dcos import cmds, config, emitting, http, util
from dcos.errors import DCOSException, DefaultError
from dcoscli.cluster.main import setup
from dcoscli.subcommand import default_command_info, default_doc
from dcoscli.util import decorate_docopt_usage
@@ -84,12 +85,29 @@ def _set(name, value):
:rtype: int
"""
if name == "core.dcos_url":
return _cluster_setup(value)
toml, msg = config.set_val(name, value)
emitter.publish(DefaultError(msg))
return 0
def _cluster_setup(dcos_url):
"""
Setup a cluster using "cluster" directory instead "global" directory, until
we deprecate "global" config command: `dcos config set core.dcos_url x`
"""
notice = ("This config property is being deprecated. "
"To setup the CLI to talk to your cluster, please run "
"`dcos cluster setup <dcos_url>`.")
emitter.publish(DefaultError(notice))
return setup(dcos_url)
def _unset(name):
"""
:returns: process status

64
cli/dcoscli/data/help/cluster.txt Normal file
View File

@@ -0,0 +1,64 @@
Description:
Manage your DC/OS clusters
Usage:
dcos cluster --help
dcos cluster --info
dcos cluster --version
dcos cluster attach <name>
dcos cluster list [--attached --json]
dcos cluster remove <name>
dcos cluster rename <name> <new_name>
dcos cluster setup <dcos_url>
[--insecure | --no-check | --ca-certs=<ca-certs>]
[--provider=<provider_id>] [--username=<username>]
[--password=<password> | --password-file=<password_file>
| --password-env=<password_env> | --private-key=<key_path>]
Commands:
attach
List only the currently attached cluster.
list
List CLI configured clusters.
rename
Rename a cluster name in the CLI.
remove
Remove a configured cluster from the CLI.
setup
Setup the CLI to talk to your DC/OS cluster.
Options:
--attached
List only attached cluster.
--ca-certs=<ca-certs>
Specify the path to a list of trusted CAs to verify requests against.
-h, --help
Print usage.
--info
Print a short description of this subcommand.
--insecure
Allow requests to bypass SSL certificate verification (insecure).
--no-check
Do not check CA certficate downloaded from cluster (insecure). Applies to Enterprise DC/OS only.
--password=<password>
Specify password on the command line (insecure).
--password-env=<password_env>
Specify an environment variable name that contains the password.
--password-file=<password_file>
Specify the path to a file that contains the password.
--provider=<provider_id>
Specify the authentication provider to use for login.
--private-key=<key_path>
Specify the path to a file that contains the private key.
--username=<username>
Specify the username for login.
--version
Print version information.
Positional Arguments:
dcos_url
The public master of your DC/OS cluster.
name
The name of the cluster.
new_name
New name of cluster.

4
cli/dcoscli/data/help/package.txt
View File

@@ -10,7 +10,7 @@ Usage:
[--package-version=<package-version>]
dcos package describe <package-name> --package-versions
dcos package install <package-name>
[--cli | [--app --app-id=<app-id>]]
[(--cli [--global]) | [--app --app-id=<app-id>]]
[--package-version=<package-version>]
[--options=<file>]
[--yes]
@@ -53,6 +53,8 @@ Options:
Command line only.
--config
Print the configurable properties of the `marathon.json` file.
--global
Install a subcommand for all configured clusters
--index=<index>
The numerical position in the package repository list. Package
repositories are searched in descending order. By default, the Universe

13
cli/dcoscli/main.py
View File

@@ -6,7 +6,8 @@ import docopt
from six.moves import urllib
import dcoscli
from dcos import config, constants, emitting, errors, http, subcommand, util
from dcos import (cluster, config, constants, emitting, errors, http,
subcommand, util)
from dcos.errors import DCOSException
from dcoscli.help.main import dcos_help
from dcoscli.subcommand import default_doc, SubcommandMain
@@ -23,17 +24,16 @@ def main():
return 1
def _get_versions(dcos_url):
def _get_versions():
"""Print DC/OS and DC/OS-CLI versions
:param dcos_url: url to DC/OS cluster
:type dcos_url: str
:returns: Process status
:rtype: int
"""
dcos_info = {}
try:
dcos_url = config.get_config_val("core.dcos_url")
url = urllib.parse.urljoin(
dcos_url, 'dcos-metadata/dcos-version.json')
res = http.get(url, timeout=1)
@@ -69,8 +69,11 @@ def _main():
util.configure_process_from_environ()
if config.uses_deprecated_config():
cluster.move_to_cluster_config()
if args['--version']:
return _get_versions(config.get_config_val("core.dcos_url"))
return _get_versions()
command = args['<command>']

9
cli/dcoscli/node/main.py
View File

@@ -11,9 +11,8 @@ from dcos import (cmds, config, emitting, errors,
from dcos.cosmos import get_cosmos_url
from dcos.errors import DCOSException, DefaultError
from dcoscli import log, metrics, tables
from dcoscli.package.main import confirm
from dcoscli.subcommand import default_command_info, default_doc
from dcoscli.util import decorate_docopt_usage
from dcoscli.util import confirm, decorate_docopt_usage
logger = util.get_logger(__name__)
@@ -370,9 +369,9 @@ def _bundle_download(bundle, location):
bundle_location = location
if bundle_size > BUNDLE_WARN_SIZE:
if not confirm('Diagnostics bundle size is {}, are you sure you want '
'to download it?'.format(sizeof_fmt(bundle_size)),
False):
msg = ('Diagnostics bundle size is {}, '
'are you sure you want to download it?')
if not confirm(msg.format(sizeof_fmt(bundle_size)), False):
return 0
r = _do_request(url, 'GET', stream=True)

40
cli/dcoscli/package/main.py
View File

@@ -1,6 +1,5 @@
import json
import os
import sys
import docopt
import pkg_resources
@@ -11,9 +10,10 @@ from dcos.errors import DCOSException
from dcos.package import get_package_manager
from dcoscli import tables
from dcoscli.subcommand import default_command_info, default_doc
from dcoscli.util import decorate_docopt_usage
from dcoscli.util import confirm, decorate_docopt_usage
logger = util.get_logger(__name__)
emitter = emitting.FlatEmitter()
@@ -73,7 +73,7 @@ def _cmds():
cmds.Command(
hierarchy=['package', 'install'],
arg_keys=['<package-name>', '--package-version', '--options',
'--app-id', '--cli', '--app', '--yes'],
'--app-id', '--cli', '--global', '--app', '--yes'],
function=_install),
cmds.Command(
@@ -298,34 +298,8 @@ def _describe(package_name,
return 0
def confirm(prompt, yes):
"""
:param prompt: message to display to the terminal
:type prompt: str
:param yes: whether to assume that the user responded with yes
:type yes: bool
:returns: True if the user responded with yes; False otherwise
:rtype: bool
"""
if yes:
return True
else:
while True:
sys.stdout.write('{} [yes/no] '.format(prompt))
sys.stdout.flush()
response = sys.stdin.readline().strip().lower()
if response == 'yes' or response == 'y':
return True
elif response == 'no' or response == 'n':
return False
else:
emitter.publish(
"'{}' is not a valid response.".format(response))
def _install(package_name, package_version, options_path, app_id, cli, app,
yes):
def _install(package_name, package_version, options_path, app_id, cli,
global_, app, yes):
"""Install the specified package.
:param package_name: the package to install
@@ -338,6 +312,8 @@ def _install(package_name, package_version, options_path, app_id, cli, app,
:type app_id: str
:param cli: indicates if the cli should be installed
:type cli: bool
:param global_: indicates that the cli should be installed globally
:type global_: bool
:param app: indicate if the application should be installed
:type app: bool
:param yes: automatically assume yes to all prompts
@@ -386,7 +362,7 @@ def _install(package_name, package_version, options_path, app_id, cli, app,
pkg.name(), pkg.version())
emitter.publish(msg)
subcommand.install(pkg)
subcommand.install(pkg, global_)
subcommand_paths = subcommand.get_package_commands(package_name)
new_commands = [os.path.basename(p).replace('-', ' ', 1)

2
cli/dcoscli/subcommand.py
View File

@@ -13,6 +13,7 @@ def _default_modules():
# avoid circular imports
from dcoscli.auth import main as auth_main
from dcoscli.cluster import main as cluster_main
from dcoscli.config import main as config_main
from dcoscli.experimental import main as experimental_main
from dcoscli.help import main as help_main
@@ -24,6 +25,7 @@ def _default_modules():
from dcoscli.task import main as task_main
return {'auth': auth_main,
'cluster': cluster_main,
'config': config_main,
'experimental': experimental_main,
'help': help_main,

26
cli/dcoscli/tables.py
View File

@@ -870,6 +870,32 @@ def auth_provider_table(providers):
return tb
def clusters_table(clusters):
"""Returns a PrettyTable representation of the configured clusters
:param clusters: configured clusters
:type clusters: [Cluster]
:rtype: PrettyTable
"""
def print_name(c):
msg = c['name']
if c['attached']:
msg += "*"
return msg
fields = OrderedDict([
('NAME', lambda c: print_name(c)),
('CLUSTER ID', lambda c: c['cluster_id']),
('VERSION', lambda c: c['version']),
('URL', lambda c: c['url'] or "N/A")
])
tb = table(fields, clusters, sortby="CLUSTER ID")
return tb
def node_table(nodes, field_names=()):
"""Returns a PrettyTable representation of the provided DC/OS nodes

28
cli/dcoscli/util.py
View File

@@ -1,3 +1,5 @@
import sys
from functools import wraps
import docopt
@@ -26,3 +28,29 @@ def decorate_docopt_usage(func):
return 1
return result
return wrapper
def confirm(prompt, yes):
"""
:param prompt: message to display to the terminal
:type prompt: str
:param yes: whether to assume that the user responded with yes
:type yes: bool
:returns: True if the user responded with yes; False otherwise
:rtype: bool
"""
if yes:
return True
else:
while True:
sys.stdout.write('{} [yes/no] '.format(prompt))
sys.stdout.flush()
response = sys.stdin.readline().strip().lower()
if response == 'yes' or response == 'y':
return True
elif response == 'no' or response == 'n':
return False
else:
msg = "'{}' is not a valid response.".format(response)
emitter.publish(msg)

2
cli/tests/data/config/invalid_section.toml
View File

@@ -1,2 +0,0 @@
[foo]
bar = "baz"

2
cli/tests/data/config/parse_error.toml
View File

@@ -1,2 +0,0 @@
header]
key=value

1
cli/tests/data/help/default.txt
View File

@@ -6,6 +6,7 @@ for easy management of a DC/OS installation.
Available DC/OS commands:
auth Authenticate to DC/OS cluster
cluster Manage your DC/OS clusters
config Manage the DC/OS configuration file
experimental Manage commands that are under development
help Display help information about DC/OS

14
cli/tests/fixtures/clusters.py vendored Normal file
View File

@@ -0,0 +1,14 @@
def cluster_list_fixture():
"""clusters fixture
:rtype: dict
"""
return {
"cluster_id": "8c4f77ff-849c-456d-a480-c5cb6766c3f2",
"name": "tamar-ytck1ge",
"url": "https://52.25.204.103",
"version": "1.9-dev",
"attached": False
}

5
cli/tests/integrations/test_auth.py
View File

@@ -10,10 +10,7 @@ from .helpers.common import assert_command, exec_command, update_config
@pytest.fixture
def env():
r = os.environ.copy()
r.update({
constants.PATH_ENV: os.environ[constants.PATH_ENV],
constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "dcos.toml"),
})
r.update({constants.PATH_ENV: os.environ[constants.PATH_ENV]})
return r

45
cli/tests/integrations/test_cluster.py Normal file
View File

@@ -0,0 +1,45 @@
import json
from .helpers.common import assert_command, exec_command
def test_info():
stdout = b'Manage your DC/OS clusters\n'
assert_command(['dcos', 'cluster', '--info'],
stdout=stdout)
def test_version():
stdout = b'dcos-cluster version SNAPSHOT\n'
assert_command(['dcos', 'cluster', '--version'],
stdout=stdout)
def test_list():
returncode, stdout, stderr = exec_command(
['dcos', 'cluster', 'list', '--json'])
assert returncode == 0
assert stderr == b''
cluster_list = json.loads(stdout.decode('utf-8'))
assert len(cluster_list) == 1
info = cluster_list[0]
assert info.get("attached")
keys = ["attached", "cluster_id", "name", "url", "version"]
assert sorted(info.keys()) == keys
def test_rename():
_, stdout, _ = exec_command(
['dcos', 'cluster', 'list', '--json'])
info = json.loads(stdout.decode('utf-8'))[0]
name = info.get("name")
new_name = "test"
assert_command(['dcos', 'cluster', 'rename', name, new_name])
returncode, stdout, stderr = exec_command(
['dcos', 'cluster', 'list', '--json'])
assert json.loads(stdout.decode('utf-8'))[0].get("name") == new_name
# rename back to original name
assert_command(['dcos', 'cluster', 'rename', new_name, name])

105
cli/tests/integrations/test_config.py
View File

@@ -6,17 +6,14 @@ import six
from dcos import config, constants
from .helpers.common import (assert_command, config_set, config_unset,
exec_command, update_config)
from .helpers.common import (assert_command, config_set, exec_command,
update_config)
@pytest.fixture
def env():
r = os.environ.copy()
r.update({
constants.PATH_ENV: os.environ[constants.PATH_ENV],
constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "dcos.toml"),
})
r.update({constants.PATH_ENV: os.environ[constants.PATH_ENV]})
return r
@@ -33,44 +30,20 @@ def test_version():
stdout=stdout)
def _test_list_property(env):
stdout = b"""core.dcos_url http://dcos.snakeoil.mesosphere.com
core.reporting False
core.ssl_verify false
core.timeout 5
"""
assert_command(['dcos', 'config', 'show'],
stdout=stdout,
env=env)
def test_get_existing_string_property(env):
_get_value('core.dcos_url', 'http://dcos.snakeoil.mesosphere.com', env)
def test_get_existing_boolean_property(env):
_get_value('core.reporting', False, env)
with update_config("core.reporting", "false", env):
_get_value('core.reporting', False, env)
def test_get_existing_number_property(env):
_get_value('core.timeout', 5, env)
with update_config("core.timeout", "5", env):
_get_value('core.timeout', 5, env)
def test_get_missing_property(env):
_get_missing_value('missing.property', env)
def test_dcos_url_without_scheme(env):
with update_config("core.dcos_url", None, env):
new = b"abc.com"
out = b"[core.dcos_url]: set to 'https://%b'\n" % (new)
assert_command(
['dcos', 'config', 'set', 'core.dcos_url', new.decode('utf-8')],
stderr=out,
returncode=0,
env=env)
def test_get_top_property(env):
returncode, stdout, stderr = exec_command(
['dcos', 'config', 'show', 'core'], env=env)
@@ -82,13 +55,6 @@ def test_get_top_property(env):
b"possible properties are:\n")
def test_set_existing_string_property(env):
new_value = 'http://dcos.snakeoil.mesosphere.com:5081'
with update_config('core.dcos_url', new_value, env):
_get_value('core.dcos_url',
'http://dcos.snakeoil.mesosphere.com:5081', env)
def test_set_existing_boolean_property(env):
config_set('core.reporting', 'true', env)
_get_value('core.reporting', True, env)
@@ -116,16 +82,6 @@ def test_set_same_output(env):
env=env)
def test_set_new_output(env):
with update_config("core.dcos_url", None, env):
assert_command(
['dcos', 'config', 'set', 'core.dcos_url',
'http://dcos.snakeoil.mesosphere.com:5081'],
stderr=(b"[core.dcos_url]: set to "
b"'http://dcos.snakeoil.mesosphere.com:5081'\n"),
env=env)
def test_set_nonexistent_subcommand(env):
assert_command(
['dcos', 'config', 'set', 'foo.bar', 'baz'],
@@ -135,15 +91,6 @@ def test_set_nonexistent_subcommand(env):
env=env)
def test_set_when_extra_section(env):
path = os.path.join('tests', 'data', 'config', 'invalid_section.toml')
env['DCOS_CONFIG'] = path
os.chmod(path, 0o600)
config_set('core.dcos_url', 'http://dcos.snakeoil.mesosphere.com', env)
config_unset('core.dcos_url', env)
def test_unset_property(env):
with update_config("core.reporting", None, env):
_get_missing_value('core.reporting', env)
@@ -193,9 +140,9 @@ def test_set_property_key(env):
def test_set_missing_property(env):
with update_config("core.dcos_url", None, env=env):
config_set('core.dcos_url', 'http://localhost:8080', env)
_get_value('core.dcos_url', 'http://localhost:8080', env)
with update_config("package.cosmos_url", None, env=env):
config_set('package.cosmos_url', 'http://localhost:8080', env)
_get_value('package.cosmos_url', 'http://localhost:8080', env)
def test_set_core_property(env):
@@ -205,33 +152,30 @@ def test_set_core_property(env):
def test_url_validation(env):
with update_config('core.dcos_url', None, env):
key = 'core.dcos_url'
key2 = 'package.cosmos_url'
key = 'package.cosmos_url'
with update_config(key, None, env):
config_set(key, 'http://localhost', env)
config_set(key, 'https://localhost', env)
config_set(key, 'http://dcos-1234', env)
config_set(key2, 'http://dcos-1234.mydomain.com', env)
config_set(key, 'http://dcos-1234.mydomain.com', env)
config_set(key, 'http://localhost:5050', env)
config_set(key, 'https://localhost:5050', env)
config_set(key, 'http://mesos-1234:5050', env)
config_set(key2, 'http://mesos-1234.mydomain.com:5050', env)
config_set(key, 'http://mesos-1234.mydomain.com:5050', env)
config_set(key, 'http://localhost:8080', env)
config_set(key, 'https://localhost:8080', env)
config_set(key, 'http://marathon-1234:8080', env)
config_set(key2, 'http://marathon-1234.mydomain.com:5050', env)
config_set(key, 'http://marathon-1234.mydomain.com:5050', env)
config_set(key, 'http://user@localhost:8080', env)
config_set(key, 'http://u-ser@localhost:8080', env)
config_set(key, 'http://user123_@localhost:8080', env)
config_set(key, 'http://user:p-ssw_rd@localhost:8080', env)
config_set(key, 'http://user123:password321@localhost:8080', env)
config_set(key2, 'http://us%r1$3:pa#sw*rd321@localhost:8080', env)
config_unset(key2, env)
config_set(key, 'http://us%r1$3:pa#sw*rd321@localhost:8080', env)
def test_fail_url_validation(env):
@@ -285,27 +229,14 @@ def test_timeout(env):
assert "(connect timeout=1)".encode('utf-8') in stderr
def test_parse_error(env):
path = os.path.join('tests', 'data', 'config', 'parse_error.toml')
os.chmod(path, 0o600)
env['DCOS_CONFIG'] = path
assert_command(['dcos', 'config', 'show'],
returncode=1,
stderr=six.b(("Error parsing config file at [{}]: Found "
"invalid character in key name: ']'. "
"Try quoting the key name.\n").format(path)),
env=env)
def _fail_url_validation(command, key, value, env):
returncode_, stdout_, stderr_ = exec_command(
['dcos', 'config', command, key, value], env=env)
assert returncode_ == 1
assert stdout_ == b''
assert stderr_.startswith(str(
'Unable to parse {!r} as a url'.format(value)).encode('utf-8'))
err = str('Unable to parse {!r} as a url'.format(value)).encode('utf-8')
assert err in stderr_
def _get_value(key, value, env):

17
cli/tests/integrations/test_job.py
View File

@@ -6,7 +6,7 @@ import pytest
from dcos import constants
from .helpers.common import assert_command, exec_command, update_config
from .helpers.common import assert_command, exec_command
from .helpers.job import job, show_job, show_job_schedule
@@ -29,24 +29,11 @@ def test_info():
@pytest.fixture
def env():
r = os.environ.copy()
r.update({
constants.PATH_ENV: os.environ[constants.PATH_ENV],
constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "dcos.toml"),
})
r.update({constants.PATH_ENV: os.environ[constants.PATH_ENV]})
return r
def test_missing_config(env):
with update_config("core.dcos_url", None, env):
assert_command(
['dcos', 'job', 'list'],
returncode=1,
stderr=(b'Missing required config parameter: "core.dcos_url". '
b'Please run `dcos config set core.dcos_url <value>`.\n'),
env=env)
def test_empty_list():
_list_jobs()

15
cli/tests/integrations/test_marathon.py
View File

@@ -50,24 +50,11 @@ def test_about():
@pytest.fixture
def env():
r = os.environ.copy()
r.update({
constants.PATH_ENV: os.environ[constants.PATH_ENV],
constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "dcos.toml"),
})
r.update({constants.PATH_ENV: os.environ[constants.PATH_ENV]})
return r
def test_missing_config(env):
with update_config("core.dcos_url", None, env):
assert_command(
['dcos', 'marathon', 'app', 'list'],
returncode=1,
stderr=(b'Missing required config parameter: "core.dcos_url". '
b'Please run `dcos config set core.dcos_url <value>`.\n'),
env=env)
def test_empty_list():
list_apps()

26
cli/tests/integrations/test_package.py
View File

@@ -23,10 +23,7 @@ from ..common import file_bytes
@pytest.fixture
def env():
r = os.environ.copy()
r.update({
constants.PATH_ENV: os.environ[constants.PATH_ENV],
constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "dcos.toml"),
})
r.update({constants.PATH_ENV: os.environ[constants.PATH_ENV]})
return r
@@ -684,7 +681,7 @@ def test_list_cli_only(env):
helloworld_json = file_json(helloworld_path)
with _helloworld_cli(), \
update_config('core.dcos_url', 'http://nohost', env):
update_config('package.cosmos_url', 'http://nohost', env):
assert_command(
cmd=['dcos', 'package', 'list', '--json', '--cli'],
stdout=helloworld_json)
@@ -699,6 +696,18 @@ def test_list_cli_only(env):
stdout=helloworld_json)
def test_cli_global():
helloworld_path = 'tests/data/package/json/test_list_helloworld_cli.json'
helloworld_json = file_json(helloworld_path)
with _helloworld_cli(global_=True):
assert os.path.exists(subcommand.global_package_dir("helloworld"))
assert_command(
cmd=['dcos', 'package', 'list', '--json', '--cli'],
stdout=helloworld_json)
def test_uninstall_multiple_frameworknames(zk_znode):
_install_chronos(
args=['--yes', '--options=tests/data/package/chronos-1.json'])
@@ -991,9 +1000,12 @@ def _helloworld():
uninstall_stderr=stderr)
def _helloworld_cli():
def _helloworld_cli(global_=False):
args = ['--yes', '--cli']
if global_:
args += ['--global']
return _package(name='helloworld',
args=['--yes', '--cli'],
args=args,
stdout=HELLOWORLD_CLI_STDOUT,
uninstall_stderr=b'')

21
cli/tests/integrations/test_ssl.py
View File

@@ -2,9 +2,9 @@ import os
import pytest
from dcos import config, constants
from dcos import constants
from .helpers.common import config_set, exec_command, update_config
from .helpers.common import exec_command, update_config
@pytest.fixture
@@ -12,27 +12,14 @@ def env():
r = os.environ.copy()
r.update({
constants.PATH_ENV: os.environ[constants.PATH_ENV],
constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "dcos.toml"),
'DCOS_SNAKEOIL_CRT_PATH': os.environ.get(
"DCOS_SNAKEOIL_CRT_PATH", "/dcos-cli/adminrouter/snakeoil.crt")
"DCOS_SNAKEOIL_CRT_PATH", "/dcos-cli/adminrouter/snakeoil.crt"),
'DCOS_URL': 'https://dcos.snakeoil.mesosphere.com'
})
return r
@pytest.yield_fixture(autouse=True)
def setup_env(env):
# token will be removed when we change dcos_url
token = config.get_config_val('core.dcos_acs_token')
config_set("core.dcos_url", "https://dcos.snakeoil.mesosphere.com", env)
config_set("core.dcos_acs_token", token, env)
try:
yield
finally:
config_set("core.dcos_url", "http://dcos.snakeoil.mesosphere.com", env)
config_set("core.dcos_acs_token", token, env)
def test_dont_verify_ssl_with_env_var(env):
env['DCOS_SSL_VERIFY'] = 'false'

2
cli/tests/unit/data/cluster.txt Normal file
View File

@@ -0,0 +1,2 @@
NAME CLUSTER ID VERSION URL
tamar-ytck1ge 8c4f77ff-849c-456d-a480-c5cb6766c3f2 1.9-dev https://52.25.204.103

9
cli/tests/unit/test_tables.py
View File

@@ -7,6 +7,7 @@ from dcos.mesos import Slave
from dcoscli import tables
from ..fixtures.auth_provider import auth_provider_fixture
from ..fixtures.clusters import cluster_list_fixture
from ..fixtures.marathon import (app_fixture, app_task_fixture,
deployment_fixture_app_post_pods,
deployment_fixture_app_pre_pods,
@@ -145,6 +146,12 @@ def test_node_table():
'tests/unit/data/node.txt')
def test_clusters_tables():
_test_table(tables.clusters_table,
[cluster_list_fixture()],
'tests/unit/data/cluster.txt')
def test_ls_long_table():
with mock.patch('dcoscli.tables._format_unix_timestamp',
lambda ts: datetime.datetime.fromtimestamp(
@@ -177,4 +184,4 @@ def test_metrics_details_no_tags_table():
def _test_table(table_fn, fixture_fn, path):
table = table_fn(fixture_fn)
with open(path) as f:
assert str(table) == f.read()
assert str(table) == f.read().strip('\n')

299
dcos/cluster.py Normal file
View File

@@ -0,0 +1,299 @@
import contextlib
import os
import shutil
import ssl
import urllib
from urllib.request import urlopen
from dcos import config, constants, http, util
from dcos.errors import DCOSException
logger = util.get_logger(__name__)
def move_to_cluster_config():
"""Create a cluster specific config file + directory
from a global config file. This will move users from global config
structure (~/.dcos/dcos.toml) to the cluster specific one
(~/.dcos/clusters/CLUSTER_ID/dcos.toml) and set that cluster as
the "attached" cluster.
:rtype: None
"""
global_config = config.get_global_config()
dcos_url = config.get_config_val("core.dcos_url", global_config)
# if no cluster is set, do not move the cluster yet
if dcos_url is None:
return
try:
# find cluster id
cluster_url = dcos_url.rstrip('/') + '/metadata'
res = http.get(cluster_url, timeout=1)
cluster_id = res.json().get("CLUSTER_ID")
# don't move cluster if dcos_url is not valid
except DCOSException as e:
logger.error(
"Error trying to find cluster id: {}".format(e))
return
# create cluster id dir
cluster_path = os.path.join(config.get_config_dir_path(),
constants.DCOS_CLUSTERS_SUBDIR,
cluster_id)
util.ensure_dir_exists(cluster_path)
# move config file to new location
global_config_path = config.get_global_config_path()
util.sh_copy(global_config_path, cluster_path)
# set cluster as attached
util.ensure_file_exists(os.path.join(
cluster_path, constants.DCOS_CLUSTER_ATTACHED_FILE))
@contextlib.contextmanager
def setup_directory():
"""
A context manager for the temporary setup directory created as a
placeholder before we find the cluster's CLUSTER_ID.
:returns: path of setup directory
:rtype: str
"""
try:
temp_path = os.path.join(config.get_config_dir_path(),
constants.DCOS_CLUSTERS_SUBDIR,
"setup")
util.ensure_dir_exists(temp_path)
yield temp_path
finally:
shutil.rmtree(temp_path, ignore_errors=True)
def setup_cluster_config(dcos_url, temp_path, stored_cert):
"""
Create a cluster directory for cluster specified in "temp_path"
directory.
:param dcos_url: url to DC/OS cluster
:type dcos_url: str
:param temp_path: path to temporary config dir
:type temp_path: str
:param stored_cert: whether we stored cert bundle in 'setup' dir
:type stored_cert: bool
:returns: path to cluster specific directory
:rtype: str
"""
try:
# find cluster id
cluster_url = dcos_url.rstrip('/') + '/metadata'
res = http.get(cluster_url, timeout=1)
cluster_id = res.json().get("CLUSTER_ID")
except DCOSException as e:
msg = ("Error trying to find cluster id: {}\n "
"Please make sure the provided DC/OS URL is valid: {}".format(
e, dcos_url))
raise DCOSException(msg)
# create cluster id dir
cluster_path = os.path.join(config.get_config_dir_path(),
constants.DCOS_CLUSTERS_SUBDIR,
cluster_id)
if os.path.exists(cluster_path):
raise DCOSException("Cluster [{}] is already setup".format(dcos_url))
util.ensure_dir_exists(cluster_path)
# move contents of setup dir to new location
for (path, dirnames, filenames) in os.walk(temp_path):
for f in filenames:
util.sh_copy(os.path.join(path, f), cluster_path)
if stored_cert:
config.set_val("core.ssl_verify", os.path.join(
cluster_path, "dcos_ca.crt"))
cluster_name = cluster_id
try:
url = dcos_url.rstrip('/') + '/mesos/state-summary'
cluster_name = http.get(url, timeout=1).json().get("cluster")
except DCOSException:
pass
config.set_val("cluster.name", cluster_name)
return cluster_path
def set_attached(cluster_path):
"""
Set the cluster specified in `cluster_path` as the attached cluster
:param cluster_path: path to cluster directory
:type cluster_path: str
:rtype: None
"""
# get currently attached cluster
attached_cluster_path = config.get_attached_cluster_path()
if attached_cluster_path and attached_cluster_path != cluster_path:
# set cluster as attached
attached_file = os.path.join(
attached_cluster_path, constants.DCOS_CLUSTER_ATTACHED_FILE)
try:
util.sh_move(attached_file, cluster_path)
except DCOSException as e:
msg = "Failed to attach cluster: {}".format(e)
raise DCOSException(msg)
else:
util.ensure_file_exists(os.path.join(
cluster_path, constants.DCOS_CLUSTER_ATTACHED_FILE))
def get_cluster_cert(dcos_url):
"""Get CA bundle from specified cluster.
This is an insecure request.
:param dcos_url: url to DC/OS cluster
:type dcos_url: str
:returns: cert
:rtype: str
"""
cert_bundle_url = dcos_url.rstrip() + "/ca/dcos-ca.crt"
unverified = ssl.create_default_context()
unverified.check_hostname = False
unverified.verify_mode = ssl.CERT_NONE
error_msg = ("Error downloading CA certificate from cluster. "
"Please check the provided DC/OS URL.")
try:
with urlopen(cert_bundle_url, context=unverified) as f:
return f.read().decode('utf-8')
except urllib.error.HTTPError as e:
# Open source DC/OS does not currently expose its root CA certificate
if e.code == 404:
return False
else:
logger.debug(e)
raise DCOSException(error_msg)
except Exception as e:
logger.debug(e)
raise DCOSException(error_msg)
def get_clusters():
"""
:returns: list of configured Clusters
:rtype: [Clusters]
"""
clusters_path = config.get_clusters_path()
util.ensure_dir_exists(clusters_path)
clusters = os.listdir(clusters_path)
return [Cluster(cluster_id) for cluster_id in clusters]
def get_cluster(name):
"""
:param name: name of cluster
:type name: str
:returns: Cluster identified by name
:rtype: Cluster
"""
return next((c for c in get_clusters()
if c.get_cluster_id() == name or c.get_name() == name), None)
def remove(name):
"""
Remove cluster `name` from the CLI.
:param name: name of cluster
:type name: str
:rtype: None
"""
def onerror(func, path, excinfo):
raise DCOSException("Error trying to remove cluster")
cluster = get_cluster(name)
if cluster:
shutil.rmtree(cluster.get_cluster_path(), onerror)
return
else:
raise DCOSException("Cluster [{}] does not exist".format(name))
class Cluster():
"""Interface for a configured cluster"""
def __init__(self, cluster_id):
self.cluster_id = cluster_id
self.cluster_path = os.path.join(
config.get_clusters_path(), cluster_id)
def get_cluster_path(self):
return self.cluster_path
def get_cluster_id(self):
return self.cluster_id
def get_config_path(self):
return os.path.join(self.cluster_path, "dcos.toml")
def get_config(self, mutable=False):
return config.load_from_path(self.get_config_path(), mutable)
def get_name(self):
return config.get_config_val(
"cluster.name", self.get_config()) or self.cluster_id
def get_url(self):
return config.get_config_val("core.dcos_url", self.get_config())
def get_dcos_version(self):
dcos_url = self.get_url()
if dcos_url:
url = os.path.join(
self.get_url(), "dcos-metadata/dcos-version.json")
try:
resp = http.get(url, timeout=1, toml_config=self.get_config())
return resp.json().get("version", "N/A")
except DCOSException:
pass
return "N/A"
def is_attached(self):
return os.path.exists(os.path.join(
self.cluster_path, constants.DCOS_CLUSTER_ATTACHED_FILE))
def __eq__(self, other):
return isinstance(other, Cluster) and \
other.get_cluster_id() == self.get_cluster_id()
def dict(self):
return {
"cluster_id": self.get_cluster_id(),
"name": self.get_name(),
"url": self.get_url(),
"version": self.get_dcos_version(),
"attached": self.is_attached()
}

139
dcos/config.py
View File

@@ -12,14 +12,92 @@ from dcos.errors import DCOSException
logger = util.get_logger(__name__)
def get_config_path():
""" Returns the path to the DCOS config file.
def uses_deprecated_config():
"""Returns True if the configuration for the user's CLI
is the deprecated 'global' config instead of the cluster
specific config
"""
global_config = get_global_config_path()
cluster_config = get_clusters_path()
return not os.path.exists(cluster_config) and os.path.exists(global_config)
def get_global_config_path():
"""Returns the path to the deprecated global DCOS config file.
:returns: path to the DCOS config file
:rtype: str
"""
return os.environ.get(constants.DCOS_CONFIG_ENV, get_default_config_path())
default_path = os.path.join(get_config_dir_path(), "dcos.toml")
return os.environ.get(constants.DCOS_CONFIG_ENV, default_path)
def get_global_config(mutable=False):
"""Returns the deprecated global DCOS config file
:param mutable: True if the returned Toml object should be mutable
:type mutable: boolean
:returns: Configuration object
:rtype: Toml | MutableToml
"""
return load_from_path(get_global_config_path(), mutable)
def get_attached_cluster_path():
"""
The attached cluster is denoted by a file named "attached" in one of the
cluster directories. Ex: $DCOS_DIR/clusters/CLUSTER_ID/attached
:returns: path to the director of the attached cluster
:rtype: str | None
"""
path = get_clusters_path()
if not os.path.exists(path):
return None
clusters = os.listdir(get_clusters_path())
for c in clusters:
cluster_path = os.path.join(path, c)
if os.path.exists(os.path.join(
cluster_path, constants.DCOS_CLUSTER_ATTACHED_FILE)):
return cluster_path
# if only one cluster, set as attached
if len(clusters) == 1:
cluster_path = os.path.join(path, clusters[0])
util.ensure_file_exists(os.path.join(
cluster_path, constants.DCOS_CLUSTER_ATTACHED_FILE))
return cluster_path
return None
def get_clusters_path():
"""
:returns: path to the directory of cluster configs
:rtype: str
"""
return os.path.join(get_config_dir_path(), constants.DCOS_CLUSTERS_SUBDIR)
def get_config_path():
"""Returns the path to the DCOS config file of the attached cluster.
If still using "global" config return that toml instead
:returns: path to the DCOS config file
:rtype: str
"""
if uses_deprecated_config():
return get_global_config_path()
else:
cluster_path = get_attached_cluster_path()
return os.path.join(cluster_path, "dcos.toml")
def get_config_dir_path():
@@ -33,16 +111,6 @@ def get_config_dir_path():
return os.path.expanduser(config_dir)
def get_default_config_path():
"""Returns the default path to the DCOS config file.
:returns: path to the DCOS config file
:rtype: str
"""
return os.path.join(get_config_dir_path(), 'dcos.toml')
def get_config(mutable=False):
"""Returns the DCOS configuration object and creates config file is None
found and `DCOS_CONFIG` set to default value. Only use to get the config,
@@ -55,11 +123,18 @@ def get_config(mutable=False):
:rtype: Toml | MutableToml
"""
path = get_config_path()
default = get_default_config_path()
cluster_path = get_attached_cluster_path()
if cluster_path is None:
if uses_deprecated_config():
return get_global_config(mutable)
if path == default:
util.ensure_dir_exists(os.path.dirname(default))
msg = ("No cluster is attached. "
"Please run `dcos cluster attach <cluster-name>`")
raise DCOSException(msg)
util.ensure_dir_exists(os.path.dirname(cluster_path))
path = os.path.join(cluster_path, "dcos.toml")
return load_from_path(path, mutable)
@@ -115,7 +190,8 @@ def get_config_val(name, config=None):
:returns: value of 'name' parameter
:rtype: str | None
"""
val, _ = get_config_val_envvar(name, config=None)
val, _ = get_config_val_envvar(name, config)
return val
@@ -135,17 +211,22 @@ def missing_config_exception(keys):
return DCOSException(msg)
def set_val(name, value):
def set_val(name, value, config_path=None):
"""
:param name: name of paramater
:type name: str
:param value: value to set to paramater `name`
:type param: str
:param config_path: path to config to use
:type config_path: str
:returns: Toml config, message of change
:rtype: Toml, str
"""
toml_config = get_config(True)
if config_path:
toml_config = load_from_path(config_path, True)
else:
toml_config = get_config(True)
section, subkey = split_key(name)
@@ -169,7 +250,7 @@ def set_val(name, value):
check_config(toml_config_pre, toml_config, section)
save(toml_config)
save(toml_config, config_path)
msg = "[{}]: ".format(name)
if name == "core.dcos_acs_token":
@@ -187,8 +268,7 @@ def set_val(name, value):
msg += "changed from '{}' to '{}'".format(old_value, new_value)
if token_unset:
msg += ("\n[core.dcos_acs_token]: removed\n"
"Please run `dcos auth login` to authenticate to new dcos_url")
msg += "\n[core.dcos_acs_token]: removed"
return toml_config, msg
@@ -215,18 +295,21 @@ def load_from_path(path, mutable=False):
return (MutableToml if mutable else Toml)(toml_obj)
def save(toml_config):
def save(toml_config, config_path=None):
"""
:param toml_config: TOML configuration object
:type toml_config: MutableToml or Toml
:param config_path: path to config to use
:type config_path: str
"""
serial = toml.dumps(toml_config._dictionary)
path = get_config_path()
if config_path is None:
config_path = get_config_path()
util.ensure_file_exists(path)
util.enforce_file_permissions(path)
with util.open_file(path, 'w') as config_file:
util.ensure_file_exists(config_path)
util.enforce_file_permissions(config_path)
with util.open_file(config_path, 'w') as config_file:
config_file.write(serial)

7
dcos/constants.py
View File

@@ -4,6 +4,13 @@ DCOS_DIR = ".dcos"
DCOS_DIR_ENV = 'DCOS_DIR'
"""Name of the environment variable pointing to the DC/OS data directory"""
DCOS_CLUSTERS_SUBDIR = "clusters"
"""Name of the subdirectory containing the configuration of all configured
clusters"""
DCOS_CLUSTER_ATTACHED_FILE = "attached"
"""Name of the file indicating the current attached cluster"""
DCOS_SUBCOMMAND_ENV_SUBDIR = 'env'
"""In a package's directory, this is the cli contents subdirectory."""

12
dcos/data/config-schema/cluster.json Normal file
View File

@@ -0,0 +1,12 @@
{
"$schema": "http://json-schema.org/schema#",
"type": "object",
"properties": {
"name": {
"type": "string",
"title": "Cluster name",
"description": "Human readable name of cluster"
}
},
"additionalProperties": false
}

5
dcos/emitting.py
View File

@@ -164,7 +164,10 @@ def _page(output, pager_command=None):
if pager_command is None:
pager_command = 'less -R'
paginate = config.get_config_val("core.pagination") or True
try:
paginate = config.get_config_val("core.pagination")
except:
paginate = True
if exceeds_tty_height and paginate and \
spawn.find_executable(pager_command.split(' ')[0]) is not None:
pydoc.pipepager(output, cmd=pager_command)

25
dcos/http.py
View File

@@ -28,17 +28,22 @@ def _default_is_success(status_code):
return 200 <= status_code < 300
def _verify_ssl(verify=None):
def _verify_ssl(verify=None, toml_config=None):
"""Returns whether to verify ssl
:param verify: whether to verify SSL certs or path to cert(s)
:type verify: bool | str
:param toml_config: cluster config to use
:type toml_config: Toml
:return: whether to verify SSL certs or path to cert(s)
:rtype: bool | str
"""
if toml_config is None:
toml_config = config.get_config()
if verify is None:
verify = config.get_config_val("core.ssl_verify")
verify = config.get_config_val("core.ssl_verify", toml_config)
if verify and verify.lower() == "true":
verify = True
elif verify and verify.lower() == "false":
@@ -54,6 +59,7 @@ def _request(method,
timeout=DEFAULT_TIMEOUT,
auth=None,
verify=None,
toml_config=None,
**kwargs):
"""Sends an HTTP request.
@@ -69,6 +75,8 @@ def _request(method,
:type auth: AuthBase
:param verify: whether to verify SSL certs or path to cert(s)
:type verify: bool | str
:param toml_config: cluster config to use
:type toml_config: Toml
:param kwargs: Additional arguments to requests.request
(see http://docs.python-requests.org/en/latest/api/#requests.request)
:type kwargs: dict
@@ -78,7 +86,7 @@ def _request(method,
if 'headers' not in kwargs:
kwargs['headers'] = {'Accept': 'application/json'}
verify = _verify_ssl(verify)
verify = _verify_ssl(verify, toml_config)
# Silence 'Unverified HTTPS request' and 'SecurityWarning' for bad certs
if verify is not None:
@@ -128,6 +136,7 @@ def request(method,
is_success=_default_is_success,
timeout=None,
verify=None,
toml_config=None,
**kwargs):
"""Sends an HTTP request. If the server responds with a 401, ask the
user for their credentials, and try request again (up to 3 times).
@@ -142,13 +151,17 @@ def request(method,
:type timeout: int
:param verify: whether to verify SSL certs or path to cert(s)
:type verify: bool | str
:param toml_config: cluster config to use
:type toml_config: Toml
:param kwargs: Additional arguments to requests.request
(see http://docs.python-requests.org/en/latest/api/#requests.request)
:type kwargs: dict
:rtype: Response
"""
toml_config = config.get_config()
if toml_config is None:
toml_config = config.get_config()
auth_token = config.get_config_val("core.dcos_acs_token", toml_config)
prompt_login = config.get_config_val("core.prompt_login", toml_config)
dcos_url = urlparse(config.get_config_val("core.dcos_url", toml_config))
@@ -162,8 +175,10 @@ def request(method,
auth = DCOSAcsAuth(auth_token)
else:
auth = None
response = _request(method, url, is_success, timeout,
auth=auth, verify=verify, **kwargs)
auth=auth, verify=verify, toml_config=toml_config,
**kwargs)
if is_success(response.status_code):
return response

134
dcos/subcommand.py
View File

@@ -113,16 +113,15 @@ def _is_executable(path):
not util.is_windows_platform() or path.endswith('.exe'))
def distributions():
"""List all of the installed subcommand packages
:returns: a list of packages
:rtype: list of str
def _find_distributions(subcommand_dir):
"""
:param subcommand_dir: directory to find packaged in
:type subcommand_dir: path
:returns: list of all installed subcommands in given directory
:rtype: [str]
"""
subcommand_dir = _subcommand_dir()
if os.path.isdir(subcommand_dir):
if subcommand_dir and os.path.isdir(subcommand_dir):
return [
subdir for subdir in os.listdir(subcommand_dir)
if os.path.isdir(
@@ -135,6 +134,18 @@ def distributions():
return []
def distributions():
"""Set of all of the installed subcommand packages
:returns: a set of packages
:rtype: Set[str]
"""
cluster_packages = _find_distributions(_cluster_subcommand_dir())
global_packages = _find_distributions(global_subcommand_dir())
return set(cluster_packages + global_packages)
# must also add subcommand name to dcoscli.subcommand._default_modules
def default_subcommands():
"""List the default dcos cli subcommands
@@ -142,8 +153,9 @@ def default_subcommands():
:returns: list of all the default dcos cli subcommands
:rtype: [str]
"""
return ["auth", "config", "experimental", "help", "job", "marathon",
"node", "package", "service", "task"]
return ["auth", "cluster", "config", "experimental", "help", "job",
"marathon", "node", "package", "service", "task"]
def documentation(executable_path):
@@ -210,16 +222,16 @@ def noun(executable_path):
return noun
def _write_package_json(pkg):
def _write_package_json(pkg, pkg_dir):
""" Write package.json locally.
:param pkg: the package being installed
:type pkg: PackageVersion
:param pkg_dir: directory to install package
:type pkg_dir: str
:rtype: None
"""
pkg_dir = _package_dir(pkg.name())
package_path = os.path.join(pkg_dir, 'package.json')
package_json = pkg.package_json()
@@ -305,15 +317,17 @@ def _get_cli_binary_info(cli_resources):
cli_resources))
def _install_cli(pkg):
def _install_cli(pkg, pkg_dir):
"""Install subcommand cli
:param pkg: the package to install
:type pkg: PackageVersion
:param pkg_dir: directory to install package
:type pkg_dir: str
:rtype: None
"""
with util.remove_path_on_error(_package_dir(pkg.name())) as pkg_dir:
with util.remove_path_on_error(pkg_dir) as pkg_dir:
env_dir = os.path.join(pkg_dir, constants.DCOS_SUBCOMMAND_ENV_SUBDIR)
resources = pkg.resource_json()
@@ -341,38 +355,90 @@ def _install_cli(pkg):
"Could not find a CLI subcommand for your platform")
def install(pkg):
def install(pkg, global_=False):
"""Installs the dcos cli subcommand
:param pkg: the package to install
:type pkg: Package
:param global_: whether to install the CLI globally
:type global_: bool
:rtype: None
"""
pkg_dir = _package_dir(pkg.name())
if global_ or config.uses_deprecated_config():
pkg_dir = global_package_dir(pkg.name())
else:
pkg_dir = _cluster_package_dir(pkg.name())
util.ensure_dir_exists(pkg_dir)
_write_package_json(pkg)
_write_package_json(pkg, pkg_dir)
_install_cli(pkg)
_install_cli(pkg, pkg_dir)
def _subcommand_dir():
""" Returns subcommand dir. defaults to ~/.dcos/subcommands """
def global_subcommand_dir():
""" Returns global subcommand dir. defaults to ~/.dcos/subcommands """
return os.path.join(config.get_config_dir_path(),
constants.DCOS_SUBCOMMAND_SUBDIR)
def _package_dir(name):
""" Returns ~/.dcos/subcommands/<name>
def _cluster_subcommand_dir():
"""
:returns: cluster specific subcommand dir or None
:rtype: str | None
"""
attached_cluster = config.get_attached_cluster_path()
if attached_cluster is not None:
return os.path.join(
attached_cluster, constants.DCOS_SUBCOMMAND_SUBDIR)
else:
return None
def _cluster_package_dir(name):
"""Returns path to package directory for attached cluster
:param name: package name
:type name: str
:returns: path to package directory
:rtype: str | None
"""
subcommand_dir = _cluster_subcommand_dir()
if subcommand_dir is not None:
return os.path.join(subcommand_dir, name)
else:
return None
def global_package_dir(name):
"""Returns path to package directory in global config
:param name: package name
:type name: str
:rtype: str
"""
return os.path.join(_subcommand_dir(),
name)
return os.path.join(global_subcommand_dir(), name)
def _package_dir(name):
"""Returns cluster subcommand dir for name if exists, and if not
returns path to global subcommand dir.
:param name: package name
:type name: str
:rtype: str
"""
cluster_subcommand = _cluster_package_dir(name)
if cluster_subcommand and os.path.exists(cluster_subcommand):
return cluster_subcommand
else:
return global_package_dir(name)
def uninstall(package_name):
@@ -626,24 +692,6 @@ class InstalledSubcommand(object):
return _package_dir(self.name)
def package_revision(self):
"""
:returns: this subcommand's version.
:rtype: str
"""
version_path = os.path.join(self._dir(), 'version')
return util.read_file(version_path)
def package_source(self):
"""
:returns: this subcommand's source.
:rtype: str
"""
source_path = os.path.join(self._dir(), 'source')
return util.read_file(source_path)
def package_json(self):
"""
:returns: contents of this subcommand's package.json file.

25
dcos/util.py
View File

@@ -117,6 +117,31 @@ def sh_copy(src, dst):
raise DCOSException(e)
def sh_move(src, dst):
"""Move file src to the file or directory dst.
:param src: source file
:type src: str
:param dst: destination file or directory
:type dst: str
:rtype: None
"""
try:
shutil.move(src, dst)
except EnvironmentError as e:
logger.exception('Unable to move [%s] to [%s]', src, dst)
if e.strerror:
if e.filename:
raise DCOSException("{}: {}".format(e.strerror, e.filename))
else:
raise DCOSException(e.strerror)
else:
raise DCOSException(e)
except Exception as e:
logger.exception('Unknown error while moving [%s] to [%s]', src, dst)
raise DCOSException(e)
def ensure_dir_exists(directory):
"""If `directory` does not exist, create it.

113
tests/test_cluster.py Normal file
View File

@@ -0,0 +1,113 @@
import os
from unittest.mock import MagicMock
import mock
from mock import patch
from test_util import add_cluster_dir, create_global_config, env
from dcos import cluster, config, constants, util
def _cluster(cluster_id):
c = cluster.Cluster(cluster_id)
c.get_name = MagicMock(return_value="cluster-{}".format(cluster_id))
return c
def _test_cluster_list():
return [_cluster("a"), _cluster("b"), _cluster("c")]
@patch('dcos.cluster.get_clusters')
def test_get_cluster(clusters):
clusters.return_value = _test_cluster_list()
assert cluster.get_cluster("a") == _cluster("a")
assert cluster.get_cluster("cluster-b") == _cluster("b")
def test_get_clusters():
with env(), util.tempdir() as tempdir:
os.environ[constants.DCOS_DIR_ENV] = tempdir
# no config file of any type
assert cluster.get_clusters() == []
# cluster dir exists, no cluster
clusters_dir = os.path.join(tempdir, constants.DCOS_CLUSTERS_SUBDIR)
util.ensure_dir_exists(clusters_dir)
assert cluster.get_clusters() == []
# one cluster
cluster_id = "fake_cluster"
add_cluster_dir(cluster_id, tempdir)
assert cluster.get_clusters() == [_cluster(cluster_id)]
def test_set_attached():
with env(), util.tempdir() as tempdir:
os.environ[constants.DCOS_DIR_ENV] = tempdir
cluster_path = add_cluster_dir("a", tempdir)
# no attached_cluster
assert cluster.set_attached(cluster_path) is None
assert config.get_attached_cluster_path() == cluster_path
assert cluster.set_attached(cluster_path) is None
assert config.get_attached_cluster_path() == cluster_path
cluster_path2 = add_cluster_dir("b", tempdir)
# attach cluster already attached
assert cluster.set_attached(cluster_path2) is None
assert config.get_attached_cluster_path() == cluster_path2
@patch('dcos.http.get')
def test_setup_cluster_config(mock_get):
with env(), util.tempdir() as tempdir:
os.environ[constants.DCOS_DIR_ENV] = tempdir
with cluster.setup_directory() as setup_temp:
cluster.set_attached(setup_temp)
cluster_id = "fake"
mock_resp = mock.Mock()
mock_resp.json.return_value = {
"CLUSTER_ID": cluster_id,
"cluster": cluster_id
}
mock_get.return_value = mock_resp
path = cluster.setup_cluster_config("fake_url", setup_temp, False)
expected_path = os.path.join(
tempdir, constants.DCOS_CLUSTERS_SUBDIR + "/" + cluster_id)
assert path == expected_path
assert os.path.exists(path)
assert os.path.exists(os.path.join(path, "dcos.toml"))
assert not os.path.exists(setup_temp)
@patch('dcos.config.get_config_val')
@patch('dcos.http.get')
def test_move_to_cluster_config(mock_get, mock_config):
with env(), util.tempdir() as tempdir:
os.environ[constants.DCOS_DIR_ENV] = tempdir
create_global_config(tempdir)
mock_config.return_value = "fake-url"
cluster_id = "fake"
mock_resp = mock.Mock()
mock_resp.json.return_value = {"CLUSTER_ID": cluster_id}
mock_get.return_value = mock_resp
assert config.get_config_dir_path() == tempdir
cluster.move_to_cluster_config()
clusters_path = os.path.join(tempdir, constants.DCOS_CLUSTERS_SUBDIR)
assert os.path.exists(clusters_path)
cluster_path = os.path.join(clusters_path, cluster_id)
assert os.path.exists(os.path.join(cluster_path, "dcos.toml"))
assert os.path.exists(os.path.join(
cluster_path, constants.DCOS_CLUSTER_ATTACHED_FILE))

86
tests/test_config.py
View File

@@ -1,6 +1,13 @@
import os
import pytest
from dcos import config
from mock import patch
from test_util import add_cluster_dir, create_global_config, env
from dcos import config, constants, util
from dcos.errors import DCOSException
@pytest.fixture
@@ -110,3 +117,80 @@ def _conf():
'repo_uri': 'git://localhost/mesosphere/package-repo.git'
}
}
def test_uses_deprecated_config():
with env(), util.tempdir() as tempdir:
os.environ.pop('DCOS_CONFIG', None)
os.environ[constants.DCOS_DIR_ENV] = tempdir
assert config.get_config_dir_path() == tempdir
# create old global config toml
global_toml = create_global_config(tempdir)
assert config.get_global_config_path() == global_toml
assert config.uses_deprecated_config() is True
# create clusters subdir
_create_clusters_dir(tempdir)
assert config.uses_deprecated_config() is False
def test_get_attached_cluster_path():
with env(), util.tempdir() as tempdir:
os.environ[constants.DCOS_DIR_ENV] = tempdir
# no clusters dir
assert config.get_attached_cluster_path() is None
# clusters dir, no clusters
_create_clusters_dir(tempdir)
assert config.get_attached_cluster_path() is None
# 1 cluster, not attached
cluster_id = "fake-cluster"
cluster_path = add_cluster_dir(cluster_id, tempdir)
assert config.get_attached_cluster_path() == cluster_path
attached_path = os.path.join(
cluster_path, constants.DCOS_CLUSTER_ATTACHED_FILE)
assert os.path.exists(attached_path)
# attached cluster
assert config.get_attached_cluster_path() == cluster_path
@patch('dcos.config.load_from_path')
def test_get_config(load_path_mock):
with env(), util.tempdir() as tempdir:
os.environ.pop('DCOS_CONFIG', None)
os.environ[constants.DCOS_DIR_ENV] = tempdir
# no config file of any type
with pytest.raises(DCOSException) as e:
config.get_config()
msg = ("No cluster is attached. "
"Please run `dcos cluster attach <cluster-name>`")
assert str(e.value) == msg
load_path_mock.assert_not_called()
# create old global config toml
global_toml = create_global_config(tempdir)
config.get_config()
load_path_mock.assert_called_once_with(global_toml, False)
# clusters dir, no clusters
_create_clusters_dir(tempdir)
with pytest.raises(DCOSException) as e:
config.get_config()
assert str(e.value) == msg
cluster_id = "fake-cluster"
cluster_path = add_cluster_dir(cluster_id, tempdir)
cluster_toml = os.path.join(cluster_path, "dcos.toml")
config.get_config(True)
load_path_mock.assert_any_call(cluster_toml, True)
def _create_clusters_dir(dcos_dir):
clusters_dir = os.path.join(dcos_dir, constants.DCOS_CLUSTERS_SUBDIR)
util.ensure_dir_exists(clusters_dir)

34
tests/test_util.py
View File

@@ -1,6 +1,9 @@
import contextlib
import os
import pytest
from dcos import util
from dcos import constants, util
from dcos.errors import DCOSException
@@ -11,3 +14,32 @@ def test_open_file():
pass
assert 'Error opening file [{}]: No such file or directory'.format(path) \
in str(excinfo.value)
@contextlib.contextmanager
def env():
"""Context manager for altering env vars in tests """
try:
old_env = dict(os.environ)
yield
finally:
os.environ.clear()
os.environ.update(old_env)
def add_cluster_dir(cluster_id, dcos_dir):
clusters_dir = os.path.join(dcos_dir, constants.DCOS_CLUSTERS_SUBDIR)
util.ensure_dir_exists(clusters_dir)
cluster_path = os.path.join(clusters_dir, cluster_id)
util.ensure_dir_exists(cluster_path)
os.path.join(cluster_path, "dcos.toml")
return cluster_path
def create_global_config(dcos_dir):
global_toml = os.path.join(dcos_dir, "dcos.toml")
util.ensure_file_exists(global_toml)
return global_toml