diff --git a/README.rst b/README.rst index 69671e1..16c08bf 100644 --- a/README.rst +++ b/README.rst @@ -106,8 +106,6 @@ make sure you set owner only permissions on these files: :code:`chmod 600 cli/tests/data/dcos.toml` -:code:`chmod 600 cli/tests/data/config/parse_error.toml` - The :code:`node` integration tests use :code:`CLI_TEST_SSH_KEY_PATH` for ssh credentials to your cluster. @@ -124,10 +122,6 @@ Running Tox will run unit and integration tests in Python 3.5 using a temporarily created virtualenv. -You can set :code:`DCOS_CONFIG` to a config file that points to a DC/OS -cluster you want to use for integration tests. This defaults to -:code:`~/.dcos/dcos.toml` - Note that in order for all the integration tests to pass, your DC/OS cluster must have the experimental packaging features enabled. In order to enable these features the :code:`staged_package_storage_uri` and :code:`package_storage_uri` diff --git a/cli/bin/test.sh b/cli/bin/test.sh index 57f14bc..478c1e9 100755 --- a/cli/bin/test.sh +++ b/cli/bin/test.sh @@ -11,6 +11,5 @@ fi echo "Virtualenv activated." chmod 600 $BASEDIR/tests/data/dcos.toml -chmod 600 $BASEDIR/tests/data/config/parse_error.toml tox diff --git a/cli/dcoscli/auth/main.py b/cli/dcoscli/auth/main.py index daeeeb9..2a4cbfa 100644 --- a/cli/dcoscli/auth/main.py +++ b/cli/dcoscli/auth/main.py @@ -144,37 +144,8 @@ def _login(password_str, password_env, password_file, # every call to login will generate a new token if applicable _logout() - password = _get_password(password_str, password_env, password_file) - if provider is None: - if username and password: - auth.dcos_uid_password_auth(dcos_url, username, password) - elif username and key_path: - auth.servicecred_auth(dcos_url, username, key_path) - else: - try: - providers = auth.get_providers() - # Let users know if they have non-default providers configured - # This is a weak check, we should check default versions per - # DC/OS version since defaults will change. jj - if len(providers) > 2: - msg = ("\nYour cluster has several authentication " - "providers enabled. Run `dcos auth " - "list-providers` to see all providers and `dcos " - "auth login --provider ` to " - "authenticate with a specific provider\n") - emitter.publish(DefaultError(msg)) - except DCOSException: - pass - finally: - auth.header_challenge_auth(dcos_url) - else: - providers = auth.get_providers() - if providers.get(provider): - _trigger_client_method( - provider, providers[provider], username, password, key_path) - else: - msg = "Provider [{}] not configured on your cluster" - raise DCOSException(msg.format(provider)) + login(dcos_url, password_str, password_env, password_file, + provider, username, key_path) emitter.publish("Login successful!") return 0 @@ -232,3 +203,58 @@ def _logout(): if config.get_config_val("core.dcos_acs_token") is not None: config.unset("core.dcos_acs_token") return 0 + + +def login(dcos_url, password_str, password_env, password_file, + provider, username, key_path): + """ + :param dcos_url: URL of DC/OS cluster + :type dcos_url: str + :param password_str: password + :type password_str: str + :param password_env: name of environment variable with password + :type password_env: str + :param password_file: path to file with password + :type password_file: bool + :param provider: name of provider to authentication with + :type provider: str + :param username: username + :type username: str + :param key_path: path to file with private key + :type param: str + :rtype: int + """ + + password = _get_password(password_str, password_env, password_file) + if provider is None: + if username and password: + auth.dcos_uid_password_auth(dcos_url, username, password) + elif username and key_path: + auth.servicecred_auth(dcos_url, username, key_path) + else: + try: + providers = auth.get_providers() + # Let users know if they have non-default providers configured + # This is a weak check, we should check default versions per + # DC/OS version since defaults will change. jj + if len(providers) > 2: + msg = ("\nYour cluster has several authentication " + "providers enabled. Run `dcos auth " + "list-providers` to see all providers and `dcos " + "auth login --provider ` to " + "authenticate with a specific provider\n") + emitter.publish(DefaultError(msg)) + except DCOSException: + pass + finally: + auth.header_challenge_auth(dcos_url) + else: + providers = auth.get_providers() + if providers.get(provider): + _trigger_client_method( + provider, providers[provider], username, password, key_path) + else: + msg = "Provider [{}] not configured on your cluster" + raise DCOSException(msg.format(provider)) + + return 0 diff --git a/cli/dcoscli/cluster/__init__.py b/cli/dcoscli/cluster/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cli/dcoscli/cluster/main.py b/cli/dcoscli/cluster/main.py new file mode 100644 index 0000000..385c747 --- /dev/null +++ b/cli/dcoscli/cluster/main.py @@ -0,0 +1,290 @@ +import os + +import docopt + +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes + +import dcoscli + +from dcos import cluster, cmds, config, emitting, http, util +from dcos.errors import DCOSAuthenticationException, DCOSException +from dcoscli.auth.main import login +from dcoscli.subcommand import default_command_info, default_doc +from dcoscli.tables import clusters_table +from dcoscli.util import confirm, decorate_docopt_usage + + +emitter = emitting.FlatEmitter() +logger = util.get_logger(__name__) + + +def main(argv): + try: + return _main(argv) + except DCOSException as e: + emitter.publish(e) + return 1 + + +@decorate_docopt_usage +def _main(argv): + args = docopt.docopt( + default_doc("cluster"), + argv=argv, + version='dcos-cluster version {}'.format(dcoscli.version)) + + http.silence_requests_warnings() + + return cmds.execute(_cmds(), args) + + +def _cmds(): + """ + :returns: all the supported commands + :rtype: list of dcos.cmds.Command + """ + + return [ + cmds.Command( + hierarchy=['cluster', 'setup'], + arg_keys=['', + '--insecure', '--no-check', '--ca-certs', + '--password', '--password-env', '--password-file', + '--provider', '--username', '--private-key'], + function=setup), + + cmds.Command( + hierarchy=['cluster', 'list'], + arg_keys=['--json', '--attached'], + function=_list), + + cmds.Command( + hierarchy=['cluster', 'remove'], + arg_keys=[''], + function=_remove), + + cmds.Command( + hierarchy=['cluster', 'attach'], + arg_keys=[''], + function=_attach), + + cmds.Command( + hierarchy=['cluster', 'rename'], + arg_keys=['', ''], + function=_rename), + + cmds.Command( + hierarchy=['cluster'], + arg_keys=['--info'], + function=_info), + ] + + +def _info(info): + """ + :param info: Whether to output a description of this subcommand + :type info: boolean + :returns: process status + :rtype: int + """ + + emitter.publish(default_command_info("cluster")) + return 0 + + +def _list(json_, attached): + """ + List configured clusters. + + :param json_: output json if True + :type json_: bool + :param attached: return only attached cluster + :type attached: True + :rtype: None + """ + + clusters = [c.dict() for c in cluster.get_clusters() + if not attached or c.is_attached()] + if json_: + emitter.publish(clusters) + elif len(clusters) == 0: + if attached: + msg = ("No cluster is attached. " + "Please run `dcos cluster attach ") + else: + msg = ("No clusters are currently configured. " + "To configure one, run `dcos cluster setup `") + raise DCOSException(msg) + else: + emitter.publish(clusters_table(clusters)) + + return + + +def _remove(name): + """ + :param name: name of cluster + :type name: str + :rtype: None + """ + + return cluster.remove(name) + + +def _attach(name): + """ + :param name: name of cluster + :type name: str + :rtype: None + """ + + c = cluster.get_cluster(name) + if c is not None: + return cluster.set_attached(c.get_cluster_path()) + else: + raise DCOSException("Cluster [{}] does not exist".format(name)) + + +def _rename(name, new_name): + """ + :param name: name of cluster + :type name: str + :param new_name: new_name of cluster + :type new_name: str + :rtype: None + """ + + c = cluster.get_cluster(name) + other = cluster.get_cluster(new_name) + if c is None: + raise DCOSException("Cluster [{}] does not exist".format(name)) + elif other and other != c: + msg = "A cluster with name [{}] already exists" + raise DCOSException(msg.format(new_name)) + else: + config.set_val("cluster.name", new_name, c.get_config_path()) + + +def setup(dcos_url, + insecure=False, no_check=False, ca_certs=None, + password_str=None, password_env=None, password_file=None, + provider=None, username=None, key_path=None): + """ + Setup the CLI to talk to your DC/OS cluster. + + :param dcos_url: master ip of cluster + :type dcos_url: str + :param insecure: whether or not to verify ssl certs + :type insecure: bool + :param no_check: whether or not to verify downloaded ca cert + :type no_check: bool + :param ca_certs: path to root CA to verify requests + :type ca_certs: str + :param password_str: password + :type password_str: str + :param password_env: name of environment variable with password + :type password_env: str + :param password_file: path to file with password + :type password_file: bool + :param provider: name of provider to authentication with + :type provider: str + :param username: username + :type username: str + :param key_path: path to file with private key + :type param: str + :returns: process status + :rtype: int + """ + + with cluster.setup_directory() as temp_path: + + # set cluster as attached + cluster.set_attached(temp_path) + + # authenticate + config.set_val("core.dcos_url", dcos_url) + # get validated dcos_url + dcos_url = config.get_config_val("core.dcos_url") + + # configure ssl settings + stored_cert = False + if insecure: + config.set_val("core.ssl_verify", "false") + elif ca_certs: + config.set_val("core.ssl_verify", ca_certs) + else: + cert = cluster.get_cluster_cert(dcos_url) + # if we don't have a cert don't try to verify one + if cert is False: + config.set_val("core.ssl_verify", "false") + else: + stored_cert = _store_cluster_cert(cert, no_check) + + try: + login(dcos_url, + password_str, password_env, password_file, + provider, username, key_path) + except DCOSAuthenticationException: + msg = ("Authentication failed. " + "Please run `dcos cluster setup `") + raise DCOSException(msg) + + # configure cluster directory + cluster.setup_cluster_config(dcos_url, temp_path, stored_cert) + + return 0 + + +def _user_cert_validation(cert_str): + """Prompt user for validation of certification from cluster + + :param cert_str: cluster certificate bundle + :type cert_str: str + :returns whether or not user validated cert + :rtype: bool + """ + + cert = x509.load_pem_x509_certificate( + cert_str.encode('utf-8'), default_backend()) + fingerprint = cert.fingerprint(hashes.SHA256()) + pp_fingerprint = ":".join("{:02x}".format(c) for c in fingerprint).upper() + + msg = "SHA256 fingerprint of cluster certificate bundle:\n{}".format( + pp_fingerprint) + + return confirm(msg, False) + + +def _store_cluster_cert(cert, no_check): + """Store cluster certificate bundle downloaded from cluster and store + settings in core.ssl_verify + + :param cert: ca cert from cluster + :type cert: str + :param no_check: whether to verify downloaded cert + :type no_check: bool + :returns: whether or not we are storing the downloaded cert bundle + :rtype: bool + """ + + if not no_check: + if not _user_cert_validation(cert): + # we don't have a cert, but we still want to validate SSL + config.set_val("core.ssl_verify", "true") + return False + + with util.temptext() as temp_file: + _, temp_path = temp_file + + with open(temp_path, 'w') as f: + f.write(cert) + + cert_path = os.path.join( + config.get_attached_cluster_path(), "dcos_ca.crt") + + util.sh_copy(temp_path, cert_path) + + config.set_val("core.ssl_verify", cert_path) + return True diff --git a/cli/dcoscli/config/main.py b/cli/dcoscli/config/main.py index ea83817..4345121 100644 --- a/cli/dcoscli/config/main.py +++ b/cli/dcoscli/config/main.py @@ -5,6 +5,7 @@ import docopt import dcoscli from dcos import cmds, config, emitting, http, util from dcos.errors import DCOSException, DefaultError +from dcoscli.cluster.main import setup from dcoscli.subcommand import default_command_info, default_doc from dcoscli.util import decorate_docopt_usage @@ -84,12 +85,29 @@ def _set(name, value): :rtype: int """ + if name == "core.dcos_url": + return _cluster_setup(value) + toml, msg = config.set_val(name, value) emitter.publish(DefaultError(msg)) return 0 +def _cluster_setup(dcos_url): + """ + Setup a cluster using "cluster" directory instead "global" directory, until + we deprecate "global" config command: `dcos config set core.dcos_url x` + """ + + notice = ("This config property is being deprecated. " + "To setup the CLI to talk to your cluster, please run " + "`dcos cluster setup `.") + emitter.publish(DefaultError(notice)) + + return setup(dcos_url) + + def _unset(name): """ :returns: process status diff --git a/cli/dcoscli/data/help/cluster.txt b/cli/dcoscli/data/help/cluster.txt new file mode 100644 index 0000000..70959ac --- /dev/null +++ b/cli/dcoscli/data/help/cluster.txt @@ -0,0 +1,64 @@ +Description: + Manage your DC/OS clusters + +Usage: + dcos cluster --help + dcos cluster --info + dcos cluster --version + dcos cluster attach + dcos cluster list [--attached --json] + dcos cluster remove + dcos cluster rename + dcos cluster setup + [--insecure | --no-check | --ca-certs=] + [--provider=] [--username=] + [--password= | --password-file= + | --password-env= | --private-key=] + +Commands: + attach + List only the currently attached cluster. + list + List CLI configured clusters. + rename + Rename a cluster name in the CLI. + remove + Remove a configured cluster from the CLI. + setup + Setup the CLI to talk to your DC/OS cluster. + +Options: + --attached + List only attached cluster. + --ca-certs= + Specify the path to a list of trusted CAs to verify requests against. + -h, --help + Print usage. + --info + Print a short description of this subcommand. + --insecure + Allow requests to bypass SSL certificate verification (insecure). + --no-check + Do not check CA certficate downloaded from cluster (insecure). Applies to Enterprise DC/OS only. + --password= + Specify password on the command line (insecure). + --password-env= + Specify an environment variable name that contains the password. + --password-file= + Specify the path to a file that contains the password. + --provider= + Specify the authentication provider to use for login. + --private-key= + Specify the path to a file that contains the private key. + --username= + Specify the username for login. + --version + Print version information. + +Positional Arguments: + dcos_url + The public master of your DC/OS cluster. + name + The name of the cluster. + new_name + New name of cluster. diff --git a/cli/dcoscli/data/help/package.txt b/cli/dcoscli/data/help/package.txt index 0f2bf28..932faa4 100644 --- a/cli/dcoscli/data/help/package.txt +++ b/cli/dcoscli/data/help/package.txt @@ -10,7 +10,7 @@ Usage: [--package-version=] dcos package describe --package-versions dcos package install - [--cli | [--app --app-id=]] + [(--cli [--global]) | [--app --app-id=]] [--package-version=] [--options=] [--yes] @@ -53,6 +53,8 @@ Options: Command line only. --config Print the configurable properties of the `marathon.json` file. + --global + Install a subcommand for all configured clusters --index= The numerical position in the package repository list. Package repositories are searched in descending order. By default, the Universe diff --git a/cli/dcoscli/main.py b/cli/dcoscli/main.py index 06145ee..32c4f5e 100644 --- a/cli/dcoscli/main.py +++ b/cli/dcoscli/main.py @@ -6,7 +6,8 @@ import docopt from six.moves import urllib import dcoscli -from dcos import config, constants, emitting, errors, http, subcommand, util +from dcos import (cluster, config, constants, emitting, errors, http, + subcommand, util) from dcos.errors import DCOSException from dcoscli.help.main import dcos_help from dcoscli.subcommand import default_doc, SubcommandMain @@ -23,17 +24,16 @@ def main(): return 1 -def _get_versions(dcos_url): +def _get_versions(): """Print DC/OS and DC/OS-CLI versions - :param dcos_url: url to DC/OS cluster - :type dcos_url: str :returns: Process status :rtype: int """ dcos_info = {} try: + dcos_url = config.get_config_val("core.dcos_url") url = urllib.parse.urljoin( dcos_url, 'dcos-metadata/dcos-version.json') res = http.get(url, timeout=1) @@ -69,8 +69,11 @@ def _main(): util.configure_process_from_environ() + if config.uses_deprecated_config(): + cluster.move_to_cluster_config() + if args['--version']: - return _get_versions(config.get_config_val("core.dcos_url")) + return _get_versions() command = args[''] diff --git a/cli/dcoscli/node/main.py b/cli/dcoscli/node/main.py index dfb793a..a012bbe 100644 --- a/cli/dcoscli/node/main.py +++ b/cli/dcoscli/node/main.py @@ -11,9 +11,8 @@ from dcos import (cmds, config, emitting, errors, from dcos.cosmos import get_cosmos_url from dcos.errors import DCOSException, DefaultError from dcoscli import log, metrics, tables -from dcoscli.package.main import confirm from dcoscli.subcommand import default_command_info, default_doc -from dcoscli.util import decorate_docopt_usage +from dcoscli.util import confirm, decorate_docopt_usage logger = util.get_logger(__name__) @@ -370,9 +369,9 @@ def _bundle_download(bundle, location): bundle_location = location if bundle_size > BUNDLE_WARN_SIZE: - if not confirm('Diagnostics bundle size is {}, are you sure you want ' - 'to download it?'.format(sizeof_fmt(bundle_size)), - False): + msg = ('Diagnostics bundle size is {}, ' + 'are you sure you want to download it?') + if not confirm(msg.format(sizeof_fmt(bundle_size)), False): return 0 r = _do_request(url, 'GET', stream=True) diff --git a/cli/dcoscli/package/main.py b/cli/dcoscli/package/main.py index 46bf5ab..ddfb702 100644 --- a/cli/dcoscli/package/main.py +++ b/cli/dcoscli/package/main.py @@ -1,6 +1,5 @@ import json import os -import sys import docopt import pkg_resources @@ -11,9 +10,10 @@ from dcos.errors import DCOSException from dcos.package import get_package_manager from dcoscli import tables from dcoscli.subcommand import default_command_info, default_doc -from dcoscli.util import decorate_docopt_usage +from dcoscli.util import confirm, decorate_docopt_usage logger = util.get_logger(__name__) + emitter = emitting.FlatEmitter() @@ -73,7 +73,7 @@ def _cmds(): cmds.Command( hierarchy=['package', 'install'], arg_keys=['', '--package-version', '--options', - '--app-id', '--cli', '--app', '--yes'], + '--app-id', '--cli', '--global', '--app', '--yes'], function=_install), cmds.Command( @@ -298,34 +298,8 @@ def _describe(package_name, return 0 -def confirm(prompt, yes): - """ - :param prompt: message to display to the terminal - :type prompt: str - :param yes: whether to assume that the user responded with yes - :type yes: bool - :returns: True if the user responded with yes; False otherwise - :rtype: bool - """ - - if yes: - return True - else: - while True: - sys.stdout.write('{} [yes/no] '.format(prompt)) - sys.stdout.flush() - response = sys.stdin.readline().strip().lower() - if response == 'yes' or response == 'y': - return True - elif response == 'no' or response == 'n': - return False - else: - emitter.publish( - "'{}' is not a valid response.".format(response)) - - -def _install(package_name, package_version, options_path, app_id, cli, app, - yes): +def _install(package_name, package_version, options_path, app_id, cli, + global_, app, yes): """Install the specified package. :param package_name: the package to install @@ -338,6 +312,8 @@ def _install(package_name, package_version, options_path, app_id, cli, app, :type app_id: str :param cli: indicates if the cli should be installed :type cli: bool + :param global_: indicates that the cli should be installed globally + :type global_: bool :param app: indicate if the application should be installed :type app: bool :param yes: automatically assume yes to all prompts @@ -386,7 +362,7 @@ def _install(package_name, package_version, options_path, app_id, cli, app, pkg.name(), pkg.version()) emitter.publish(msg) - subcommand.install(pkg) + subcommand.install(pkg, global_) subcommand_paths = subcommand.get_package_commands(package_name) new_commands = [os.path.basename(p).replace('-', ' ', 1) diff --git a/cli/dcoscli/subcommand.py b/cli/dcoscli/subcommand.py index 8a8005d..de90726 100644 --- a/cli/dcoscli/subcommand.py +++ b/cli/dcoscli/subcommand.py @@ -13,6 +13,7 @@ def _default_modules(): # avoid circular imports from dcoscli.auth import main as auth_main + from dcoscli.cluster import main as cluster_main from dcoscli.config import main as config_main from dcoscli.experimental import main as experimental_main from dcoscli.help import main as help_main @@ -24,6 +25,7 @@ def _default_modules(): from dcoscli.task import main as task_main return {'auth': auth_main, + 'cluster': cluster_main, 'config': config_main, 'experimental': experimental_main, 'help': help_main, diff --git a/cli/dcoscli/tables.py b/cli/dcoscli/tables.py index b08c866..7ee8983 100644 --- a/cli/dcoscli/tables.py +++ b/cli/dcoscli/tables.py @@ -870,6 +870,32 @@ def auth_provider_table(providers): return tb +def clusters_table(clusters): + """Returns a PrettyTable representation of the configured clusters + + :param clusters: configured clusters + :type clusters: [Cluster] + :rtype: PrettyTable + """ + + def print_name(c): + msg = c['name'] + if c['attached']: + msg += "*" + return msg + + fields = OrderedDict([ + ('NAME', lambda c: print_name(c)), + ('CLUSTER ID', lambda c: c['cluster_id']), + ('VERSION', lambda c: c['version']), + ('URL', lambda c: c['url'] or "N/A") + ]) + + tb = table(fields, clusters, sortby="CLUSTER ID") + + return tb + + def node_table(nodes, field_names=()): """Returns a PrettyTable representation of the provided DC/OS nodes diff --git a/cli/dcoscli/util.py b/cli/dcoscli/util.py index 68019b0..4c60406 100644 --- a/cli/dcoscli/util.py +++ b/cli/dcoscli/util.py @@ -1,3 +1,5 @@ +import sys + from functools import wraps import docopt @@ -26,3 +28,29 @@ def decorate_docopt_usage(func): return 1 return result return wrapper + + +def confirm(prompt, yes): + """ + :param prompt: message to display to the terminal + :type prompt: str + :param yes: whether to assume that the user responded with yes + :type yes: bool + :returns: True if the user responded with yes; False otherwise + :rtype: bool + """ + + if yes: + return True + else: + while True: + sys.stdout.write('{} [yes/no] '.format(prompt)) + sys.stdout.flush() + response = sys.stdin.readline().strip().lower() + if response == 'yes' or response == 'y': + return True + elif response == 'no' or response == 'n': + return False + else: + msg = "'{}' is not a valid response.".format(response) + emitter.publish(msg) diff --git a/cli/tests/data/config/invalid_section.toml b/cli/tests/data/config/invalid_section.toml deleted file mode 100644 index e8655ba..0000000 --- a/cli/tests/data/config/invalid_section.toml +++ /dev/null @@ -1,2 +0,0 @@ -[foo] -bar = "baz" diff --git a/cli/tests/data/config/parse_error.toml b/cli/tests/data/config/parse_error.toml deleted file mode 100644 index 1809800..0000000 --- a/cli/tests/data/config/parse_error.toml +++ /dev/null @@ -1,2 +0,0 @@ -header] -key=value diff --git a/cli/tests/data/help/default.txt b/cli/tests/data/help/default.txt index 14ef9e6..f759485 100644 --- a/cli/tests/data/help/default.txt +++ b/cli/tests/data/help/default.txt @@ -6,6 +6,7 @@ for easy management of a DC/OS installation. Available DC/OS commands: auth Authenticate to DC/OS cluster + cluster Manage your DC/OS clusters config Manage the DC/OS configuration file experimental Manage commands that are under development help Display help information about DC/OS diff --git a/cli/tests/fixtures/clusters.py b/cli/tests/fixtures/clusters.py new file mode 100644 index 0000000..5f277c1 --- /dev/null +++ b/cli/tests/fixtures/clusters.py @@ -0,0 +1,14 @@ + +def cluster_list_fixture(): + """clusters fixture + + :rtype: dict + """ + + return { + "cluster_id": "8c4f77ff-849c-456d-a480-c5cb6766c3f2", + "name": "tamar-ytck1ge", + "url": "https://52.25.204.103", + "version": "1.9-dev", + "attached": False + } diff --git a/cli/tests/integrations/test_auth.py b/cli/tests/integrations/test_auth.py index e9f5b85..ecce672 100644 --- a/cli/tests/integrations/test_auth.py +++ b/cli/tests/integrations/test_auth.py @@ -10,10 +10,7 @@ from .helpers.common import assert_command, exec_command, update_config @pytest.fixture def env(): r = os.environ.copy() - r.update({ - constants.PATH_ENV: os.environ[constants.PATH_ENV], - constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "dcos.toml"), - }) + r.update({constants.PATH_ENV: os.environ[constants.PATH_ENV]}) return r diff --git a/cli/tests/integrations/test_cluster.py b/cli/tests/integrations/test_cluster.py new file mode 100644 index 0000000..be37cdc --- /dev/null +++ b/cli/tests/integrations/test_cluster.py @@ -0,0 +1,45 @@ +import json + +from .helpers.common import assert_command, exec_command + + +def test_info(): + stdout = b'Manage your DC/OS clusters\n' + assert_command(['dcos', 'cluster', '--info'], + stdout=stdout) + + +def test_version(): + stdout = b'dcos-cluster version SNAPSHOT\n' + assert_command(['dcos', 'cluster', '--version'], + stdout=stdout) + + +def test_list(): + returncode, stdout, stderr = exec_command( + ['dcos', 'cluster', 'list', '--json']) + assert returncode == 0 + assert stderr == b'' + cluster_list = json.loads(stdout.decode('utf-8')) + assert len(cluster_list) == 1 + info = cluster_list[0] + assert info.get("attached") + keys = ["attached", "cluster_id", "name", "url", "version"] + assert sorted(info.keys()) == keys + + +def test_rename(): + _, stdout, _ = exec_command( + ['dcos', 'cluster', 'list', '--json']) + info = json.loads(stdout.decode('utf-8'))[0] + name = info.get("name") + + new_name = "test" + assert_command(['dcos', 'cluster', 'rename', name, new_name]) + + returncode, stdout, stderr = exec_command( + ['dcos', 'cluster', 'list', '--json']) + assert json.loads(stdout.decode('utf-8'))[0].get("name") == new_name + + # rename back to original name + assert_command(['dcos', 'cluster', 'rename', new_name, name]) diff --git a/cli/tests/integrations/test_config.py b/cli/tests/integrations/test_config.py index f30bb24..3e3bd73 100644 --- a/cli/tests/integrations/test_config.py +++ b/cli/tests/integrations/test_config.py @@ -6,17 +6,14 @@ import six from dcos import config, constants -from .helpers.common import (assert_command, config_set, config_unset, - exec_command, update_config) +from .helpers.common import (assert_command, config_set, exec_command, + update_config) @pytest.fixture def env(): r = os.environ.copy() - r.update({ - constants.PATH_ENV: os.environ[constants.PATH_ENV], - constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "dcos.toml"), - }) + r.update({constants.PATH_ENV: os.environ[constants.PATH_ENV]}) return r @@ -33,44 +30,20 @@ def test_version(): stdout=stdout) -def _test_list_property(env): - stdout = b"""core.dcos_url http://dcos.snakeoil.mesosphere.com -core.reporting False -core.ssl_verify false -core.timeout 5 -""" - assert_command(['dcos', 'config', 'show'], - stdout=stdout, - env=env) - - -def test_get_existing_string_property(env): - _get_value('core.dcos_url', 'http://dcos.snakeoil.mesosphere.com', env) - - def test_get_existing_boolean_property(env): - _get_value('core.reporting', False, env) + with update_config("core.reporting", "false", env): + _get_value('core.reporting', False, env) def test_get_existing_number_property(env): - _get_value('core.timeout', 5, env) + with update_config("core.timeout", "5", env): + _get_value('core.timeout', 5, env) def test_get_missing_property(env): _get_missing_value('missing.property', env) -def test_dcos_url_without_scheme(env): - with update_config("core.dcos_url", None, env): - new = b"abc.com" - out = b"[core.dcos_url]: set to 'https://%b'\n" % (new) - assert_command( - ['dcos', 'config', 'set', 'core.dcos_url', new.decode('utf-8')], - stderr=out, - returncode=0, - env=env) - - def test_get_top_property(env): returncode, stdout, stderr = exec_command( ['dcos', 'config', 'show', 'core'], env=env) @@ -82,13 +55,6 @@ def test_get_top_property(env): b"possible properties are:\n") -def test_set_existing_string_property(env): - new_value = 'http://dcos.snakeoil.mesosphere.com:5081' - with update_config('core.dcos_url', new_value, env): - _get_value('core.dcos_url', - 'http://dcos.snakeoil.mesosphere.com:5081', env) - - def test_set_existing_boolean_property(env): config_set('core.reporting', 'true', env) _get_value('core.reporting', True, env) @@ -116,16 +82,6 @@ def test_set_same_output(env): env=env) -def test_set_new_output(env): - with update_config("core.dcos_url", None, env): - assert_command( - ['dcos', 'config', 'set', 'core.dcos_url', - 'http://dcos.snakeoil.mesosphere.com:5081'], - stderr=(b"[core.dcos_url]: set to " - b"'http://dcos.snakeoil.mesosphere.com:5081'\n"), - env=env) - - def test_set_nonexistent_subcommand(env): assert_command( ['dcos', 'config', 'set', 'foo.bar', 'baz'], @@ -135,15 +91,6 @@ def test_set_nonexistent_subcommand(env): env=env) -def test_set_when_extra_section(env): - path = os.path.join('tests', 'data', 'config', 'invalid_section.toml') - env['DCOS_CONFIG'] = path - os.chmod(path, 0o600) - - config_set('core.dcos_url', 'http://dcos.snakeoil.mesosphere.com', env) - config_unset('core.dcos_url', env) - - def test_unset_property(env): with update_config("core.reporting", None, env): _get_missing_value('core.reporting', env) @@ -193,9 +140,9 @@ def test_set_property_key(env): def test_set_missing_property(env): - with update_config("core.dcos_url", None, env=env): - config_set('core.dcos_url', 'http://localhost:8080', env) - _get_value('core.dcos_url', 'http://localhost:8080', env) + with update_config("package.cosmos_url", None, env=env): + config_set('package.cosmos_url', 'http://localhost:8080', env) + _get_value('package.cosmos_url', 'http://localhost:8080', env) def test_set_core_property(env): @@ -205,33 +152,30 @@ def test_set_core_property(env): def test_url_validation(env): - with update_config('core.dcos_url', None, env): - key = 'core.dcos_url' - key2 = 'package.cosmos_url' + key = 'package.cosmos_url' + with update_config(key, None, env): config_set(key, 'http://localhost', env) config_set(key, 'https://localhost', env) config_set(key, 'http://dcos-1234', env) - config_set(key2, 'http://dcos-1234.mydomain.com', env) + config_set(key, 'http://dcos-1234.mydomain.com', env) config_set(key, 'http://localhost:5050', env) config_set(key, 'https://localhost:5050', env) config_set(key, 'http://mesos-1234:5050', env) - config_set(key2, 'http://mesos-1234.mydomain.com:5050', env) + config_set(key, 'http://mesos-1234.mydomain.com:5050', env) config_set(key, 'http://localhost:8080', env) config_set(key, 'https://localhost:8080', env) config_set(key, 'http://marathon-1234:8080', env) - config_set(key2, 'http://marathon-1234.mydomain.com:5050', env) + config_set(key, 'http://marathon-1234.mydomain.com:5050', env) config_set(key, 'http://user@localhost:8080', env) config_set(key, 'http://u-ser@localhost:8080', env) config_set(key, 'http://user123_@localhost:8080', env) config_set(key, 'http://user:p-ssw_rd@localhost:8080', env) config_set(key, 'http://user123:password321@localhost:8080', env) - config_set(key2, 'http://us%r1$3:pa#sw*rd321@localhost:8080', env) - - config_unset(key2, env) + config_set(key, 'http://us%r1$3:pa#sw*rd321@localhost:8080', env) def test_fail_url_validation(env): @@ -285,27 +229,14 @@ def test_timeout(env): assert "(connect timeout=1)".encode('utf-8') in stderr -def test_parse_error(env): - path = os.path.join('tests', 'data', 'config', 'parse_error.toml') - os.chmod(path, 0o600) - env['DCOS_CONFIG'] = path - - assert_command(['dcos', 'config', 'show'], - returncode=1, - stderr=six.b(("Error parsing config file at [{}]: Found " - "invalid character in key name: ']'. " - "Try quoting the key name.\n").format(path)), - env=env) - - def _fail_url_validation(command, key, value, env): returncode_, stdout_, stderr_ = exec_command( ['dcos', 'config', command, key, value], env=env) assert returncode_ == 1 assert stdout_ == b'' - assert stderr_.startswith(str( - 'Unable to parse {!r} as a url'.format(value)).encode('utf-8')) + err = str('Unable to parse {!r} as a url'.format(value)).encode('utf-8') + assert err in stderr_ def _get_value(key, value, env): diff --git a/cli/tests/integrations/test_job.py b/cli/tests/integrations/test_job.py index c2b3e88..7bbb400 100644 --- a/cli/tests/integrations/test_job.py +++ b/cli/tests/integrations/test_job.py @@ -6,7 +6,7 @@ import pytest from dcos import constants -from .helpers.common import assert_command, exec_command, update_config +from .helpers.common import assert_command, exec_command from .helpers.job import job, show_job, show_job_schedule @@ -29,24 +29,11 @@ def test_info(): @pytest.fixture def env(): r = os.environ.copy() - r.update({ - constants.PATH_ENV: os.environ[constants.PATH_ENV], - constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "dcos.toml"), - }) + r.update({constants.PATH_ENV: os.environ[constants.PATH_ENV]}) return r -def test_missing_config(env): - with update_config("core.dcos_url", None, env): - assert_command( - ['dcos', 'job', 'list'], - returncode=1, - stderr=(b'Missing required config parameter: "core.dcos_url". ' - b'Please run `dcos config set core.dcos_url `.\n'), - env=env) - - def test_empty_list(): _list_jobs() diff --git a/cli/tests/integrations/test_marathon.py b/cli/tests/integrations/test_marathon.py index 9e91579..c4b4d72 100644 --- a/cli/tests/integrations/test_marathon.py +++ b/cli/tests/integrations/test_marathon.py @@ -50,24 +50,11 @@ def test_about(): @pytest.fixture def env(): r = os.environ.copy() - r.update({ - constants.PATH_ENV: os.environ[constants.PATH_ENV], - constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "dcos.toml"), - }) + r.update({constants.PATH_ENV: os.environ[constants.PATH_ENV]}) return r -def test_missing_config(env): - with update_config("core.dcos_url", None, env): - assert_command( - ['dcos', 'marathon', 'app', 'list'], - returncode=1, - stderr=(b'Missing required config parameter: "core.dcos_url". ' - b'Please run `dcos config set core.dcos_url `.\n'), - env=env) - - def test_empty_list(): list_apps() diff --git a/cli/tests/integrations/test_package.py b/cli/tests/integrations/test_package.py index 49dd098..b96bd6c 100644 --- a/cli/tests/integrations/test_package.py +++ b/cli/tests/integrations/test_package.py @@ -23,10 +23,7 @@ from ..common import file_bytes @pytest.fixture def env(): r = os.environ.copy() - r.update({ - constants.PATH_ENV: os.environ[constants.PATH_ENV], - constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "dcos.toml"), - }) + r.update({constants.PATH_ENV: os.environ[constants.PATH_ENV]}) return r @@ -684,7 +681,7 @@ def test_list_cli_only(env): helloworld_json = file_json(helloworld_path) with _helloworld_cli(), \ - update_config('core.dcos_url', 'http://nohost', env): + update_config('package.cosmos_url', 'http://nohost', env): assert_command( cmd=['dcos', 'package', 'list', '--json', '--cli'], stdout=helloworld_json) @@ -699,6 +696,18 @@ def test_list_cli_only(env): stdout=helloworld_json) +def test_cli_global(): + helloworld_path = 'tests/data/package/json/test_list_helloworld_cli.json' + helloworld_json = file_json(helloworld_path) + + with _helloworld_cli(global_=True): + assert os.path.exists(subcommand.global_package_dir("helloworld")) + + assert_command( + cmd=['dcos', 'package', 'list', '--json', '--cli'], + stdout=helloworld_json) + + def test_uninstall_multiple_frameworknames(zk_znode): _install_chronos( args=['--yes', '--options=tests/data/package/chronos-1.json']) @@ -991,9 +1000,12 @@ def _helloworld(): uninstall_stderr=stderr) -def _helloworld_cli(): +def _helloworld_cli(global_=False): + args = ['--yes', '--cli'] + if global_: + args += ['--global'] return _package(name='helloworld', - args=['--yes', '--cli'], + args=args, stdout=HELLOWORLD_CLI_STDOUT, uninstall_stderr=b'') diff --git a/cli/tests/integrations/test_ssl.py b/cli/tests/integrations/test_ssl.py index fb55bff..03bf612 100644 --- a/cli/tests/integrations/test_ssl.py +++ b/cli/tests/integrations/test_ssl.py @@ -2,9 +2,9 @@ import os import pytest -from dcos import config, constants +from dcos import constants -from .helpers.common import config_set, exec_command, update_config +from .helpers.common import exec_command, update_config @pytest.fixture @@ -12,27 +12,14 @@ def env(): r = os.environ.copy() r.update({ constants.PATH_ENV: os.environ[constants.PATH_ENV], - constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "dcos.toml"), 'DCOS_SNAKEOIL_CRT_PATH': os.environ.get( - "DCOS_SNAKEOIL_CRT_PATH", "/dcos-cli/adminrouter/snakeoil.crt") + "DCOS_SNAKEOIL_CRT_PATH", "/dcos-cli/adminrouter/snakeoil.crt"), + 'DCOS_URL': 'https://dcos.snakeoil.mesosphere.com' }) return r -@pytest.yield_fixture(autouse=True) -def setup_env(env): - # token will be removed when we change dcos_url - token = config.get_config_val('core.dcos_acs_token') - config_set("core.dcos_url", "https://dcos.snakeoil.mesosphere.com", env) - config_set("core.dcos_acs_token", token, env) - try: - yield - finally: - config_set("core.dcos_url", "http://dcos.snakeoil.mesosphere.com", env) - config_set("core.dcos_acs_token", token, env) - - def test_dont_verify_ssl_with_env_var(env): env['DCOS_SSL_VERIFY'] = 'false' diff --git a/cli/tests/unit/data/cluster.txt b/cli/tests/unit/data/cluster.txt new file mode 100644 index 0000000..f3e64ed --- /dev/null +++ b/cli/tests/unit/data/cluster.txt @@ -0,0 +1,2 @@ + NAME CLUSTER ID VERSION URL +tamar-ytck1ge 8c4f77ff-849c-456d-a480-c5cb6766c3f2 1.9-dev https://52.25.204.103 diff --git a/cli/tests/unit/test_tables.py b/cli/tests/unit/test_tables.py index a1e2367..7426973 100644 --- a/cli/tests/unit/test_tables.py +++ b/cli/tests/unit/test_tables.py @@ -7,6 +7,7 @@ from dcos.mesos import Slave from dcoscli import tables from ..fixtures.auth_provider import auth_provider_fixture +from ..fixtures.clusters import cluster_list_fixture from ..fixtures.marathon import (app_fixture, app_task_fixture, deployment_fixture_app_post_pods, deployment_fixture_app_pre_pods, @@ -145,6 +146,12 @@ def test_node_table(): 'tests/unit/data/node.txt') +def test_clusters_tables(): + _test_table(tables.clusters_table, + [cluster_list_fixture()], + 'tests/unit/data/cluster.txt') + + def test_ls_long_table(): with mock.patch('dcoscli.tables._format_unix_timestamp', lambda ts: datetime.datetime.fromtimestamp( @@ -177,4 +184,4 @@ def test_metrics_details_no_tags_table(): def _test_table(table_fn, fixture_fn, path): table = table_fn(fixture_fn) with open(path) as f: - assert str(table) == f.read() + assert str(table) == f.read().strip('\n') diff --git a/dcos/cluster.py b/dcos/cluster.py new file mode 100644 index 0000000..dda0fe4 --- /dev/null +++ b/dcos/cluster.py @@ -0,0 +1,299 @@ +import contextlib +import os +import shutil +import ssl +import urllib + +from urllib.request import urlopen + +from dcos import config, constants, http, util +from dcos.errors import DCOSException + + +logger = util.get_logger(__name__) + + +def move_to_cluster_config(): + """Create a cluster specific config file + directory + from a global config file. This will move users from global config + structure (~/.dcos/dcos.toml) to the cluster specific one + (~/.dcos/clusters/CLUSTER_ID/dcos.toml) and set that cluster as + the "attached" cluster. + + :rtype: None + """ + + global_config = config.get_global_config() + dcos_url = config.get_config_val("core.dcos_url", global_config) + + # if no cluster is set, do not move the cluster yet + if dcos_url is None: + return + + try: + # find cluster id + cluster_url = dcos_url.rstrip('/') + '/metadata' + res = http.get(cluster_url, timeout=1) + cluster_id = res.json().get("CLUSTER_ID") + + # don't move cluster if dcos_url is not valid + except DCOSException as e: + logger.error( + "Error trying to find cluster id: {}".format(e)) + return + + # create cluster id dir + cluster_path = os.path.join(config.get_config_dir_path(), + constants.DCOS_CLUSTERS_SUBDIR, + cluster_id) + + util.ensure_dir_exists(cluster_path) + + # move config file to new location + global_config_path = config.get_global_config_path() + util.sh_copy(global_config_path, cluster_path) + + # set cluster as attached + util.ensure_file_exists(os.path.join( + cluster_path, constants.DCOS_CLUSTER_ATTACHED_FILE)) + + +@contextlib.contextmanager +def setup_directory(): + """ + A context manager for the temporary setup directory created as a + placeholder before we find the cluster's CLUSTER_ID. + + :returns: path of setup directory + :rtype: str + """ + + try: + temp_path = os.path.join(config.get_config_dir_path(), + constants.DCOS_CLUSTERS_SUBDIR, + "setup") + util.ensure_dir_exists(temp_path) + + yield temp_path + finally: + shutil.rmtree(temp_path, ignore_errors=True) + + +def setup_cluster_config(dcos_url, temp_path, stored_cert): + """ + Create a cluster directory for cluster specified in "temp_path" + directory. + + :param dcos_url: url to DC/OS cluster + :type dcos_url: str + :param temp_path: path to temporary config dir + :type temp_path: str + :param stored_cert: whether we stored cert bundle in 'setup' dir + :type stored_cert: bool + :returns: path to cluster specific directory + :rtype: str + """ + + try: + # find cluster id + cluster_url = dcos_url.rstrip('/') + '/metadata' + res = http.get(cluster_url, timeout=1) + cluster_id = res.json().get("CLUSTER_ID") + + except DCOSException as e: + msg = ("Error trying to find cluster id: {}\n " + "Please make sure the provided DC/OS URL is valid: {}".format( + e, dcos_url)) + raise DCOSException(msg) + + # create cluster id dir + cluster_path = os.path.join(config.get_config_dir_path(), + constants.DCOS_CLUSTERS_SUBDIR, + cluster_id) + if os.path.exists(cluster_path): + raise DCOSException("Cluster [{}] is already setup".format(dcos_url)) + + util.ensure_dir_exists(cluster_path) + + # move contents of setup dir to new location + for (path, dirnames, filenames) in os.walk(temp_path): + for f in filenames: + util.sh_copy(os.path.join(path, f), cluster_path) + + if stored_cert: + config.set_val("core.ssl_verify", os.path.join( + cluster_path, "dcos_ca.crt")) + + cluster_name = cluster_id + try: + url = dcos_url.rstrip('/') + '/mesos/state-summary' + cluster_name = http.get(url, timeout=1).json().get("cluster") + except DCOSException: + pass + + config.set_val("cluster.name", cluster_name) + + return cluster_path + + +def set_attached(cluster_path): + """ + Set the cluster specified in `cluster_path` as the attached cluster + + :param cluster_path: path to cluster directory + :type cluster_path: str + :rtype: None + """ + + # get currently attached cluster + attached_cluster_path = config.get_attached_cluster_path() + + if attached_cluster_path and attached_cluster_path != cluster_path: + # set cluster as attached + attached_file = os.path.join( + attached_cluster_path, constants.DCOS_CLUSTER_ATTACHED_FILE) + try: + util.sh_move(attached_file, cluster_path) + except DCOSException as e: + msg = "Failed to attach cluster: {}".format(e) + raise DCOSException(msg) + + else: + util.ensure_file_exists(os.path.join( + cluster_path, constants.DCOS_CLUSTER_ATTACHED_FILE)) + + +def get_cluster_cert(dcos_url): + """Get CA bundle from specified cluster. + + This is an insecure request. + + :param dcos_url: url to DC/OS cluster + :type dcos_url: str + :returns: cert + :rtype: str + """ + + cert_bundle_url = dcos_url.rstrip() + "/ca/dcos-ca.crt" + + unverified = ssl.create_default_context() + unverified.check_hostname = False + unverified.verify_mode = ssl.CERT_NONE + + error_msg = ("Error downloading CA certificate from cluster. " + "Please check the provided DC/OS URL.") + try: + with urlopen(cert_bundle_url, context=unverified) as f: + return f.read().decode('utf-8') + except urllib.error.HTTPError as e: + # Open source DC/OS does not currently expose its root CA certificate + if e.code == 404: + return False + else: + logger.debug(e) + raise DCOSException(error_msg) + except Exception as e: + logger.debug(e) + raise DCOSException(error_msg) + + +def get_clusters(): + """ + :returns: list of configured Clusters + :rtype: [Clusters] + """ + + clusters_path = config.get_clusters_path() + util.ensure_dir_exists(clusters_path) + clusters = os.listdir(clusters_path) + return [Cluster(cluster_id) for cluster_id in clusters] + + +def get_cluster(name): + """ + :param name: name of cluster + :type name: str + :returns: Cluster identified by name + :rtype: Cluster + """ + + return next((c for c in get_clusters() + if c.get_cluster_id() == name or c.get_name() == name), None) + + +def remove(name): + """ + Remove cluster `name` from the CLI. + + :param name: name of cluster + :type name: str + :rtype: None + """ + + def onerror(func, path, excinfo): + raise DCOSException("Error trying to remove cluster") + + cluster = get_cluster(name) + if cluster: + shutil.rmtree(cluster.get_cluster_path(), onerror) + return + else: + raise DCOSException("Cluster [{}] does not exist".format(name)) + + +class Cluster(): + """Interface for a configured cluster""" + + def __init__(self, cluster_id): + self.cluster_id = cluster_id + self.cluster_path = os.path.join( + config.get_clusters_path(), cluster_id) + + def get_cluster_path(self): + return self.cluster_path + + def get_cluster_id(self): + return self.cluster_id + + def get_config_path(self): + return os.path.join(self.cluster_path, "dcos.toml") + + def get_config(self, mutable=False): + return config.load_from_path(self.get_config_path(), mutable) + + def get_name(self): + return config.get_config_val( + "cluster.name", self.get_config()) or self.cluster_id + + def get_url(self): + return config.get_config_val("core.dcos_url", self.get_config()) + + def get_dcos_version(self): + dcos_url = self.get_url() + if dcos_url: + url = os.path.join( + self.get_url(), "dcos-metadata/dcos-version.json") + try: + resp = http.get(url, timeout=1, toml_config=self.get_config()) + return resp.json().get("version", "N/A") + except DCOSException: + pass + + return "N/A" + + def is_attached(self): + return os.path.exists(os.path.join( + self.cluster_path, constants.DCOS_CLUSTER_ATTACHED_FILE)) + + def __eq__(self, other): + return isinstance(other, Cluster) and \ + other.get_cluster_id() == self.get_cluster_id() + + def dict(self): + return { + "cluster_id": self.get_cluster_id(), + "name": self.get_name(), + "url": self.get_url(), + "version": self.get_dcos_version(), + "attached": self.is_attached() + } diff --git a/dcos/config.py b/dcos/config.py index 4203646..c60df49 100644 --- a/dcos/config.py +++ b/dcos/config.py @@ -12,14 +12,92 @@ from dcos.errors import DCOSException logger = util.get_logger(__name__) -def get_config_path(): - """ Returns the path to the DCOS config file. +def uses_deprecated_config(): + """Returns True if the configuration for the user's CLI + is the deprecated 'global' config instead of the cluster + specific config + """ + + global_config = get_global_config_path() + cluster_config = get_clusters_path() + return not os.path.exists(cluster_config) and os.path.exists(global_config) + + +def get_global_config_path(): + """Returns the path to the deprecated global DCOS config file. :returns: path to the DCOS config file :rtype: str """ - return os.environ.get(constants.DCOS_CONFIG_ENV, get_default_config_path()) + default_path = os.path.join(get_config_dir_path(), "dcos.toml") + return os.environ.get(constants.DCOS_CONFIG_ENV, default_path) + + +def get_global_config(mutable=False): + """Returns the deprecated global DCOS config file + + :param mutable: True if the returned Toml object should be mutable + :type mutable: boolean + :returns: Configuration object + :rtype: Toml | MutableToml + """ + + return load_from_path(get_global_config_path(), mutable) + + +def get_attached_cluster_path(): + """ + The attached cluster is denoted by a file named "attached" in one of the + cluster directories. Ex: $DCOS_DIR/clusters/CLUSTER_ID/attached + + :returns: path to the director of the attached cluster + :rtype: str | None + """ + + path = get_clusters_path() + if not os.path.exists(path): + return None + + clusters = os.listdir(get_clusters_path()) + for c in clusters: + cluster_path = os.path.join(path, c) + if os.path.exists(os.path.join( + cluster_path, constants.DCOS_CLUSTER_ATTACHED_FILE)): + return cluster_path + + # if only one cluster, set as attached + if len(clusters) == 1: + cluster_path = os.path.join(path, clusters[0]) + util.ensure_file_exists(os.path.join( + cluster_path, constants.DCOS_CLUSTER_ATTACHED_FILE)) + return cluster_path + + return None + + +def get_clusters_path(): + """ + :returns: path to the directory of cluster configs + :rtype: str + """ + + return os.path.join(get_config_dir_path(), constants.DCOS_CLUSTERS_SUBDIR) + + +def get_config_path(): + """Returns the path to the DCOS config file of the attached cluster. + If still using "global" config return that toml instead + + :returns: path to the DCOS config file + :rtype: str + """ + + if uses_deprecated_config(): + return get_global_config_path() + else: + cluster_path = get_attached_cluster_path() + return os.path.join(cluster_path, "dcos.toml") def get_config_dir_path(): @@ -33,16 +111,6 @@ def get_config_dir_path(): return os.path.expanduser(config_dir) -def get_default_config_path(): - """Returns the default path to the DCOS config file. - - :returns: path to the DCOS config file - :rtype: str - """ - - return os.path.join(get_config_dir_path(), 'dcos.toml') - - def get_config(mutable=False): """Returns the DCOS configuration object and creates config file is None found and `DCOS_CONFIG` set to default value. Only use to get the config, @@ -55,11 +123,18 @@ def get_config(mutable=False): :rtype: Toml | MutableToml """ - path = get_config_path() - default = get_default_config_path() + cluster_path = get_attached_cluster_path() + if cluster_path is None: + if uses_deprecated_config(): + return get_global_config(mutable) - if path == default: - util.ensure_dir_exists(os.path.dirname(default)) + msg = ("No cluster is attached. " + "Please run `dcos cluster attach `") + raise DCOSException(msg) + + util.ensure_dir_exists(os.path.dirname(cluster_path)) + + path = os.path.join(cluster_path, "dcos.toml") return load_from_path(path, mutable) @@ -115,7 +190,8 @@ def get_config_val(name, config=None): :returns: value of 'name' parameter :rtype: str | None """ - val, _ = get_config_val_envvar(name, config=None) + + val, _ = get_config_val_envvar(name, config) return val @@ -135,17 +211,22 @@ def missing_config_exception(keys): return DCOSException(msg) -def set_val(name, value): +def set_val(name, value, config_path=None): """ :param name: name of paramater :type name: str :param value: value to set to paramater `name` :type param: str + :param config_path: path to config to use + :type config_path: str :returns: Toml config, message of change :rtype: Toml, str """ - toml_config = get_config(True) + if config_path: + toml_config = load_from_path(config_path, True) + else: + toml_config = get_config(True) section, subkey = split_key(name) @@ -169,7 +250,7 @@ def set_val(name, value): check_config(toml_config_pre, toml_config, section) - save(toml_config) + save(toml_config, config_path) msg = "[{}]: ".format(name) if name == "core.dcos_acs_token": @@ -187,8 +268,7 @@ def set_val(name, value): msg += "changed from '{}' to '{}'".format(old_value, new_value) if token_unset: - msg += ("\n[core.dcos_acs_token]: removed\n" - "Please run `dcos auth login` to authenticate to new dcos_url") + msg += "\n[core.dcos_acs_token]: removed" return toml_config, msg @@ -215,18 +295,21 @@ def load_from_path(path, mutable=False): return (MutableToml if mutable else Toml)(toml_obj) -def save(toml_config): +def save(toml_config, config_path=None): """ :param toml_config: TOML configuration object :type toml_config: MutableToml or Toml + :param config_path: path to config to use + :type config_path: str """ serial = toml.dumps(toml_config._dictionary) - path = get_config_path() + if config_path is None: + config_path = get_config_path() - util.ensure_file_exists(path) - util.enforce_file_permissions(path) - with util.open_file(path, 'w') as config_file: + util.ensure_file_exists(config_path) + util.enforce_file_permissions(config_path) + with util.open_file(config_path, 'w') as config_file: config_file.write(serial) diff --git a/dcos/constants.py b/dcos/constants.py index 1a09a0e..51e3a76 100644 --- a/dcos/constants.py +++ b/dcos/constants.py @@ -4,6 +4,13 @@ DCOS_DIR = ".dcos" DCOS_DIR_ENV = 'DCOS_DIR' """Name of the environment variable pointing to the DC/OS data directory""" +DCOS_CLUSTERS_SUBDIR = "clusters" +"""Name of the subdirectory containing the configuration of all configured +clusters""" + +DCOS_CLUSTER_ATTACHED_FILE = "attached" +"""Name of the file indicating the current attached cluster""" + DCOS_SUBCOMMAND_ENV_SUBDIR = 'env' """In a package's directory, this is the cli contents subdirectory.""" diff --git a/dcos/data/config-schema/cluster.json b/dcos/data/config-schema/cluster.json new file mode 100644 index 0000000..cafd961 --- /dev/null +++ b/dcos/data/config-schema/cluster.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "name": { + "type": "string", + "title": "Cluster name", + "description": "Human readable name of cluster" + } + }, + "additionalProperties": false +} diff --git a/dcos/emitting.py b/dcos/emitting.py index 7c64123..19b71f7 100644 --- a/dcos/emitting.py +++ b/dcos/emitting.py @@ -164,7 +164,10 @@ def _page(output, pager_command=None): if pager_command is None: pager_command = 'less -R' - paginate = config.get_config_val("core.pagination") or True + try: + paginate = config.get_config_val("core.pagination") + except: + paginate = True if exceeds_tty_height and paginate and \ spawn.find_executable(pager_command.split(' ')[0]) is not None: pydoc.pipepager(output, cmd=pager_command) diff --git a/dcos/http.py b/dcos/http.py index 0d1190c..9b7aa9e 100644 --- a/dcos/http.py +++ b/dcos/http.py @@ -28,17 +28,22 @@ def _default_is_success(status_code): return 200 <= status_code < 300 -def _verify_ssl(verify=None): +def _verify_ssl(verify=None, toml_config=None): """Returns whether to verify ssl :param verify: whether to verify SSL certs or path to cert(s) :type verify: bool | str + :param toml_config: cluster config to use + :type toml_config: Toml :return: whether to verify SSL certs or path to cert(s) :rtype: bool | str """ + if toml_config is None: + toml_config = config.get_config() + if verify is None: - verify = config.get_config_val("core.ssl_verify") + verify = config.get_config_val("core.ssl_verify", toml_config) if verify and verify.lower() == "true": verify = True elif verify and verify.lower() == "false": @@ -54,6 +59,7 @@ def _request(method, timeout=DEFAULT_TIMEOUT, auth=None, verify=None, + toml_config=None, **kwargs): """Sends an HTTP request. @@ -69,6 +75,8 @@ def _request(method, :type auth: AuthBase :param verify: whether to verify SSL certs or path to cert(s) :type verify: bool | str + :param toml_config: cluster config to use + :type toml_config: Toml :param kwargs: Additional arguments to requests.request (see http://docs.python-requests.org/en/latest/api/#requests.request) :type kwargs: dict @@ -78,7 +86,7 @@ def _request(method, if 'headers' not in kwargs: kwargs['headers'] = {'Accept': 'application/json'} - verify = _verify_ssl(verify) + verify = _verify_ssl(verify, toml_config) # Silence 'Unverified HTTPS request' and 'SecurityWarning' for bad certs if verify is not None: @@ -128,6 +136,7 @@ def request(method, is_success=_default_is_success, timeout=None, verify=None, + toml_config=None, **kwargs): """Sends an HTTP request. If the server responds with a 401, ask the user for their credentials, and try request again (up to 3 times). @@ -142,13 +151,17 @@ def request(method, :type timeout: int :param verify: whether to verify SSL certs or path to cert(s) :type verify: bool | str + :param toml_config: cluster config to use + :type toml_config: Toml :param kwargs: Additional arguments to requests.request (see http://docs.python-requests.org/en/latest/api/#requests.request) :type kwargs: dict :rtype: Response """ - toml_config = config.get_config() + if toml_config is None: + toml_config = config.get_config() + auth_token = config.get_config_val("core.dcos_acs_token", toml_config) prompt_login = config.get_config_val("core.prompt_login", toml_config) dcos_url = urlparse(config.get_config_val("core.dcos_url", toml_config)) @@ -162,8 +175,10 @@ def request(method, auth = DCOSAcsAuth(auth_token) else: auth = None + response = _request(method, url, is_success, timeout, - auth=auth, verify=verify, **kwargs) + auth=auth, verify=verify, toml_config=toml_config, + **kwargs) if is_success(response.status_code): return response diff --git a/dcos/subcommand.py b/dcos/subcommand.py index 32eb5b9..72413b6 100644 --- a/dcos/subcommand.py +++ b/dcos/subcommand.py @@ -113,16 +113,15 @@ def _is_executable(path): not util.is_windows_platform() or path.endswith('.exe')) -def distributions(): - """List all of the installed subcommand packages - - :returns: a list of packages - :rtype: list of str +def _find_distributions(subcommand_dir): + """ + :param subcommand_dir: directory to find packaged in + :type subcommand_dir: path + :returns: list of all installed subcommands in given directory + :rtype: [str] """ - subcommand_dir = _subcommand_dir() - - if os.path.isdir(subcommand_dir): + if subcommand_dir and os.path.isdir(subcommand_dir): return [ subdir for subdir in os.listdir(subcommand_dir) if os.path.isdir( @@ -135,6 +134,18 @@ def distributions(): return [] +def distributions(): + """Set of all of the installed subcommand packages + + :returns: a set of packages + :rtype: Set[str] + """ + + cluster_packages = _find_distributions(_cluster_subcommand_dir()) + global_packages = _find_distributions(global_subcommand_dir()) + return set(cluster_packages + global_packages) + + # must also add subcommand name to dcoscli.subcommand._default_modules def default_subcommands(): """List the default dcos cli subcommands @@ -142,8 +153,9 @@ def default_subcommands(): :returns: list of all the default dcos cli subcommands :rtype: [str] """ - return ["auth", "config", "experimental", "help", "job", "marathon", - "node", "package", "service", "task"] + + return ["auth", "cluster", "config", "experimental", "help", "job", + "marathon", "node", "package", "service", "task"] def documentation(executable_path): @@ -210,16 +222,16 @@ def noun(executable_path): return noun -def _write_package_json(pkg): +def _write_package_json(pkg, pkg_dir): """ Write package.json locally. :param pkg: the package being installed :type pkg: PackageVersion + :param pkg_dir: directory to install package + :type pkg_dir: str :rtype: None """ - pkg_dir = _package_dir(pkg.name()) - package_path = os.path.join(pkg_dir, 'package.json') package_json = pkg.package_json() @@ -305,15 +317,17 @@ def _get_cli_binary_info(cli_resources): cli_resources)) -def _install_cli(pkg): +def _install_cli(pkg, pkg_dir): """Install subcommand cli :param pkg: the package to install :type pkg: PackageVersion + :param pkg_dir: directory to install package + :type pkg_dir: str :rtype: None """ - with util.remove_path_on_error(_package_dir(pkg.name())) as pkg_dir: + with util.remove_path_on_error(pkg_dir) as pkg_dir: env_dir = os.path.join(pkg_dir, constants.DCOS_SUBCOMMAND_ENV_SUBDIR) resources = pkg.resource_json() @@ -341,38 +355,90 @@ def _install_cli(pkg): "Could not find a CLI subcommand for your platform") -def install(pkg): +def install(pkg, global_=False): """Installs the dcos cli subcommand :param pkg: the package to install :type pkg: Package + :param global_: whether to install the CLI globally + :type global_: bool :rtype: None """ - pkg_dir = _package_dir(pkg.name()) + if global_ or config.uses_deprecated_config(): + pkg_dir = global_package_dir(pkg.name()) + else: + pkg_dir = _cluster_package_dir(pkg.name()) + util.ensure_dir_exists(pkg_dir) - _write_package_json(pkg) + _write_package_json(pkg, pkg_dir) - _install_cli(pkg) + _install_cli(pkg, pkg_dir) -def _subcommand_dir(): - """ Returns subcommand dir. defaults to ~/.dcos/subcommands """ +def global_subcommand_dir(): + """ Returns global subcommand dir. defaults to ~/.dcos/subcommands """ return os.path.join(config.get_config_dir_path(), constants.DCOS_SUBCOMMAND_SUBDIR) -def _package_dir(name): - """ Returns ~/.dcos/subcommands/ +def _cluster_subcommand_dir(): + """ + :returns: cluster specific subcommand dir or None + :rtype: str | None + """ + + attached_cluster = config.get_attached_cluster_path() + if attached_cluster is not None: + return os.path.join( + attached_cluster, constants.DCOS_SUBCOMMAND_SUBDIR) + else: + return None + + +def _cluster_package_dir(name): + """Returns path to package directory for attached cluster + + :param name: package name + :type name: str + :returns: path to package directory + :rtype: str | None + """ + + subcommand_dir = _cluster_subcommand_dir() + if subcommand_dir is not None: + return os.path.join(subcommand_dir, name) + else: + return None + + +def global_package_dir(name): + """Returns path to package directory in global config :param name: package name :type name: str :rtype: str """ - return os.path.join(_subcommand_dir(), - name) + + return os.path.join(global_subcommand_dir(), name) + + +def _package_dir(name): + """Returns cluster subcommand dir for name if exists, and if not + returns path to global subcommand dir. + + :param name: package name + :type name: str + :rtype: str + """ + + cluster_subcommand = _cluster_package_dir(name) + if cluster_subcommand and os.path.exists(cluster_subcommand): + return cluster_subcommand + else: + return global_package_dir(name) def uninstall(package_name): @@ -626,24 +692,6 @@ class InstalledSubcommand(object): return _package_dir(self.name) - def package_revision(self): - """ - :returns: this subcommand's version. - :rtype: str - """ - - version_path = os.path.join(self._dir(), 'version') - return util.read_file(version_path) - - def package_source(self): - """ - :returns: this subcommand's source. - :rtype: str - """ - - source_path = os.path.join(self._dir(), 'source') - return util.read_file(source_path) - def package_json(self): """ :returns: contents of this subcommand's package.json file. diff --git a/dcos/util.py b/dcos/util.py index e6ea589..a2febb0 100644 --- a/dcos/util.py +++ b/dcos/util.py @@ -117,6 +117,31 @@ def sh_copy(src, dst): raise DCOSException(e) +def sh_move(src, dst): + """Move file src to the file or directory dst. + + :param src: source file + :type src: str + :param dst: destination file or directory + :type dst: str + :rtype: None + """ + try: + shutil.move(src, dst) + except EnvironmentError as e: + logger.exception('Unable to move [%s] to [%s]', src, dst) + if e.strerror: + if e.filename: + raise DCOSException("{}: {}".format(e.strerror, e.filename)) + else: + raise DCOSException(e.strerror) + else: + raise DCOSException(e) + except Exception as e: + logger.exception('Unknown error while moving [%s] to [%s]', src, dst) + raise DCOSException(e) + + def ensure_dir_exists(directory): """If `directory` does not exist, create it. diff --git a/tests/test_cluster.py b/tests/test_cluster.py new file mode 100644 index 0000000..22ffe90 --- /dev/null +++ b/tests/test_cluster.py @@ -0,0 +1,113 @@ +import os + +from unittest.mock import MagicMock + +import mock +from mock import patch + +from test_util import add_cluster_dir, create_global_config, env + +from dcos import cluster, config, constants, util + + +def _cluster(cluster_id): + c = cluster.Cluster(cluster_id) + c.get_name = MagicMock(return_value="cluster-{}".format(cluster_id)) + return c + + +def _test_cluster_list(): + return [_cluster("a"), _cluster("b"), _cluster("c")] + + +@patch('dcos.cluster.get_clusters') +def test_get_cluster(clusters): + clusters.return_value = _test_cluster_list() + assert cluster.get_cluster("a") == _cluster("a") + assert cluster.get_cluster("cluster-b") == _cluster("b") + + +def test_get_clusters(): + with env(), util.tempdir() as tempdir: + os.environ[constants.DCOS_DIR_ENV] = tempdir + + # no config file of any type + assert cluster.get_clusters() == [] + + # cluster dir exists, no cluster + clusters_dir = os.path.join(tempdir, constants.DCOS_CLUSTERS_SUBDIR) + util.ensure_dir_exists(clusters_dir) + assert cluster.get_clusters() == [] + + # one cluster + cluster_id = "fake_cluster" + add_cluster_dir(cluster_id, tempdir) + assert cluster.get_clusters() == [_cluster(cluster_id)] + + +def test_set_attached(): + with env(), util.tempdir() as tempdir: + os.environ[constants.DCOS_DIR_ENV] = tempdir + + cluster_path = add_cluster_dir("a", tempdir) + # no attached_cluster + assert cluster.set_attached(cluster_path) is None + assert config.get_attached_cluster_path() == cluster_path + + assert cluster.set_attached(cluster_path) is None + assert config.get_attached_cluster_path() == cluster_path + + cluster_path2 = add_cluster_dir("b", tempdir) + # attach cluster already attached + assert cluster.set_attached(cluster_path2) is None + assert config.get_attached_cluster_path() == cluster_path2 + + +@patch('dcos.http.get') +def test_setup_cluster_config(mock_get): + with env(), util.tempdir() as tempdir: + os.environ[constants.DCOS_DIR_ENV] = tempdir + with cluster.setup_directory() as setup_temp: + + cluster.set_attached(setup_temp) + + cluster_id = "fake" + mock_resp = mock.Mock() + mock_resp.json.return_value = { + "CLUSTER_ID": cluster_id, + "cluster": cluster_id + } + mock_get.return_value = mock_resp + path = cluster.setup_cluster_config("fake_url", setup_temp, False) + expected_path = os.path.join( + tempdir, constants.DCOS_CLUSTERS_SUBDIR + "/" + cluster_id) + assert path == expected_path + assert os.path.exists(path) + assert os.path.exists(os.path.join(path, "dcos.toml")) + + assert not os.path.exists(setup_temp) + + +@patch('dcos.config.get_config_val') +@patch('dcos.http.get') +def test_move_to_cluster_config(mock_get, mock_config): + with env(), util.tempdir() as tempdir: + os.environ[constants.DCOS_DIR_ENV] = tempdir + + create_global_config(tempdir) + mock_config.return_value = "fake-url" + + cluster_id = "fake" + mock_resp = mock.Mock() + mock_resp.json.return_value = {"CLUSTER_ID": cluster_id} + mock_get.return_value = mock_resp + + assert config.get_config_dir_path() == tempdir + cluster.move_to_cluster_config() + + clusters_path = os.path.join(tempdir, constants.DCOS_CLUSTERS_SUBDIR) + assert os.path.exists(clusters_path) + cluster_path = os.path.join(clusters_path, cluster_id) + assert os.path.exists(os.path.join(cluster_path, "dcos.toml")) + assert os.path.exists(os.path.join( + cluster_path, constants.DCOS_CLUSTER_ATTACHED_FILE)) diff --git a/tests/test_config.py b/tests/test_config.py index d6f9a6c..eecedb8 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,6 +1,13 @@ +import os + import pytest -from dcos import config +from mock import patch + +from test_util import add_cluster_dir, create_global_config, env + +from dcos import config, constants, util +from dcos.errors import DCOSException @pytest.fixture @@ -110,3 +117,80 @@ def _conf(): 'repo_uri': 'git://localhost/mesosphere/package-repo.git' } } + + +def test_uses_deprecated_config(): + with env(), util.tempdir() as tempdir: + os.environ.pop('DCOS_CONFIG', None) + os.environ[constants.DCOS_DIR_ENV] = tempdir + assert config.get_config_dir_path() == tempdir + + # create old global config toml + global_toml = create_global_config(tempdir) + assert config.get_global_config_path() == global_toml + assert config.uses_deprecated_config() is True + + # create clusters subdir + _create_clusters_dir(tempdir) + assert config.uses_deprecated_config() is False + + +def test_get_attached_cluster_path(): + with env(), util.tempdir() as tempdir: + os.environ[constants.DCOS_DIR_ENV] = tempdir + + # no clusters dir + assert config.get_attached_cluster_path() is None + + # clusters dir, no clusters + _create_clusters_dir(tempdir) + assert config.get_attached_cluster_path() is None + + # 1 cluster, not attached + cluster_id = "fake-cluster" + cluster_path = add_cluster_dir(cluster_id, tempdir) + assert config.get_attached_cluster_path() == cluster_path + attached_path = os.path.join( + cluster_path, constants.DCOS_CLUSTER_ATTACHED_FILE) + assert os.path.exists(attached_path) + + # attached cluster + assert config.get_attached_cluster_path() == cluster_path + + +@patch('dcos.config.load_from_path') +def test_get_config(load_path_mock): + with env(), util.tempdir() as tempdir: + os.environ.pop('DCOS_CONFIG', None) + os.environ[constants.DCOS_DIR_ENV] = tempdir + + # no config file of any type + with pytest.raises(DCOSException) as e: + config.get_config() + + msg = ("No cluster is attached. " + "Please run `dcos cluster attach `") + assert str(e.value) == msg + load_path_mock.assert_not_called() + + # create old global config toml + global_toml = create_global_config(tempdir) + config.get_config() + load_path_mock.assert_called_once_with(global_toml, False) + + # clusters dir, no clusters + _create_clusters_dir(tempdir) + with pytest.raises(DCOSException) as e: + config.get_config() + assert str(e.value) == msg + + cluster_id = "fake-cluster" + cluster_path = add_cluster_dir(cluster_id, tempdir) + cluster_toml = os.path.join(cluster_path, "dcos.toml") + config.get_config(True) + load_path_mock.assert_any_call(cluster_toml, True) + + +def _create_clusters_dir(dcos_dir): + clusters_dir = os.path.join(dcos_dir, constants.DCOS_CLUSTERS_SUBDIR) + util.ensure_dir_exists(clusters_dir) diff --git a/tests/test_util.py b/tests/test_util.py index d271af5..18a2555 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,6 +1,9 @@ +import contextlib +import os + import pytest -from dcos import util +from dcos import constants, util from dcos.errors import DCOSException @@ -11,3 +14,32 @@ def test_open_file(): pass assert 'Error opening file [{}]: No such file or directory'.format(path) \ in str(excinfo.value) + + +@contextlib.contextmanager +def env(): + """Context manager for altering env vars in tests """ + + try: + old_env = dict(os.environ) + yield + finally: + os.environ.clear() + os.environ.update(old_env) + + +def add_cluster_dir(cluster_id, dcos_dir): + clusters_dir = os.path.join(dcos_dir, constants.DCOS_CLUSTERS_SUBDIR) + util.ensure_dir_exists(clusters_dir) + + cluster_path = os.path.join(clusters_dir, cluster_id) + util.ensure_dir_exists(cluster_path) + + os.path.join(cluster_path, "dcos.toml") + return cluster_path + + +def create_global_config(dcos_dir): + global_toml = os.path.join(dcos_dir, "dcos.toml") + util.ensure_file_exists(global_toml) + return global_toml