diff --git a/cli/dcoscli/analytics.py b/cli/dcoscli/analytics.py index 80f8a75..2352eda 100644 --- a/cli/dcoscli/analytics.py +++ b/cli/dcoscli/analytics.py @@ -6,71 +6,29 @@ import dcoscli import docopt import rollbar import six -from concurrent.futures import ThreadPoolExecutor from dcos import http, util from dcoscli.constants import (ROLLBAR_SERVER_POST_KEY, SEGMENT_IO_CLI_ERROR_EVENT, SEGMENT_IO_CLI_EVENT, SEGMENT_IO_WRITE_KEY_PROD, SEGMENT_URL) +from dcoscli.subcommand import default_doc from requests.auth import HTTPBasicAuth logger = util.get_logger(__name__) session_id = uuid.uuid4().hex -def wait_and_track(subproc, cluster_id): +def _track(conf): """ - Run a command and report it to analytics services. + Whether or not to send reporting information - :param subproc: Subprocess to capture - :type subproc: Popen - :param cluster_id: dcos cluster id to send to segment - :type cluster_id: str - :returns: exit code of subproc - :rtype: int + :param conf: dcos config file + :type conf: Toml + :returns: whether to send reporting information + :rtype: bool """ - rollbar.init(ROLLBAR_SERVER_POST_KEY, 'prod') - - conf = util.get_config() - report = conf.get('core.reporting', True) - with ThreadPoolExecutor(max_workers=2) as pool: - if report: - _segment_track_cli(pool, conf, cluster_id) - - exit_code, err = wait_and_capture(subproc) - - # We only want to catch exceptions, not other stderr messages - # (such as "task does not exist", so we look for the 'Traceback' - # string. This only works for python, so we'll need to revisit - # this in the future when we support subcommands written in other - # languages. - if report and 'Traceback' in err: - _track_err(pool, exit_code, err, conf, cluster_id) - - return exit_code - - -def wait_and_capture(subproc): - """ - Run a subprocess and capture its stderr. - - :param subproc: Subprocess to capture - :type subproc: Popen - :returns: exit code of subproc - :rtype: int - """ - - err = '' - while subproc.poll() is None: - line = subproc.stderr.readline().decode('utf-8') - err += line - sys.stderr.write(line) - sys.stderr.flush() - - exit_code = subproc.poll() - - return exit_code, err + return dcoscli.version != 'SNAPSHOT' and conf.get('core.reporting', True) def _segment_track(event, conf, properties): @@ -135,7 +93,7 @@ def _segment_request(path, data): logger.exception(e) -def _track_err(pool, exit_code, err, conf, cluster_id): +def track_err(pool, exit_code, err, conf, cluster_id): """ Report error details to analytics services. @@ -152,13 +110,16 @@ def _track_err(pool, exit_code, err, conf, cluster_id): :rtype: None """ + if not _track(conf): + return + # Segment.io calls are async, but rollbar is not, so for # parallelism, we must call segment first. _segment_track_err(pool, conf, cluster_id, err, exit_code) _rollbar_track_err(conf, cluster_id, err, exit_code) -def _segment_track_cli(pool, conf, cluster_id): +def segment_track_cli(pool, conf, cluster_id): """ Send segment.io cli event. @@ -171,6 +132,9 @@ def _segment_track_cli(pool, conf, cluster_id): :rtype: None """ + if not _track(conf): + return + props = _base_properties(conf, cluster_id) pool.submit(_segment_track, SEGMENT_IO_CLI_EVENT, conf, props) @@ -213,6 +177,8 @@ def _rollbar_track_err(conf, cluster_id, err, exit_code): :rtype: None """ + rollbar.init(ROLLBAR_SERVER_POST_KEY, 'prod') + props = _base_properties(conf, cluster_id) props['exit_code'] = exit_code @@ -236,13 +202,10 @@ def _command(): :rtype: str """ - # avoid circular import - import dcoscli.main - - args = docopt.docopt(dcoscli.main._doc(), + args = docopt.docopt(default_doc("dcos"), help=False, options_first=True) - return args[''] + return args.get('', "") def _base_properties(conf=None, cluster_id=None): diff --git a/cli/dcoscli/common.py b/cli/dcoscli/common.py deleted file mode 100644 index e3048ac..0000000 --- a/cli/dcoscli/common.py +++ /dev/null @@ -1,39 +0,0 @@ -import subprocess - - -def exec_command(cmd, env=None, stdin=None): - """Execute CLI command - - :param cmd: Program and arguments - :type cmd: [str] - :param env: Environment variables - :type env: dict - :param stdin: File to use for stdin - :type stdin: file - :returns: A tuple with the returncode, stdout and stderr - :rtype: (int, bytes, bytes) - """ - - process = subprocess.Popen( - cmd, - stdin=stdin, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env) - - # This is needed to get rid of '\r' from Windows's lines endings. - stdout, stderr = [std_stream.replace(b'\r', b'') - for std_stream in process.communicate()] - - return (process.returncode, stdout, stderr) - - -def command_info(doc): - """Get description from doc text - :param doc: command help text - :type doc: str - :returns: one line description of command - :rtype: str - """ - - return doc.split('\n')[1].strip(".").lstrip() diff --git a/cli/dcoscli/config/main.py b/cli/dcoscli/config/main.py index 2dce239..8e963a8 100644 --- a/cli/dcoscli/config/main.py +++ b/cli/dcoscli/config/main.py @@ -2,31 +2,29 @@ import collections import dcoscli import docopt -import pkg_resources from dcos import cmds, config, emitting, http, util from dcos.errors import DCOSException from dcoscli import analytics -from dcoscli.common import command_info -from dcoscli.main import decorate_docopt_usage +from dcoscli.subcommand import default_command_info, default_doc +from dcoscli.util import decorate_docopt_usage emitter = emitting.FlatEmitter() logger = util.get_logger(__name__) -def main(): +def main(argv): try: - return _main() + return _main(argv) except DCOSException as e: emitter.publish(e) return 1 @decorate_docopt_usage -def _main(): - util.configure_process_from_environ() - +def _main(argv): args = docopt.docopt( - _doc(), + default_doc("config"), + argv=argv, version='dcos-config version {}'.format(dcoscli.version)) http.silence_requests_warnings() @@ -34,15 +32,6 @@ def _main(): return cmds.execute(_cmds(), args) -def _doc(): - """ - :rtype: str - """ - return pkg_resources.resource_string( - 'dcoscli', - 'data/help/config.txt').decode('utf-8') - - def _cmds(): """ :returns: all the supported commands @@ -85,7 +74,7 @@ def _info(info): :rtype: int """ - emitter.publish(command_info(_doc())) + emitter.publish(default_command_info("config")) return 0 diff --git a/cli/dcoscli/help/main.py b/cli/dcoscli/help/main.py index 4c4573b..124dbaf 100644 --- a/cli/dcoscli/help/main.py +++ b/cli/dcoscli/help/main.py @@ -2,45 +2,35 @@ import subprocess import dcoscli import docopt -import pkg_resources from concurrent.futures import ThreadPoolExecutor from dcos import cmds, emitting, options, subcommand, util from dcos.errors import DCOSException -from dcoscli.common import command_info -from dcoscli.main import decorate_docopt_usage +from dcoscli.subcommand import (default_command_documentation, + default_command_info, default_doc) +from dcoscli.util import decorate_docopt_usage emitter = emitting.FlatEmitter() logger = util.get_logger(__name__) -def main(): +def main(argv): try: - return _main() + return _main(argv) except DCOSException as e: emitter.publish(e) return 1 @decorate_docopt_usage -def _main(): - util.configure_process_from_environ() - +def _main(argv): args = docopt.docopt( - _doc(), + default_doc("help"), + argv=argv, version='dcos-help version {}'.format(dcoscli.version)) return cmds.execute(_cmds(), args) -def _doc(): - """ - :rtype: str - """ - return pkg_resources.resource_string( - 'dcoscli', - 'data/help/help.txt').decode('utf-8') - - def _cmds(): """ :returns: All of the supported commands @@ -66,7 +56,7 @@ def _info(): :rtype: int """ - emitter.publish(command_info(_doc())) + emitter.publish(default_command_info("help")) return 0 @@ -83,9 +73,11 @@ def _help(command): else: logger.debug("DCOS bin path: {!r}".format(util.dcos_bin_path())) + results = [(c, default_command_info(c)) + for c in subcommand.default_subcommands()] paths = subcommand.list_paths() with ThreadPoolExecutor(max_workers=len(paths)) as executor: - results = executor.map(subcommand.documentation, paths) + results += list(executor.map(subcommand.documentation, paths)) commands_message = options\ .make_command_summary_string(sorted(results)) @@ -110,5 +102,9 @@ def _help_command(command): :rtype: int """ - executable = subcommand.command_executables(command) - return subprocess.call([executable, command, '--help']) + if command in subcommand.default_subcommands(): + emitter.publish(default_command_documentation(command)) + return 0 + else: + executable = subcommand.command_executables(command) + return subprocess.call([executable, command, '--help']) diff --git a/cli/dcoscli/main.py b/cli/dcoscli/main.py index b68d58c..37eb5f8 100644 --- a/cli/dcoscli/main.py +++ b/cli/dcoscli/main.py @@ -1,16 +1,15 @@ import os import signal import sys -from functools import wraps -from subprocess import PIPE, Popen import dcoscli import docopt -import pkg_resources +from concurrent.futures import ThreadPoolExecutor from dcos import (auth, constants, emitting, errors, http, mesos, subcommand, util) from dcos.errors import DCOSAuthenticationException, DCOSException from dcoscli import analytics +from dcoscli.subcommand import SubcommandMain, default_doc logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() @@ -28,7 +27,7 @@ def _main(): signal.signal(signal.SIGINT, signal_handler) args = docopt.docopt( - _doc(), + default_doc("dcos"), version='dcos version {}'.format(dcoscli.version), options_first=True) @@ -54,8 +53,6 @@ def _main(): if not command: command = "help" - executable = subcommand.command_executables(command) - cluster_id = None if dcoscli.version != 'SNAPSHOT' and command and \ command not in ["config", "help"]: @@ -67,26 +64,28 @@ def _main(): msg = 'Unable to get the cluster_id of the cluster.' logger.exception(msg) - # the call to retrieve cluster_id must happen before we run the subcommand - # so that if you have auth enabled we don't ask for user/pass multiple - # times (with the text being out of order) before we can cache the auth - # token - subproc = Popen([executable, command] + args[''], - stderr=PIPE) + # send args call to segment.io + with ThreadPoolExecutor(max_workers=2) as reporting_executor: + analytics.segment_track_cli(reporting_executor, config, cluster_id) - if dcoscli.version != 'SNAPSHOT': - return analytics.wait_and_track(subproc, cluster_id) - else: - return analytics.wait_and_capture(subproc)[0] + # the call to retrieve cluster_id must happen before we run the + # subcommand so that if you have auth enabled we don't ask for + # user/pass multiple times (with the text being out of order) + # before we can cache the auth token + if command in subcommand.default_subcommands(): + sc = SubcommandMain(command, args['']) + else: + executable = subcommand.command_executables(command) + sc = subcommand.SubcommandProcess( + executable, command, args['']) + exitcode, err = sc.run_and_capture() -def _doc(): - """ - :rtype: str - """ - return pkg_resources.resource_string( - 'dcoscli', - 'data/help/dcos.txt').decode('utf-8') + if err: + analytics.track_err( + reporting_executor, exitcode, err, config, cluster_id) + + return exitcode def _config_log_level_environ(log_level): @@ -115,27 +114,6 @@ def signal_handler(signal, frame): sys.exit(0) -def decorate_docopt_usage(func): - """Handle DocoptExit exception - - :param func: function - :type func: function - :return: wrapped function - :rtype: function - """ - - @wraps(func) - def wrapper(*args, **kwargs): - try: - result = func(*args, **kwargs) - except docopt.DocoptExit as e: - emitter.publish("Command not recognized\n") - emitter.publish(e) - return 1 - return result - return wrapper - - def set_ssl_info_env_vars(config): """Set SSL info from config to environment variable if enviornment variable doesn't exist diff --git a/cli/dcoscli/marathon/main.py b/cli/dcoscli/marathon/main.py index 5bc3fee..46c3280 100644 --- a/cli/dcoscli/marathon/main.py +++ b/cli/dcoscli/marathon/main.py @@ -9,41 +9,31 @@ import pkg_resources from dcos import cmds, emitting, http, jsonitem, marathon, options, util from dcos.errors import DCOSException from dcoscli import tables -from dcoscli.common import command_info -from dcoscli.main import decorate_docopt_usage +from dcoscli.subcommand import default_command_info, default_doc +from dcoscli.util import decorate_docopt_usage logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() -def main(): +def main(argv): try: - return _main() + return _main(argv) except DCOSException as e: emitter.publish(e) return 1 @decorate_docopt_usage -def _main(): - util.configure_process_from_environ() - +def _main(argv): args = docopt.docopt( - _doc(), + default_doc("marathon"), + argv=argv, version='dcos-marathon version {}'.format(dcoscli.version)) return cmds.execute(_cmds(), args) -def _doc(): - """ - :rtype: str - """ - return pkg_resources.resource_string( - 'dcoscli', - 'data/help/marathon.txt').decode('utf-8') - - def _cmds(): """ :returns: all the supported commands @@ -189,7 +179,8 @@ def _marathon(config_schema, info): elif info: _info() else: - emitter.publish(options.make_generic_usage_message(_doc())) + doc = default_command_info("marathon") + emitter.publish(options.make_generic_usage_message(doc)) return 1 return 0 @@ -201,7 +192,7 @@ def _info(): :rtype: int """ - emitter.publish(command_info(_doc())) + emitter.publish(default_command_info("marathon")) return 0 diff --git a/cli/dcoscli/node/main.py b/cli/dcoscli/node/main.py index 805db77..c26d4e7 100644 --- a/cli/dcoscli/node/main.py +++ b/cli/dcoscli/node/main.py @@ -3,31 +3,29 @@ import subprocess import dcoscli import docopt -import pkg_resources from dcos import cmds, emitting, errors, mesos, util from dcos.errors import DCOSException, DefaultError from dcoscli import log, tables -from dcoscli.common import command_info -from dcoscli.main import decorate_docopt_usage +from dcoscli.subcommand import default_command_info, default_doc +from dcoscli.util import decorate_docopt_usage logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() -def main(): +def main(argv): try: - return _main() + return _main(argv) except DCOSException as e: emitter.publish(e) return 1 @decorate_docopt_usage -def _main(): - util.configure_process_from_environ() - +def _main(argv): args = docopt.docopt( - _doc(), + default_doc("node"), + argv=argv, version="dcos-node version {}".format(dcoscli.version)) if args.get('--master'): @@ -42,15 +40,6 @@ def _main(): return cmds.execute(_cmds(), args) -def _doc(): - """ - :rtype: str - """ - return pkg_resources.resource_string( - 'dcoscli', - 'data/help/node.txt').decode('utf-8') - - def _cmds(): """ :returns: All of the supported commands @@ -88,7 +77,7 @@ def _info(): :rtype: int """ - emitter.publish(command_info(_doc())) + emitter.publish(default_command_info("node")) return 0 diff --git a/cli/dcoscli/package/main.py b/cli/dcoscli/package/main.py index a3fddf7..0e198a8 100644 --- a/cli/dcoscli/package/main.py +++ b/cli/dcoscli/package/main.py @@ -13,34 +13,27 @@ from dcos import (cmds, cosmospackage, emitting, errors, http, options, package, subcommand, util) from dcos.errors import DCOSException from dcoscli import tables -from dcoscli.common import command_info -from dcoscli.main import decorate_docopt_usage +from dcoscli.subcommand import default_command_info, default_doc +from dcoscli.util import decorate_docopt_usage from six import iteritems logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() -def main(): +def main(argv): try: - return _main() + return _main(argv) except DCOSException as e: emitter.publish(e) return 1 -def _doc(): - return pkg_resources.resource_string( - 'dcoscli', - 'data/help/package.txt').decode('utf-8') - - @decorate_docopt_usage -def _main(): - util.configure_process_from_environ() - +def _main(argv): args = docopt.docopt( - _doc(), + default_doc("package"), + argv=argv, version='dcos-package version {}'.format(dcoscli.version)) http.silence_requests_warnings() @@ -127,7 +120,8 @@ def _package(config_schema, info): elif info: _info() else: - emitter.publish(options.make_generic_usage_message(_doc())) + doc = default_doc("package") + emitter.publish(options.make_generic_usage_message(doc)) return 1 return 0 @@ -140,7 +134,7 @@ def _info(): :rtype: int """ - emitter.publish(command_info(_doc())) + emitter.publish(default_command_info("package")) return 0 diff --git a/cli/dcoscli/service/main.py b/cli/dcoscli/service/main.py index 74935f4..e3db0bc 100644 --- a/cli/dcoscli/service/main.py +++ b/cli/dcoscli/service/main.py @@ -2,45 +2,34 @@ import subprocess import dcoscli import docopt -import pkg_resources from dcos import cmds, emitting, marathon, mesos, util from dcos.errors import DCOSException, DefaultError from dcoscli import log, tables -from dcoscli.common import command_info -from dcoscli.main import decorate_docopt_usage +from dcoscli.subcommand import default_command_info, default_doc +from dcoscli.util import decorate_docopt_usage logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() -def main(): +def main(argv): try: - return _main() + return _main(argv) except DCOSException as e: emitter.publish(e) return 1 @decorate_docopt_usage -def _main(): - util.configure_process_from_environ() - +def _main(argv): args = docopt.docopt( - _doc(), + default_doc("service"), + argv=argv, version="dcos-service version {}".format(dcoscli.version)) return cmds.execute(_cmds(), args) -def _doc(): - """ - :rtype: str - """ - return pkg_resources.resource_string( - 'dcoscli', - 'data/help/service.txt').decode('utf-8') - - def _cmds(): """ :returns: All of the supported commands @@ -79,7 +68,7 @@ def _info(): :rtype: int """ - emitter.publish(command_info(_doc())) + emitter.publish(default_command_info("service")) return 0 diff --git a/cli/dcoscli/subcommand.py b/cli/dcoscli/subcommand.py new file mode 100644 index 0000000..b37ef43 --- /dev/null +++ b/cli/dcoscli/subcommand.py @@ -0,0 +1,99 @@ +import traceback + +import pkg_resources + + +def _default_modules(): + """Dict of the default dcos cli subcommands and their main methods + + :returns: default subcommand -> main method + :rtype: {} + """ + + # avoid circular imports + from dcoscli.config import main as config_main + from dcoscli.help import main as help_main + from dcoscli.marathon import main as marathon_main + from dcoscli.node import main as node_main + from dcoscli.package import main as package_main + from dcoscli.service import main as service_main + from dcoscli.task import main as task_main + + return {'config': config_main, + 'help': help_main, + 'marathon': marathon_main, + 'node': node_main, + 'package': package_main, + 'service': service_main, + 'task': task_main + } + + +def default_doc(command): + """Returns documentation of command + + :param command: default DCOS-CLI command + :type command: str + :returns: config schema of command + :rtype: dict + """ + + resource = "data/help/{}.txt".format(command) + return pkg_resources.resource_string( + 'dcoscli', + resource).decode('utf-8') + + +def default_command_info(command): + """top level documentation of default DCOS-CLI command + + :param command: name of command + :param type: str + :returns: command summary + :rtype: str + """ + + doc = default_command_documentation(command) + return doc.split('\n')[1].strip(".").lstrip() + + +def default_command_documentation(command): + """documentation of default DCOS-CLI command + + :param command: name of command + :param type: str + :returns: command summary + :rtype: str + """ + + return default_doc(command).rstrip('\n') + + +class SubcommandMain(): + + def __init__(self, command, args): + """Representes a subcommand running in the main thread + + :param commad: function to run in thread + :type command: str + """ + + self._command = command + self._args = args + + def run_and_capture(self): + """ + Run a command and capture exceptions. This is a blocking call + :returns: tuple of exitcode, error (or None) + :rtype: int, str | None + """ + + m = _default_modules()[self._command] + err = None + try: + exit_code = m.main([self._command] + self._args) + except Exception: + err = traceback.format_exc() + traceback.print_exc() + exit_code = 1 + return exit_code, err diff --git a/cli/dcoscli/task/main.py b/cli/dcoscli/task/main.py index bc95fb4..b7c9a26 100644 --- a/cli/dcoscli/task/main.py +++ b/cli/dcoscli/task/main.py @@ -2,45 +2,34 @@ import posixpath import dcoscli import docopt -import pkg_resources from dcos import cmds, emitting, mesos, util from dcos.errors import DCOSException, DCOSHTTPException, DefaultError from dcoscli import log, tables -from dcoscli.common import command_info -from dcoscli.main import decorate_docopt_usage +from dcoscli.subcommand import default_command_info, default_doc +from dcoscli.util import decorate_docopt_usage logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() -def main(): +def main(argv): try: - return _main() + return _main(argv) except DCOSException as e: emitter.publish(e) return 1 @decorate_docopt_usage -def _main(): - util.configure_process_from_environ() - +def _main(argv): args = docopt.docopt( - _doc(), + default_doc("task"), + argv=argv, version="dcos-task version {}".format(dcoscli.version)) return cmds.execute(_cmds(), args) -def _doc(): - """ - :rtype: str - """ - return pkg_resources.resource_string( - 'dcoscli', - 'data/help/task.txt').decode('utf-8') - - def _cmds(): """ :returns: All of the supported commands @@ -78,7 +67,7 @@ def _info(): :rtype: int """ - emitter.publish(command_info(_doc())) + emitter.publish(default_command_info("task")) return 0 diff --git a/cli/dcoscli/util.py b/cli/dcoscli/util.py new file mode 100644 index 0000000..9b3ede4 --- /dev/null +++ b/cli/dcoscli/util.py @@ -0,0 +1,27 @@ +from functools import wraps + +import docopt +from dcos import emitting + +emitter = emitting.FlatEmitter() + + +def decorate_docopt_usage(func): + """Handle DocoptExit exception + + :param func: function + :type func: function + :return: wrapped function + :rtype: function + """ + + @wraps(func) + def wrapper(*args, **kwargs): + try: + result = func(*args, **kwargs) + except docopt.DocoptExit as e: + emitter.publish("Command not recognized\n") + emitter.publish(e) + return 1 + return result + return wrapper diff --git a/cli/setup.py b/cli/setup.py index 7fc59dc..da31fad 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -92,14 +92,7 @@ setup( # pip to create the appropriate form of executable for the target platform. entry_points={ 'console_scripts': [ - 'dcos=dcoscli.main:main', - 'dcos-help=dcoscli.help.main:main', - 'dcos-config=dcoscli.config.main:main', - 'dcos-marathon=dcoscli.marathon.main:main', - 'dcos-package=dcoscli.package.main:main', - 'dcos-service=dcoscli.service.main:main', - 'dcos-task=dcoscli.task.main:main', - 'dcos-node=dcoscli.node.main:main' + 'dcos=dcoscli.main:main' ], }, diff --git a/cli/tests/integrations/common.py b/cli/tests/integrations/common.py index 5890e39..0242a5d 100644 --- a/cli/tests/integrations/common.py +++ b/cli/tests/integrations/common.py @@ -101,7 +101,7 @@ def exec_mock(main, args): print('MOCK ARGS: {}'.format(' '.join(args))) with mock_args(args) as (stdout, stderr): - returncode = main() + returncode = main(args) stdout_val = six.b(stdout.getvalue()) stderr_val = six.b(stderr.getvalue()) @@ -554,7 +554,7 @@ def ssh_output(cmd): proc, master = popen_tty(cmd) # wait for the ssh connection - time.sleep(8) + time.sleep(3) proc.poll() returncode = proc.returncode diff --git a/cli/tests/integrations/test_analytics.py b/cli/tests/integrations/test_analytics.py index e2e09ac..9402e51 100644 --- a/cli/tests/integrations/test_analytics.py +++ b/cli/tests/integrations/test_analytics.py @@ -36,11 +36,11 @@ def test_config_set(): '''Tests that a `dcos config set core.email ` makes a segment.io identify call''' - args = [util.which('dcos'), 'config', 'set', 'core.email', 'test@mail.com'] + argv = ['config', 'set', 'core.email', 'test@mail.com'] env = _env_reporting() - with patch('sys.argv', args), patch.dict(os.environ, env): - assert config_main() == 0 + with patch.dict(os.environ, env): + assert config_main(argv) == 0 # segment.io assert mock_called_some_args(http.post, @@ -98,12 +98,12 @@ def test_exc(): with patch('sys.argv', args), \ patch('dcoscli.version', version), \ patch.dict(os.environ, env), \ - patch('dcoscli.analytics.wait_and_capture', - return_value=(1, 'Traceback')), \ - patch('dcoscli.analytics._segment_track_cli') as track: + patch('dcoscli.subcommand.SubcommandMain.run_and_capture', + return_value=(1, "Traceback")), \ + patch('dcoscli.analytics._segment_track') as track: assert main() == 1 - assert track.call_count == 1 + assert track.call_count == 2 assert rollbar.report_message.call_count == 1 @@ -118,9 +118,9 @@ def test_config_reporting_false(): with patch('sys.argv', args), \ patch('dcoscli.version', version), \ patch.dict(os.environ, env), \ - patch('dcoscli.analytics.wait_and_capture', - return_value=(1, 'Traceback')), \ - patch('dcoscli.analytics._segment_track_cli') as track: + patch('dcoscli.subcommand.SubcommandMain.run_and_capture', + return_value=(1, "Traceback")), \ + patch('dcoscli.analytics._segment_track') as track: assert main() == 1 assert track.call_count == 0 diff --git a/cli/tests/integrations/test_node.py b/cli/tests/integrations/test_node.py index 262ba27..dde54ca 100644 --- a/cli/tests/integrations/test_node.py +++ b/cli/tests/integrations/test_node.py @@ -180,13 +180,7 @@ def _node_ssh(args): stdout, stderr, returncode = _node_ssh_output(args) assert returncode is None - assert stdout assert b"Running `" in stderr - num_lines = len(stderr.decode().split('\n')) - expected_num_lines = 2 if '--master-proxy' in args else 3 - assert (num_lines == expected_num_lines or - (num_lines == (expected_num_lines + 1) and - b'Warning: Permanently added' in stderr)) def _get_schema(slave): diff --git a/dcos/config.py b/dcos/config.py index a5708df..6384b6d 100644 --- a/dcos/config.py +++ b/dcos/config.py @@ -205,7 +205,7 @@ def get_config_schema(command): 'data/config-schema/core.json').decode('utf-8')) executable = subcommand.command_executables(command) - return subcommand.config_schema(executable) + return subcommand.config_schema(executable, command) def check_config(toml_config_pre, toml_config_post): diff --git a/dcos/subcommand.py b/dcos/subcommand.py index ef7ca36..d25dcd5 100644 --- a/dcos/subcommand.py +++ b/dcos/subcommand.py @@ -4,11 +4,14 @@ import json import os import shutil import subprocess +import sys +from subprocess import PIPE, Popen -from dcos import constants, util +from dcos import constants, emitting, util from dcos.errors import DCOSException logger = util.get_logger(__name__) +emitter = emitting.FlatEmitter() def command_executables(subcommand): @@ -20,7 +23,11 @@ def command_executables(subcommand): :rtype: str """ - executables = [ + executables = [] + if subcommand in default_subcommands(): + executables += [default_list_paths()] + + executables += [ command_path for command_path in list_paths() if noun(command_path) == subcommand @@ -61,6 +68,18 @@ def get_package_commands(package_name): return executables +def default_list_paths(): + """List the real path to dcos executable + + :returns: list dcos program path + :rtype: str + """ + + # Let's get all the default subcommands + binpath = util.dcos_bin_path() + return os.path.join(binpath, "dcos") + + def list_paths(): """List the real path to executable dcos subcommand programs. @@ -68,20 +87,11 @@ def list_paths(): :rtype: [str] """ - # Let's get all the default subcommands - binpath = util.dcos_bin_path() - commands = [ - os.path.join(binpath, filename) - for filename in os.listdir(binpath) - if (filename.startswith(constants.DCOS_COMMAND_PREFIX) and - _is_executable(os.path.join(binpath, filename))) - ] - subcommands = [] for package in distributions(): subcommands += get_package_commands(package) - return commands + subcommands + return subcommands def _is_executable(path): @@ -118,6 +128,16 @@ def distributions(): return [] +def default_subcommands(): + """List the default dcos cli subcommands + + :returns: list of all the default dcos cli subcommands + :rtype: [str] + """ + + return ["config", "help", "marathon", "node", "package", "service", "task"] + + def documentation(executable_path): """Gather subcommand summary @@ -148,17 +168,20 @@ def info(executable_path, path_noun): return out.decode('utf-8').strip() -def config_schema(executable_path): +def config_schema(executable_path, noun=None): """Collects subcommand config schema :param executable_path: real path to the dcos subcommand :type executable_path: str + :param noun: name of subcommand + :type noun: str :returns: the subcommand config schema :rtype: dict """ - + if noun is None: + noun = noun(executable_path) out = subprocess.check_output( - [executable_path, noun(executable_path), '--config-schema']) + [executable_path, noun, '--config-schema']) return json.loads(out.decode('utf-8')) @@ -435,3 +458,47 @@ class InstalledSubcommand(object): package_json_path = os.path.join(self._dir(), 'package.json') with util.open_file(package_json_path) as package_json_file: return util.load_json(package_json_file) + + +class SubcommandProcess(): + + def __init__(self, executable, command, args): + """Representes a subcommand running by a forked process + + :param executable: executable to run + :type executable: executable + :param command: command to run by executable + :type command: str + :param args: arguments for command + :type args: [str] + """ + + self._executable = executable + self._command = command + self._args = args + + def run_and_capture(self): + """ + Run a command and capture exceptions. This is a blocking call + :returns: tuple of exitcode, error (or None) + :rtype: int, str | None + """ + + subproc = Popen([self._executable, self._command] + self._args, + stderr=PIPE) + err = '' + while subproc.poll() is None: + line = subproc.stderr.readline().decode('utf-8') + err += line + sys.stderr.write(line) + sys.stderr.flush() + + exitcode = subproc.poll() + # We only want to catch exceptions, not other stderr messages + # (such as "task does not exist", so we look for the 'Traceback' + # string. This only works for python, so we'll need to revisit + # this in the future when we support subcommands written in other + # languages. + err = ('Traceback' in err and err) or None + + return exitcode, err