From 9d6220a6b6c32cfcb26560f74c96dc5db77c2655 Mon Sep 17 00:00:00 2001 From: Michael Gummelt Date: Mon, 18 May 2015 11:46:34 -0700 Subject: [PATCH] dcos services --- cli/dcoscli/service/__init__.py | 0 cli/dcoscli/service/main.py | 146 ++++++++++++++++ cli/dcoscli/task/main.py | 4 +- cli/setup.py | 1 + cli/tests/integrations/cli/common.py | 13 ++ cli/tests/integrations/cli/test_dcos.py | 1 + cli/tests/integrations/cli/test_help.py | 1 + cli/tests/integrations/cli/test_package.py | 3 +- cli/tests/integrations/cli/test_service.py | 183 +++++++++++++++++++++ cli/tests/integrations/cli/test_task.py | 13 +- dcos/mesos.py | 40 +++-- dcos/util.py | 13 +- 12 files changed, 385 insertions(+), 33 deletions(-) create mode 100644 cli/dcoscli/service/__init__.py create mode 100644 cli/dcoscli/service/main.py create mode 100644 cli/tests/integrations/cli/test_service.py diff --git a/cli/dcoscli/service/__init__.py b/cli/dcoscli/service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cli/dcoscli/service/main.py b/cli/dcoscli/service/main.py new file mode 100644 index 0000000..b443c48 --- /dev/null +++ b/cli/dcoscli/service/main.py @@ -0,0 +1,146 @@ +"""Get the status of DCOS services + +Usage: + dcos service --info + dcos service [--inactive --json] + +Options: + -h, --help Show this screen + + --info Show a short description of this subcommand + + --json Print json-formatted services + + --inactive Show inactive services in addition to active ones. + Inactive services are those that have been disconnected from + master, but haven't yet reached their failover timeout. + + --version Show version +""" + + +from collections import OrderedDict + +import blessings +import dcoscli +import docopt +import prettytable +from dcos import cmds, emitting, mesos, util +from dcos.errors import DCOSException + +logger = util.get_logger(__name__) +emitter = emitting.FlatEmitter() + + +def main(): + try: + return _main() + except DCOSException as e: + emitter.publish(e) + return 1 + + +def _main(): + util.configure_logger_from_environ() + + args = docopt.docopt( + __doc__, + version="dcos-service version {}".format(dcoscli.version)) + + return cmds.execute(_cmds(), args) + + +def _cmds(): + """ + :returns: All of the supported commands + :rtype: [Command] + """ + + return [ + cmds.Command( + hierarchy=['service', '--info'], + arg_keys=[], + function=_info), + + cmds.Command( + hierarchy=['service'], + arg_keys=['--inactive', '--json'], + function=_service), + ] + + +def _info(): + """Print services cli information. + + :returns: process return code + :rtype: int + """ + + emitter.publish(__doc__.split('\n')[0]) + return 0 + + +def _service_table(services): + """Returns a PrettyTable representation of the provided services. + + :param services: services to render + :type services: [Framework] + :rtype: TaskTable + """ + + term = blessings.Terminal() + + table_generator = OrderedDict([ + ("name", lambda s: s['name']), + ("host", lambda s: s['hostname']), + ("active", lambda s: s['active']), + ("tasks", lambda s: len(s['tasks'])), + ("cpu", lambda s: s['resources']['cpus']), + ("mem", lambda s: s['resources']['mem']), + ("disk", lambda s: s['resources']['disk']), + ("ID", lambda s: s['id']), + ]) + + tb = prettytable.PrettyTable( + [k.upper() for k in table_generator.keys()], + border=False, + max_table_width=term.width, + hrules=prettytable.NONE, + vrules=prettytable.NONE, + left_padding_width=0, + right_padding_width=1 + ) + + for service in services: + row = [fn(service) for fn in table_generator.values()] + tb.add_row(row) + + return tb + + +# TODO (mgummelt): support listing completed services as well. +# blocked on framework shutdown. +def _service(inactive, is_json): + """List dcos services + + :param inactive: If True, include completed tasks + :type completed: bool + :param is_json: If true, output json. + Otherwise, output a human readable table. + :type is_json: bool + :returns: process return code + :rtype: int + """ + + master = mesos.get_master() + services = master.frameworks(inactive=inactive) + + if is_json: + emitter.publish([service.dict() for service in services]) + else: + table = _service_table(services) + output = str(table) + if output: + emitter.publish(output) + + return 0 diff --git a/cli/dcoscli/task/main.py b/cli/dcoscli/task/main.py index ef72bf6..f417cc5 100644 --- a/cli/dcoscli/task/main.py +++ b/cli/dcoscli/task/main.py @@ -7,7 +7,7 @@ Usage: Options: -h, --help Show this screen --info Show a short description of this subcommand - --json Print json-formatted task data + --json Print json-formatted tasks --completed Show completed tasks as well --version Show version @@ -130,7 +130,7 @@ def _task(fltr, completed, is_json): fltr = "" master = mesos.get_master() - tasks = sorted(master.tasks(active_only=(not completed), fltr=fltr), + tasks = sorted(master.tasks(completed=completed, fltr=fltr), key=lambda task: task['name']) if is_json: diff --git a/cli/setup.py b/cli/setup.py index 50473b1..dd831bf 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -98,6 +98,7 @@ setup( 'dcos-marathon=dcoscli.marathon.main:main', 'dcos-package=dcoscli.package.main:main', 'dcos-task=dcoscli.task.main:main', + 'dcos-service=dcoscli.service.main:main', ], }, diff --git a/cli/tests/integrations/cli/common.py b/cli/tests/integrations/cli/common.py index 32a918b..7789e4d 100644 --- a/cli/tests/integrations/cli/common.py +++ b/cli/tests/integrations/cli/common.py @@ -109,6 +109,19 @@ def watch_deployment(deployment_id, count): assert stderr == b'' +def watch_all_deployments(count=60): + """ Wait for all deployments to complete. + + :param count: max number of seconds to wait + :type count: int + :rtype: None + """ + + deps = list_deployments() + for dep in deps: + watch_deployment(dep['id'], count) + + def list_deployments(expected_count=None, app_id=None): """Get all active deployments. diff --git a/cli/tests/integrations/cli/test_dcos.py b/cli/tests/integrations/cli/test_dcos.py index 8b3f205..e5ec459 100644 --- a/cli/tests/integrations/cli/test_dcos.py +++ b/cli/tests/integrations/cli/test_dcos.py @@ -21,6 +21,7 @@ Available DCOS commands: \thelp \tDisplay command line usage information \tmarathon \tDeploy and manage applications on the DCOS \tpackage \tInstall and manage DCOS software packages +\tservice \tGet the status of DCOS services \ttask \tGet the status of DCOS tasks Get detailed command description with 'dcos --help'. diff --git a/cli/tests/integrations/cli/test_help.py b/cli/tests/integrations/cli/test_help.py index d108f4b..a876a33 100644 --- a/cli/tests/integrations/cli/test_help.py +++ b/cli/tests/integrations/cli/test_help.py @@ -40,6 +40,7 @@ Available DCOS commands: \thelp \tDisplay command line usage information \tmarathon \tDeploy and manage applications on the DCOS \tpackage \tInstall and manage DCOS software packages +\tservice \tGet the status of DCOS services \ttask \tGet the status of DCOS tasks Get detailed command description with 'dcos --help'. diff --git a/cli/tests/integrations/cli/test_package.py b/cli/tests/integrations/cli/test_package.py index b44c137..fc5251e 100644 --- a/cli/tests/integrations/cli/test_package.py +++ b/cli/tests/integrations/cli/test_package.py @@ -260,6 +260,7 @@ Please create a JSON file with the appropriate options, and pass the \ def test_install(): _install_chronos() + _uninstall_chronos() def test_package_metadata(): @@ -591,7 +592,7 @@ def test_search(): for registry in registries: # assert the number of packages is gte the number at the time # this test was written - assert len(registry['packages']) >= 7 + assert len(registry['packages']) >= 5 assert returncode == 0 assert stderr == b'' diff --git a/cli/tests/integrations/cli/test_service.py b/cli/tests/integrations/cli/test_service.py new file mode 100644 index 0000000..451b695 --- /dev/null +++ b/cli/tests/integrations/cli/test_service.py @@ -0,0 +1,183 @@ +import collections +import json +import time + +import dcos.util as util +from dcos.mesos import Framework +from dcos.util import create_schema +from dcoscli.service.main import _service_table + +import pytest +from common import assert_command, exec_command, watch_all_deployments + + +@pytest.fixture +def service(): + service = Framework({ + "active": True, + "checkpoint": True, + "completed_tasks": [], + "failover_timeout": 604800, + "hostname": "mesos.vm", + "id": "20150502-231327-16842879-5050-3889-0000", + "name": "marathon", + "offered_resources": { + "cpus": 0.0, + "disk": 0.0, + "mem": 0.0, + "ports": "[1379-1379, 10000-10000]" + }, + "offers": [], + "pid": + "scheduler-a58cd5ba-f566-42e0-a283-b5f39cb66e88@172.17.8.101:55130", + "registered_time": 1431543498.31955, + "reregistered_time": 1431543498.31959, + "resources": { + "cpus": 0.2, + "disk": 0, + "mem": 32, + "ports": "[1379-1379, 10000-10000]" + }, + "role": "*", + "tasks": [], + "unregistered_time": 0, + "used_resources": { + "cpus": 0.2, + "disk": 0, + "mem": 32, + "ports": "[1379-1379, 10000-10000]" + }, + "user": "root", + "webui_url": "http://mesos:8080" + }) + + return service + + +def test_help(): + stdout = b"""Get the status of DCOS services + +Usage: + dcos service --info + dcos service [--inactive --json] + +Options: + -h, --help Show this screen + + --info Show a short description of this subcommand + + --json Print json-formatted services + + --inactive Show inactive services in addition to active ones. + Inactive services are those that have been disconnected from + master, but haven't yet reached their failover timeout. + + --version Show version +""" + assert_command(['dcos', 'service', '--help'], stdout=stdout) + + +def test_info(): + stdout = b"Get the status of DCOS services\n" + assert_command(['dcos', 'service', '--info'], stdout=stdout) + + +def test_service(service): + returncode, stdout, stderr = exec_command(['dcos', 'service', '--json']) + + services = _get_services(1) + + schema = _get_schema(service) + for srv in services: + assert not util.validate_json(srv, schema) + + +def _get_schema(service): + schema = create_schema(service.dict()) + schema['required'].remove('reregistered_time') + schema['required'].remove('pid') + schema['properties']['offered_resources']['required'].remove('ports') + schema['properties']['resources']['required'].remove('ports') + schema['properties']['used_resources']['required'].remove('ports') + + return schema + + +def test_service_inactive(): + # install cassandra + stdout = b"""Installing package [cassandra] version \ +[0.1.0-SNAPSHOT-447-master-3ad1bbf8f7] +The Apache Cassandra DCOS Service implementation is alpha and there may \ +be bugs, incomplete features, incorrect documentation or other discrepancies. +In order for Cassandra to start successfully, all resources must be \ +available in the cluster, including ports, CPU shares, RAM and disk. + +\tDocumentation: http://mesosphere.github.io/cassandra-mesos/ +\tIssues: https://github.com/mesosphere/cassandra-mesos/issues +""" + assert_command(['dcos', 'package', 'install', 'cassandra'], + stdout=stdout) + + # wait for it to deploy + watch_all_deployments(300) + + # wait long enough for it to register + time.sleep(5) + + # assert marathon and cassandra are listed + _get_services(2) + + # uninstall cassandra. For now, need to explicitly remove the + # group that is left by cassandra. See MARATHON-144 + assert_command(['dcos', 'package', 'uninstall', 'cassandra']) + assert_command(['dcos', 'marathon', 'group', 'remove', '/cassandra']) + + watch_all_deployments(300) + + # I'm not quite sure why we have to sleep, but it seems cassandra + # only transitions to "inactive" after a few seconds. + time.sleep(5) + + # assert only marathon is active + _get_services(1) + # assert marathon and cassandra are listed with --inactive + services = _get_services(None, ['--inactive']) + assert len(services) >= 2 + + +# not an integration test +def test_task_table(service): + table = _service_table([service]) + + stdout = """\ + NAME HOST ACTIVE TASKS CPU MEM DISK ID\ + \n\ + marathon mesos.vm True 0 0.2 32 0 \ +20150502-231327-16842879-5050-3889-0000 """ + assert str(table) == stdout + + +def _get_services(expected_count=None, args=[]): + """Get services + + :param expected_count: assert exactly this number of services are + running + :type expected_count: int + :param args: cli arguments + :type args: [str] + :returns: services + :rtype: [dict] + """ + + returncode, stdout, stderr = exec_command( + ['dcos', 'service', '--json'] + args) + + assert returncode == 0 + assert stderr == b'' + + services = json.loads(stdout.decode('utf-8')) + assert isinstance(services, collections.Sequence) + if expected_count is not None: + assert len(services) == expected_count + + return services diff --git a/cli/tests/integrations/cli/test_task.py b/cli/tests/integrations/cli/test_task.py index fb61444..f39f0be 100644 --- a/cli/tests/integrations/cli/test_task.py +++ b/cli/tests/integrations/cli/test_task.py @@ -8,8 +8,7 @@ from dcoscli.task.main import _task_table import mock import pytest -from common import (assert_command, exec_command, list_deployments, - watch_deployment) +from common import assert_command, exec_command, watch_all_deployments SLEEP1 = 'tests/data/marathon/apps/sleep.json' SLEEP2 = 'tests/data/marathon/apps/sleep2.json' @@ -53,7 +52,7 @@ Usage: Options: -h, --help Show this screen --info Show a short description of this subcommand - --json Print json-formatted task data + --json Print json-formatted tasks --completed Show completed tasks as well --version Show version @@ -143,13 +142,7 @@ def _install_sleep_task(app_path=SLEEP1, app_name='test-app'): # install helloworld app args = ['dcos', 'marathon', 'app', 'add', app_path] assert_command(args) - _wait_for_deployment() - - -def _wait_for_deployment(): - deps = list_deployments() - if deps: - watch_deployment(deps[0]['id'], 60) + watch_all_deployments() def _uninstall_helloworld(args=[]): diff --git a/dcos/mesos.py b/dcos/mesos.py index 2ff9d13..e04890d 100644 --- a/dcos/mesos.py +++ b/dcos/mesos.py @@ -125,25 +125,25 @@ class MesosMaster(object): return tasks[0] # TODO (thomas): need to filter on task state as well as id - def tasks(self, fltr="", active_only=False): + def tasks(self, fltr="", completed=False): """Returns tasks running under the master :param fltr: May be a substring or unix glob pattern. Only return tasks whose 'id' matches `fltr`. :type fltr: str - :param active_only: don't include completed tasks - :type active_only: bool + :param completed: also include completed tasks + :type completed: bool :returns: a list of tasks :rtype: [Task] """ keys = ['tasks'] - if not active_only: + if completed: keys = ['completed_tasks'] tasks = [] - for framework in self._framework_dicts(active_only): + for framework in self._framework_dicts(completed, completed): tasks += \ [Task(task, self) for task in _merge(framework, *keys) @@ -161,35 +161,42 @@ class MesosMaster(object): :rtype: Framework """ - for f in self._framework_dicts(active_only=False): + for f in self._framework_dicts(inactive=True): if f['id'] == framework_id: return Framework(f) raise DCOSException('No Framework with id [{}]'.format(framework_id)) - def frameworks(self, active_only=False): + def frameworks(self, inactive=False, completed=False): """Returns a list of all frameworks - :param active_only: only include active frameworks - :type active_only: bool + :param inactive: also include inactive frameworks + :type inactive: bool + :param completed: also include completed frameworks + :type completed: bool :returns: a list of frameworks :rtype: [Framework] """ - return [Framework(f) for f in self._framework_dicts(active_only)] + return [Framework(f) + for f in self._framework_dicts(inactive, completed)] - def _framework_dicts(self, active_only=False): + def _framework_dicts(self, inactive=False, completed=False): """Returns a list of all frameworks as their raw dictionaries - :param active_only: only include active frameworks - :type active_only: bool + :param inactive: also include inactive frameworks + :type inactive: bool + :param completed: also include completed frameworks + :type completed: bool :returns: a list of frameworks :rtype: [dict] """ keys = ['frameworks'] - if not active_only: + if completed: keys.append('completed_frameworks') - return _merge(self.state(), *keys) + for framework in _merge(self.state(), *keys): + if inactive or framework['active']: + yield framework @util.duration def fetch(self, path, **kwargs): @@ -234,6 +241,9 @@ class Framework(object): def __init__(self, framework): self._framework = framework + def dict(self): + return self._framework + def __getitem__(self, name): return self._framework[name] diff --git a/dcos/util.py b/dcos/util.py index 60e3fd2..38d945d 100644 --- a/dcos/util.py +++ b/dcos/util.py @@ -316,20 +316,23 @@ def create_schema(obj): :rtype: dict """ - if isinstance(obj, six.string_types): - return {'type': 'string'} + if isinstance(obj, bool): + return {'type': 'boolean'} + + elif isinstance(obj, float): + return {'type': 'number'} elif isinstance(obj, six.integer_types): return {'type': 'integer'} - elif isinstance(obj, float): - return {'type': 'number'} + elif isinstance(obj, six.string_types): + return {'type': 'string'} elif isinstance(obj, collections.Mapping): schema = {'type': 'object', 'properties': {}, 'additionalProperties': False, - 'required': obj.keys()} + 'required': list(obj.keys())} for key, val in obj.items(): schema['properties'][key] = create_schema(val)