From 46ac0b82b7678a20b9f957bd33201d571fedbdb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Armando=20Garc=C3=ADa=20Sancio?= Date: Mon, 18 May 2015 17:31:17 -0700 Subject: [PATCH] dcos-1254 Shutdown framework after uninstall --- README.rst | 17 +- cli/dcoscli/marathon/main.py | 2 +- cli/dcoscli/service/main.py | 25 ++- cli/dcoscli/task/main.py | 7 +- cli/tests/data/analytics/dcos_reporting.toml | 2 +- cli/tests/data/package/chronos-1.json | 7 + cli/tests/data/package/chronos-2.json | 7 + cli/tests/integrations/cli/common.py | 61 +++++++ cli/tests/integrations/cli/test_marathon.py | 2 +- cli/tests/integrations/cli/test_package.py | 161 +++++++++++++++---- cli/tests/integrations/cli/test_service.py | 79 ++++----- dcos/http.py | 1 + dcos/mesos.py | 125 +++++++++----- dcos/package.py | 58 +++++-- 14 files changed, 401 insertions(+), 153 deletions(-) create mode 100644 cli/tests/data/package/chronos-1.json create mode 100644 cli/tests/data/package/chronos-2.json diff --git a/README.rst b/README.rst index 43b6342..7a17760 100644 --- a/README.rst +++ b/README.rst @@ -72,18 +72,6 @@ environments. If you're using OS X, be sure to use the officially distributed Python 3.4 installer_ since the Homebrew version is missing a necessary library. -To support subcommand integration tests, you'll need to clone, package and -configure your environment to point to the packaged `dcos-helloworld` account. - -#. Check out the dcos-helloworld_ project - -#. :code:`cd dcos-helloworld` - -#. :code:`make packages` - -#. Set the :code:`DCOS_TEST_WHEEL` environment variable to the path of the created - wheel package: :code:`export DCOS_TEST_WHEEL=$(pwd)/dist/dcos_helloworld-0.1.0-py2.py3-none-any.whl` - Running ####### @@ -97,6 +85,11 @@ instance running on localhost, set :code:`DCOS_CONFIG` as follows:: export DCOS_CONFIG=$(pwd)/tests/data/dcos.toml +If you are testing against the DCOS Image you can configure the URL to the +Exhibitor:: + + export EXHIBITOR=http://:8181/ + There are two ways to run tests, you can either use the virtualenv created by :code:`make env` above:: diff --git a/cli/dcoscli/marathon/main.py b/cli/dcoscli/marathon/main.py index 9ed56d7..4de6f90 100644 --- a/cli/dcoscli/marathon/main.py +++ b/cli/dcoscli/marathon/main.py @@ -57,7 +57,7 @@ Options: and return --interval= Number of seconds to wait between actions -Positional arguments: +Positional Arguments: The application id The application resource; for a detailed description see (https://mesosphere.github.io/ diff --git a/cli/dcoscli/service/main.py b/cli/dcoscli/service/main.py index b443c48..6f1bab5 100644 --- a/cli/dcoscli/service/main.py +++ b/cli/dcoscli/service/main.py @@ -3,6 +3,7 @@ Usage: dcos service --info dcos service [--inactive --json] + dcos service shutdown Options: -h, --help Show this screen @@ -16,6 +17,9 @@ Options: master, but haven't yet reached their failover timeout. --version Show version + +Positional Arguments: + The ID for the DCOS Service """ @@ -57,6 +61,11 @@ def _cmds(): """ return [ + cmds.Command( + hierarchy=['service', 'shutdown'], + arg_keys=[''], + function=_shutdown), + cmds.Command( hierarchy=['service', '--info'], arg_keys=[], @@ -132,8 +141,7 @@ def _service(inactive, is_json): :rtype: int """ - master = mesos.get_master() - services = master.frameworks(inactive=inactive) + services = mesos.get_master().frameworks(inactive=inactive) if is_json: emitter.publish([service.dict() for service in services]) @@ -144,3 +152,16 @@ def _service(inactive, is_json): emitter.publish(output) return 0 + + +def _shutdown(service_id): + """Shuts down a service + + :param service_id: the id for the service + :type service_id: str + :returns: process return code + :rtype: int + """ + + mesos.get_master_client().shutdown_framework(service_id) + return 0 diff --git a/cli/dcoscli/task/main.py b/cli/dcoscli/task/main.py index f417cc5..72480a5 100644 --- a/cli/dcoscli/task/main.py +++ b/cli/dcoscli/task/main.py @@ -120,8 +120,8 @@ def _task(fltr, completed, is_json): :type fltr: str :param completed: If True, include completed tasks :type completed: bool - :param is_json: If true, output json. - Otherwise, output a human readable table. + :param is_json: If True, output json. Otherwise, output a human readable + table. :type is_json: bool :returns: process return code """ @@ -129,8 +129,7 @@ def _task(fltr, completed, is_json): if fltr is None: fltr = "" - master = mesos.get_master() - tasks = sorted(master.tasks(completed=completed, fltr=fltr), + tasks = sorted(mesos.get_master().tasks(completed=completed, fltr=fltr), key=lambda task: task['name']) if is_json: diff --git a/cli/tests/data/analytics/dcos_reporting.toml b/cli/tests/data/analytics/dcos_reporting.toml index 637ea68..590a344 100644 --- a/cli/tests/data/analytics/dcos_reporting.toml +++ b/cli/tests/data/analytics/dcos_reporting.toml @@ -1,3 +1,3 @@ [core] -email = "test@mail.com" reporting = true +email = "test@mail.com" diff --git a/cli/tests/data/package/chronos-1.json b/cli/tests/data/package/chronos-1.json new file mode 100644 index 0000000..cc8d626 --- /dev/null +++ b/cli/tests/data/package/chronos-1.json @@ -0,0 +1,7 @@ +{ + "chronos": { + "id": "chronos-user-1", + "framework-name": "chronos-user", + "zk-path": "/universe/chronos-user-1" + } +} diff --git a/cli/tests/data/package/chronos-2.json b/cli/tests/data/package/chronos-2.json new file mode 100644 index 0000000..d210dc3 --- /dev/null +++ b/cli/tests/data/package/chronos-2.json @@ -0,0 +1,7 @@ +{ + "chronos": { + "id": "chronos-user-2", + "framework-name": "chronos-user", + "zk-path": "/universe/chronos-user-2" + } +} diff --git a/cli/tests/integrations/cli/common.py b/cli/tests/integrations/cli/common.py index 7789e4d..0e5b954 100644 --- a/cli/tests/integrations/cli/common.py +++ b/cli/tests/integrations/cli/common.py @@ -1,6 +1,12 @@ +import collections import json +import os import subprocess +import requests + +from six.moves import urllib + def exec_command(cmd, env=None, stdin=None): """Execute CLI command @@ -148,3 +154,58 @@ def list_deployments(expected_count=None, app_id=None): assert stderr == b'' return result + + +def get_services(expected_count=None, args=[]): + """Get services + + :param expected_count: assert exactly this number of services are + running + :type expected_count: int | None + :param args: cli arguments + :type args: [str] + :returns: services + :rtype: [dict] + """ + + returncode, stdout, stderr = exec_command( + ['dcos', 'service', '--json'] + args) + + assert returncode == 0 + assert stderr == b'' + + services = json.loads(stdout.decode('utf-8')) + assert isinstance(services, collections.Sequence) + if expected_count is not None: + assert len(services) == expected_count + + return services + + +def service_shutdown(service_id): + """Shuts down a service using the command line program + + :param service_id: the id of the service + :type: service_id: str + :rtype: None + """ + + assert_command(['dcos', 'service', 'shutdown', service_id]) + + +def delete_zk_nodes(): + """Delete Zookeeper nodes that were created during the tests + + :rtype: None + """ + + base_url = os.environ.get('EXHIBITOR_URL') + if base_url: + base_path = 'exhibitor/v1/explorer/znode/{}' + + for znode in ['universe', 'cassandra-mesos', 'chronos']: + znode_url = urllib.parse.urljoin( + base_url, + base_path.format(znode)) + + requests.delete(znode_url) diff --git a/cli/tests/integrations/cli/test_marathon.py b/cli/tests/integrations/cli/test_marathon.py index eb4651c..b58fe3f 100644 --- a/cli/tests/integrations/cli/test_marathon.py +++ b/cli/tests/integrations/cli/test_marathon.py @@ -68,7 +68,7 @@ Options: and return --interval= Number of seconds to wait between actions -Positional arguments: +Positional Arguments: The application id The application resource; for a detailed description see (https://mesosphere.github.io/ diff --git a/cli/tests/integrations/cli/test_package.py b/cli/tests/integrations/cli/test_package.py index fe7d4be..ef0c1b2 100644 --- a/cli/tests/integrations/cli/test_package.py +++ b/cli/tests/integrations/cli/test_package.py @@ -4,7 +4,15 @@ import os import six from dcos import subcommand -from common import assert_command, exec_command +import pytest +from common import (assert_command, delete_zk_nodes, exec_command, + get_services, service_shutdown, watch_all_deployments) + + +@pytest.fixture(scope="module") +def zk_znode(request): + request.addfinalizer(delete_zk_nodes) + return request def test_package(): @@ -120,8 +128,23 @@ icon-service-marathon-medium.png", "icon-small": "https://downloads.mesosphere.io/marathon/assets/\ icon-service-marathon-small.png" }, + "licenses": [ + { + "name": "Apache License Version 2.0", + "url": "https://github.com/mesosphere/marathon/blob/master/LICENSE" + } + ], "maintainer": "support@mesosphere.io", "name": "marathon", + "postInstallNotes": "Marathon DCOS Service has been successfully \ +installed!\\n\\n\\tDocumentation: https://\ +mesosphere.github.io/marathon\\n\\tIssues: https:/github.com/mesosphere/\ +marathon/issues\\n", + "postUninstallNotes": "The Marathon DCOS Service has been uninstalled and \ +will no longer run.\\nPlease follow the instructions at http://beta-docs.\ +mesosphere.com/services/marathon/#uninstall to clean up any persisted state", + "preInstallNotes": "We recommend a minimum of one node with at least 2 \ +CPU's and 1GB of RAM available for the Marathon Service.", "scm": "https://github.com/mesosphere/marathon.git", "tags": [ "mesosphere", @@ -183,7 +206,10 @@ marathon-user --http_port $PORT0 ", }, "type": "DOCKER" }, - "cpus": 1.0, + "cpus": 2.0, + "env": { + "JVM_OPTS": "-Xms256m -Xmx768m" + }, "id": "marathon-user", "instances": 1, "labels": { @@ -196,17 +222,29 @@ RwczovL2Rvd25sb2Fkcy5tZXNvc3BoZXJlLmlvL21hcmF0aG9uL2Fzc2V0cy9pY29uLXNlcnZpY2Ut\ bWFyYXRob24tbGFyZ2UucG5nIiwgImljb24tbWVkaXVtIjogImh0dHBzOi8vZG93bmxvYWRzLm1lc2\ 9zcGhlcmUuaW8vbWFyYXRob24vYXNzZXRzL2ljb24tc2VydmljZS1tYXJhdGhvbi1tZWRpdW0ucG5n\ IiwgImljb24tc21hbGwiOiAiaHR0cHM6Ly9kb3dubG9hZHMubWVzb3NwaGVyZS5pby9tYXJhdGhvbi\ -9hc3NldHMvaWNvbi1zZXJ2aWNlLW1hcmF0aG9uLXNtYWxsLnBuZyJ9LCAibWFpbnRhaW5lciI6ICJz\ -dXBwb3J0QG1lc29zcGhlcmUuaW8iLCAibmFtZSI6ICJtYXJhdGhvbiIsICJzY20iOiAiaHR0cHM6Ly\ -9naXRodWIuY29tL21lc29zcGhlcmUvbWFyYXRob24uZ2l0IiwgInRhZ3MiOiBbIm1lc29zcGhlcmUi\ -LCAiZnJhbWV3b3JrIl0sICJ2ZXJzaW9uIjogIjAuOC4xIn0=", +9hc3NldHMvaWNvbi1zZXJ2aWNlLW1hcmF0aG9uLXNtYWxsLnBuZyJ9LCAibGljZW5zZXMiOiBbeyJu\ +YW1lIjogIkFwYWNoZSBMaWNlbnNlIFZlcnNpb24gMi4wIiwgInVybCI6ICJodHRwczovL2dpdGh1Yi\ +5jb20vbWVzb3NwaGVyZS9tYXJhdGhvbi9ibG9iL21hc3Rlci9MSUNFTlNFIn1dLCAibWFpbnRhaW5l\ +ciI6ICJzdXBwb3J0QG1lc29zcGhlcmUuaW8iLCAibmFtZSI6ICJtYXJhdGhvbiIsICJwb3N0SW5zdG\ +FsbE5vdGVzIjogIk1hcmF0aG9uIERDT1MgU2VydmljZSBoYXMgYmVlbiBzdWNjZXNzZnVsbHkgaW5z\ +dGFsbGVkIVxuXG5cdERvY3VtZW50YXRpb246IGh0dHBzOi8vbWVzb3NwaGVyZS5naXRodWIuaW8vbW\ +FyYXRob25cblx0SXNzdWVzOiBodHRwczovZ2l0aHViLmNvbS9tZXNvc3BoZXJlL21hcmF0aG9uL2lz\ +c3Vlc1xuIiwgInBvc3RVbmluc3RhbGxOb3RlcyI6ICJUaGUgTWFyYXRob24gRENPUyBTZXJ2aWNlIG\ +hhcyBiZWVuIHVuaW5zdGFsbGVkIGFuZCB3aWxsIG5vIGxvbmdlciBydW4uXG5QbGVhc2UgZm9sbG93\ +IHRoZSBpbnN0cnVjdGlvbnMgYXQgaHR0cDovL2JldGEtZG9jcy5tZXNvc3BoZXJlLmNvbS9zZXJ2aW\ +Nlcy9tYXJhdGhvbi8jdW5pbnN0YWxsIHRvIGNsZWFuIHVwIGFueSBwZXJzaXN0ZWQgc3RhdGUiLCAi\ +cHJlSW5zdGFsbE5vdGVzIjogIldlIHJlY29tbWVuZCBhIG1pbmltdW0gb2Ygb25lIG5vZGUgd2l0aC\ +BhdCBsZWFzdCAyIENQVSdzIGFuZCAxR0Igb2YgUkFNIGF2YWlsYWJsZSBmb3IgdGhlIE1hcmF0aG9u\ +IFNlcnZpY2UuIiwgInNjbSI6ICJodHRwczovL2dpdGh1Yi5jb20vbWVzb3NwaGVyZS9tYXJhdGhvbi\ +5naXQiLCAidGFncyI6IFsibWVzb3NwaGVyZSIsICJmcmFtZXdvcmsiXSwgInZlcnNpb24iOiAiMC44\ +LjEifQ==", "DCOS_PACKAGE_NAME": "marathon", "DCOS_PACKAGE_REGISTRY_VERSION": "0.1.0-alpha", "DCOS_PACKAGE_RELEASE": "0", "DCOS_PACKAGE_SOURCE": "git://github.com/mesosphere/universe.git", "DCOS_PACKAGE_VERSION": "0.8.1" }, - "mem": 512.0, + "mem": 1024.0, "ports": [ 0, 0 @@ -224,8 +262,23 @@ icon-service-marathon-medium.png", "icon-small": "https://downloads.mesosphere.io/marathon/assets/\ icon-service-marathon-small.png" }, + "licenses": [ + { + "name": "Apache License Version 2.0", + "url": "https://github.com/mesosphere/marathon/blob/master/LICENSE" + } + ], "maintainer": "support@mesosphere.io", "name": "marathon", + "postInstallNotes": "Marathon DCOS Service has been successfully \ +installed!\\n\\n\\tDocumentation: https://\ +mesosphere.github.io/marathon\\n\\tIssues: https:/github.com/mesosphere/\ +marathon/issues\\n", + "postUninstallNotes": "The Marathon DCOS Service has been uninstalled and \ +will no longer run.\\nPlease follow the instructions at http://beta-docs.\ +mesosphere.com/services/marathon/#uninstall to clean up any persisted state", + "preInstallNotes": "We recommend a minimum of one node with at least 2 \ +CPU's and 1GB of RAM available for the Marathon Service.", "scm": "https://github.com/mesosphere/marathon.git", "tags": [ "mesosphere", @@ -258,17 +311,22 @@ Please create a JSON file with the appropriate options, and pass the \ postInstallNotes=b'') -def test_install(): +def test_install(zk_znode): _install_chronos() + watch_all_deployments() _uninstall_chronos() + get_services(expected_count=1, args=['--inactive']) def test_install_missing_options_file(): """Test that a missing options file results in the expected stderr message.""" assert_command( - ['dcos', 'package', 'install', 'chronos', '--options=asdf.json'], + ['dcos', 'package', 'install', 'chronos', '--yes', + '--options=asdf.json'], returncode=1, + stdout=b'We recommend a minimum of one node with at least 1 CPU and ' + b'2GB of RAM available for the Chronos Service.\n', stderr=b"No such file: asdf.json\n") @@ -300,7 +358,7 @@ CJdfQ==""" 'DCOS_PACKAGE_RELEASE': b'0', } - app_labels = get_app_labels('helloworld') + app_labels = _get_app_labels('helloworld') for label, value in expected_labels.items(): assert value == six.b(app_labels.get(label)) @@ -338,15 +396,15 @@ CJdfQ==""" _uninstall_helloworld() -def test_install_with_id(): +def test_install_with_id(zk_znode): args = ['--app-id=chronos-1', '--yes'] - stdout = (b"""Installing package [chronos] version [2.3.4] with app """ - b"""id [chronos-1]\n""") + stdout = (b'Installing package [chronos] version [2.3.4] with app id ' + b'[chronos-1]\n') _install_chronos(args=args, stdout=stdout) args = ['--app-id=chronos-2', '--yes'] - stdout = (b"""Installing package [chronos] version [2.3.4] with app """ - b"""id [chronos-2]\n""") + stdout = (b'Installing package [chronos] version [2.3.4] with app id ' + b'[chronos-2]\n') _install_chronos(args=args, stdout=stdout) @@ -359,19 +417,20 @@ You may need to run 'dcos package update' to update your repositories stderr=stderr) -def test_uninstall_with_id(): +def test_uninstall_with_id(zk_znode): _uninstall_chronos(args=['--app-id=chronos-1']) -def test_uninstall_all(): +def test_uninstall_all(zk_znode): _uninstall_chronos(args=['--all']) + get_services(expected_count=1, args=['--inactive']) def test_uninstall_missing(): - stderr = b'Package [chronos] is not installed.\n' + stderr = 'Package [chronos] is not installed.\n' _uninstall_chronos(returncode=1, stderr=stderr) - stderr = b'Package [chronos] with id [chronos-1] is not installed.\n' + stderr = 'Package [chronos] with id [chronos-1] is not installed.\n' _uninstall_chronos( args=['--app-id=chronos-1'], returncode=1, @@ -417,7 +476,7 @@ def test_uninstall_cli(): _uninstall_helloworld() -def test_list_installed(): +def test_list_installed(zk_znode): assert_command(['dcos', 'package', 'list-installed'], stdout=b'[]\n') @@ -446,14 +505,23 @@ service-chronos-medium.png", "icon-small": "https://downloads.mesosphere.io/chronos/assets/icon-\ service-chronos-small.png" }, + "licenses": [ + { + "name": "Apache License Version 2.0", + "url": "https://github.com/mesos/chronos/blob/master/LICENSE" + } + ], "maintainer": "support@mesosphere.io", "name": "chronos", "packageSource": "git://github.com/mesosphere/universe.git", "postInstallNotes": "Chronos DCOS Service has been successfully installed!\ -\\nWe recommend a minimum of one node with at least 1 CPU and 2GB of RAM \ -available for the Chronos Service.\\n\\n\\tDocumentation: \ -http://mesos.github.io/chronos\\n\\tIssues: https://github.com/mesos/\ -chronos/issues", +\\n\\n\\tDocumentation: http://mesos.github.io/chronos\\n\\tIssues: https://\ +github.com/mesos/chronos/issues", + "postUninstallNotes": "The Chronos DCOS Service has been uninstalled and \ +will no longer run.\\nPlease follow the instructions at http://beta-docs.\ +mesosphere.com/services/chronos/#uninstall to clean up any persisted state", + "preInstallNotes": "We recommend a minimum of one node with at least 1 \ +CPU and 2GB of RAM available for the Chronos Service.", "releaseVersion": "0", "scm": "https://github.com/mesos/chronos.git", "tags": [ @@ -570,6 +638,30 @@ def test_list_installed_cli(): _uninstall_helloworld() +def test_uninstall_multiple_frameworknames(zk_znode): + _install_chronos( + args=['--yes', '--options=tests/data/package/chronos-1.json']) + _install_chronos( + args=['--yes', '--options=tests/data/package/chronos-2.json']) + + watch_all_deployments() + + _uninstall_chronos( + args=['--app-id=chronos-user-1'], + returncode=1, + stderr='Unable to shutdown the framework for [chronos-user] because ' + 'there are multiple frameworks with the same name: ') + _uninstall_chronos( + args=['--app-id=chronos-user-2'], + returncode=1, + stderr='Unable to shutdown the framework for [chronos-user] because ' + 'there are multiple frameworks with the same name: ') + + for framework in get_services(args=['--inactive']): + if framework['name'] == 'chronos-user': + service_shutdown(framework['id']) + + def test_search(): returncode, stdout, stderr = exec_command( ['dcos', @@ -607,7 +699,7 @@ def test_search(): assert stderr == b'' -def get_app_labels(app_id): +def _get_app_labels(app_id): returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'show', app_id]) @@ -635,9 +727,13 @@ def _uninstall_helloworld(args=[]): assert_command(['dcos', 'package', 'uninstall', 'helloworld'] + args) -def _uninstall_chronos(args=[], returncode=0, stdout=b'', stderr=b''): - cmd = ['dcos', 'package', 'uninstall', 'chronos'] + args - assert_command(cmd, returncode, stdout, stderr) +def _uninstall_chronos(args=[], returncode=0, stdout=b'', stderr=''): + result_returncode, result_stdout, result_stderr = exec_command( + ['dcos', 'package', 'uninstall', 'chronos'] + args) + + assert result_returncode == returncode + assert result_stdout == stdout + assert result_stderr.decode('utf-8').startswith(stderr) def _install_chronos( @@ -645,10 +741,11 @@ def _install_chronos( returncode=0, stdout=b'Installing package [chronos] version [2.3.4]\n', stderr=b'', + preInstallNotes=b'We recommend a minimum of one node with at least 1 ' + b'CPU and 2GB of RAM available for the Chronos ' + b'Service.\n', postInstallNotes=b'Chronos DCOS Service has been successfully ' - b'installed!\nWe recommend a minimum of one node ' - b'with at least 1 CPU and 2GB of RAM available for ' - b'''the Chronos Service. + b'''installed! \tDocumentation: http://mesos.github.io/chronos \tIssues: https://github.com/mesos/chronos/issues\n''', @@ -658,6 +755,6 @@ def _install_chronos( assert_command( cmd, returncode, - stdout + postInstallNotes, + preInstallNotes + stdout + postInstallNotes, stderr, stdin=stdin) diff --git a/cli/tests/integrations/cli/test_service.py b/cli/tests/integrations/cli/test_service.py index 451b695..038e769 100644 --- a/cli/tests/integrations/cli/test_service.py +++ b/cli/tests/integrations/cli/test_service.py @@ -1,5 +1,3 @@ -import collections -import json import time import dcos.util as util @@ -8,7 +6,14 @@ from dcos.util import create_schema from dcoscli.service.main import _service_table import pytest -from common import assert_command, exec_command, watch_all_deployments +from common import (assert_command, delete_zk_nodes, exec_command, + get_services, service_shutdown, watch_all_deployments) + + +@pytest.fixture(scope="module") +def zk_znode(request): + request.addfinalizer(delete_zk_nodes) + return request @pytest.fixture @@ -60,6 +65,7 @@ def test_help(): Usage: dcos service --info dcos service [--inactive --json] + dcos service shutdown Options: -h, --help Show this screen @@ -73,6 +79,9 @@ Options: master, but haven't yet reached their failover timeout. --version Show version + +Positional Arguments: + The ID for the DCOS Service """ assert_command(['dcos', 'service', '--help'], stdout=stdout) @@ -85,7 +94,7 @@ def test_info(): def test_service(service): returncode, stdout, stderr = exec_command(['dcos', 'service', '--json']) - services = _get_services(1) + services = get_services(1) schema = _get_schema(service) for srv in services: @@ -103,19 +112,20 @@ def _get_schema(service): return schema -def test_service_inactive(): +def test_service_inactive(zk_znode): # install cassandra - stdout = b"""Installing package [cassandra] version \ -[0.1.0-SNAPSHOT-447-master-3ad1bbf8f7] -The Apache Cassandra DCOS Service implementation is alpha and there may \ -be bugs, incomplete features, incorrect documentation or other discrepancies. -In order for Cassandra to start successfully, all resources must be \ -available in the cluster, including ports, CPU shares, RAM and disk. + stdout = b"""The Apache Cassandra DCOS Service implementation is alpha \ +and there may be bugs, incomplete features, incorrect documentation or other \ +discrepancies. +The default configuration requires 3 nodes each with 0.3 CPU shares, 1184MB \ +of memory and 272MB of disk. +Installing package [cassandra] version [0.1.0-SNAPSHOT-447-master-3ad1bbf8f7] +Thank you for installing the Apache Cassandra DCOS Service. \tDocumentation: http://mesosphere.github.io/cassandra-mesos/ \tIssues: https://github.com/mesosphere/cassandra-mesos/issues """ - assert_command(['dcos', 'package', 'install', 'cassandra'], + assert_command(['dcos', 'package', 'install', 'cassandra', '--yes'], stdout=stdout) # wait for it to deploy @@ -125,11 +135,10 @@ available in the cluster, including ports, CPU shares, RAM and disk. time.sleep(5) # assert marathon and cassandra are listed - _get_services(2) + get_services(2) - # uninstall cassandra. For now, need to explicitly remove the - # group that is left by cassandra. See MARATHON-144 - assert_command(['dcos', 'package', 'uninstall', 'cassandra']) + # uninstall cassandra using marathon. For now, need to explicitly remove + # the group that is left by cassandra. See MARATHON-144 assert_command(['dcos', 'marathon', 'group', 'remove', '/cassandra']) watch_all_deployments(300) @@ -139,11 +148,19 @@ available in the cluster, including ports, CPU shares, RAM and disk. time.sleep(5) # assert only marathon is active - _get_services(1) + get_services(1) # assert marathon and cassandra are listed with --inactive - services = _get_services(None, ['--inactive']) + services = get_services(None, ['--inactive']) assert len(services) >= 2 + # shutdown the cassandra framework + for framework in get_services(args=['--inactive']): + if framework['name'] == 'cassandra.dcos': + service_shutdown(framework['id']) + + # assert marathon is only listed with --inactive + get_services(1, ['--inactive']) + # not an integration test def test_task_table(service): @@ -155,29 +172,3 @@ def test_task_table(service): marathon mesos.vm True 0 0.2 32 0 \ 20150502-231327-16842879-5050-3889-0000 """ assert str(table) == stdout - - -def _get_services(expected_count=None, args=[]): - """Get services - - :param expected_count: assert exactly this number of services are - running - :type expected_count: int - :param args: cli arguments - :type args: [str] - :returns: services - :rtype: [dict] - """ - - returncode, stdout, stderr = exec_command( - ['dcos', 'service', '--json'] + args) - - assert returncode == 0 - assert stderr == b'' - - services = json.loads(stdout.decode('utf-8')) - assert isinstance(services, collections.Sequence) - if expected_count is not None: - assert len(services) == expected_count - - return services diff --git a/dcos/http.py b/dcos/http.py index fb40476..9bf7f9a 100644 --- a/dcos/http.py +++ b/dcos/http.py @@ -31,6 +31,7 @@ def _default_to_error(response): return DefaultError('{}: {}'.format(response.status_code, response.text)) +@util.duration def request(method, url, timeout=3.0, diff --git a/dcos/mesos.py b/dcos/mesos.py index e04890d..4bddf6d 100644 --- a/dcos/mesos.py +++ b/dcos/mesos.py @@ -1,8 +1,7 @@ import fnmatch import itertools -import dcos.http -from dcos import util +from dcos import http, util from dcos.errors import DCOSException from six.moves import urllib @@ -11,23 +10,43 @@ logger = util.get_logger(__name__) def get_master(config=None): - """Create a MesosMaster object using the url stored in the - 'core.master' property of the user's config. + """Create a Master object using the URLs stored in the user's + configuration. :param config: config :type config: Toml - :returns: MesosMaster object - :rtype: MesosMaster - + :returns: master state object + :rtype: Master """ + + return Master(get_master_client(config).get_state()) + + +def get_master_client(config=None): + """Create a Mesos master client using the URLs stored in the user's + configuration. + + :param config: config + :type config: Toml + :returns: mesos master client + :rtype: MasterClient + """ + if config is None: config = util.get_config() - mesos_url = get_mesos_url(config) - return MesosMaster(mesos_url) + mesos_url = _get_mesos_url(config) + return MasterClient(mesos_url) -def get_mesos_url(config): +def _get_mesos_url(config): + """ + :param config: configuration + :type config: Toml + :returns: url for the Mesos master + :rtype: str + """ + mesos_master_url = config.get('core.mesos_master_url') if mesos_master_url is None: dcos_url = util.get_config_vals(config, ['core.dcos_url'])[0] @@ -36,30 +55,65 @@ def get_mesos_url(config): return mesos_master_url -MESOS_TIMEOUT = 3 +class MasterClient: + """Client for communicating with the Mesos master - -class MesosMaster(object): - """Mesos Master Model - - :param url: master url (e.g. "http://localhost:5050") + :param url: URL for the Mesos master :type url: str """ def __init__(self, url): - self._url = url - self._state = None + self._base_url = url + + def _create_url(self, path): + """Creates the url from the provided path. + + :param path: url path + :type path: str + :returns: constructed url + :rtype: str + """ + + return urllib.parse.urljoin(self._base_url, path) + + def get_state(self): + """Get the Mesos master state json object + + :returns: Mesos' master state json object + :rtype: dict + """ + + return http.get(self._create_url('master/state.json')).json() + + def shutdown_framework(self, framework_id): + """Shuts down a Mesos framework + + :returns: None + """ + + logger.info('Shutting down framework {}'.format(framework_id)) + + data = 'frameworkId={}'.format(framework_id) + http.post(self._create_url('master/shutdown'), data=data) + + +class Master(object): + """Mesos Master Model + + :param state: Mesos master state json + :type state: dict + """ + + def __init__(self, state): + self._state = state def state(self): - """Returns master's /master/state.json. Fetches and saves it if we - haven't already. + """Returns master's master/state.json. :returns: state.json :rtype: dict """ - if not self._state: - self._state = self.fetch('master/state.json').json() return self._state def slave(self, fltr): @@ -70,7 +124,7 @@ class MesosMaster(object): :param fltr: filter string :type fltr: str :returns: the slave that has `fltr` in its id - :rtype: MesosSlave + :rtype: Slave """ slaves = self.slaves(fltr) @@ -93,10 +147,10 @@ class MesosMaster(object): :param fltr: filter string :type fltr: str :returns: Those slaves that have `fltr` in their 'id' - :rtype: [MesosSlave] + :rtype: [Slave] """ - return [MesosSlave(slave) + return [Slave(slave) for slave in self.state()['slaves'] if fltr in slave['id']] @@ -198,25 +252,8 @@ class MesosMaster(object): if inactive or framework['active']: yield framework - @util.duration - def fetch(self, path, **kwargs): - """GET the resource located at `path` - :param path: the URL path - :type path: str - :param **kwargs: requests.get kwargs - :type **kwargs: dict - :returns: the response object - :rtype: Response - """ - - url = urllib.parse.urljoin(self._url, path) - return dcos.http.get(url, - timeout=MESOS_TIMEOUT, - **kwargs) - - -class MesosSlave(object): +class Slave(object): """Mesos Slave Model :param slave: dictionary representing the slave. @@ -256,7 +293,7 @@ class Task(object): :param task: task properties :type task: dict :param master: mesos master - :type master: MesosMaster + :type master: Master """ def __init__(self, task, master): diff --git a/dcos/package.py b/dcos/package.py index abad2fa..148c47e 100644 --- a/dcos/package.py +++ b/dcos/package.py @@ -14,7 +14,7 @@ import git import portalocker import pystache import six -from dcos import constants, emitting, errors, marathon, subcommand, util +from dcos import constants, emitting, errors, marathon, mesos, subcommand, util from dcos.errors import DCOSException from six.moves import urllib @@ -159,12 +159,12 @@ def uninstall(package_name, remove_all, app_id, cli, app): uninstalled = True if app: - init_client = marathon.create_client() - - num_apps = uninstall_app(package_name, - remove_all, - app_id, - init_client) + num_apps = uninstall_app( + package_name, + remove_all, + app_id, + marathon.create_client(), + mesos.get_master_client()) if num_apps > 0: uninstalled = True @@ -191,7 +191,7 @@ def uninstall_subcommand(distribution_name): return subcommand.uninstall(distribution_name) -def uninstall_app(app_name, remove_all, app_id, init_client): +def uninstall_app(app_name, remove_all, app_id, init_client, master_client): """Uninstalls an app. :param app_name: The app to uninstall @@ -202,6 +202,8 @@ def uninstall_app(app_name, remove_all, app_id, init_client): :type app_id: str :param init_client: The program to use to run the app :type init_client: object + :param master_client: the mesos master client + :type master_client: dcos.mesos.MasterClient :returns: number of apps uninstalled :rtype: int """ @@ -226,14 +228,46 @@ def uninstall_app(app_name, remove_all, app_id, init_client): if not remove_all and len(matching_apps) > 1: app_ids = [a.get('id') for a in matching_apps] - raise DCOSException("""Multiple instances of app [{}] are installed. \ -Please specify the app id of the instance to uninstall or uninstall all. \ -The app ids of the installed package instances are: [{}].""".format( - app_name, ', '.join(app_ids))) + raise DCOSException( + ("Multiple instances of app [{}] are installed. Please specify " + "the app id of the instance to uninstall or uninstall all. The " + "app ids of the installed package instances are: [{}].").format( + app_name, + ', '.join(app_ids))) for app in matching_apps: + # First, remove the app from Marathon init_client.remove_app(app['id'], force=True) + # Second, shutdown the framework with Mesos + framework_name = app.get('labels', {}).get(PACKAGE_FRAMEWORK_NAME_KEY) + if framework_name is not None: + logger.info( + 'Trying to shutdown framework {}'.format(framework_name)) + frameworks = mesos.Master(master_client.get_state()).frameworks( + inactive=True) + + # Look up all the framework names + framework_ids = [ + framework['id'] + for framework in frameworks + if framework['name'] == framework_name + ] + + logger.info( + 'Found the following frameworks: {}'.format(framework_ids)) + + if len(framework_ids) == 1: + master_client.shutdown_framework(framework_ids[0]) + elif len(framework_ids) > 1: + raise DCOSException( + "Unable to shutdown the framework for [{}] because there " + "are multiple frameworks with the same name: [{}]. " + "Manually shut them down using 'dcos service " + "shutdown'.".format( + framework_name, + ', '.join(framework_ids))) + return len(matching_apps)