From 5d13dfce0ef991a3fcd4a4c0942b37a72292bb1b Mon Sep 17 00:00:00 2001 From: Sargun Dhillon Date: Mon, 2 Nov 2015 14:47:50 -0800 Subject: [PATCH] JIRA: DCOS-2941 Add kill tasks to dcos/marathon.py and CLI; Documented in Marathon API here: https://mesosphere.github.io/marathon/docs/rest-api.html#delete-v2-apps-appid-tasks Add tests, and CLI integration Add scale, and kill parameters to app kill command / API call Add test for host-based kill, but test requires >=2 hosts to succeed - skip if not true [TODO: FIX] Cleanup --- cli/dcoscli/marathon/main.py | 41 +++++++++++ cli/tests/data/help/marathon.txt | 7 ++ cli/tests/integrations/test_marathon.py | 92 ++++++++++++++++++++++++- dcos/marathon.py | 23 +++++++ 4 files changed, 161 insertions(+), 2 deletions(-) diff --git a/cli/dcoscli/marathon/main.py b/cli/dcoscli/marathon/main.py index a0f0fbc..dce8c9f 100644 --- a/cli/dcoscli/marathon/main.py +++ b/cli/dcoscli/marathon/main.py @@ -11,6 +11,7 @@ Usage: dcos marathon app show [--app-version=] dcos marathon app start [--force] [] dcos marathon app stop [--force] + dcos marathon app kill [--scale] [--host=] dcos marathon app update [--force] [...] dcos marathon app version list [--max-count=] dcos marathon deployment list [--json ] @@ -69,6 +70,12 @@ Options: --interval= Number of seconds to wait between actions + --scale Scale the app down after performing the + the operation. + + --host= The host name to isolate your command to + + Positional Arguments: The application id @@ -218,6 +225,11 @@ def _cmds(): arg_keys=['', '--force'], function=_restart), + cmds.Command( + hierarchy=['marathon', 'app', 'kill'], + arg_keys=['', '--scale', '--host'], + function=_kill), + cmds.Command( hierarchy=['marathon', 'group', 'add'], arg_keys=[''], @@ -714,6 +726,35 @@ def _restart(app_id, force): return 0 +def _kill(app_id, scale, host): + """ + :param app_id: the id of the application + :type app_id: str + :param: scale: Scale the app down + :type: scale: bool + :param: host: Kill only those tasks running on host specified + :type: string + :returns: process return code + :rtype: int + """ + client = marathon.create_client() + + payload = client.kill_tasks(app_id, host=host, scale=scale) + # If scale is provided, the API return a "deploymentResult" + # https://github.com/mesosphere/marathon/blob/50366c8/src/main/scala/mesosphere/marathon/api/RestResource.scala#L34-L36 + if scale: + emitter.publish("Started deployment: {}".format(payload)) + else: + if 'tasks' in payload: + emitter.publish('Killed tasks: {}'.format(payload['tasks'])) + if len(payload['tasks']) == 0: + return 1 + else: + emitter.publish('Killed tasks: []') + return 1 + return 0 + + def _version_list(app_id, max_count): """ :param app_id: the id of the application diff --git a/cli/tests/data/help/marathon.txt b/cli/tests/data/help/marathon.txt index c63e917..52de7b7 100644 --- a/cli/tests/data/help/marathon.txt +++ b/cli/tests/data/help/marathon.txt @@ -11,6 +11,7 @@ Usage: dcos marathon app show [--app-version=] dcos marathon app start [--force] [] dcos marathon app stop [--force] + dcos marathon app kill [--scale] [--host=] dcos marathon app update [--force] [...] dcos marathon app version list [--max-count=] dcos marathon deployment list [--json ] @@ -69,6 +70,12 @@ Options: --interval= Number of seconds to wait between actions + --scale Scale the app down after performing the + the operation. + + --host= The host name to isolate your command to + + Positional Arguments: The application id diff --git a/cli/tests/integrations/test_marathon.py b/cli/tests/integrations/test_marathon.py index 848f58c..1422148 100644 --- a/cli/tests/integrations/test_marathon.py +++ b/cli/tests/integrations/test_marathon.py @@ -30,6 +30,7 @@ Usage: dcos marathon app show [--app-version=] dcos marathon app start [--force] [] dcos marathon app stop [--force] + dcos marathon app kill [--scale] [--host=] dcos marathon app update [--force] [...] dcos marathon app version list [--max-count=] dcos marathon deployment list [--json ] @@ -88,6 +89,12 @@ Options: --interval= Number of seconds to wait between actions + --scale Scale the app down after performing the + the operation. + + --host= The host name to isolate your command to + + Positional Arguments: The application id @@ -465,6 +472,86 @@ def test_restarting_app(): assert stderr == b'' +def test_killing_app(): + with _zero_instance_app(): + _start_app('zero-instance-app', 3) + watch_all_deployments() + task_set_1 = set([task['id'] + for task in _list_tasks(3, 'zero-instance-app')]) + returncode, stdout, stderr = exec_command( + ['dcos', 'marathon', 'app', 'kill', 'zero-instance-app']) + assert returncode == 0 + assert stdout.decode().startswith('Killed tasks: ') + assert stderr == b'' + watch_all_deployments() + task_set_2 = set([task['id'] + for task in _list_tasks(app_id='zero-instance-app')]) + assert len(task_set_1.intersection(task_set_2)) == 0 + + +def test_killing_scaling_app(): + with _zero_instance_app(): + _start_app('zero-instance-app', 3) + watch_all_deployments() + _list_tasks(3) + command = ['dcos', 'marathon', 'app', 'kill', '--scale', + 'zero-instance-app'] + returncode, stdout, stderr = exec_command(command) + assert returncode == 0 + assert stdout.decode().startswith('Started deployment: ') + assert stdout.decode().find('version') > -1 + assert stdout.decode().find('deploymentId') > -1 + assert stderr == b'' + watch_all_deployments() + _list_tasks(0) + + +def test_killing_with_host_app(): + with _zero_instance_app(): + _start_app('zero-instance-app', 3) + watch_all_deployments() + existing_tasks = _list_tasks(3, 'zero-instance-app') + task_hosts = set([task['host'] for task in existing_tasks]) + if len(task_hosts) <= 1: + pytest.skip('test needs 2 or more agents to succeed, ' + 'only {} agents available'.format(len(task_hosts))) + assert len(task_hosts) > 1 + kill_host = list(task_hosts)[0] + expected_to_be_killed = set([task['id'] + for task in existing_tasks + if task['host'] == kill_host]) + not_to_be_killed = set([task['id'] + for task in existing_tasks + if task['host'] != kill_host]) + assert len(not_to_be_killed) > 0 + assert len(expected_to_be_killed) > 0 + command = ['dcos', 'marathon', 'app', 'kill', '--host', kill_host, + 'zero-instance-app'] + returncode, stdout, stderr = exec_command(command) + assert stdout.decode().startswith('Killed tasks: ') + assert stderr == b'' + new_tasks = set([task['id'] for task in _list_tasks()]) + assert not_to_be_killed.intersection(new_tasks) == not_to_be_killed + assert len(expected_to_be_killed.intersection(new_tasks)) == 0 + + +def test_kill_stopped_app(): + with _zero_instance_app(): + returncode, stdout, stderr = exec_command( + ['dcos', 'marathon', 'app', 'kill', 'zero-instance-app']) + assert returncode == 1 + assert stdout.decode().startswith('Killed tasks: []') + + +def test_kill_missing_app(): + returncode, stdout, stderr = exec_command( + ['dcos', 'marathon', 'app', 'kill', 'app']) + assert returncode == 1 + assert stdout.decode() == '' + stderr_expected = "Error: App '/app' does not exist" + assert stderr.decode().strip() == stderr_expected + + def test_list_version_missing_app(): assert_command( ['dcos', 'marathon', 'app', 'version', 'list', 'missing-id'], @@ -744,7 +831,7 @@ def _list_versions(app_id, expected_count, max_count=None): assert stderr == b'' -def _list_tasks(expected_count, app_id=None): +def _list_tasks(expected_count=None, app_id=None): cmd = ['dcos', 'marathon', 'task', 'list', '--json'] if app_id is not None: cmd.append(app_id) @@ -754,7 +841,8 @@ def _list_tasks(expected_count, app_id=None): result = json.loads(stdout.decode('utf-8')) assert returncode == 0 - assert len(result) == expected_count + if expected_count: + assert len(result) == expected_count assert stderr == b'' return result diff --git a/dcos/marathon.py b/dcos/marathon.py index f8e7881..7da1255 100644 --- a/dcos/marathon.py +++ b/dcos/marathon.py @@ -471,6 +471,29 @@ class Client(object): _http_req(http.delete, url, params=params, timeout=self._timeout) + def kill_tasks(self, app_id, scale=None, host=None): + """Kills the tasks for a given application, + and can target a given agent, with a future target scale + + :param app_id: the id of the application to restart + :type app_id: str + :param scale: Scale the app down after killing the specified tasks + :type scale: bool + :param host: host to target restarts on + :type host: string + """ + params = {} + app_id = self.normalize_app_id(app_id) + if host: + params['host'] = host + if scale: + params['scale'] = scale + url = self._create_url('v2/apps{}/tasks'.format(app_id)) + response = _http_req(http.delete, url, + params=params, + timeout=self._timeout) + return response.json() + def restart_app(self, app_id, force=None): """Performs a rolling restart of all of the tasks.