diff --git a/cli/dcoscli/data/marathon-group-schema.json b/cli/dcoscli/data/marathon-group-schema.json new file mode 100644 index 0000000..6b839f5 --- /dev/null +++ b/cli/dcoscli/data/marathon-group-schema.json @@ -0,0 +1,244 @@ +{ + "$schema": "http://json-schema.org/schema#", + "additionalProperties": false, + "definitions": { + "app": { + "additionalProperties": false, + "properties": { + "args": { + "items": { + "type": "string" + }, + "type": "array" + }, + "backoffFactor": { + "minimum": 1.0, + "type": "number" + }, + "backoffSeconds": { + "minimum": 0, + "type": "integer" + }, + "cmd": { + "type": "string" + }, + "constraints": {}, + "container": { + "additionalProperties": false, + "properties": { + "docker": { + "additionalProperties": false, + "properties": { + "image": { + "type": "string" + }, + "network": { + "type": "string" + }, + "portMappings": { + "items": { + "additionalProperties": false, + "properties": { + "containerPort": { + "maximum": 65535, + "minimum": 0, + "type": "integer" + }, + "hostPort": { + "maximum": 65535, + "minimum": 0, + "type": "integer" + }, + "protocol": { + "type": "string" + }, + "servicePort": { + "maximum": 65535, + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "image" + ], + "type": "object" + }, + "type": { + "type": "string" + }, + "volumes": { + "items": { + "additionalProperties": false, + "properties": { + "containerPath": { + "type": "string" + }, + "hostPath": { + "type": "string" + }, + "mode": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "cpus": { + "minimum": 0, + "type": "number" + }, + "dependencies": { + "items": { + "pattern": "^/?(([a-z0-9]|[a-z0-9][a-z0-9\\-]*[a-z0-9])\\.)*([a-z0-9]|[a-z0-9][a-z0-9\\-]*[a-z0-9])$", + "type": "string" + }, + "type": "array" + }, + "disk": { + "minimum": 0, + "type": "number" + }, + "env": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "executor": { + "type": "string" + }, + "healthChecks": { + "items": { + "additionalProperties": false, + "properties": { + "command": { + "type": "string" + }, + "gracePeriodSeconds": { + "minimum": 0, + "type": "integer" + }, + "intervalSeconds": { + "minimum": 0, + "type": "integer" + }, + "maxConsecutiveFailures": { + "minimum": 0, + "type": "integer" + }, + "path": { + "type": "string" + }, + "portIndex": { + "minimum": 0, + "type": "integer" + }, + "protocol": { + "type": "string" + }, + "timeoutSeconds": { + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "type": "array" + }, + "id": { + "pattern": "^/?(([a-z0-9]|[a-z0-9][a-z0-9\\-]*[a-z0-9])\\.)*([a-z0-9]|[a-z0-9][a-z0-9\\-]*[a-z0-9])$", + "type": "string" + }, + "instances": { + "minimum": 0, + "type": "integer" + }, + "labels": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "mem": { + "minimum": 0, + "type": "number" + }, + "ports": { + "items": { + "maximum": 65535, + "minimum": 0, + "type": "integer" + }, + "type": "array" + }, + "requirePorts": { + "type": "boolean" + }, + "required": [ + "id" + ], + "storeUrls": { + "items": { + "type": "string" + }, + "type": "array" + }, + "upgradeStrategy": { + "additionalProperties": false, + "properties": { + "minimumHealthCapacity": { + "maximum": 1.0, + "minimum": 0.0, + "type": "number" + } + }, + "type": "object" + }, + "uris": { + "items": { + "type": "string" + }, + "type": "array" + }, + "user": { + "type": "string" + } + } + } + }, + "properties": { + "apps": { + "items": { + "$ref": "#/definitions/app" + }, + "type": "array" + }, + "dependencies": { + "$ref": "#/definitions/app/properties/dependencies" + }, + "groups": { + "items": { + "$ref": "#" + }, + "type": "array" + }, + "id": { + "$ref": "#/definitions/app/properties/id" + }, + "required": [ + "id" + ] + }, + "type": "object" +} diff --git a/cli/dcoscli/data/marathon-schema.json b/cli/dcoscli/data/marathon-schema.json deleted file mode 100644 index f470304..0000000 --- a/cli/dcoscli/data/marathon-schema.json +++ /dev/null @@ -1,215 +0,0 @@ -{ - "$schema": "http://json-schema.org/schema#", - "type": "object", - "properties": { - "id": { - "type": "string", - "pattern": "^/?(([a-z0-9]|[a-z0-9][a-z0-9\\-]*[a-z0-9])\\.)*([a-z0-9]|[a-z0-9][a-z0-9\\-]*[a-z0-9])$" - }, - "cmd": { - "type": "string" - }, - "args": { - "type": "array", - "items": { - "type": "string" - } - }, - "user": { - "type": "string" - }, - "env": { - "type": "object", - "patternProperties": { - ".*": { "type": "string" } - } - }, - "instances": { - "type": "integer", - "minimum": 0 - }, - "cpus": { - "type": "number", - "minimum": 0 - }, - "mem": { - "type": "number", - "minimum": 0 - }, - "disk": { - "type": "number", - "minimum": 0 - }, - "executor": { - "type": "string" - }, - "constraints": { - }, - "uris": { - "type": "array", - "items": { - "type": "string" - } - }, - "storeUrls": { - "type": "array", - "items": { - "type": "string" - } - }, - "ports": { - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 65535 - } - }, - "requirePorts": { - "type": "boolean" - }, - "backoffSeconds": { - "type": "integer", - "minimum": 0 - }, - "backoffFactor": { - "type": "number", - "minimum": 1.0 - }, - "container": { - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "docker": { - "type": "object", - "properties": { - "image": { - "type": "string" - }, - "network": { - "type": "string" - }, - "portMappings": { - "type": "array", - "items": { - "type": "object", - "properties": { - "containerPort": { - "type": "integer", - "minimum": 0, - "maximum": 65535 - }, - "hostPort": { - "type": "integer", - "minimum": 0, - "maximum": 65535 - }, - "servicePort": { - "type": "integer", - "minimum": 0, - "maximum": 65535 - }, - "protocol": { - "type": "string" - } - }, - "additionalProperties": false - } - } - }, - "additionalProperties": false, - "required": [ - "image" - ] - }, - "volumes": { - "type": "array", - "items": { - "type": "object", - "properties": { - "containerPath": { - "type": "string" - }, - "hostPath": { - "type": "string" - }, - "mode": { - "type": "string" - } - }, - "additionalProperties": false - } - } - }, - "additionalProperties": false - }, - "healthChecks": { - "type": "array", - "items": { - "type": "object", - "properties": { - "protocol": { - "type": "string" - }, - "command": { - "type": "string" - }, - "path": { - "type": "string" - }, - "portIndex": { - "type": "integer", - "minimum": 0 - }, - "gracePeriodSeconds": { - "type": "integer", - "minimum": 0 - }, - "intervalSeconds": { - "type": "integer", - "minimum": 0 - }, - "timeoutSeconds": { - "type": "integer", - "minimum": 0 - }, - "maxConsecutiveFailures": { - "type": "integer", - "minimum": 0 - } - }, - "additionalProperties": false - } - }, - "dependencies": { - "type": "array", - "items": { - "type": "string", - "pattern": "^/?(([a-z0-9]|[a-z0-9][a-z0-9\\-]*[a-z0-9])\\.)*([a-z0-9]|[a-z0-9][a-z0-9\\-]*[a-z0-9])$" - } - }, - "upgradeStrategy": { - "type": "object", - "properties": { - "minimumHealthCapacity": { - "type": "number", - "minimum": 0.0, - "maximum": 1.0 - } - }, - "additionalProperties": false - }, - "labels": { - "type": "object", - "additionalProperties": { - "type": "string" - } - } - }, - "additionalProperties": false, - "required": [ - "id" - ] -} diff --git a/cli/dcoscli/marathon/main.py b/cli/dcoscli/marathon/main.py index bf219ce..701f3c3 100644 --- a/cli/dcoscli/marathon/main.py +++ b/cli/dcoscli/marathon/main.py @@ -20,27 +20,42 @@ Usage: [--interval=] dcos marathon task list [] dcos marathon task show + dcos marathon group add [] + dcos marathon group list + dcos marathon group show [--group-version=] + dcos marathon group remove [--force] Options: - -h, --help Show this screen - --info Show a short description of this subcommand - --version Show version - --force This flag disable checks in Marathon during - update operations - --app-version= This flag specifies the application version to - use for the command. The application version - () can be specified as an - absolute value or as relative value. Absolute - version values must be in ISO8601 date format. - Relative values must be specified as a - negative integer and they represent the - version from the currently deployed - application definition - --config-schema Show the configuration schema for the Marathon - subcommand - --max-count= Maximum number of entries to try to fetch and - return - --interval= Number of seconds to wait between actions + -h, --help Show this screen + --info Show a short description of this + subcommand + --version Show version + --force This flag disable checks in Marathon + during update operations + --app-version= This flag specifies the application + version to use for the command. The + application version () can be + specified as an absolute value or as + relative value. Absolute version values + must be in ISO8601 date format. Relative + values must be specified as a negative + integer and they represent the version + from the currently deployed application + definition + --group-version= This flag specifies the group version to + use for the command. The group version + () can be specified as an + absolute value or as relative value. + Absolute version values must be in ISO8601 + date format. Relative values must be + specified as a negative integer and they + represent the version from the currently + deployed group definition + --config-schema Show the configuration schema for the + Marathon subcommand + --max-count= Maximum number of entries to try to fetch + and return + --interval= Number of seconds to wait between actions Positional arguments: The application id @@ -48,6 +63,10 @@ Positional arguments: description see (https://mesosphere.github.io/ marathon/docs/rest-api.html#post-/v2/apps) The deployment id + The group id + The group resource; for a detailed description + see (https://mesosphere.github.io/marathon/docs + /rest-api.html#post-/v2/groups) The number of instances to start Optional key-value pairs to be included in the command. The separator between the key and @@ -168,6 +187,26 @@ def _cmds(): arg_keys=['', '--force'], function=_restart), + cmds.Command( + hierarchy=['marathon', 'group', 'add'], + arg_keys=[''], + function=_group_add), + + cmds.Command( + hierarchy=['marathon', 'group', 'list'], + arg_keys=[], + function=_group_list), + + cmds.Command( + hierarchy=['marathon', 'group', 'show'], + arg_keys=['', '--group-version'], + function=_group_show), + + cmds.Command( + hierarchy=['marathon', 'group', 'remove'], + arg_keys=['', '--force'], + function=_group_remove), + cmds.Command( hierarchy=['marathon', 'about'], arg_keys=[], @@ -191,10 +230,7 @@ def _marathon(config_schema, info): """ if config_schema: - schema = json.loads( - pkg_resources.resource_string( - 'dcoscli', - 'data/config-schema/marathon.json').decode('utf-8')) + schema = _cli_config_schema() emitter.publish(schema) elif info: _info() @@ -227,6 +263,29 @@ def _about(): return 0 +def _get_resource(resource): + """ + :param resource: optional filename for the application or group resource + :type resource: str + :returns: resource + :rtype: dict + """ + if resource is not None: + with open(resource) as fd: + return util.load_json(fd) + + # Check that stdin is not tty + if sys.stdin.isatty(): + # We don't support TTY right now. In the future we will start an + # editor + raise DCOSException( + "We currently don't support reading from the TTY. Please " + "specify an application JSON.\n" + "Usage: dcos app add < app_resource.json") + + return util.load_json(sys.stdin) + + def _add(app_resource): """ :param app_resource: optional filename for the application resource @@ -234,26 +293,8 @@ def _add(app_resource): :returns: Process status :rtype: int """ - - if app_resource is not None: - with open(app_resource) as fd: - application_resource = util.load_json(fd) - else: - # Check that stdin is not tty - if sys.stdin.isatty(): - # We don't support TTY right now. In the future we will start an - # editor - raise DCOSException( - "We currently don't support reading from the TTY. Please " - "specify an application JSON.\n" - "Usage: dcos app add < app_resource.json") - - application_resource = util.load_json(sys.stdin) - - schema = json.loads( - pkg_resources.resource_string( - 'dcoscli', - 'data/marathon-schema.json').decode('utf-8')) + application_resource = _get_resource(app_resource) + schema = _app_schema() errs = util.validate_json(application_resource, schema) if errs: @@ -290,6 +331,51 @@ def _list(): return 0 +def _group_list(): + """ + :returns: process status + :rtype: int + """ + + client = marathon.create_client() + groups = client.get_groups() + + emitter.publish(groups) + return 0 + + +def _group_add(group_resource): + """ + :param group_resource: optional filename for the group resource + :type group_resource: str + :returns: Process status + :rtype: int + """ + + group_resource = _get_resource(group_resource) + schema = _data_schema() + + errs = util.validate_json(group_resource, schema) + if errs: + raise DCOSException(util.list_to_err(errs)) + + client = marathon.create_client() + + # Check that the group doesn't exist + group_id = client.normalize_app_id(group_resource['id']) + + try: + client.get_group(group_id) + except DCOSException as e: + logger.exception(e) + else: + raise DCOSException("Group '{}' already exists".format(group_id)) + + client.create_group(group_resource) + + return 0 + + def _remove(app_id, force): """ :param app_id: ID of the app to remove @@ -305,6 +391,21 @@ def _remove(app_id, force): return 0 +def _group_remove(group_id, force): + """ + :param group_id: ID of the app to remove + :type group_id: str + :param force: Whether to override running deployments. + :type force: bool + :returns: Process status + :rtype: int + """ + + client = marathon.create_client() + client.remove_group(group_id, force) + return 0 + + def _show(app_id, version): """Show details of a Marathon application. @@ -327,6 +428,25 @@ def _show(app_id, version): return 0 +def _group_show(group_id, version=None): + """Show details of a Marathon application. + + :param group_id: The id for the application + :type group_id: str + :param version: The version, either absolute (date-time) or relative + :type version: str + :returns: Process status + :rtype: int + """ + + client = marathon.create_client() + + app = client.get_group(group_id, version=version) + + emitter.publish(app) + return 0 + + def _start(app_id, instances, force): """Start a Marathon application. @@ -352,10 +472,7 @@ def _start(app_id, instances, force): desc['instances'])) return 1 - schema = json.loads( - pkg_resources.resource_string( - 'dcoscli', - 'data/marathon-schema.json').decode('utf-8')) + schema = _app_schema() # Need to add the 'id' because it is required app_json = {'id': app_id} @@ -443,10 +560,7 @@ def _update(app_id, json_items, force): else: return _update_from_stdin(app_id, force) - schema = json.loads( - pkg_resources.resource_string( - 'dcoscli', - 'data/marathon-schema.json').decode('utf-8')) + schema = _app_schema() # Need to add the 'id' because it is required app_json = {'id': app_id} @@ -684,6 +798,37 @@ def _calculate_version(client, app_id, version): raise DCOSException(msg.format(app_id, len(versions), value)) else: return versions[value] + else: raise DCOSException( 'Relative versions must be negative: {}'.format(version)) + + +def _cli_config_schema(): + """ + :returns: schema for marathon cli config + :rtype: dict + """ + return json.loads( + pkg_resources.resource_string( + 'dcoscli', + 'data/config-schema/marathon.json').decode('utf-8')) + + +def _data_schema(): + """ + :returns: schema for marathon data + :rtype: dict + """ + return json.loads( + pkg_resources.resource_string( + 'dcoscli', + 'data/marathon-group-schema.json').decode('utf-8')) + + +def _app_schema(): + """ + :returns: schema for apps + :rtype: dict + """ + return _data_schema()['definitions']['app'] diff --git a/cli/tests/data/marathon/bad.json b/cli/tests/data/marathon/apps/bad.json similarity index 100% rename from cli/tests/data/marathon/bad.json rename to cli/tests/data/marathon/apps/bad.json diff --git a/cli/tests/data/marathon/update_zero_instance_sleep.json b/cli/tests/data/marathon/apps/update_zero_instance_sleep.json similarity index 100% rename from cli/tests/data/marathon/update_zero_instance_sleep.json rename to cli/tests/data/marathon/apps/update_zero_instance_sleep.json diff --git a/cli/tests/data/marathon/zero_instance_sleep.json b/cli/tests/data/marathon/apps/zero_instance_sleep.json similarity index 100% rename from cli/tests/data/marathon/zero_instance_sleep.json rename to cli/tests/data/marathon/apps/zero_instance_sleep.json diff --git a/cli/tests/data/marathon/zero_instance_sleep_v2.json b/cli/tests/data/marathon/apps/zero_instance_sleep_v2.json similarity index 100% rename from cli/tests/data/marathon/zero_instance_sleep_v2.json rename to cli/tests/data/marathon/apps/zero_instance_sleep_v2.json diff --git a/cli/tests/data/marathon/groups/bad_app.json b/cli/tests/data/marathon/groups/bad_app.json new file mode 100644 index 0000000..34d6419 --- /dev/null +++ b/cli/tests/data/marathon/groups/bad_app.json @@ -0,0 +1,15 @@ +{ + "groups": [ + { + "id": "notgood", + "apps": [ + { + "id": "hi", + "cmd": "sleep 0", + "badtype": 0 + } + ] + } + ], + "id": "bad-group" +} diff --git a/cli/tests/data/marathon/groups/bad_group.json b/cli/tests/data/marathon/groups/bad_group.json new file mode 100644 index 0000000..304e2d2 --- /dev/null +++ b/cli/tests/data/marathon/groups/bad_group.json @@ -0,0 +1,15 @@ +{ + "groups": [ + { + "fakeapp": [ + { + "cmds": "sleep 0", + "id": "hi", + "instances": 0 + } + ], + "id": "notgood" + } + ], + "id": "bad-group" +} diff --git a/cli/tests/data/marathon/groups/complicated.json b/cli/tests/data/marathon/groups/complicated.json new file mode 100644 index 0000000..69ba177 --- /dev/null +++ b/cli/tests/data/marathon/groups/complicated.json @@ -0,0 +1,37 @@ +{ + "apps": [ + { + "cmd": "sleep 1", + "id": "appingroups", + "instances": 0 + } + ], + "groups": [ + { + "apps": [ + { + "cmd": "sleep 10", + "id": "sleep10", + "instances": 0 + } + ], + "id": "app" + }, + { + "groups": [ + { + "apps": [ + { + "cmd": "sleep 1", + "id": "sleep1", + "instances": 0 + } + ], + "id": "moregroups" + } + ], + "id": "moregroups" + } + ], + "id": "test-group" +} diff --git a/cli/tests/data/marathon/groups/complicated_bad.json b/cli/tests/data/marathon/groups/complicated_bad.json new file mode 100644 index 0000000..b0fc719 --- /dev/null +++ b/cli/tests/data/marathon/groups/complicated_bad.json @@ -0,0 +1,30 @@ +{ + "apps": [ + { + "cmd": "sleep 1", + "id": "appingroups" + } + ], + "groups": [ + { + "apps": [ + {} + ], + "id": "appingroups" + }, + { + "groups": [ + { + "apps": [ + { + "cmd": "sleep 1", + "id": "sleep1" + } + ] + } + ], + "id": "moregroups" + } + ], + "id": "test-group" +} diff --git a/cli/tests/data/marathon/groups/good.json b/cli/tests/data/marathon/groups/good.json new file mode 100644 index 0000000..dc0bb62 --- /dev/null +++ b/cli/tests/data/marathon/groups/good.json @@ -0,0 +1,15 @@ +{ + "groups": [ + { + "apps": [ + { + "id": "goodnight", + "cmd": "sleep 1", + "instances": 0 + } + ], + "id": "sleep" + } + ], + "id": "test-group" +} diff --git a/cli/tests/data/marathon/sleep.json b/cli/tests/data/marathon/sleep.json deleted file mode 100644 index fd7c919..0000000 --- a/cli/tests/data/marathon/sleep.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "id": "test-app", - "cmd": "sleep 1000", - "cpus": 0.1, - "mem": 16, - "instances": 1, - "labels": { - "PACKAGE_ID": "test-app", - "PACKAGE_VERSION": "1.2.3" - } -} diff --git a/cli/tests/integrations/cli/marathon_common.py b/cli/tests/integrations/cli/marathon_common.py new file mode 100644 index 0000000..5cc8221 --- /dev/null +++ b/cli/tests/integrations/cli/marathon_common.py @@ -0,0 +1,29 @@ +import json + +from common import exec_command + + +def watch_deployment(deployment_id, count): + returncode, stdout, stderr = exec_command( + ['dcos', 'marathon', 'deployment', 'watch', + '--max-count={}'.format(count), deployment_id]) + + assert returncode == 0 + assert stderr == b'' + + +def list_deployments(expected_count, app_id=None): + cmd = ['dcos', 'marathon', 'deployment', 'list'] + if app_id is not None: + cmd.append(app_id) + + returncode, stdout, stderr = exec_command(cmd) + + result = json.loads(stdout.decode('utf-8')) + + assert returncode == 0 + if expected_count is not None: + assert len(result) == expected_count + assert stderr == b'' + + return result diff --git a/cli/tests/integrations/cli/test_marathon.py b/cli/tests/integrations/cli/test_marathon.py index 0443ea3..6516250 100644 --- a/cli/tests/integrations/cli/test_marathon.py +++ b/cli/tests/integrations/cli/test_marathon.py @@ -5,6 +5,7 @@ from dcos import constants import pytest from common import assert_command, exec_command +from marathon_common import list_deployments, watch_deployment def test_help(): @@ -30,27 +31,42 @@ Usage: [--interval=] dcos marathon task list [] dcos marathon task show + dcos marathon group add [] + dcos marathon group list + dcos marathon group show [--group-version=] + dcos marathon group remove [--force] Options: - -h, --help Show this screen - --info Show a short description of this subcommand - --version Show version - --force This flag disable checks in Marathon during - update operations - --app-version= This flag specifies the application version to - use for the command. The application version - () can be specified as an - absolute value or as relative value. Absolute - version values must be in ISO8601 date format. - Relative values must be specified as a - negative integer and they represent the - version from the currently deployed - application definition - --config-schema Show the configuration schema for the Marathon - subcommand - --max-count= Maximum number of entries to try to fetch and - return - --interval= Number of seconds to wait between actions + -h, --help Show this screen + --info Show a short description of this + subcommand + --version Show version + --force This flag disable checks in Marathon + during update operations + --app-version= This flag specifies the application + version to use for the command. The + application version () can be + specified as an absolute value or as + relative value. Absolute version values + must be in ISO8601 date format. Relative + values must be specified as a negative + integer and they represent the version + from the currently deployed application + definition + --group-version= This flag specifies the group version to + use for the command. The group version + () can be specified as an + absolute value or as relative value. + Absolute version values must be in ISO8601 + date format. Relative values must be + specified as a negative integer and they + represent the version from the currently + deployed group definition + --config-schema Show the configuration schema for the + Marathon subcommand + --max-count= Maximum number of entries to try to fetch + and return + --interval= Number of seconds to wait between actions Positional arguments: The application id @@ -58,6 +74,10 @@ Positional arguments: description see (https://mesosphere.github.io/ marathon/docs/rest-api.html#post-/v2/apps) The deployment id + The group id + The group resource; for a detailed description + see (https://mesosphere.github.io/marathon/docs + /rest-api.html#post-/v2/groups) The number of instances to start Optional key-value pairs to be included in the command. The separator between the key and @@ -111,27 +131,27 @@ def test_empty_list(): def test_add_app(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _list_apps('zero-instance-app') _remove_app('zero-instance-app') def test_optional_add_app(): assert_command(['dcos', 'marathon', 'app', 'add', - 'tests/data/marathon/zero_instance_sleep.json']) + 'tests/data/marathon/apps/zero_instance_sleep.json']) _list_apps('zero-instance-app') _remove_app('zero-instance-app') def test_remove_app(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _remove_app('zero-instance-app') _list_apps() def test_add_bad_json_app(): - with open('tests/data/marathon/bad.json') as fd: + with open('tests/data/marathon/apps/bad.json') as fd: returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'add'], stdin=fd) @@ -142,9 +162,9 @@ def test_add_bad_json_app(): def test_add_existing_app(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') - with open('tests/data/marathon/zero_instance_sleep_v2.json') as fd: + with open('tests/data/marathon/apps/zero_instance_sleep_v2.json') as fd: stderr = b"Application '/zero-instance-app' already exists\n" assert_command(['dcos', 'marathon', 'app', 'add'], returncode=1, @@ -155,16 +175,16 @@ def test_add_existing_app(): def test_show_app(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _show_app('zero-instance-app') _remove_app('zero-instance-app') def test_show_absolute_app_version(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _update_app( 'zero-instance-app', - 'tests/data/marathon/update_zero_instance_sleep.json') + 'tests/data/marathon/apps/update_zero_instance_sleep.json') result = _show_app('zero-instance-app') _show_app('zero-instance-app', result['version']) @@ -173,19 +193,19 @@ def test_show_absolute_app_version(): def test_show_relative_app_version(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _update_app( 'zero-instance-app', - 'tests/data/marathon/update_zero_instance_sleep.json') + 'tests/data/marathon/apps/update_zero_instance_sleep.json') _show_app('zero-instance-app', "-1") _remove_app('zero-instance-app') def test_show_missing_relative_app_version(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _update_app( 'zero-instance-app', - 'tests/data/marathon/update_zero_instance_sleep.json') + 'tests/data/marathon/apps/update_zero_instance_sleep.json') stderr = b"Application 'zero-instance-app' only has 2 version(s).\n" assert_command(['dcos', 'marathon', 'app', 'show', @@ -197,10 +217,10 @@ def test_show_missing_relative_app_version(): def test_show_missing_absolute_app_version(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _update_app( 'zero-instance-app', - 'tests/data/marathon/update_zero_instance_sleep.json') + 'tests/data/marathon/apps/update_zero_instance_sleep.json') returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'show', @@ -215,10 +235,10 @@ def test_show_missing_absolute_app_version(): def test_show_bad_app_version(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _update_app( 'zero-instance-app', - 'tests/data/marathon/update_zero_instance_sleep.json') + 'tests/data/marathon/apps/update_zero_instance_sleep.json') stderr = (b'Error: Invalid format: "20:39:32.972Z" is malformed at ' b'":39:32.972Z"\n') @@ -232,10 +252,10 @@ def test_show_bad_app_version(): def test_show_bad_relative_app_version(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _update_app( 'zero-instance-app', - 'tests/data/marathon/update_zero_instance_sleep.json') + 'tests/data/marathon/apps/update_zero_instance_sleep.json') assert_command( ['dcos', 'marathon', 'app', 'show', @@ -254,13 +274,13 @@ def test_start_missing_app(): def test_start_app(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _start_app('zero-instance-app') _remove_app('zero-instance-app') def test_start_already_started_app(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _start_app('zero-instance-app') stdout = b"Application 'zero-instance-app' already started: 1 instances.\n" @@ -278,10 +298,10 @@ def test_stop_missing_app(): def test_stop_app(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _start_app('zero-instance-app', 3) - result = _list_deployments(1, 'zero-instance-app') - _watch_deployment(result[0]['id'], 60) + result = list_deployments(1, 'zero-instance-app') + watch_deployment(result[0]['id'], 60) returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'stop', 'zero-instance-app']) @@ -294,7 +314,7 @@ def test_stop_app(): def test_stop_already_stopped_app(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') stdout = b"Application 'zero-instance-app' already stopped: 0 instances.\n" assert_command(['dcos', 'marathon', 'app', 'stop', 'zero-instance-app'], @@ -311,7 +331,7 @@ def test_update_missing_app(): def test_update_missing_field(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'update', @@ -327,7 +347,7 @@ def test_update_missing_field(): def test_update_bad_type(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'update', @@ -343,7 +363,7 @@ def test_update_bad_type(): def test_update_app(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'update', 'zero-instance-app', @@ -357,16 +377,16 @@ def test_update_app(): def test_update_app_from_stdin(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _update_app( 'zero-instance-app', - 'tests/data/marathon/update_zero_instance_sleep.json') + 'tests/data/marathon/apps/update_zero_instance_sleep.json') _remove_app('zero-instance-app') def test_restarting_stopped_app(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') stdout = (b"Unable to perform rolling restart of application '" b"/zero-instance-app' because it has no running tasks\n") @@ -384,10 +404,10 @@ def test_restarting_missing_app(): def test_restarting_app(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _start_app('zero-instance-app', 3) - result = _list_deployments(1, 'zero-instance-app') - _watch_deployment(result[0]['id'], 60) + result = list_deployments(1, 'zero-instance-app') + watch_deployment(result[0]['id'], 60) returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'restart', 'zero-instance-app']) @@ -414,22 +434,22 @@ def test_list_version_negative_max_count(): def test_list_version_app(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _list_versions('zero-instance-app', 1) _update_app( 'zero-instance-app', - 'tests/data/marathon/update_zero_instance_sleep.json') + 'tests/data/marathon/apps/update_zero_instance_sleep.json') _list_versions('zero-instance-app', 2) _remove_app('zero-instance-app') def test_list_version_max_count(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _update_app( 'zero-instance-app', - 'tests/data/marathon/update_zero_instance_sleep.json') + 'tests/data/marathon/apps/update_zero_instance_sleep.json') _list_versions('zero-instance-app', 1, 1) _list_versions('zero-instance-app', 2, 2) @@ -439,27 +459,27 @@ def test_list_version_max_count(): def test_list_empty_deployment(): - _list_deployments(0) + list_deployments(0) def test_list_deployment(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _start_app('zero-instance-app', 3) - _list_deployments(1) + list_deployments(1) _remove_app('zero-instance-app') def test_list_deployment_missing_app(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _start_app('zero-instance-app') - _list_deployments(0, 'missing-id') + list_deployments(0, 'missing-id') _remove_app('zero-instance-app') def test_list_deployment_app(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _start_app('zero-instance-app', 3) - _list_deployments(1, 'zero-instance-app') + list_deployments(1, 'zero-instance-app') _remove_app('zero-instance-app') @@ -471,9 +491,9 @@ def test_rollback_missing_deployment(): def test_rollback_deployment(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _start_app('zero-instance-app', 3) - result = _list_deployments(1, 'zero-instance-app') + result = list_deployments(1, 'zero-instance-app') returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'deployment', 'rollback', result[0]['id']]) @@ -485,33 +505,33 @@ def test_rollback_deployment(): assert 'version' in result assert stderr == b'' - _list_deployments(0) + list_deployments(0) _remove_app('zero-instance-app') def test_stop_deployment(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _start_app('zero-instance-app', 3) - result = _list_deployments(1, 'zero-instance-app') + result = list_deployments(1, 'zero-instance-app') assert_command(['dcos', 'marathon', 'deployment', 'stop', result[0]['id']]) - _list_deployments(0) + list_deployments(0) _remove_app('zero-instance-app') def test_watching_missing_deployment(): - _watch_deployment('missing-deployment', 1) + watch_deployment('missing-deployment', 1) def test_watching_deployment(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _start_app('zero-instance-app', 3) - result = _list_deployments(1, 'zero-instance-app') - _watch_deployment(result[0]['id'], 60) - _list_deployments(0, 'zero-instance-app') + result = list_deployments(1, 'zero-instance-app') + watch_deployment(result[0]['id'], 60) + list_deployments(0, 'zero-instance-app') _remove_app('zero-instance-app') @@ -520,34 +540,34 @@ def test_list_empty_task(): def test_list_empty_task_not_running_app(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _list_tasks(0) _remove_app('zero-instance-app') def test_list_tasks(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _start_app('zero-instance-app', 3) - result = _list_deployments(1, 'zero-instance-app') - _watch_deployment(result[0]['id'], 60) + result = list_deployments(1, 'zero-instance-app') + watch_deployment(result[0]['id'], 60) _list_tasks(3) _remove_app('zero-instance-app') def test_list_app_tasks(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _start_app('zero-instance-app', 3) - result = _list_deployments(1, 'zero-instance-app') - _watch_deployment(result[0]['id'], 60) + result = list_deployments(1, 'zero-instance-app') + watch_deployment(result[0]['id'], 60) _list_tasks(3, 'zero-instance-app') _remove_app('zero-instance-app') def test_list_missing_app_tasks(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _start_app('zero-instance-app', 3) - result = _list_deployments(1, 'zero-instance-app') - _watch_deployment(result[0]['id'], 60) + result = list_deployments(1, 'zero-instance-app') + watch_deployment(result[0]['id'], 60) _list_tasks(0, 'missing-id') _remove_app('zero-instance-app') @@ -565,10 +585,10 @@ def test_show_missing_task(): def test_show_task(): - _add_app('tests/data/marathon/zero_instance_sleep.json') + _add_app('tests/data/marathon/apps/zero_instance_sleep.json') _start_app('zero-instance-app', 3) - result = _list_deployments(1, 'zero-instance-app') - _watch_deployment(result[0]['id'], 60) + result = list_deployments(1, 'zero-instance-app') + watch_deployment(result[0]['id'], 60) result = _list_tasks(3, 'zero-instance-app') returncode, stdout, stderr = exec_command( @@ -628,9 +648,9 @@ def _remove_app(app_id): assert_command(['dcos', 'marathon', 'app', 'remove', app_id]) # Let's make sure that we don't return until the deployment has finished - result = _list_deployments(None, app_id) + result = list_deployments(None, app_id) if len(result) != 0: - _watch_deployment(result[0]['id'], 60) + watch_deployment(result[0]['id'], 60) def _add_app(file_path): @@ -696,32 +716,6 @@ def _list_versions(app_id, expected_count, max_count=None): assert stderr == b'' -def _list_deployments(expected_count, app_id=None): - cmd = ['dcos', 'marathon', 'deployment', 'list'] - if app_id is not None: - cmd.append(app_id) - - returncode, stdout, stderr = exec_command(cmd) - - result = json.loads(stdout.decode('utf-8')) - - assert returncode == 0 - if expected_count is not None: - assert len(result) == expected_count - assert stderr == b'' - - return result - - -def _watch_deployment(deployment_id, count): - returncode, stdout, stderr = exec_command( - ['dcos', 'marathon', 'deployment', 'watch', - '--max-count={}'.format(count), deployment_id]) - - assert returncode == 0 - assert stderr == b'' - - def _list_tasks(expected_count, app_id=None): cmd = ['dcos', 'marathon', 'task', 'list'] if app_id is not None: diff --git a/cli/tests/integrations/cli/test_marathon_groups.py b/cli/tests/integrations/cli/test_marathon_groups.py new file mode 100644 index 0000000..d1fca9e --- /dev/null +++ b/cli/tests/integrations/cli/test_marathon_groups.py @@ -0,0 +1,168 @@ +import json + +from common import assert_command, exec_command +from marathon_common import list_deployments, watch_deployment + + +def test_add_group(): + _add_group('tests/data/marathon/groups/good.json') + _list_groups('test-group/sleep/goodnight') + result = list_deployments(None, 'test-group/sleep/goodnight') + if len(result) != 0: + watch_deployment(result[0]['id'], 60) + _remove_group('test-group') + + +def test_validate_complicated_group_and_app(): + _add_group('tests/data/marathon/groups/complicated.json') + result = list_deployments(None, 'test-group/moregroups/moregroups/sleep1') + if len(result) != 0: + watch_deployment(result[0]['id'], 60) + _remove_group('test-group') + + +def test_optional_add_group(): + assert_command(['dcos', 'marathon', 'group', 'add', + 'tests/data/marathon/groups/good.json']) + + _list_groups('test-group/sleep/goodnight') + result = list_deployments(None, 'test-group/sleep/goodnight') + if len(result) != 0: + watch_deployment(result[0]['id'], 60) + _remove_group('test-group') + + +def test_add_existing_group(): + _add_group('tests/data/marathon/groups/good.json') + + result = list_deployments(None, 'test-group/sleep/goodnight') + if len(result) != 0: + watch_deployment(result[0]['id'], 60) + + with open('tests/data/marathon/groups/good.json') as fd: + stderr = b"Group '/test-group' already exists\n" + assert_command(['dcos', 'marathon', 'group', 'add'], + returncode=1, + stderr=stderr, + stdin=fd) + + result = list_deployments(None, 'test-group/sleep/goodnight') + if len(result) != 0: + watch_deployment(result[0]['id'], 60) + _remove_group('test-group') + + +def test_show_group(): + _add_group('tests/data/marathon/groups/good.json') + _list_groups('test-group/sleep/goodnight') + result = list_deployments(None, 'test-group/sleep/goodnight') + if len(result) != 0: + watch_deployment(result[0]['id'], 60) + _show_group('test-group') + _remove_group('test-group') + + +def test_add_bad_app(): + with open('tests/data/marathon/groups/bad_app.json') as fd: + returncode, stdout, stderr = exec_command( + ['dcos', 'marathon', 'group', 'add'], + stdin=fd) + + expected = "Error: Additional properties are not allowed" + \ + " ('badtype' was unexpected)" + assert returncode == 1 + assert stdout == b'' + assert stderr.decode('utf-8').startswith(expected) + + +def test_add_bad_group(): + with open('tests/data/marathon/groups/bad_group.json') as fd: + returncode, stdout, stderr = exec_command( + ['dcos', 'marathon', 'group', 'add'], + stdin=fd) + + expected = "Error: Additional properties are not allowed" + \ + " ('fakeapp' was unexpected)" + assert returncode == 1 + assert stdout == b'' + assert stderr.decode('utf-8').startswith(expected) + + +def test_add_bad_complicated_group(): + with open('tests/data/marathon/groups/complicated_bad.json') as fd: + returncode, stdout, stderr = exec_command( + ['dcos', 'marathon', 'group', 'add'], + stdin=fd) + + # id for group in test-group/more-groups + err = "Property missing which is mandatory" + # missing id in apps in appingroups + err2 = "identifier / is not child of /test-group/appingroups" + # missing cmd in appingroups + err3 = "AppDefinition must either contain one of 'cmd' or 'args', " + \ + "and/or a 'container'" + assert returncode == 1 + assert stdout == b'' + assert err in stderr.decode('utf-8') + assert err2 in stderr.decode('utf-8') + assert err3 in stderr.decode('utf-8') + + +def _list_groups(group_id=None): + returncode, stdout, stderr = exec_command( + ['dcos', 'marathon', 'group', 'list']) + + result = json.loads(stdout.decode('utf-8')) + + if group_id is None: + assert len(result) == 0 + else: + groups = None + for g in group_id.split("/")[:-1]: + if groups is None: + result = result[0] + groups = "/{}".format(g) + else: + result = result['groups'][0] + groups += g + assert result['id'] == groups + groups += "/" + assert result['apps'][0]['id'] == "/" + group_id + + assert returncode == 0 + assert stderr == b'' + + return result + + +def _remove_group(group_id): + assert_command(['dcos', 'marathon', 'group', 'remove', group_id]) + + # Let's make sure that we don't return until the deployment has finished + result = list_deployments(None, group_id) + if len(result) != 0: + watch_deployment(result[0]['id'], 60) + + +def _add_group(file_path): + with open(file_path) as fd: + assert_command(['dcos', 'marathon', 'group', 'add'], stdin=fd) + + +def _show_group(group_id, version=None): + if version is None: + cmd = ['dcos', 'marathon', 'group', 'show', group_id] + else: + cmd = ['dcos', 'marathon', 'group', 'show', + '--group-version={}'.format(version), group_id] + + returncode, stdout, stderr = exec_command(cmd) + + result = json.loads(stdout.decode('utf-8')) + + assert returncode == 0 + assert isinstance(result, dict) + assert result['id'] == '/' + group_id + assert stderr == b'' + + return result diff --git a/dcos/marathon.py b/dcos/marathon.py index 1a7ff54..c50cfc3 100644 --- a/dcos/marathon.py +++ b/dcos/marathon.py @@ -93,7 +93,6 @@ class Client(object): def _create_url(self, path): """Creates the url from the provided path. - :param path: url path :type path: str :returns: constructed url @@ -142,11 +141,49 @@ class Client(object): else: return response.json() + def get_groups(self): + """Get a list of known groups. + + :returns: list of known groups + :rtype: list of dict + """ + + url = self._create_url('v2/groups') + + response = http.get(url, to_error=_to_error) + + return response.json()['groups'] + + def get_group(self, group_id, version=None): + """Returns a representation of the requested group version. If + version is None the return the latest version. + + :param group_id: the ID of the application + :type group_id: str + :param version: application version as a ISO8601 datetime + :type version: str + :returns: the requested Marathon application + :rtype: dict + """ + + group_id = self.normalize_app_id(group_id) + if version is None: + url = self._create_url('v2/groups{}'.format(group_id)) + else: + url = self._create_url( + 'v2/groups{}/versions/{}'.format(group_id, version)) + + response = http.get(url, to_error=_to_error) + + return response.json() + def get_app_versions(self, app_id, max_count=None): """Asks Marathon for all the versions of the Application up to a maximum count. - :param app_id: the ID of the application + :param app_id: the ID of the application or group + :type app_id: str + :param id_type: type of the id (apps or groups) :type app_id: str :param max_count: the maximum number of version to fetch :type max_count: int @@ -302,6 +339,27 @@ class Client(object): http.delete(url, params=params, to_error=_to_error) + def remove_group(self, group_id, force=None): + """Completely removes the requested application. + + :param group_id: the ID of the application to remove + :type group_id: str + :param force: whether to override running deployments + :type force: bool + :rtype: None + """ + + group_id = self.normalize_app_id(group_id) + + if not force: + params = None + else: + params = {'force': 'true'} + + url = self._create_url('v2/groups{}'.format(group_id)) + + http.delete(url, params=params, to_error=_to_error) + def restart_app(self, app_id, force=None): """Performs a rolling restart of all of the tasks. @@ -480,6 +538,26 @@ class Client(object): return urllib.parse.quote('/' + app_id.strip('/')) + def create_group(self, group_resource): + """Add a new group. + + :param group_resource: grouplication resource + :type group_resource: dict, bytes or file + :returns: the group description + :rtype: dict + """ + url = self._create_url('v2/groups') + + # The file type exists only in Python 2, preventing type(...) is file. + if hasattr(group_resource, 'read'): + group_json = json.load(group_resource) + else: + group_json = group_resource + + response = http.post(url, json=group_json, to_error=_to_error) + + return response.json() + def _default_marathon_error(message=""): """ diff --git a/dcos/util.py b/dcos/util.py index ce46b23..1171ca1 100644 --- a/dcos/util.py +++ b/dcos/util.py @@ -278,7 +278,8 @@ def _format_validation_error(error): else: message = 'Error: {}\n'.format(error_message) if len(error.absolute_path) > 0: - message += 'Path: {}\n'.format('.'.join(error.absolute_path)) + message += 'Path: {}\n'.format( + '.'.join([str(path) for path in error.absolute_path])) message += 'Value: {}'.format(json.dumps(error.instance)) return message