diff --git a/cli/dcoscli/package/main.py b/cli/dcoscli/package/main.py index 002315e..16c7ead 100644 --- a/cli/dcoscli/package/main.py +++ b/cli/dcoscli/package/main.py @@ -4,6 +4,7 @@ import os import sys import tempfile import zipfile +from collections import defaultdict import dcoscli import docopt @@ -13,6 +14,7 @@ from dcos import (cmds, emitting, errors, http, marathon, options, package, from dcos.errors import DCOSException from dcoscli import tables from dcoscli.main import decorate_docopt_usage +from six import iteritems logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() @@ -682,7 +684,23 @@ def _bundle_uris(uris_directory, zip_file): :rtype: None """ - for filename in sorted(os.listdir(uris_directory)): + uris = sorted(os.listdir(uris_directory)) + + # these uris will be found through a mustache template from the property + # name so make sure each name is unique. + uri_properties = defaultdict(list) + for name in uris: + uri_properties[name.replace('.', '-')].append(name) + + collisions = [uri_list for (prop, uri_list) in iteritems(uri_properties) + if len(uri_list) > 1] + + if collisions: + raise DCOSException( + 'Error bundling package. Multiple assets map to the same property ' + 'name (periods [.] are replaced with dashes [-]): {}'.format( + collisions)) + for filename in uris: fullpath = os.path.join(uris_directory, filename) zip_file.write(fullpath, arcname='assets/uris/{}'.format(filename)) diff --git a/cli/tests/data/package/cassandra.zip b/cli/tests/data/package/cassandra.zip index 10b47b1..581ad08 100644 Binary files a/cli/tests/data/package/cassandra.zip and b/cli/tests/data/package/cassandra.zip differ diff --git a/cli/tests/data/package/create-directory/marathon.json.mustache b/cli/tests/data/package/create-directory/marathon.json.mustache index b36ef3f..ab3f247 100644 --- a/cli/tests/data/package/create-directory/marathon.json.mustache +++ b/cli/tests/data/package/create-directory/marathon.json.mustache @@ -5,8 +5,8 @@ "cpus": {{cassandra.framework.cpus}}, "mem": {{cassandra.framework.mem}}, "uris": [ - "{{package.assets.uris.cassandra-mesos.tar.gz}}", - "{{package.assets.uris.jre-linux.tar.gz}}" + "{{resource.assets.uris.cassandra-mesos-tar-gz}}", + "{{resource.assets.uris.jre-linux-tar-gz}}" ], "ports": [ 0 diff --git a/cli/tests/data/package/extra-assets-files-directory/marathon.json.mustache b/cli/tests/data/package/extra-assets-files-directory/marathon.json.mustache index b36ef3f..ab3f247 100644 --- a/cli/tests/data/package/extra-assets-files-directory/marathon.json.mustache +++ b/cli/tests/data/package/extra-assets-files-directory/marathon.json.mustache @@ -5,8 +5,8 @@ "cpus": {{cassandra.framework.cpus}}, "mem": {{cassandra.framework.mem}}, "uris": [ - "{{package.assets.uris.cassandra-mesos.tar.gz}}", - "{{package.assets.uris.jre-linux.tar.gz}}" + "{{resource.assets.uris.cassandra-mesos-tar-gz}}", + "{{resource.assets.uris.jre-linux-tar-gz}}" ], "ports": [ 0 diff --git a/cli/tests/data/package/extra-files-directory/marathon.json.mustache b/cli/tests/data/package/extra-files-directory/marathon.json.mustache index b36ef3f..ab3f247 100644 --- a/cli/tests/data/package/extra-files-directory/marathon.json.mustache +++ b/cli/tests/data/package/extra-files-directory/marathon.json.mustache @@ -5,8 +5,8 @@ "cpus": {{cassandra.framework.cpus}}, "mem": {{cassandra.framework.mem}}, "uris": [ - "{{package.assets.uris.cassandra-mesos.tar.gz}}", - "{{package.assets.uris.jre-linux.tar.gz}}" + "{{resource.assets.uris.cassandra-mesos-tar-gz}}", + "{{resource.assets.uris.jre-linux-tar-gz}}" ], "ports": [ 0 diff --git a/cli/tests/data/package/extra-icons-directory/marathon.json.mustache b/cli/tests/data/package/extra-icons-directory/marathon.json.mustache index b36ef3f..ab3f247 100644 --- a/cli/tests/data/package/extra-icons-directory/marathon.json.mustache +++ b/cli/tests/data/package/extra-icons-directory/marathon.json.mustache @@ -5,8 +5,8 @@ "cpus": {{cassandra.framework.cpus}}, "mem": {{cassandra.framework.mem}}, "uris": [ - "{{package.assets.uris.cassandra-mesos.tar.gz}}", - "{{package.assets.uris.jre-linux.tar.gz}}" + "{{resource.assets.uris.cassandra-mesos-tar-gz}}", + "{{resource.assets.uris.jre-linux-tar-gz}}" ], "ports": [ 0 diff --git a/cli/tests/data/package/invalid-asset-names-directory/assets/uris/cassandra-mesos-tar-gz b/cli/tests/data/package/invalid-asset-names-directory/assets/uris/cassandra-mesos-tar-gz new file mode 100644 index 0000000..e69de29 diff --git a/cli/tests/data/package/invalid-asset-names-directory/assets/uris/cassandra-mesos.tar.gz b/cli/tests/data/package/invalid-asset-names-directory/assets/uris/cassandra-mesos.tar.gz new file mode 100644 index 0000000..e69de29 diff --git a/cli/tests/data/package/invalid-asset-names-directory/config.json b/cli/tests/data/package/invalid-asset-names-directory/config.json new file mode 100644 index 0000000..32ec2c7 --- /dev/null +++ b/cli/tests/data/package/invalid-asset-names-directory/config.json @@ -0,0 +1,229 @@ +{ + "type": "object", + "properties": { + "mesos": { + "description": "Mesos specific configuration properties", + "type": "object", + "properties": { + "master": { + "default": "zk://master.mesos:2181/mesos", + "description": "The URL of the Mesos master. The format is a comma-delimited list of hosts like zk://host1:port,host2:port/mesos. If using ZooKeeper, pay particular attention to the leading zk:// and trailing /mesos! If not using ZooKeeper, standard host:port patterns, like localhost:5050 or 10.0.0.5:5050,10.0.0.6:5050, are also acceptable.", + "type": "string" + } + }, + "required": [ + "master" + ] + }, + "cassandra": { + "description": "Cassandra Framework Configuration Properties", + "type": "object", + "additionalProperties": false, + "properties": { + "framework": { + "description": "Framework Scheduler specific Configuration Properties", + "type": "object", + "additionalProperties": false, + "properties": { + "failover-timeout-seconds": { + "description": "The failover_timeout for Mesos in seconds. If the framework instance has not re-registered with Mesos this long after a failover, Mesos will shut down all running tasks started by the framework.", + "type": "integer", + "minimum": 0, + "default": 604800 + }, + "cpus": { + "default": 0.5, + "description": "CPU shares to allocate to each Cassandra framework instance.", + "type": "number" + }, + "mem": { + "default": 512.0, + "description": "Memory (MB) to allocate to each Cassandra framework instance.", + "minimum": 512.0, + "type": "number" + }, + "instances": { + "default": 1, + "description": "Number of Cassandra framework instances to run.", + "minimum": 0, + "type": "integer" + }, + "role": { + "description": "Mesos role for this framework.", + "type": "string", + "default": "*" + }, + "authentication": { + "description": "Framework Scheduler Authentication Configuration Properties", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Whether framework authentication should be used", + "type": "boolean", + "default": false + }, + "principal": { + "description": "The Mesos principal used for authentication.", + "type": "string" + }, + "secret": { + "description": "The path to the Mesos secret file containing the authentication secret.", + "type": "string" + } + }, + "required": [ + "enabled" + ] + } + }, + "required": [ + "instances", + "cpus", + "mem", + "failover-timeout-seconds", + "role", + "authentication" + ] + }, + "cluster-name": { + "description": "The name of the framework to register with mesos. Will also be used as the cluster name in Cassandra", + "type": "string", + "default": "dcos" + }, + "zk": { + "description": "ZooKeeper URL for storing state. Format: zk://host1:port1,host2:port2,.../path (can have nested directories)", + "type": "string" + }, + "zk-timeout-ms": { + "description": "Timeout for ZooKeeper in milliseconds.", + "type": "integer", + "minimum": 0, + "default": 10000 + }, + "node-count": { + "description": "The number of nodes in the ring for the framework to run.", + "type": "integer", + "minimum": 1, + "default": 3 + }, + "seed-count": { + "description": "The number of seed nodes in the ring for the framework to run.", + "type": "integer", + "minimum": 1, + "default": 2 + }, + "health-check-interval-seconds": { + "description": "The interval in seconds that the framework should check the health of each Cassandra Server instance.", + "type": "integer", + "minimum": 15, + "default": 60 + }, + "bootstrap-grace-time-seconds": { + "description": "The minimum number of seconds to wait between starting each node. Setting this too low could result in the ring not bootstrapping correctly.", + "type": "integer", + "minimum": 15, + "default": 120 + }, + "data-directory": { + "description": "The location on disk where Cassandra will be configured to write it's data.", + "type": "string", + "default": "." + }, + "resources": { + "description": "Cassandra Server Resources Configuration Properties", + "type": "object", + "additionalProperties": false, + "properties": { + "cpus": { + "description": "CPU shares to allocate to each Cassandra Server Instance.", + "type": "number", + "minimum": 0.0, + "default": 0.1 + }, + "mem": { + "description": "Memory (MB) to allocate to each Cassandra Server instance.", + "type": "integer", + "minimum": 0, + "default": 768 + }, + "disk": { + "description": "Disk (MB) to allocate to each Cassandra Server instance.", + "type": "integer", + "minimum": 0, + "default": 16 + }, + "heap-mb": { + "description": "The amount of memory in MB that are allocated to each Cassandra Server Instance. This value should be smaller than 'cassandra.resources.mem'. The remaining difference will be used for memory mapped files and other off-heap memory requirements.", + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "cpus", + "mem", + "disk" + ] + }, + "dc": { + "description": "Cassandra multi Datacenter Configuration Properties", + "type": "object", + "additionalProperties": false, + "properties": { + "default-dc": { + "description": "Default value to be set for dc name in the GossipingPropertyFileSnitch", + "type": "string", + "default": "DC1" + }, + "default-rack": { + "description": "Default value to be set for rack name in the GossipingPropertyFileSnitch", + "type": "string", + "default": "RAC1" + }, + "external-dcs": { + "description": "Name and URL for another instance of Cassandra DCOS Service", + "type": "array", + "additionalProperties": false, + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "name", + "url" + ] + } + } + }, + "required": [ + "default-dc", + "default-rack" + ] + } + }, + "required": [ + "framework", + "cluster-name", + "zk-timeout-ms", + "node-count", + "seed-count", + "health-check-interval-seconds", + "bootstrap-grace-time-seconds", + "data-directory", + "resources" + ] + } + }, + "required": [ + "mesos", + "cassandra" + ] + +} diff --git a/cli/tests/data/package/invalid-asset-names-directory/images/icon-large.png b/cli/tests/data/package/invalid-asset-names-directory/images/icon-large.png new file mode 100644 index 0000000..c6703ca Binary files /dev/null and b/cli/tests/data/package/invalid-asset-names-directory/images/icon-large.png differ diff --git a/cli/tests/data/package/invalid-asset-names-directory/images/icon-medium.png b/cli/tests/data/package/invalid-asset-names-directory/images/icon-medium.png new file mode 100644 index 0000000..5d84140 Binary files /dev/null and b/cli/tests/data/package/invalid-asset-names-directory/images/icon-medium.png differ diff --git a/cli/tests/data/package/invalid-asset-names-directory/images/icon-small.png b/cli/tests/data/package/invalid-asset-names-directory/images/icon-small.png new file mode 100644 index 0000000..f072c77 Binary files /dev/null and b/cli/tests/data/package/invalid-asset-names-directory/images/icon-small.png differ diff --git a/cli/tests/data/package/invalid-asset-names-directory/marathon.json.mustache b/cli/tests/data/package/invalid-asset-names-directory/marathon.json.mustache new file mode 100644 index 0000000..ab3f247 --- /dev/null +++ b/cli/tests/data/package/invalid-asset-names-directory/marathon.json.mustache @@ -0,0 +1,74 @@ +{ + "id": "/cassandra/{{cassandra.cluster-name}}", + "cmd": "$(pwd)/jre*/bin/java $JAVA_OPTS -classpath cassandra-mesos-framework.jar io.mesosphere.mesos.frameworks.cassandra.framework.Main", + "instances": {{cassandra.framework.instances}}, + "cpus": {{cassandra.framework.cpus}}, + "mem": {{cassandra.framework.mem}}, + "uris": [ + "{{resource.assets.uris.cassandra-mesos-tar-gz}}", + "{{resource.assets.uris.jre-linux-tar-gz}}" + ], + "ports": [ + 0 + ], + "healthChecks": [ + { + "gracePeriodSeconds": 120, + "intervalSeconds": 15, + "maxConsecutiveFailures": 0, + "path": "/health/cluster", + "portIndex": 0, + "protocol": "HTTP", + "timeoutSeconds": 5 + }, + { + "gracePeriodSeconds": 120, + "intervalSeconds": 30, + "maxConsecutiveFailures": 3, + "path": "/health/process", + "portIndex": 0, + "protocol": "HTTP", + "timeoutSeconds": 5 + } + ], + "labels": { + "DCOS_PACKAGE_FRAMEWORK_NAME": "cassandra.{{cassandra.cluster-name}}" + }, + "env": { + "MESOS_ZK": "{{mesos.master}}" + ,"JAVA_OPTS": "-Xms256m -Xmx256m" + ,"CASSANDRA_CLUSTER_NAME": "{{cassandra.cluster-name}}" + ,"CASSANDRA_NODE_COUNT": "{{cassandra.node-count}}" + ,"CASSANDRA_SEED_COUNT": "{{cassandra.seed-count}}" + ,"CASSANDRA_RESOURCE_CPU_CORES": "{{cassandra.resources.cpus}}" + ,"CASSANDRA_RESOURCE_MEM_MB": "{{cassandra.resources.mem}}" + ,"CASSANDRA_RESOURCE_DISK_MB": "{{cassandra.resources.disk}}" + ,"CASSANDRA_FAILOVER_TIMEOUT_SECONDS": "{{cassandra.framework.failover-timeout-seconds}}" + ,"CASSANDRA_DATA_DIRECTORY": "{{cassandra.data-directory}}" + ,"CASSANDRA_HEALTH_CHECK_INTERVAL_SECONDS": "{{cassandra.health-check-interval-seconds}}" + ,"CASSANDRA_ZK_TIMEOUT_MS": "{{cassandra.zk-timeout-ms}}" + ,"CASSANDRA_BOOTSTRAP_GRACE_TIME_SECONDS": "{{cassandra.bootstrap-grace-time-seconds}}" + ,"CASSANDRA_FRAMEWORK_MESOS_ROLE": "{{cassandra.framework.role}}" + ,"CASSANDRA_DEFAULT_DC": "{{cassandra.dc.default-dc}}" + ,"CASSANDRA_DEFAULT_RACK": "{{cassandra.dc.default-rack}}" + ,"MESOS_AUTHENTICATE": "{{cassandra.framework.authentication.enabled}}" +{{#cassandra.dc.external-dcs}} + ,"CASSANDRA_EXTERNAL_DC_{{name}}": "{{url}}" +{{/cassandra.dc.external-dcs}} +{{#cassandra.zk}} {{! if the full cassandra zk url has been specified use it }} + ,"CASSANDRA_ZK": "{{cassandra.zk}}" +{{/cassandra.zk}} +{{^cassandra.zk}} {{! else, create a url based on convention and cluster name }} + ,"CASSANDRA_ZK": "zk://master.mesos:2181/cassandra-mesos/{{cassandra.cluster-name}}" +{{/cassandra.zk}} +{{#cassandra.resource.heap-mb}} +,"CASSANDRA_RESOURCE_HEAP_MB": "{{cassandra.resource.heap-mb}}" +{{/cassandra.resource.heap-mb}} +{{#cassandra.framework.authentication.principal}} + ,"DEFAULT_PRINCIPAL": "{{cassandra.framework.authentication.principal}}" +{{/cassandra.framework.authentication.principal}} +{{#cassandra.framework.authentication.secret}} + ,"DEFAULT_SECRET": "{{cassandra.framework.authentication.secret}}" +{{/cassandra.framework.authentication.secret}} + } +} diff --git a/cli/tests/data/package/invalid-asset-names-directory/package.json b/cli/tests/data/package/invalid-asset-names-directory/package.json new file mode 100644 index 0000000..9587e9f --- /dev/null +++ b/cli/tests/data/package/invalid-asset-names-directory/package.json @@ -0,0 +1,18 @@ +{ + "description": "Apache Cassandra running on Apache Mesos", + "framework": true, + "licenses": [ + { + "name": "Apache License Version 2.0", + "url": "https://github.com/mesosphere/cassandra-mesos/blob/master/LICENSE.txt" + } + ], + "maintainer": "support@mesosphere.io", + "name": "cassandra", + "postInstallNotes": "Thank you for installing the Apache Cassandra DCOS Service.\n\n\tDocumentation: http://mesosphere.github.io/cassandra-mesos/\n\tIssues: https://github.com/mesosphere/cassandra-mesos/issues", + "postUninstallNotes": "The Apache Cassandra DCOS Service has been uninstalled and will no longer run.\nPlease follow the instructions at http://docs.mesosphere.com/services/cassandra/#uninstall to clean up any persisted state", + "preInstallNotes":"The Apache Cassandra DCOS Service implementation is alpha and there may be bugs, incomplete features, incorrect documentation or other discrepancies.\nThe default configuration requires 3 nodes each with 0.3 CPU shares, 1184MB of memory and 272MB of disk.", + "scm": "https://github.com/mesosphere/cassandra-mesos.git", + "tags": ["mesosphere", "framework", "data", "database"], + "version": "0.2.0-1" +} diff --git a/cli/tests/data/package/invalid-command-json-directory/marathon.json.mustache b/cli/tests/data/package/invalid-command-json-directory/marathon.json.mustache index b36ef3f..ab3f247 100644 --- a/cli/tests/data/package/invalid-command-json-directory/marathon.json.mustache +++ b/cli/tests/data/package/invalid-command-json-directory/marathon.json.mustache @@ -5,8 +5,8 @@ "cpus": {{cassandra.framework.cpus}}, "mem": {{cassandra.framework.mem}}, "uris": [ - "{{package.assets.uris.cassandra-mesos.tar.gz}}", - "{{package.assets.uris.jre-linux.tar.gz}}" + "{{resource.assets.uris.cassandra-mesos-tar-gz}}", + "{{resource.assets.uris.jre-linux-tar-gz}}" ], "ports": [ 0 diff --git a/cli/tests/data/package/invalid-config-json-directory/marathon.json.mustache b/cli/tests/data/package/invalid-config-json-directory/marathon.json.mustache index b36ef3f..ab3f247 100644 --- a/cli/tests/data/package/invalid-config-json-directory/marathon.json.mustache +++ b/cli/tests/data/package/invalid-config-json-directory/marathon.json.mustache @@ -5,8 +5,8 @@ "cpus": {{cassandra.framework.cpus}}, "mem": {{cassandra.framework.mem}}, "uris": [ - "{{package.assets.uris.cassandra-mesos.tar.gz}}", - "{{package.assets.uris.jre-linux.tar.gz}}" + "{{resource.assets.uris.cassandra-mesos-tar-gz}}", + "{{resource.assets.uris.jre-linux-tar-gz}}" ], "ports": [ 0 diff --git a/cli/tests/data/package/invalid-package-json-directory/marathon.json.mustache b/cli/tests/data/package/invalid-package-json-directory/marathon.json.mustache index b36ef3f..ab3f247 100644 --- a/cli/tests/data/package/invalid-package-json-directory/marathon.json.mustache +++ b/cli/tests/data/package/invalid-package-json-directory/marathon.json.mustache @@ -5,8 +5,8 @@ "cpus": {{cassandra.framework.cpus}}, "mem": {{cassandra.framework.mem}}, "uris": [ - "{{package.assets.uris.cassandra-mesos.tar.gz}}", - "{{package.assets.uris.jre-linux.tar.gz}}" + "{{resource.assets.uris.cassandra-mesos-tar-gz}}", + "{{resource.assets.uris.jre-linux-tar-gz}}" ], "ports": [ 0 diff --git a/cli/tests/data/package/missing-package-json-directory/marathon.json.mustache b/cli/tests/data/package/missing-package-json-directory/marathon.json.mustache index b36ef3f..ab3f247 100644 --- a/cli/tests/data/package/missing-package-json-directory/marathon.json.mustache +++ b/cli/tests/data/package/missing-package-json-directory/marathon.json.mustache @@ -5,8 +5,8 @@ "cpus": {{cassandra.framework.cpus}}, "mem": {{cassandra.framework.mem}}, "uris": [ - "{{package.assets.uris.cassandra-mesos.tar.gz}}", - "{{package.assets.uris.jre-linux.tar.gz}}" + "{{resource.assets.uris.cassandra-mesos-tar-gz}}", + "{{resource.assets.uris.jre-linux-tar-gz}}" ], "ports": [ 0 diff --git a/cli/tests/data/package/non-png-icons-directory/marathon.json.mustache b/cli/tests/data/package/non-png-icons-directory/marathon.json.mustache index b36ef3f..ab3f247 100644 --- a/cli/tests/data/package/non-png-icons-directory/marathon.json.mustache +++ b/cli/tests/data/package/non-png-icons-directory/marathon.json.mustache @@ -5,8 +5,8 @@ "cpus": {{cassandra.framework.cpus}}, "mem": {{cassandra.framework.mem}}, "uris": [ - "{{package.assets.uris.cassandra-mesos.tar.gz}}", - "{{package.assets.uris.jre-linux.tar.gz}}" + "{{resource.assets.uris.cassandra-mesos-tar-gz}}", + "{{resource.assets.uris.jre-linux-tar-gz}}" ], "ports": [ 0 diff --git a/cli/tests/data/package/non-png-screenshots-directory/marathon.json.mustache b/cli/tests/data/package/non-png-screenshots-directory/marathon.json.mustache index b36ef3f..ab3f247 100644 --- a/cli/tests/data/package/non-png-screenshots-directory/marathon.json.mustache +++ b/cli/tests/data/package/non-png-screenshots-directory/marathon.json.mustache @@ -5,8 +5,8 @@ "cpus": {{cassandra.framework.cpus}}, "mem": {{cassandra.framework.mem}}, "uris": [ - "{{package.assets.uris.cassandra-mesos.tar.gz}}", - "{{package.assets.uris.jre-linux.tar.gz}}" + "{{resource.assets.uris.cassandra-mesos-tar-gz}}", + "{{resource.assets.uris.jre-linux-tar-gz}}" ], "ports": [ 0 diff --git a/cli/tests/integrations/test_package_bundle.py b/cli/tests/integrations/test_package_bundle.py index 25dd9c0..5b43052 100644 --- a/cli/tests/integrations/test_package_bundle.py +++ b/cli/tests/integrations/test_package_bundle.py @@ -60,6 +60,19 @@ def test_bundle_fail_missing_package_json(): assert not glob.glob(_PACKAGE_NAME_GLOB) +def test_bundle_fail_duplicate_asset_names(): + common.assert_command( + ['dcos', 'package', 'bundle', '--output-directory=/tmp', + 'tests/data/package/invalid-asset-names-directory'], + returncode=1, + stderr=(b'Error bundling package. Multiple assets map to the same ' + b'property name (periods [.] are replaced with dashes [-]): ' + b'[[\'cassandra-mesos-tar-gz\', ' + b'\'cassandra-mesos.tar.gz\']]\n')) + + assert not glob.glob(_PACKAGE_NAME_GLOB) + + def test_bundle_fail_invalid_package_json(): common.assert_command( ['dcos', 'package', 'bundle', '--output-directory=/tmp',