openstack/deb-python-dcos
Code Issues Proposed changes

dcos-4107 check that assets don't have same filename in package bundle

Browse Source
This commit is contained in:
Tamar Ben-Shachar
2015-12-14 18:02:07 -08:00
parent 3b100344c8
commit 614741e6fe
21 changed files with 373 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
cli/dcoscli/package/main.py
View File

@@ -4,6 +4,7 @@ import os
import sys
import tempfile
import zipfile
from collections import defaultdict
import dcoscli
import docopt
@@ -13,6 +14,7 @@ from dcos import (cmds, emitting, errors, http, marathon, options, package,
from dcos.errors import DCOSException
from dcoscli import tables
from dcoscli.main import decorate_docopt_usage
from six import iteritems
logger = util.get_logger(__name__)
emitter = emitting.FlatEmitter()
@@ -682,7 +684,23 @@ def _bundle_uris(uris_directory, zip_file):
:rtype: None
"""
for filename in sorted(os.listdir(uris_directory)):
uris = sorted(os.listdir(uris_directory))
# these uris will be found through a mustache template from the property
# name so make sure each name is unique.
uri_properties = defaultdict(list)
for name in uris:
uri_properties[name.replace('.', '-')].append(name)
collisions = [uri_list for (prop, uri_list) in iteritems(uri_properties)
if len(uri_list) > 1]
if collisions:
raise DCOSException(
'Error bundling package. Multiple assets map to the same property '
'name (periods [.] are replaced with dashes [-]): {}'.format(
collisions))
for filename in uris:
fullpath = os.path.join(uris_directory, filename)
zip_file.write(fullpath, arcname='assets/uris/{}'.format(filename))

BIN
cli/tests/data/package/cassandra.zip
View File

Binary file not shown.

4
cli/tests/data/package/create-directory/marathon.json.mustache
View File

@@ -5,8 +5,8 @@
"cpus": {{cassandra.framework.cpus}},
"mem": {{cassandra.framework.mem}},
"uris": [
"{{package.assets.uris.cassandra-mesos.tar.gz}}",
"{{package.assets.uris.jre-linux.tar.gz}}"
"{{resource.assets.uris.cassandra-mesos-tar-gz}}",
"{{resource.assets.uris.jre-linux-tar-gz}}"
],
"ports": [
0

4
cli/tests/data/package/extra-assets-files-directory/marathon.json.mustache
View File

@@ -5,8 +5,8 @@
"cpus": {{cassandra.framework.cpus}},
"mem": {{cassandra.framework.mem}},
"uris": [
"{{package.assets.uris.cassandra-mesos.tar.gz}}",
"{{package.assets.uris.jre-linux.tar.gz}}"
"{{resource.assets.uris.cassandra-mesos-tar-gz}}",
"{{resource.assets.uris.jre-linux-tar-gz}}"
],
"ports": [
0

4
cli/tests/data/package/extra-files-directory/marathon.json.mustache
View File

@@ -5,8 +5,8 @@
"cpus": {{cassandra.framework.cpus}},
"mem": {{cassandra.framework.mem}},
"uris": [
"{{package.assets.uris.cassandra-mesos.tar.gz}}",
"{{package.assets.uris.jre-linux.tar.gz}}"
"{{resource.assets.uris.cassandra-mesos-tar-gz}}",
"{{resource.assets.uris.jre-linux-tar-gz}}"
],
"ports": [
0

4
cli/tests/data/package/extra-icons-directory/marathon.json.mustache
View File

@@ -5,8 +5,8 @@
"cpus": {{cassandra.framework.cpus}},
"mem": {{cassandra.framework.mem}},
"uris": [
"{{package.assets.uris.cassandra-mesos.tar.gz}}",
"{{package.assets.uris.jre-linux.tar.gz}}"
"{{resource.assets.uris.cassandra-mesos-tar-gz}}",
"{{resource.assets.uris.jre-linux-tar-gz}}"
],
"ports": [
0

0
cli/tests/data/package/invalid-asset-names-directory/assets/uris/cassandra-mesos-tar-gz Normal file
View File

0
cli/tests/data/package/invalid-asset-names-directory/assets/uris/cassandra-mesos.tar.gz Normal file
View File

229
cli/tests/data/package/invalid-asset-names-directory/config.json Normal file
View File

@@ -0,0 +1,229 @@
{
"type": "object",
"properties": {
"mesos": {
"description": "Mesos specific configuration properties",
"type": "object",
"properties": {
"master": {
"default": "zk://master.mesos:2181/mesos",
"description": "The URL of the Mesos master. The format is a comma-delimited list of hosts like zk://host1:port,host2:port/mesos. If using ZooKeeper, pay particular attention to the leading zk:// and trailing /mesos! If not using ZooKeeper, standard host:port patterns, like localhost:5050 or 10.0.0.5:5050,10.0.0.6:5050, are also acceptable.",
"type": "string"
}
},
"required": [
"master"
]
},
"cassandra": {
"description": "Cassandra Framework Configuration Properties",
"type": "object",
"additionalProperties": false,
"properties": {
"framework": {
"description": "Framework Scheduler specific Configuration Properties",
"type": "object",
"additionalProperties": false,
"properties": {
"failover-timeout-seconds": {
"description": "The failover_timeout for Mesos in seconds. If the framework instance has not re-registered with Mesos this long after a failover, Mesos will shut down all running tasks started by the framework.",
"type": "integer",
"minimum": 0,
"default": 604800
},
"cpus": {
"default": 0.5,
"description": "CPU shares to allocate to each Cassandra framework instance.",
"type": "number"
},
"mem": {
"default": 512.0,
"description": "Memory (MB) to allocate to each Cassandra framework instance.",
"minimum": 512.0,
"type": "number"
},
"instances": {
"default": 1,
"description": "Number of Cassandra framework instances to run.",
"minimum": 0,
"type": "integer"
},
"role": {
"description": "Mesos role for this framework.",
"type": "string",
"default": "*"
},
"authentication": {
"description": "Framework Scheduler Authentication Configuration Properties",
"type": "object",
"additionalProperties": false,
"properties": {
"enabled": {
"description": "Whether framework authentication should be used",
"type": "boolean",
"default": false
},
"principal": {
"description": "The Mesos principal used for authentication.",
"type": "string"
},
"secret": {
"description": "The path to the Mesos secret file containing the authentication secret.",
"type": "string"
}
},
"required": [
"enabled"
]
}
},
"required": [
"instances",
"cpus",
"mem",
"failover-timeout-seconds",
"role",
"authentication"
]
},
"cluster-name": {
"description": "The name of the framework to register with mesos. Will also be used as the cluster name in Cassandra",
"type": "string",
"default": "dcos"
},
"zk": {
"description": "ZooKeeper URL for storing state. Format: zk://host1:port1,host2:port2,.../path (can have nested directories)",
"type": "string"
},
"zk-timeout-ms": {
"description": "Timeout for ZooKeeper in milliseconds.",
"type": "integer",
"minimum": 0,
"default": 10000
},
"node-count": {
"description": "The number of nodes in the ring for the framework to run.",
"type": "integer",
"minimum": 1,
"default": 3
},
"seed-count": {
"description": "The number of seed nodes in the ring for the framework to run.",
"type": "integer",
"minimum": 1,
"default": 2
},
"health-check-interval-seconds": {
"description": "The interval in seconds that the framework should check the health of each Cassandra Server instance.",
"type": "integer",
"minimum": 15,
"default": 60
},
"bootstrap-grace-time-seconds": {
"description": "The minimum number of seconds to wait between starting each node. Setting this too low could result in the ring not bootstrapping correctly.",
"type": "integer",
"minimum": 15,
"default": 120
},
"data-directory": {
"description": "The location on disk where Cassandra will be configured to write it's data.",
"type": "string",
"default": "."
},
"resources": {
"description": "Cassandra Server Resources Configuration Properties",
"type": "object",
"additionalProperties": false,
"properties": {
"cpus": {
"description": "CPU shares to allocate to each Cassandra Server Instance.",
"type": "number",
"minimum": 0.0,
"default": 0.1
},
"mem": {
"description": "Memory (MB) to allocate to each Cassandra Server instance.",
"type": "integer",
"minimum": 0,
"default": 768
},
"disk": {
"description": "Disk (MB) to allocate to each Cassandra Server instance.",
"type": "integer",
"minimum": 0,
"default": 16
},
"heap-mb": {
"description": "The amount of memory in MB that are allocated to each Cassandra Server Instance. This value should be smaller than 'cassandra.resources.mem'. The remaining difference will be used for memory mapped files and other off-heap memory requirements.",
"type": "integer",
"minimum": 0
}
},
"required": [
"cpus",
"mem",
"disk"
]
},
"dc": {
"description": "Cassandra multi Datacenter Configuration Properties",
"type": "object",
"additionalProperties": false,
"properties": {
"default-dc": {
"description": "Default value to be set for dc name in the GossipingPropertyFileSnitch",
"type": "string",
"default": "DC1"
},
"default-rack": {
"description": "Default value to be set for rack name in the GossipingPropertyFileSnitch",
"type": "string",
"default": "RAC1"
},
"external-dcs": {
"description": "Name and URL for another instance of Cassandra DCOS Service",
"type": "array",
"additionalProperties": false,
"items": {
"type": "object",
"additionalProperties": false,
"properties": {
"name": {
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"name",
"url"
]
}
}
},
"required": [
"default-dc",
"default-rack"
]
}
},
"required": [
"framework",
"cluster-name",
"zk-timeout-ms",
"node-count",
"seed-count",
"health-check-interval-seconds",
"bootstrap-grace-time-seconds",
"data-directory",
"resources"
]
}
},
"required": [
"mesos",
"cassandra"
]
}

BIN
cli/tests/data/package/invalid-asset-names-directory/images/icon-large.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 18 KiB

BIN
cli/tests/data/package/invalid-asset-names-directory/images/icon-medium.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 5.0 KiB

BIN
cli/tests/data/package/invalid-asset-names-directory/images/icon-small.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.9 KiB

74
cli/tests/data/package/invalid-asset-names-directory/marathon.json.mustache Normal file
View File

@@ -0,0 +1,74 @@
{
"id": "/cassandra/{{cassandra.cluster-name}}",
"cmd": "$(pwd)/jre*/bin/java $JAVA_OPTS -classpath cassandra-mesos-framework.jar io.mesosphere.mesos.frameworks.cassandra.framework.Main",
"instances": {{cassandra.framework.instances}},
"cpus": {{cassandra.framework.cpus}},
"mem": {{cassandra.framework.mem}},
"uris": [
"{{resource.assets.uris.cassandra-mesos-tar-gz}}",
"{{resource.assets.uris.jre-linux-tar-gz}}"
],
"ports": [
0
],
"healthChecks": [
{
"gracePeriodSeconds": 120,
"intervalSeconds": 15,
"maxConsecutiveFailures": 0,
"path": "/health/cluster",
"portIndex": 0,
"protocol": "HTTP",
"timeoutSeconds": 5
},
{
"gracePeriodSeconds": 120,
"intervalSeconds": 30,
"maxConsecutiveFailures": 3,
"path": "/health/process",
"portIndex": 0,
"protocol": "HTTP",
"timeoutSeconds": 5
}
],
"labels": {
"DCOS_PACKAGE_FRAMEWORK_NAME": "cassandra.{{cassandra.cluster-name}}"
},
"env": {
"MESOS_ZK": "{{mesos.master}}"
,"JAVA_OPTS": "-Xms256m -Xmx256m"
,"CASSANDRA_CLUSTER_NAME": "{{cassandra.cluster-name}}"
,"CASSANDRA_NODE_COUNT": "{{cassandra.node-count}}"
,"CASSANDRA_SEED_COUNT": "{{cassandra.seed-count}}"
,"CASSANDRA_RESOURCE_CPU_CORES": "{{cassandra.resources.cpus}}"
,"CASSANDRA_RESOURCE_MEM_MB": "{{cassandra.resources.mem}}"
,"CASSANDRA_RESOURCE_DISK_MB": "{{cassandra.resources.disk}}"
,"CASSANDRA_FAILOVER_TIMEOUT_SECONDS": "{{cassandra.framework.failover-timeout-seconds}}"
,"CASSANDRA_DATA_DIRECTORY": "{{cassandra.data-directory}}"
,"CASSANDRA_HEALTH_CHECK_INTERVAL_SECONDS": "{{cassandra.health-check-interval-seconds}}"
,"CASSANDRA_ZK_TIMEOUT_MS": "{{cassandra.zk-timeout-ms}}"
,"CASSANDRA_BOOTSTRAP_GRACE_TIME_SECONDS": "{{cassandra.bootstrap-grace-time-seconds}}"
,"CASSANDRA_FRAMEWORK_MESOS_ROLE": "{{cassandra.framework.role}}"
,"CASSANDRA_DEFAULT_DC": "{{cassandra.dc.default-dc}}"
,"CASSANDRA_DEFAULT_RACK": "{{cassandra.dc.default-rack}}"
,"MESOS_AUTHENTICATE": "{{cassandra.framework.authentication.enabled}}"
{{#cassandra.dc.external-dcs}}
,"CASSANDRA_EXTERNAL_DC_{{name}}": "{{url}}"
{{/cassandra.dc.external-dcs}}
{{#cassandra.zk}} {{! if the full cassandra zk url has been specified use it }}
,"CASSANDRA_ZK": "{{cassandra.zk}}"
{{/cassandra.zk}}
{{^cassandra.zk}} {{! else, create a url based on convention and cluster name }}
,"CASSANDRA_ZK": "zk://master.mesos:2181/cassandra-mesos/{{cassandra.cluster-name}}"
{{/cassandra.zk}}
{{#cassandra.resource.heap-mb}}
,"CASSANDRA_RESOURCE_HEAP_MB": "{{cassandra.resource.heap-mb}}"
{{/cassandra.resource.heap-mb}}
{{#cassandra.framework.authentication.principal}}
,"DEFAULT_PRINCIPAL": "{{cassandra.framework.authentication.principal}}"
{{/cassandra.framework.authentication.principal}}
{{#cassandra.framework.authentication.secret}}
,"DEFAULT_SECRET": "{{cassandra.framework.authentication.secret}}"
{{/cassandra.framework.authentication.secret}}
}
}

18
cli/tests/data/package/invalid-asset-names-directory/package.json Normal file
View File

@@ -0,0 +1,18 @@
{
"description": "Apache Cassandra running on Apache Mesos",
"framework": true,
"licenses": [
{
"name": "Apache License Version 2.0",
"url": "https://github.com/mesosphere/cassandra-mesos/blob/master/LICENSE.txt"
}
],
"maintainer": "support@mesosphere.io",
"name": "cassandra",
"postInstallNotes": "Thank you for installing the Apache Cassandra DCOS Service.\n\n\tDocumentation: http://mesosphere.github.io/cassandra-mesos/\n\tIssues: https://github.com/mesosphere/cassandra-mesos/issues",
"postUninstallNotes": "The Apache Cassandra DCOS Service has been uninstalled and will no longer run.\nPlease follow the instructions at http://docs.mesosphere.com/services/cassandra/#uninstall to clean up any persisted state",
"preInstallNotes":"The Apache Cassandra DCOS Service implementation is alpha and there may be bugs, incomplete features, incorrect documentation or other discrepancies.\nThe default configuration requires 3 nodes each with 0.3 CPU shares, 1184MB of memory and 272MB of disk.",
"scm": "https://github.com/mesosphere/cassandra-mesos.git",
"tags": ["mesosphere", "framework", "data", "database"],
"version": "0.2.0-1"
}

4
cli/tests/data/package/invalid-command-json-directory/marathon.json.mustache
View File

@@ -5,8 +5,8 @@
"cpus": {{cassandra.framework.cpus}},
"mem": {{cassandra.framework.mem}},
"uris": [
"{{package.assets.uris.cassandra-mesos.tar.gz}}",
"{{package.assets.uris.jre-linux.tar.gz}}"
"{{resource.assets.uris.cassandra-mesos-tar-gz}}",
"{{resource.assets.uris.jre-linux-tar-gz}}"
],
"ports": [
0

4
cli/tests/data/package/invalid-config-json-directory/marathon.json.mustache
View File

@@ -5,8 +5,8 @@
"cpus": {{cassandra.framework.cpus}},
"mem": {{cassandra.framework.mem}},
"uris": [
"{{package.assets.uris.cassandra-mesos.tar.gz}}",
"{{package.assets.uris.jre-linux.tar.gz}}"
"{{resource.assets.uris.cassandra-mesos-tar-gz}}",
"{{resource.assets.uris.jre-linux-tar-gz}}"
],
"ports": [
0

4
cli/tests/data/package/invalid-package-json-directory/marathon.json.mustache
View File

@@ -5,8 +5,8 @@
"cpus": {{cassandra.framework.cpus}},
"mem": {{cassandra.framework.mem}},
"uris": [
"{{package.assets.uris.cassandra-mesos.tar.gz}}",
"{{package.assets.uris.jre-linux.tar.gz}}"
"{{resource.assets.uris.cassandra-mesos-tar-gz}}",
"{{resource.assets.uris.jre-linux-tar-gz}}"
],
"ports": [
0

4
cli/tests/data/package/missing-package-json-directory/marathon.json.mustache
View File

@@ -5,8 +5,8 @@
"cpus": {{cassandra.framework.cpus}},
"mem": {{cassandra.framework.mem}},
"uris": [
"{{package.assets.uris.cassandra-mesos.tar.gz}}",
"{{package.assets.uris.jre-linux.tar.gz}}"
"{{resource.assets.uris.cassandra-mesos-tar-gz}}",
"{{resource.assets.uris.jre-linux-tar-gz}}"
],
"ports": [
0

4
cli/tests/data/package/non-png-icons-directory/marathon.json.mustache
View File

@@ -5,8 +5,8 @@
"cpus": {{cassandra.framework.cpus}},
"mem": {{cassandra.framework.mem}},
"uris": [
"{{package.assets.uris.cassandra-mesos.tar.gz}}",
"{{package.assets.uris.jre-linux.tar.gz}}"
"{{resource.assets.uris.cassandra-mesos-tar-gz}}",
"{{resource.assets.uris.jre-linux-tar-gz}}"
],
"ports": [
0

4
cli/tests/data/package/non-png-screenshots-directory/marathon.json.mustache
View File

@@ -5,8 +5,8 @@
"cpus": {{cassandra.framework.cpus}},
"mem": {{cassandra.framework.mem}},
"uris": [
"{{package.assets.uris.cassandra-mesos.tar.gz}}",
"{{package.assets.uris.jre-linux.tar.gz}}"
"{{resource.assets.uris.cassandra-mesos-tar-gz}}",
"{{resource.assets.uris.jre-linux-tar-gz}}"
],
"ports": [
0

13
cli/tests/integrations/test_package_bundle.py
View File

@@ -60,6 +60,19 @@ def test_bundle_fail_missing_package_json():
assert not glob.glob(_PACKAGE_NAME_GLOB)
def test_bundle_fail_duplicate_asset_names():
common.assert_command(
['dcos', 'package', 'bundle', '--output-directory=/tmp',
'tests/data/package/invalid-asset-names-directory'],
returncode=1,
stderr=(b'Error bundling package. Multiple assets map to the same '
b'property name (periods [.] are replaced with dashes [-]): '
b'[[\'cassandra-mesos-tar-gz\', '
b'\'cassandra-mesos.tar.gz\']]\n'))
assert not glob.glob(_PACKAGE_NAME_GLOB)
def test_bundle_fail_invalid_package_json():
common.assert_command(
['dcos', 'package', 'bundle', '--output-directory=/tmp',