Merge pull request #281 from mesosphere/dcos-2137-dcos-task-ls

[DCOS-2137] Added a command to list files in a task's sandbox: `dcos task ls`
This commit is contained in:
mgummelt
2015-08-05 10:43:46 -07:00
31 changed files with 502 additions and 318 deletions

View File

@@ -13,8 +13,8 @@ Options:
-h, --help Show this screen
--info Show a short description of this subcommand
--json Print json-formatted nodes
--follow Output data as the file grows
--lines=N Output the last N lines [default: 10]
--follow Print data as the file grows
--lines=N Print the last N lines [default: 10]
--master Access the leading master
--slave=<slave-id> Access the slave with the provided ID
--option SSHOPT=VAL SSH option (see `man ssh_config`)

View File

@@ -15,7 +15,7 @@ Options:
--ssh-config-file=<path> Path to SSH config file. Used to access
marathon logs.
--follow Output data as the file grows
--follow Print data as the file grows
--inactive Show inactive services in addition to active
ones. Inactive services are those that have
@@ -24,7 +24,7 @@ Options:
--json Print json-formatted services
--lines=N Output the last N lines [default: 10]
--lines=N Print the last N lines [default: 10]
--version Show version

View File

@@ -1,6 +1,9 @@
import copy
import datetime
import posixpath
from collections import OrderedDict
import prettytable
from dcos import mesos, util
EMPTY_ENTRY = '---'
@@ -31,7 +34,7 @@ def task_table(tasks):
("ID", lambda t: t["id"]),
])
tb = util.table(fields, tasks, sortby="NAME")
tb = table(fields, tasks, sortby="NAME")
tb.align["NAME"] = "l"
tb.align["HOST"] = "l"
tb.align["ID"] = "l"
@@ -101,7 +104,7 @@ def app_table(apps, deployments):
("CMD", get_cmd)
])
tb = util.table(fields, apps, sortby="ID")
tb = table(fields, apps, sortby="ID")
tb.align["CMD"] = "l"
tb.align["ID"] = "l"
@@ -125,7 +128,7 @@ def app_task_table(tasks):
("ID", lambda t: t["id"])
])
tb = util.table(fields, tasks, sortby="APP")
tb = table(fields, tasks, sortby="APP")
tb.align["APP"] = "l"
tb.align["ID"] = "l"
@@ -172,7 +175,7 @@ def deployment_table(deployments):
('ID', lambda d: d['id'])
])
tb = util.table(fields, deployments, sortby="APP")
tb = table(fields, deployments, sortby="APP")
tb.align['APP'] = 'l'
tb.align['ACTION'] = 'l'
tb.align['ID'] = 'l'
@@ -199,7 +202,7 @@ def service_table(services):
("ID", lambda s: s['id']),
])
tb = util.table(fields, services, sortby="NAME")
tb = table(fields, services, sortby="NAME")
tb.align["ID"] = 'l'
tb.align["NAME"] = 'l'
@@ -248,7 +251,7 @@ def group_table(groups):
('APPS', lambda g: g[1]),
])
tb = util.table(fields, group_dict.values(), sortby="ID")
tb = table(fields, group_dict.values(), sortby="ID")
tb.align['ID'] = 'l'
return tb
@@ -273,7 +276,7 @@ def package_table(packages):
('DESCRIPTION', lambda p: p['description'])
])
tb = util.table(fields, packages, sortby="NAME")
tb = table(fields, packages, sortby="NAME")
tb.align['NAME'] = 'l'
tb.align['VERSION'] = 'l'
tb.align['APP'] = 'l'
@@ -309,7 +312,7 @@ def package_search_table(search_results):
package_['source'] = result['source']
packages.append(package_)
tb = util.table(fields, packages, sortby="NAME")
tb = table(fields, packages, sortby="NAME")
tb.align['NAME'] = 'l'
tb.align['VERSION'] = 'l'
tb.align['FRAMEWORK'] = 'l'
@@ -333,5 +336,77 @@ def slave_table(slaves):
('ID', lambda s: s['id'])
])
tb = util.table(fields, slaves, sortby="HOSTNAME")
tb = table(fields, slaves, sortby="HOSTNAME")
return tb
def _format_unix_timestamp(ts):
""" Formats a unix timestamp in a `dcos task ls --long` format.
:param ts: unix timestamp
:type ts: int
:rtype: str
"""
return datetime.datetime.fromtimestamp(ts).strftime('%b %d %H:%M')
def ls_long_table(files):
"""Returns a PrettyTable representation of `files`
:param files: Files to render. Of the form returned from the
mesos /files/browse.json endpoint.
:param files: [dict]
:rtype: PrettyTable
"""
fields = OrderedDict([
('MODE', lambda f: f['mode']),
('NLINK', lambda f: f['nlink']),
('UID', lambda f: f['uid']),
('GID', lambda f: f['gid']),
('SIZE', lambda f: f['size']),
('DATE', lambda f: _format_unix_timestamp(int(f['mtime']))),
('PATH', lambda f: posixpath.basename(f['path']))])
tb = table(fields, files, sortby="PATH", header=False)
tb.align = 'r'
return tb
def table(fields, objs, **kwargs):
"""Returns a PrettyTable. `fields` represents the header schema of
the table. `objs` represents the objects to be rendered into
rows.
:param fields: An OrderedDict, where each element represents a
column. The key is the column header, and the
value is the function that transforms an element of
`objs` into a value for that column.
:type fields: OrderdDict(str, function)
:param objs: objects to render into rows
:type objs: [object]
:param **kwargs: kwargs to pass to `prettytable.PrettyTable`
:type **kwargs: dict
:rtype: PrettyTable
"""
tb = prettytable.PrettyTable(
[k.upper() for k in fields.keys()],
border=False,
hrules=prettytable.NONE,
vrules=prettytable.NONE,
left_padding_width=0,
right_padding_width=1,
**kwargs
)
# Set these explicitly due to a bug in prettytable where
# '0' values are not honored.
tb._left_padding_width = 0
tb._right_padding_width = 2
for obj in objs:
row = [fn(obj) for fn in fields.values()]
tb.add_row(row)
return tb

View File

@@ -4,28 +4,31 @@ Usage:
dcos task --info
dcos task [--completed --json <task>]
dcos task log [--completed --follow --lines=N] <task> [<file>]
dcos task ls [--long] <task> [<path>]
Options:
-h, --help Show this screen
--info Show a short description of this subcommand
--completed Include completed tasks as well
--follow Output data as the file grows
--follow Print data as the file grows
--json Print json-formatted tasks
--lines=N Output the last N lines [default: 10]
--lines=N Print the last N lines [default: 10]
--long Use a long listing format
--version Show version
Positional Arguments:
<file> Print this file. [default: stdout]
<path> List this directory. [default: '.']
<task> Only match tasks whose ID matches <task>. <task> may be
a substring of the ID, or a unix glob pattern.
<file> Output this file. [default: stdout]
"""
import posixpath
import dcoscli
import docopt
from dcos import cmds, emitting, mesos, util
from dcos.errors import DCOSException, DefaultError
from dcos.errors import DCOSException, DCOSHTTPException, DefaultError
from dcoscli import log, tables
logger = util.get_logger(__name__)
@@ -68,6 +71,11 @@ def _cmds():
'<file>'],
function=_log),
cmds.Command(
hierarchy=['task', 'ls'],
arg_keys=['<task>', '<path>', '--long'],
function=_ls),
cmds.Command(
hierarchy=['task'],
arg_keys=['<task>', '--completed', '--json'],
@@ -151,6 +159,46 @@ def _log(follow, completed, lines, task, file_):
return 0
def _ls(task, path, long_):
""" List files in a task's sandbox.
:param task: task pattern to match
:type task: str
:param path: file path to read
:type path: str
:param long_: whether to use a long listing format
:type long_: bool
:returns: process return code
:rtype: int
"""
if path is None:
path = '.'
if path.startswith('/'):
path = path[1:]
dcos_client = mesos.DCOSClient()
task_obj = mesos.get_master(dcos_client).task(task)
dir_ = posixpath.join(task_obj.directory(), path)
try:
files = dcos_client.browse(task_obj.slave(), dir_)
except DCOSHTTPException as e:
if e.response.status_code == 404:
raise DCOSException(
'Cannot access [{}]: No such file or directory'.format(path))
else:
raise
if files:
if long_:
emitter.publish(tables.ls_long_table(files))
else:
emitter.publish(
' '.join(posixpath.basename(file_['path'])
for file_ in files))
def _mesos_files(completed, fltr, file_):
"""Return MesosFile objects for the specified files. Only include
files that satisfy all of the following:

View File

@@ -0,0 +1,11 @@
{
"id": "test-app-completed",
"cmd": "sleep 1000",
"cpus": 0.1,
"mem": 16,
"instances": 1,
"labels": {
"PACKAGE_ID": "test-app",
"PACKAGE_VERSION": "1.2.3"
}
}

View File

@@ -0,0 +1,11 @@
{
"id": "test-app1",
"cmd": "sleep 1000",
"cpus": 0.1,
"mem": 16,
"instances": 1,
"labels": {
"PACKAGE_ID": "test-app",
"PACKAGE_VERSION": "1.2.3"
}
}

View File

@@ -0,0 +1,7 @@
{
"id": "ls-app",
"cmd": "mkdir test && touch test/test1 && touch test/test2 && sleep 1000",
"cpus": 0.1,
"mem": 32,
"instances": 1
}

View File

@@ -35,3 +35,31 @@ def task_fixture():
slave = Slave({"hostname": "mock-hostname"}, None, None)
task.slave = mock.Mock(return_value=slave)
return task
def browse_fixture():
return [
{u'uid': u'root',
u'mtime': 1437089500,
u'nlink': 1,
u'mode': u'-rw-r--r--',
u'gid': u'root',
u'path': (u'/var/lib/mesos/slave/slaves/' +
u'20150716-183440-1695027628-5050-2710-S0/frameworks/' +
u'20150716-183440-1695027628-5050-2710-0000/executors/' +
u'chronos.8810d396-2c09-11e5-af1a-080027d3e806/runs/' +
u'aaecec57-7c7c-4030-aca3-d7aac2f9fd29/stderr'),
u'size': 4507},
{u'uid': u'root',
u'mtime': 1437089604,
u'nlink': 1,
u'mode': u'-rw-r--r--',
u'gid': u'root',
u'path': (u'/var/lib/mesos/slave/slaves/' +
u'20150716-183440-1695027628-5050-2710-S0/frameworks/' +
u'20150716-183440-1695027628-5050-2710-0000/executors/' +
u'chronos.8810d396-2c09-11e5-af1a-080027d3e806/runs/' +
u'aaecec57-7c7c-4030-aca3-d7aac2f9fd29/stdout'),
u'size': 353857}
]

View File

@@ -630,9 +630,8 @@ def test_bad_configuration():
assert returncode == 1
assert stdout == b''
assert stderr.decode().startswith(
"Marathon likely misconfigured. Please check your proxy or "
"Marathon URL settings. See dcos config --help. ")
assert stderr.startswith(
b"URL [http://localhost:88888/v2/info] is unreachable")
assert_command(['dcos', 'config', 'unset', 'marathon.url'])

View File

@@ -25,8 +25,8 @@ Options:
-h, --help Show this screen
--info Show a short description of this subcommand
--json Print json-formatted nodes
--follow Output data as the file grows
--lines=N Output the last N lines [default: 10]
--follow Print data as the file grows
--lines=N Print the last N lines [default: 10]
--master Access the leading master
--slave=<slave-id> Access the slave with the provided ID
--option SSHOPT=VAL SSH option (see `man ssh_config`)

View File

@@ -47,7 +47,7 @@ Options:
--ssh-config-file=<path> Path to SSH config file. Used to access
marathon logs.
--follow Output data as the file grows
--follow Print data as the file grows
--inactive Show inactive services in addition to active
ones. Inactive services are those that have
@@ -56,7 +56,7 @@ Options:
--json Print json-formatted services
--lines=N Output the last N lines [default: 10]
--lines=N Print the last N lines [default: 10]
--version Show version

View File

@@ -15,14 +15,35 @@ from dcoscli.task.main import _mesos_files, main
from mock import MagicMock, patch
from ..fixtures.task import task_fixture
from .common import (app, assert_command, assert_lines, assert_mock,
exec_command, watch_all_deployments)
from .common import (add_app, app, assert_command, assert_lines, assert_mock,
exec_command, remove_app, watch_all_deployments)
SLEEP1 = 'tests/data/marathon/apps/sleep.json'
SLEEP_COMPLETED = 'tests/data/marathon/apps/sleep-completed.json'
SLEEP1 = 'tests/data/marathon/apps/sleep1.json'
SLEEP2 = 'tests/data/marathon/apps/sleep2.json'
FOLLOW = 'tests/data/file/follow.json'
TWO_TASKS = 'tests/data/file/two_tasks.json'
TWO_TASKS_FOLLOW = 'tests/data/file/two_tasks_follow.json'
LS = 'tests/data/tasks/ls-app.json'
INIT_APPS = ((LS, 'ls-app'),
(SLEEP1, 'test-app1'),
(SLEEP2, 'test-app2'))
NUM_TASKS = len(INIT_APPS)
def setup_module():
# create a completed task
with app(SLEEP_COMPLETED, 'test-app-completed', True):
pass
for app_ in INIT_APPS:
add_app(app_[0], True)
def teardown_module():
for app_ in INIT_APPS:
remove_app(app_[1])
def test_help():
@@ -32,22 +53,23 @@ Usage:
dcos task --info
dcos task [--completed --json <task>]
dcos task log [--completed --follow --lines=N] <task> [<file>]
dcos task ls [--long] <task> [<path>]
Options:
-h, --help Show this screen
--info Show a short description of this subcommand
--completed Include completed tasks as well
--follow Output data as the file grows
--follow Print data as the file grows
--json Print json-formatted tasks
--lines=N Output the last N lines [default: 10]
--lines=N Print the last N lines [default: 10]
--long Use a long listing format
--version Show version
Positional Arguments:
<file> Print this file. [default: stdout]
<path> List this directory. [default: '.']
<task> Only match tasks whose ID matches <task>. <task> may be
a substring of the ID, or a unix glob pattern.
<file> Output this file. [default: stdout]
"""
assert_command(['dcos', 'task', '--help'], stdout=stdout)
@@ -58,8 +80,6 @@ def test_info():
def test_task():
_install_sleep_task()
# test `dcos task` output
returncode, stdout, stderr = exec_command(['dcos', 'task', '--json'])
@@ -68,50 +88,37 @@ def test_task():
tasks = json.loads(stdout.decode('utf-8'))
assert isinstance(tasks, collections.Sequence)
assert len(tasks) == 1
assert len(tasks) == NUM_TASKS
schema = create_schema(task_fixture().dict())
for task in tasks:
assert not util.validate_json(task, schema)
_uninstall_sleep()
def test_task_table():
_install_sleep_task()
assert_lines(['dcos', 'task'], 2)
_uninstall_sleep()
assert_lines(['dcos', 'task'], NUM_TASKS+1)
def test_task_completed():
_install_sleep_task()
_uninstall_sleep()
_install_sleep_task()
returncode, stdout, stderr = exec_command(
['dcos', 'task', '--completed', '--json'])
assert returncode == 0
assert stderr == b''
assert len(json.loads(stdout.decode('utf-8'))) > 1
assert len(json.loads(stdout.decode('utf-8'))) > NUM_TASKS
returncode, stdout, stderr = exec_command(
['dcos', 'task', '--json'])
assert returncode == 0
assert stderr == b''
assert len(json.loads(stdout.decode('utf-8'))) == 1
_uninstall_sleep()
assert len(json.loads(stdout.decode('utf-8'))) == NUM_TASKS
def test_task_none():
assert_command(['dcos', 'task', '--json'],
assert_command(['dcos', 'task', 'bogus', '--json'],
stdout=b'[]\n')
def test_filter():
_install_sleep_task()
_install_sleep_task(SLEEP2, 'test-app2')
returncode, stdout, stderr = exec_command(
['dcos', 'task', 'test-app2', '--json'])
@@ -119,48 +126,42 @@ def test_filter():
assert stderr == b''
assert len(json.loads(stdout.decode('utf-8'))) == 1
_uninstall_sleep()
_uninstall_sleep('test-app2')
def test_log_no_files():
""" Tail stdout on nonexistant task """
assert_command(['dcos', 'task', 'log', 'asdf'],
assert_command(['dcos', 'task', 'log', 'bogus'],
returncode=1,
stderr=b'No matching tasks. Exiting.\n')
def test_log_single_file():
""" Tail a single file on a single task """
with app(SLEEP1, 'test-app', True):
returncode, stdout, stderr = exec_command(
['dcos', 'task', 'log', 'test-app'])
returncode, stdout, stderr = exec_command(
['dcos', 'task', 'log', 'test-app1'])
assert returncode == 0
assert stderr == b''
assert len(stdout.decode('utf-8').split('\n')) == 5
assert returncode == 0
assert stderr == b''
assert len(stdout.decode('utf-8').split('\n')) == 5
def test_log_missing_file():
""" Tail a single file on a single task """
with app(SLEEP1, 'test-app', True):
returncode, stdout, stderr = exec_command(
['dcos', 'task', 'log', 'test-app', 'asdf'])
returncode, stdout, stderr = exec_command(
['dcos', 'task', 'log', 'test-app', 'bogus'])
assert returncode == 1
assert stdout == b''
assert stderr == b'No files exist. Exiting.\n'
assert returncode == 1
assert stdout == b''
assert stderr == b'No files exist. Exiting.\n'
def test_log_lines():
""" Test --lines """
with app(SLEEP1, 'test-app', True):
assert_lines(['dcos', 'task', 'log', 'test-app', '--lines=2'], 2)
assert_lines(['dcos', 'task', 'log', 'test-app1', '--lines=2'], 2)
def test_log_lines_invalid():
""" Test invalid --lines value """
assert_command(['dcos', 'task', 'log', 'test-app', '--lines=bogus'],
assert_command(['dcos', 'task', 'log', 'test-app1', '--lines=bogus'],
stdout=b'',
stderr=b'Error parsing string as int\n',
returncode=1)
@@ -168,8 +169,8 @@ def test_log_lines_invalid():
def test_log_follow():
""" Test --follow """
# verify output
with app(FOLLOW, 'follow', True):
# verify output
proc = subprocess.Popen(['dcos', 'task', 'log', 'follow', '--follow'],
stdout=subprocess.PIPE)
@@ -190,17 +191,16 @@ def test_log_follow():
def test_log_two_tasks():
""" Test tailing a single file on two separate tasks """
with app(TWO_TASKS, 'two-tasks', True):
returncode, stdout, stderr = exec_command(
['dcos', 'task', 'log', 'two-tasks'])
returncode, stdout, stderr = exec_command(
['dcos', 'task', 'log', 'test-app'])
assert returncode == 0
assert stderr == b''
assert returncode == 0
assert stderr == b''
lines = stdout.decode('utf-8').split('\n')
assert len(lines) == 11
assert re.match('===>.*<===', lines[0])
assert re.match('===>.*<===', lines[5])
lines = stdout.decode('utf-8').split('\n')
assert len(lines) == 11
assert re.match('===>.*<===', lines[0])
assert re.match('===>.*<===', lines[5])
def test_log_two_tasks_follow():
@@ -231,20 +231,17 @@ def test_log_two_tasks_follow():
def test_log_completed():
""" Test --completed """
""" Test `dcos task log --completed` """
# create a completed task
# ensure that tail lists nothing
# ensure that tail --completed lists a completed task
with app(SLEEP1, 'test-app', True):
pass
assert_command(['dcos', 'task', 'log', 'test-app'],
assert_command(['dcos', 'task', 'log', 'test-app-completed'],
returncode=1,
stderr=b'No matching tasks. Exiting.\n',
stdout=b'')
returncode, stdout, stderr = exec_command(
['dcos', 'task', 'log', '--completed', 'test-app'])
['dcos', 'task', 'log', '--completed', 'test-app-completed'])
assert returncode == 0
assert stderr == b''
assert len(stdout.decode('utf-8').split('\n')) > 4
@@ -262,28 +259,58 @@ def test_log_master_unavailable():
def test_log_slave_unavailable():
""" Test slave's state.json being unavailable """
with app(SLEEP1, 'test-app', True):
client = mesos.DCOSClient()
client.get_slave_state = _mock_exception()
client = mesos.DCOSClient()
client.get_slave_state = _mock_exception()
with patch('dcos.mesos.DCOSClient', return_value=client):
args = ['task', 'log', 'test-app']
stderr = (b"""Error accessing slave: exception\n"""
b"""No matching tasks. Exiting.\n""")
assert_mock(main, args, returncode=1, stderr=stderr)
with patch('dcos.mesos.DCOSClient', return_value=client):
args = ['task', 'log', 'test-app1']
stderr = (b"""Error accessing slave: exception\n"""
b"""No matching tasks. Exiting.\n""")
assert_mock(main, args, returncode=1, stderr=stderr)
def test_log_file_unavailable():
""" Test a file's read.json being unavailable """
with app(SLEEP1, 'test-app', True):
files = _mesos_files(False, "", "stdout")
assert len(files) == 1
files[0].read = _mock_exception('exception')
fltr = "test-app1"
files = _mesos_files(False, fltr, "stdout")
assert len(files) == 1
files[0].read = _mock_exception('exception')
with patch('dcoscli.task.main._mesos_files', return_value=files):
args = ['task', 'log', 'test-app']
stderr = b"No files exist. Exiting.\n"
assert_mock(main, args, returncode=1, stderr=stderr)
with patch('dcoscli.task.main._mesos_files', return_value=files):
args = ['task', 'log', fltr]
stderr = b"No files exist. Exiting.\n"
assert_mock(main, args, returncode=1, stderr=stderr)
def test_ls():
assert_command(['dcos', 'task', 'ls', 'test-app1'],
stdout=b'stderr stdout\n')
def test_ls_multiple_tasks():
returncode, stdout, stderr = exec_command(
['dcos', 'task', 'ls', 'test-app'])
assert returncode == 1
assert stdout == b''
assert stderr.startswith(b'There are multiple tasks with ID matching '
b'[test-app]. Please choose one:\n\t')
def test_ls_long():
assert_lines(['dcos', 'task', 'ls', '--long', 'test-app1'], 2)
def test_ls_path():
assert_command(['dcos', 'task', 'ls', 'ls-app', 'test'],
stdout=b'test1 test2\n')
def test_ls_bad_path():
assert_command(
['dcos', 'task', 'ls', 'test-app1', 'bogus'],
stderr=b'Cannot access [bogus]: No such file or directory\n',
returncode=1)
def _mock_exception(contents='exception'):
@@ -295,7 +322,6 @@ def _mark_non_blocking(file_):
def _install_sleep_task(app_path=SLEEP1, app_name='test-app'):
# install helloworld app
args = ['dcos', 'marathon', 'app', 'add', app_path]
assert_command(args)
watch_all_deployments()

View File

@@ -1,2 +1,2 @@
ID MEM CPUS TASKS HEALTH DEPLOYMENT CONTAINER CMD
/test-app 16.0 0.1 1/1 --- --- mesos sleep 1000
ID MEM CPUS TASKS HEALTH DEPLOYMENT CONTAINER CMD
/test-app 16.0 0.1 1/1 --- --- mesos sleep 1000

View File

@@ -1,2 +1,2 @@
APP HEALTHY STARTED HOST ID
/zero-instance-app True 2015-05-29T19:58:01.114Z dcos-01 zero-instance-app.027b3a83-063d-11e5-84a3-56847afe9799
APP HEALTHY STARTED HOST ID
/zero-instance-app True 2015-05-29T19:58:01.114Z dcos-01 zero-instance-app.027b3a83-063d-11e5-84a3-56847afe9799

View File

@@ -1,2 +1,2 @@
APP ACTION PROGRESS ID
/cassandra/dcos scale 1/2 bebb8ffd-118e-4067-8fcb-d19e44126911
APP ACTION PROGRESS ID
/cassandra/dcos scale 1/2 bebb8ffd-118e-4067-8fcb-d19e44126911

View File

@@ -1,3 +1,3 @@
ID APPS
/test-group 1
/test-group/sleep 1
ID APPS
/test-group 1
/test-group/sleep 1

View File

@@ -0,0 +1,2 @@
-rw-r--r-- 1 root root 4507 Jul 16 23:31 stderr
-rw-r--r-- 1 root root 353857 Jul 16 23:33 stdout

View File

@@ -1,2 +1,2 @@
HOSTNAME IP ID
dcos-01 172.17.8.101 20150630-004309-1695027628-5050-1649-S0
HOSTNAME IP ID
dcos-01 172.17.8.101 20150630-004309-1695027628-5050-1649-S0

View File

@@ -1,2 +1,2 @@
NAME VERSION APP COMMAND DESCRIPTION
helloworld 0.1.0 /helloworld helloworld Example DCOS application package
NAME VERSION APP COMMAND DESCRIPTION
helloworld 0.1.0 /helloworld helloworld Example DCOS application package

View File

@@ -1,8 +1,8 @@
NAME VERSION FRAMEWORK SOURCE DESCRIPTION
cassandra 0.1.0-SNAPSHOT-447-master-3ad1bbf8f7 True https://github.com/mesosphere/universe/archive/master.zip Apache Cassandra running on Apache Mesos
chronos 2.3.4 True https://github.com/mesosphere/universe/archive/master.zip A fault tolerant job scheduler for Mesos which handles dependencies and ISO8601 based schedules.
hdfs 0.1.1 True https://github.com/mesosphere/universe/archive/master.zip Hadoop Distributed File System (HDFS), Highly Available
helloworld 0.1.0 False https://github.com/mesosphere/universe/archive/master.zip Example DCOS application package
kafka 0.9.0-beta True https://github.com/mesosphere/universe/archive/master.zip Apache Kafka running on top of Apache Mesos
marathon 0.8.1 True https://github.com/mesosphere/universe/archive/master.zip A cluster-wide init and control system for services in cgroups or Docker containers.
spark 1.4.0-SNAPSHOT True https://github.com/mesosphere/universe/archive/master.zip Spark is a fast and general cluster computing system for Big Data
NAME VERSION FRAMEWORK SOURCE DESCRIPTION
cassandra 0.1.0-SNAPSHOT-447-master-3ad1bbf8f7 True https://github.com/mesosphere/universe/archive/master.zip Apache Cassandra running on Apache Mesos
chronos 2.3.4 True https://github.com/mesosphere/universe/archive/master.zip A fault tolerant job scheduler for Mesos which handles dependencies and ISO8601 based schedules.
hdfs 0.1.1 True https://github.com/mesosphere/universe/archive/master.zip Hadoop Distributed File System (HDFS), Highly Available
helloworld 0.1.0 False https://github.com/mesosphere/universe/archive/master.zip Example DCOS application package
kafka 0.9.0-beta True https://github.com/mesosphere/universe/archive/master.zip Apache Kafka running on top of Apache Mesos
marathon 0.8.1 True https://github.com/mesosphere/universe/archive/master.zip A cluster-wide init and control system for services in cgroups or Docker containers.
spark 1.4.0-SNAPSHOT True https://github.com/mesosphere/universe/archive/master.zip Spark is a fast and general cluster computing system for Big Data

View File

@@ -1,2 +1,2 @@
NAME HOST ACTIVE TASKS CPU MEM DISK ID
marathon mesos.vm True 0 0.2 32 0 20150502-231327-16842879-5050-3889-0000
NAME HOST ACTIVE TASKS CPU MEM DISK ID
marathon mesos.vm True 0 0.2 32 0 20150502-231327-16842879-5050-3889-0000

View File

@@ -1,2 +1,2 @@
NAME HOST USER STATE ID
test-app mock-hostname root R test-app.d44dd7f2-f9b7-11e4-bb43-56847afe9799
NAME HOST USER STATE ID
test-app mock-hostname root R test-app.d44dd7f2-f9b7-11e4-bb43-56847afe9799

View File

@@ -1,16 +1,21 @@
import datetime
from dcoscli import tables
import mock
import pytz
from ..fixtures.marathon import (app_fixture, app_task_fixture,
deployment_fixture, group_fixture)
from ..fixtures.node import slave_fixture
from ..fixtures.package import package_fixture, search_result_fixture
from ..fixtures.service import framework_fixture
from ..fixtures.task import task_fixture
from ..fixtures.task import browse_fixture, task_fixture
def test_task_table():
_test_table(tables.task_table,
task_fixture,
[task_fixture()],
'tests/unit/data/task.txt')
@@ -24,47 +29,56 @@ def test_app_table():
def test_deployment_table():
_test_table(tables.deployment_table,
deployment_fixture,
[deployment_fixture()],
'tests/unit/data/deployment.txt')
def test_app_task_table():
_test_table(tables.app_task_table,
app_task_fixture,
[app_task_fixture()],
'tests/unit/data/app_task.txt')
def test_service_table():
_test_table(tables.service_table,
framework_fixture,
[framework_fixture()],
'tests/unit/data/service.txt')
def test_group_table():
_test_table(tables.group_table,
group_fixture,
[group_fixture()],
'tests/unit/data/group.txt')
def test_package_table():
_test_table(tables.package_table,
package_fixture,
[package_fixture()],
'tests/unit/data/package.txt')
def test_package_search_table():
_test_table(tables.package_search_table,
search_result_fixture,
[search_result_fixture()],
'tests/unit/data/package_search.txt')
def test_node_table():
_test_table(tables.slave_table,
slave_fixture,
[slave_fixture()],
'tests/unit/data/node.txt')
def test_ls_long_table():
with mock.patch('dcoscli.tables._format_unix_timestamp',
lambda ts: datetime.datetime.fromtimestamp(
ts, pytz.utc).strftime('%b %d %H:%M')):
_test_table(tables.ls_long_table,
browse_fixture(),
'tests/unit/data/ls_long.txt')
def _test_table(table_fn, fixture_fn, path):
table = table_fn([fixture_fn()])
table = table_fn(fixture_fn)
with open(path) as f:
assert str(table) == f.read()

View File

@@ -7,6 +7,7 @@ deps =
pytest
pytest-cov
mock
pytz
-e..
[testenv:syntax]
@@ -22,16 +23,16 @@ commands =
[testenv:py27-integration]
commands =
py.test -x -vv {env:CI_FLAGS:} tests/integrations{posargs}
py.test -vv {env:CI_FLAGS:} tests/integrations{posargs}
[testenv:py34-integration]
commands =
py.test -x -vv {env:CI_FLAGS:} tests/integrations{posargs}
py.test -vv {env:CI_FLAGS:} tests/integrations{posargs}
[testenv:py27-unit]
commands =
py.test -x -vv {env:CI_FLAGS:} tests/unit{posargs}
py.test -vv {env:CI_FLAGS:} tests/unit{posargs}
[testenv:py34-unit]
commands =
py.test -x -vv {env:CI_FLAGS:} tests/unit{posargs}
py.test -vv {env:CI_FLAGS:} tests/unit{posargs}

View File

@@ -11,6 +11,7 @@ deps =
pytest-cov
mock
pypiwin32
pytz
-e..
[testenv:syntax]

View File

@@ -5,6 +5,22 @@ class DCOSException(Exception):
pass
class DCOSHTTPException(DCOSException):
""" A wrapper around Response objects for HTTP error codes.
:param response: requests Response object
:type response: Response
"""
def __init__(self, response):
self.response = response
def __str__(self):
return 'Error while fetching [{0}]: HTTP {1}: {2}'.format(
self.response.request.url,
self.response.status_code,
self.response.reason)
class Error(object):
"""Abstract class for describing errors."""

View File

@@ -1,6 +1,6 @@
import requests
from dcos import util
from dcos.errors import DCOSException
from dcos.errors import DCOSException, DCOSHTTPException
logger = util.get_logger(__name__)
@@ -21,40 +21,13 @@ def _default_is_success(status_code):
:rtype: bool
"""
return status_code >= 200 and status_code < 300
def _default_to_exception(response):
"""
:param response: HTTP response object or Exception
:type response: requests.Response | Exception
:returns: exception
:rtype: Exception
"""
if isinstance(response, Exception) and \
not isinstance(response, requests.exceptions.RequestException):
return response
if isinstance(response, requests.exceptions.ConnectionError):
return DCOSException('URL [{0}] is unreachable: {1}'
.format(response.request.url, response))
elif isinstance(response, requests.exceptions.Timeout):
return DCOSException(
'Request to URL [{0}] timed out'.format(response.request.url))
elif isinstance(response, requests.exceptions.RequestException):
return response
else:
return DCOSException(
'Error while fetching [{0}]: HTTP {1}: {2}'.format(
response.request.url, response.status_code, response.reason))
return 200 <= status_code < 300
@util.duration
def request(method,
url,
is_success=_default_is_success,
to_exception=_default_to_exception,
timeout=None,
**kwargs):
"""Sends an HTTP request.
@@ -90,10 +63,14 @@ def request(method,
url=url,
timeout=timeout,
**kwargs)
except Exception as ex:
logger.exception('Error making HTTP request: %r', url)
raise to_exception(ex)
except requests.exceptions.ConnectionError as e:
logger.exception("HTTP Connection Error")
raise DCOSException('URL [{0}] is unreachable: {1}'.format(
e.request.url, e))
except requests.exceptions.Timeout as e:
logger.exception("HTTP Timeout")
raise DCOSException('Request to URL [{0}] timed out.'.format(
e.request.url))
logger.info('Received HTTP response [%r]: %r',
response.status_code,
@@ -102,10 +79,10 @@ def request(method,
if is_success(response.status_code):
return response
else:
raise to_exception(response)
raise DCOSHTTPException(response)
def head(url, to_exception=_default_to_exception, **kwargs):
def head(url, **kwargs):
"""Sends a HEAD request.
:param url: URL for the new Request object
@@ -119,7 +96,7 @@ def head(url, to_exception=_default_to_exception, **kwargs):
return request('head', url, **kwargs)
def get(url, to_exception=_default_to_exception, **kwargs):
def get(url, **kwargs):
"""Sends a GET request.
:param url: URL for the new Request object
@@ -130,11 +107,10 @@ def get(url, to_exception=_default_to_exception, **kwargs):
:rtype: Response
"""
return request('get', url, to_exception=to_exception, **kwargs)
return request('get', url, **kwargs)
def post(url, to_exception=_default_to_exception,
data=None, json=None, **kwargs):
def post(url, data=None, json=None, **kwargs):
"""Sends a POST request.
:param url: URL for the new Request object
@@ -149,11 +125,10 @@ def post(url, to_exception=_default_to_exception,
:rtype: Response
"""
return request('post', url,
to_exception=to_exception, data=data, json=json, **kwargs)
return request('post', url, data=data, json=json, **kwargs)
def put(url, to_exception=_default_to_exception, data=None, **kwargs):
def put(url, data=None, **kwargs):
"""Sends a PUT request.
:param url: URL for the new Request object
@@ -166,10 +141,10 @@ def put(url, to_exception=_default_to_exception, data=None, **kwargs):
:rtype: Response
"""
return request('put', url, to_exception=to_exception, data=data, **kwargs)
return request('put', url, data=data, **kwargs)
def patch(url, to_exception=_default_to_exception, data=None, **kwargs):
def patch(url, data=None, **kwargs):
"""Sends a PATCH request.
:param url: URL for the new Request object
@@ -182,11 +157,10 @@ def patch(url, to_exception=_default_to_exception, data=None, **kwargs):
:rtype: Response
"""
return request('patch', url,
to_exception=to_exception, data=data, **kwargs)
return request('patch', url, data=data, **kwargs)
def delete(url, to_exception=_default_to_exception, **kwargs):
def delete(url, **kwargs):
"""Sends a DELETE request.
:param url: URL for the new Request object
@@ -197,7 +171,7 @@ def delete(url, to_exception=_default_to_exception, **kwargs):
:rtype: Response
"""
return request('delete', url, to_exception=to_exception, **kwargs)
return request('delete', url, **kwargs)
def silence_requests_warnings():

View File

@@ -2,7 +2,7 @@ import json
from distutils.version import LooseVersion
from dcos import http, util
from dcos.errors import DCOSException
from dcos.errors import DCOSException, DCOSHTTPException
from six.moves import urllib
@@ -25,7 +25,7 @@ def create_client(config=None):
timeout = config.get('core.timeout', http.DEFAULT_TIMEOUT)
logger.info('Creating marathon client with: %r', marathon_url)
return Client(marathon_url, timeout)
return Client(marathon_url, timeout=timeout)
def _get_marathon_url(config):
@@ -52,9 +52,6 @@ def _to_exception(response):
:rtype: Exception
"""
if isinstance(response, Exception):
return DCOSException(_default_marathon_error(str(response)))
if response.status_code == 400:
return DCOSException(
'Error while fetching [{0}]: HTTP {1}: {2}'.format(
@@ -82,7 +79,27 @@ def _to_exception(response):
msg = '\n'.join(error['error'] for error in errs)
return DCOSException(_default_marathon_error(msg))
return DCOSException('Error: {}'.format(response_json['message']))
return DCOSException('Error: {}'.format(message))
def _http_req(fn, *args, **kwargs):
"""Make an HTTP request, and raise a marathon-specific exception for
HTTP error codes.
:param fn: function to call
:type fn: function
:param args: args to pass to `fn`
:type args: [object]
:param kwargs: kwargs to pass to `fn`
:type kwargs: dict
:returns: `fn` return value
:rtype: object
"""
try:
return fn(*args, **kwargs)
except DCOSHTTPException as e:
raise _to_exception(e.response)
class Client(object):
@@ -92,7 +109,7 @@ class Client(object):
:type marathon_url: str
"""
def __init__(self, marathon_url, timeout):
def __init__(self, marathon_url, timeout=http.DEFAULT_TIMEOUT):
self._base_url = marathon_url
self._timeout = timeout
@@ -132,10 +149,7 @@ class Client(object):
"""
url = self._create_url('v2/info')
response = http.get(url,
to_exception=_to_exception,
timeout=self._timeout)
response = _http_req(http.get, url, timeout=self._timeout)
return response.json()
@@ -158,9 +172,7 @@ class Client(object):
url = self._create_url(
'v2/apps{}/versions/{}'.format(app_id, version))
response = http.get(url,
to_exception=_to_exception,
timeout=self._timeout)
response = _http_req(http.get, url, timeout=self._timeout)
# Looks like Marathon return different JSON for versions
if version is None:
@@ -176,11 +188,7 @@ class Client(object):
"""
url = self._create_url('v2/groups')
response = http.get(url,
to_exception=_to_exception,
timeout=self._timeout)
response = _http_req(http.get, url, timeout=self._timeout)
return response.json()['groups']
def get_group(self, group_id, version=None):
@@ -202,10 +210,7 @@ class Client(object):
url = self._create_url(
'v2/groups{}/versions/{}'.format(group_id, version))
response = http.get(url,
to_exception=_to_exception,
timeout=self._timeout)
response = _http_req(http.get, url, timeout=self._timeout)
return response.json()
def get_app_versions(self, app_id, max_count=None):
@@ -231,9 +236,7 @@ class Client(object):
url = self._create_url('v2/apps{}/versions'.format(app_id))
response = http.get(url,
to_exception=_to_exception,
timeout=self._timeout)
response = _http_req(http.get, url, timeout=self._timeout)
if max_count is None:
return response.json()['versions']
@@ -248,11 +251,7 @@ class Client(object):
"""
url = self._create_url('v2/apps')
response = http.get(url,
to_exception=_to_exception,
timeout=self._timeout)
response = _http_req(http.get, url, timeout=self._timeout)
return response.json()['apps']
def add_app(self, app_resource):
@@ -272,9 +271,8 @@ class Client(object):
else:
app_json = app_resource
response = http.post(url,
response = _http_req(http.post, url,
json=app_json,
to_exception=_to_exception,
timeout=self._timeout)
return response.json()
@@ -303,11 +301,10 @@ class Client(object):
url = self._create_url('v2/{}{}'.format(url_endpoint, resource_id))
response = http.put(url,
params=params,
json=payload,
to_exception=_to_exception,
timeout=self._timeout)
response = _http_req(http.put, url,
params=params,
json=payload,
timeout=self._timeout)
return response.json().get('deploymentId')
@@ -351,7 +348,7 @@ class Client(object):
:param force: whether to override running deployments
:type force: bool
:returns: the resulting deployment ID
:rtype: bool
:rtype: str
"""
app_id = self.normalize_app_id(app_id)
@@ -363,14 +360,14 @@ class Client(object):
url = self._create_url('v2/apps{}'.format(app_id))
response = http.put(url,
params=params,
json={'instances': int(instances)},
to_exception=_to_exception,
timeout=self._timeout)
response = _http_req(http.put,
url,
params=params,
json={'instances': int(instances)},
timeout=self._timeout)
deployment = response.json()['deploymentId']
return (deployment, None)
return deployment
def stop_app(self, app_id, force=None):
"""Scales an application to zero instances.
@@ -403,11 +400,7 @@ class Client(object):
params = {'force': 'true'}
url = self._create_url('v2/apps{}'.format(app_id))
http.delete(url,
params=params,
to_exception=_to_exception,
timeout=self._timeout)
_http_req(http.delete, url, params=params, timeout=self._timeout)
def remove_group(self, group_id, force=None):
"""Completely removes the requested application.
@@ -428,10 +421,7 @@ class Client(object):
url = self._create_url('v2/groups{}'.format(group_id))
http.delete(url,
params=params,
to_exception=_to_exception,
timeout=self._timeout)
_http_req(http.delete, url, params=params, timeout=self._timeout)
def restart_app(self, app_id, force=None):
"""Performs a rolling restart of all of the tasks.
@@ -453,11 +443,9 @@ class Client(object):
url = self._create_url('v2/apps{}/restart'.format(app_id))
response = http.post(url,
response = _http_req(http.post, url,
params=params,
to_exception=_to_exception,
timeout=self._timeout)
return response.json()
def get_deployment(self, deployment_id):
@@ -471,10 +459,7 @@ class Client(object):
url = self._create_url('v2/deployments')
response = http.get(url,
to_exception=_to_exception,
timeout=self._timeout)
response = _http_req(http.get, url, timeout=self._timeout)
deployment = next(
(deployment for deployment in response.json()
if deployment_id == deployment['id']),
@@ -493,9 +478,7 @@ class Client(object):
url = self._create_url('v2/deployments')
response = http.get(url,
to_exception=_to_exception,
timeout=self._timeout)
response = _http_req(http.get, url, timeout=self._timeout)
if app_id is not None:
app_id = self.normalize_app_id(app_id)
@@ -529,11 +512,9 @@ class Client(object):
url = self._create_url('v2/deployments/{}'.format(deployment_id))
response = http.delete(
url,
params=params,
to_exception=_to_exception,
timeout=self._timeout)
response = _http_req(http.delete, url,
params=params,
timeout=self._timeout)
if force:
return None
@@ -572,9 +553,7 @@ class Client(object):
url = self._create_url('v2/tasks')
response = http.get(url,
to_exception=_to_exception,
timeout=self._timeout)
response = _http_req(http.get, url, timeout=self._timeout)
if app_id is not None:
app_id = self.normalize_app_id(app_id)
@@ -598,9 +577,7 @@ class Client(object):
url = self._create_url('v2/tasks')
response = http.get(url,
to_exception=_to_exception,
timeout=self._timeout)
response = _http_req(http.get, url, timeout=self._timeout)
task = next(
(task for task in response.json()['tasks']
@@ -622,7 +599,7 @@ class Client(object):
return None
url = self._create_url('v2/schemas/app')
response = http.get(url, timeout=self._timeout)
response = _http_req(http.get, url, timeout=self._timeout)
return response.json()
@@ -653,11 +630,9 @@ class Client(object):
else:
group_json = group_resource
response = http.post(url,
response = _http_req(http.post, url,
json=group_json,
to_exception=_to_exception,
timeout=self._timeout)
return response.json()
def get_leader(self):
@@ -668,8 +643,7 @@ class Client(object):
"""
url = self._create_url('v2/leader')
response = http.get(url, timeout=self._timeout)
response = _http_req(http.get, url, timeout=self._timeout)
return response.json()['leader']

View File

@@ -195,7 +195,7 @@ class DCOSClient(object):
http.post(url, data=data, timeout=self._timeout)
def metadata(self):
""" Get /metadata
""" GET /metadata
:returns: /metadata content
:rtype: dict
@@ -203,6 +203,34 @@ class DCOSClient(object):
url = self.get_dcos_url('metadata')
return http.get(url, timeout=self._timeout).json()
def browse(self, slave, path):
""" GET /files/browse.json
Request
path:... # path to run ls on
Response
[
{
path: # full path to file
nlink:
size:
mtime:
mode:
uid:
gid:
}
]
:param slave: slave to issue the request on
:type slave: Slave
:returns: /files/browse.json response
:rtype: dict
"""
url = self.slave_url(slave['id'], 'files/browse.json')
return http.get(url, params={'path': path}).json()
class MesosDNSClient(object):
""" Mesos-DNS client
@@ -273,12 +301,12 @@ class Master(object):
'slave/{}/'.format(slave['id']))
def slave(self, fltr):
"""Returns the slave that has `fltr` in its id. Raises a
"""Returns the slave that has `fltr` in its ID. Raises a
DCOSException if there is not exactly one such slave.
:param fltr: filter string
:type fltr: str
:returns: the slave that has `fltr` in its id
:returns: the slave that has `fltr` in its ID
:rtype: Slave
"""
@@ -290,19 +318,19 @@ class Master(object):
elif len(slaves) > 1:
matches = ['\t{0}'.format(slave.id) for slave in slaves]
raise DCOSException(
"There are multiple slaves with that id. " +
"There are multiple slaves with that ID. " +
"Please choose one: {}".format('\n'.join(matches)))
else:
return slaves[0]
def task(self, fltr):
"""Returns the task with `fltr` in its id. Raises a DCOSException if
"""Returns the task with `fltr` in its ID. Raises a DCOSException if
there is not exactly one such task.
:param fltr: filter string
:type fltr: str
:returns: the task that has `fltr` in its id
:returns: the task that has `fltr` in its ID
:rtype: Task
"""
@@ -310,10 +338,11 @@ class Master(object):
if len(tasks) == 0:
raise DCOSException(
'Cannot find a task containing "{}"'.format(fltr))
'Cannot find a task with ID containing "{}"'.format(fltr))
elif len(tasks) > 1:
msg = ["There are multiple tasks with that id. Please choose one:"]
msg = [("There are multiple tasks with ID matching [{}]. " +
"Please choose one:").format(fltr)]
msg += ["\t{0}".format(t["id"]) for t in tasks]
raise DCOSException('\n'.join(msg))
@@ -321,9 +350,9 @@ class Master(object):
return tasks[0]
def framework(self, framework_id):
"""Returns a framework by id
"""Returns a framework by ID
:param framework_id: the framework's id
:param framework_id: the framework's ID
:type framework_id: str
:returns: the framework
:rtype: Framework

View File

@@ -13,7 +13,6 @@ import time
import concurrent.futures
import jsonschema
import prettytable
import pystache
import six
from dcos import constants
@@ -514,37 +513,6 @@ def humanize_bytes(b):
return "{0:.2f} {1}".format(b/float(factor), suffix)
def table(fields, objs, sortby=None):
"""Returns a PrettyTable. `fields` represents the header schema of
the table. `objs` represents the objects to be rendered into
rows.
:param fields: An OrderedDict, where each element represents a
column. The key is the column header, and the
value is the function that transforms an element of
`objs` into a value for that column.
:type fields: OrderdDict(str, function)
:param objs: objects to render into rows
:type objs: [object]
"""
tb = prettytable.PrettyTable(
[k.upper() for k in fields.keys()],
border=False,
hrules=prettytable.NONE,
vrules=prettytable.NONE,
left_padding_width=0,
right_padding_width=1,
sortby=sortby
)
for obj in objs:
row = [fn(obj) for fn in fields.values()]
tb.add_row(row)
return tb
@contextlib.contextmanager
def open_file(path, *args):
"""Context manager that opens a file, and raises a DCOSException if

View File

@@ -19,8 +19,8 @@ commands =
[testenv:py27-unit]
commands =
py.test -x -vv {env:CI_FLAGS:} --cov {envsitepackagesdir}/dcos tests
py.test -vv {env:CI_FLAGS:} --cov {envsitepackagesdir}/dcos tests
[testenv:py34-unit]
commands =
py.test -x -vv {env:CI_FLAGS:} --cov {envsitepackagesdir}/dcos tests
py.test -vv {env:CI_FLAGS:} --cov {envsitepackagesdir}/dcos tests