openstack/deb-python-dcos
Code Issues Proposed changes

add snapshot report to dcos cli (#661)

Browse Source
This commit is contained in:
Maksym Naboka
2016-06-30 14:00:42 -07:00
committed by tamarrow
parent c5faea4ba1
commit 3712b47e1c
6 changed files with 615 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
cli/dcoscli/data/help/node.txt
View File

@@ -12,6 +12,11 @@ Usage:
[--master-proxy]
(--leader | --master | --mesos-id=<mesos-id> | --slave=<slave-id>)
[<command>]
dcos node snapshot create (<nodes>)...
dcos node snapshot delete <snapshot>
dcos node snapshot download <snapshot> [--location=<location>]
dcos node snapshot (--list | --status | --cancel)
[--json]
Commands:
log
@@ -19,6 +24,12 @@ Commands:
ssh
Establish an SSH connection to the master or agent nodes of your DC/OS
cluster.
snapshot create
Create a snapshot. Nodes can be: ip address, hostname, mesos ID or key words "all", "masters", "agents".
snapshot download
Download a snapshot.
snapshot delete
Delete a snapshot.
Options:
--config-file=<path>
@@ -50,6 +61,14 @@ Options:
Agent node with the provided ID.
--user=<user>
The SSH user, where the default user [default: core].
--list
List available snapshots.
--status
Print snapshot job status.
--cancel
Cancel a running snapshot job.
--location=<location>
Download a snapshot to a particular location. If not set default to present working directory.
--version
Print version information.

359
cli/dcoscli/node/main.py
View File

@@ -1,18 +1,28 @@
import functools
import os
import subprocess
import dcoscli
import docopt
import six
from dcos import cmds, emitting, errors, mesos, util
from dcos import (cmds, config, cosmospackage, emitting, errors, http, mesos,
util)
from dcos.errors import DCOSException, DefaultError
from dcoscli import log, tables
from dcoscli.package.main import confirm, get_cosmos_url
from dcoscli.subcommand import default_command_info, default_doc
from dcoscli.util import decorate_docopt_usage
from six.moves import urllib
logger = util.get_logger(__name__)
emitter = emitting.FlatEmitter()
SNAPSHOT_BASE_URL = '/system/health/v1/report/snapshot/'
# if snapshot size if more then 100Mb then warn user.
SNAPSHOT_WARN_SIZE = 1000000
def main(argv):
try:
@@ -64,13 +74,358 @@ def _cmds():
'--user', '--master-proxy', '<command>'],
function=_ssh),
cmds.Command(
hierarchy=['node', 'snapshot', 'create'],
arg_keys=['<nodes>'],
function=_snapshot_create),
cmds.Command(
hierarchy=['node', 'snapshot', 'delete'],
arg_keys=['<snapshot>'],
function=_snapshot_delete),
cmds.Command(
hierarchy=['node', 'snapshot', 'download'],
arg_keys=['<snapshot>', '--location'],
function=_snapshot_download),
cmds.Command(
hierarchy=['node', 'snapshot'],
arg_keys=['--list', '--status', '--cancel', '--json'],
function=_snapshot_manage),
cmds.Command(
hierarchy=['node'],
arg_keys=['--json'],
function=_list),
function=_list)
]
def snapshot_error(fn):
@functools.wraps(fn)
def check_for_snapshot_error(*args, **kwargs):
response = fn(*args, **kwargs)
if response.status_code != 200:
err_msg = ('Error making {} request\nURL: '
'{}, status_code: {}.'.format(args[1], args[0],
response.status_code))
if not kwargs.get('stream'):
err_status = _read_http_response_body(response).get('status')
if err_status:
err_msg = err_status
raise DCOSException(err_msg)
return response
return check_for_snapshot_error
def _check_3dt_version():
"""
The function checks if cluster has snapshot capability.
:raises: DCOSException if cluster does not have snapshot capability
"""
cosmos = cosmospackage.Cosmos(get_cosmos_url())
if not cosmos.has_capability('SUPPORT_CLUSTER_REPORT'):
raise DCOSException(
'DC/OS backend does not support snapshot capabilities in this '
'version. Must be DC/OS >= 1.8')
# http://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size # noqa
def sizeof_fmt(num, suffix='B'):
for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
if abs(num) < 1024.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f%s%s" % (num, 'Yi', suffix)
def _get_snapshots_json():
"""
Get a json with a list of snapshots
:return: available snapshots on a cluster.
:rtype: dict
"""
return _do_snapshot_request(
urllib.parse.urljoin(SNAPSHOT_BASE_URL, 'list/all'),
'GET')
def _get_snapshots_list():
"""
Get a list of tuples (snapshot_file_name, file_size), ..
:return: list of snapshots
:rtype: list of tuples
"""
available_snapshots = []
for _, snapshot_files in _get_snapshots_json().items():
if snapshot_files is None:
continue
for snapshot_file_obj in snapshot_files:
if ('file_name' not in snapshot_file_obj
or 'file_size' not in snapshot_file_obj):
raise DCOSException(
'Request to get a list of available snapshot returned '
'unexpected response {}'.format(snapshot_file_obj))
available_snapshots.append(
(os.path.basename(snapshot_file_obj['file_name']),
snapshot_file_obj['file_size']))
return available_snapshots
def _snapshot_manage(list_snapshots, status, cancel, json):
"""
Manage snapshots
:param list_snapshots: a list of available snapshots
:type list_snapshots: bool
:param status: show snapshot job status
:type status: bool
:param cancel: cancel snapshot job
:type cancel: bool
:return: process return code
:rtype: int
"""
_check_3dt_version()
if list_snapshots:
if json:
emitter.publish(_get_snapshots_json())
return 0
available_snapshots = _get_snapshots_list()
if not available_snapshots:
emitter.publish("No snapshots")
return 0
emitter.publish("Available snapshots:")
for available_snapshot in sorted(available_snapshots,
key=lambda t: t[0]):
emitter.publish('{} {}'.format(available_snapshot[0],
sizeof_fmt(available_snapshot[1])))
return 0
elif status:
url = urllib.parse.urljoin(SNAPSHOT_BASE_URL, 'status/all')
snapshot_response = _do_snapshot_request(url, 'GET')
if json:
emitter.publish(snapshot_response)
return 0
for host, props in sorted(snapshot_response.items()):
emitter.publish(host)
for key, value in sorted(props.items()):
emitter.publish(' {}: {}'.format(key, value))
emitter.publish('\n')
return 0
elif cancel:
url = urllib.parse.urljoin(SNAPSHOT_BASE_URL, 'cancel')
snapshot_response = _do_snapshot_request(url, 'POST')
if json:
emitter.publish(snapshot_response)
return 0
if 'status' not in snapshot_response:
raise DCOSException(
'Request to cancel a snapshot job {} returned '
'an unexpected response {}'.format(url, snapshot_response))
emitter.publish(snapshot_response['status'])
return 0
else:
raise DCOSException(
'Must specify one of list_snapshots, status, cancel')
@snapshot_error
def _do_request(url, method, timeout=None, stream=False, **kwargs):
"""
make HTTP request
:param url: url
:type url: string
:param method: HTTP method, GET or POST
:type method: string
:param timeout: HTTP request timeout, default 3 seconds
:type timeout: integer
:param stream: stream parameter for requests lib
:type stream: bool
:return: http response
:rtype: requests.Response
"""
def _is_success(status_code):
# consider 400 and 503 to be successful status codes.
# API will return the error message.
if status_code in [200, 400, 503]:
return True
return False
# if timeout is not passed, try to read `core.timeout`
# if `core.timeout` is not set, default to 3 min.
if timeout is None:
timeout = config.get_config_val('core.timeout')
if not timeout:
timeout = 180
# POST to snapshot api
base_url = config.get_config_val("core.dcos_url")
if not base_url:
raise config.missing_config_exception(['core.dcos_url'])
url = urllib.parse.urljoin(base_url, url)
if method.lower() == 'get':
http_response = http.get(url, is_success=_is_success, timeout=timeout,
**kwargs)
elif method.lower() == 'post':
http_response = http.post(url, is_success=_is_success, timeout=timeout,
stream=stream, **kwargs)
else:
raise DCOSException('Unsupported HTTP method: ' + method)
return http_response
def _do_snapshot_request(url, method, **kwargs):
"""
Make HTTP request and expect a JSON response.
:param url: url
:type url: string
:param method: HTTP method, GET or POST
:type method: string
:return: snapshot JSON repsponse
:rtype: dict
"""
http_response = _do_request(url, method, **kwargs)
return _read_http_response_body(http_response)
def _read_http_response_body(http_response):
"""
Get an requests HTTP response, read it and deserialize to json.
:param http_response: http response
:type http_response: requests.Response onject
:return: deserialized json
:rtype: dict
"""
data = b''
try:
for chunk in http_response.iter_content(1024):
data += chunk
snapshot_response = util.load_jsons(data.decode('utf-8'))
return snapshot_response
except DCOSException:
raise
def _snapshot_download(snapshot, location):
"""
Download snapshot and put in the the current directory.
:param snapshot: snapshot file name.
:type snapshot: string
:return: status code
:rtype: int
"""
# make sure the requested snapshot exists
snapshot_size = 0
for available_snapshot in _get_snapshots_list():
# _get_snapshot_list must return a list of tuples
# where first element is file name and second is its size.
if len(available_snapshot) != 2:
raise DCOSException(
'Request to get a list of snapshots returned an '
'unexpected response: {}'.format(available_snapshot))
# available_snapshot[0] is a snapshot file name
# available_snapshot[1] is a snapshot file size
if available_snapshot[0] == snapshot:
snapshot_size = available_snapshot[1]
url = urllib.parse.urljoin(SNAPSHOT_BASE_URL, 'serve/' + snapshot)
snapshot_location = os.path.join(os.getcwd(), snapshot)
if location:
if os.path.isdir(location):
snapshot_location = os.path.join(location, snapshot)
else:
snapshot_location = location
if snapshot_size > SNAPSHOT_WARN_SIZE:
if not confirm('Snapshot size is {}, are you sure you want '
'to download it?'.format(sizeof_fmt(snapshot_size)),
False):
return 0
r = _do_request(url, 'GET', stream=True)
try:
with open(snapshot_location, 'wb') as f:
for chunk in r.iter_content(1024):
f.write(chunk)
except Exception as e:
raise DCOSException(e)
emitter.publish('Snapshot downloaded to ' + snapshot_location)
return 0
def _snapshot_delete(snapshot):
"""
Delete a snapshot
:param snapshot: snapshot file name
:type: str
:return: status code
:rtype: int
"""
_check_3dt_version()
url = urllib.parse.urljoin(
SNAPSHOT_BASE_URL, 'delete/' + snapshot)
response = _do_snapshot_request(url, 'POST')
if 'status' not in response:
raise DCOSException(
'Request to delete the snapshot {} returned an '
'unexpected response {}'.format(url, response))
emitter.publish(response['status'])
return 0
def _snapshot_create(nodes):
"""
Create a snapshot.
:param nodes: a list of nodes to collect the logs from.
:type nodes: list
:returns: process return code
:rtype: int
"""
_check_3dt_version()
url = urllib.parse.urljoin(SNAPSHOT_BASE_URL, 'create')
response = _do_snapshot_request(url,
'POST',
json={'nodes': nodes})
if ('status' not in response or 'extra' not in response
or 'snapshot_name' not in response['extra']):
raise DCOSException(
'Request to create snapshot {} returned an '
'unexpected response {}'.format(url, response))
emitter.publish('\n{}, available snapshot: {}'.format(
response['status'],
response['extra']['snapshot_name']))
return 0
def _info():
"""Print node cli information.

8
cli/dcoscli/package/main.py
View File

@@ -301,7 +301,7 @@ def _user_options(path):
return util.load_json(options_file)
def _confirm(prompt, yes):
def confirm(prompt, yes):
"""
:param prompt: message to display to the terminal
:type prompt: str
@@ -365,7 +365,7 @@ def _install(package_name, package_version, options_path, app_id, cli, app,
pre_install_notes = pkg_json.get('preInstallNotes')
if app and pre_install_notes:
emitter.publish(pre_install_notes)
if not _confirm('Continue installing?', yes):
if not confirm('Continue installing?', yes):
emitter.publish('Exiting installation.')
return 0
@@ -519,7 +519,7 @@ def _uninstall(package_name, remove_all, app_id, cli, app):
return 0
def _get_cosmos_url():
def get_cosmos_url():
"""
:returns: cosmos base url
:rtype: str
@@ -540,7 +540,7 @@ def _get_package_manager():
:rtype: PackageManager
"""
cosmos_url = _get_cosmos_url()
cosmos_url = get_cosmos_url()
cosmos_manager = cosmospackage.Cosmos(cosmos_url)
if cosmos_manager.enabled():
return cosmos_manager

19
cli/tests/data/help/node.txt
View File

@@ -12,6 +12,11 @@ Usage:
[--master-proxy]
(--leader | --master | --mesos-id=<mesos-id> | --slave=<slave-id>)
[<command>]
dcos node snapshot create (<nodes>)...
dcos node snapshot delete <snapshot>
dcos node snapshot download <snapshot> [--location=<location>]
dcos node snapshot (--list | --status | --cancel)
[--json]
Commands:
log
@@ -19,6 +24,12 @@ Commands:
ssh
Establish an SSH connection to the master or agent nodes of your DC/OS
cluster.
snapshot create
Create a snapshot. Nodes can be: ip address, hostname, mesos ID or key words "all", "masters", "agents".
snapshot download
Download a snapshot.
snapshot delete
Delete a snapshot.
Options:
--config-file=<path>
@@ -50,6 +61,14 @@ Options:
Agent node with the provided ID.
--user=<user>
The SSH user, where the default user [default: core].
--list
List available snapshots.
--status
Print snapshot job status.
--cancel
Cancel a running snapshot job.
--location=<location>
Download a snapshot to a particular location. If not set default to present working directory.
--version
Print version information.

186
cli/tests/unit/test_node.py Normal file
View File

@@ -0,0 +1,186 @@
import dcoscli.node.main as main
from dcos.errors import DCOSException
import mock
import pytest
@mock.patch('dcos.cosmospackage.Cosmos')
def test_check_version_fail(mock_cosmos):
"""
Test _check_3dt_version(), should throw DCOSException exception.
"""
mock_cosmos().enabled.return_value = True
mock_cosmos().has_capability.return_value = False
with pytest.raises(DCOSException) as excinfo:
main._check_3dt_version()
assert str(excinfo.value) == (
'DC/OS backend does not support snapshot capabilities in this version.'
' Must be DC/OS >= 1.8')
@mock.patch('dcos.cosmospackage.Cosmos')
@mock.patch('dcos.http.get')
def test_check_version_success(mock_get, mock_cosmos):
"""
Test _check_3dt_version(), should not fail.
"""
mock_cosmos().enabled.return_value = True
m = mock.MagicMock()
m.json.return_value = {
'capabilities': [{'name': 'SUPPORT_CLUSTER_REPORT'}]
}
mock_get.return_value = m
main._check_3dt_version()
@mock.patch('dcos.cosmospackage.Cosmos')
@mock.patch('dcos.http.get')
@mock.patch('dcoscli.node.main._do_snapshot_request')
def test_node_snapshot_create(mock_do_snapshot_request, mock_get, mock_cosmos):
"""
Test _snapshot_create(), should not fail.
"""
mock_cosmos().enabled.return_value = True
m = mock.MagicMock()
m.json.return_value = {
'capabilities': [{'name': 'SUPPORT_CLUSTER_REPORT'}]
}
mock_get.return_value = m
mock_do_snapshot_request.return_value = {
'status': 'OK',
'extra': {
'snapshot_name': 'snapshot.zip'
}
}
main._snapshot_create(['10.10.0.1'])
mock_do_snapshot_request.assert_called_once_with(
'/system/health/v1/report/snapshot/create',
'POST',
json={'nodes': ['10.10.0.1']})
@mock.patch('dcos.cosmospackage.Cosmos')
@mock.patch('dcos.http.get')
@mock.patch('dcoscli.node.main._do_snapshot_request')
def test_node_snapshot_delete(mock_do_snapshot_request, mock_get, mock_cosmos):
"""
Test _snapshot_delete(), should not fail
"""
mock_cosmos().enabled.return_value = True
m = mock.MagicMock()
m.json.return_value = {
'capabilities': [{'name': 'SUPPORT_CLUSTER_REPORT'}]
}
mock_get.return_value = m
mock_do_snapshot_request.return_value = {
'status': 'OK'
}
main._snapshot_delete('snapshot.zip')
mock_do_snapshot_request.assert_called_once_with(
'/system/health/v1/report/snapshot/delete/snapshot.zip',
'POST'
)
@mock.patch('dcos.cosmospackage.Cosmos')
@mock.patch('dcos.http.get')
@mock.patch('dcoscli.node.main._do_snapshot_request')
def test_node_snapshot_list(mock_do_snapshot_request, mock_get, mock_cosmos):
"""
Test _snapshot_manage(), should not fail
"""
mock_cosmos().enabled.return_value = True
m = mock.MagicMock()
m.json.return_value = {
'capabilities': [{'name': 'SUPPORT_CLUSTER_REPORT'}]
}
mock_get.return_value = m
mock_do_snapshot_request.return_value = {
'127.0.0.1': [
{
'file_name': 'snapshot.zip',
'file_size': 123
}
]
}
# _snapshot_manage(list_snapshots, status, cancel, json)
main._snapshot_manage(True, False, False, False)
mock_do_snapshot_request.assert_called_once_with(
'/system/health/v1/report/snapshot/list/all',
'GET'
)
@mock.patch('dcos.cosmospackage.Cosmos')
@mock.patch('dcos.http.get')
@mock.patch('dcoscli.node.main._do_snapshot_request')
def test_node_snapshot_status(mock_do_snapshot_request, mock_get, mock_cosmos):
"""
Test _snapshot_manage(), should not fail
"""
mock_cosmos().enabled.return_value = True
m = mock.MagicMock()
m.json.return_value = {
'capabilities': [{'name': 'SUPPORT_CLUSTER_REPORT'}]
}
mock_get.return_value = m
mock_do_snapshot_request.return_value = {
'host1': {
'prop1': 'value1'
}
}
# _snapshot_manage(list_snapshots, status, cancel, json)
main._snapshot_manage(False, True, False, False)
mock_do_snapshot_request.assert_called_once_with(
'/system/health/v1/report/snapshot/status/all',
'GET'
)
@mock.patch('dcos.cosmospackage.Cosmos')
@mock.patch('dcos.http.get')
@mock.patch('dcoscli.node.main._do_snapshot_request')
def test_node_snapshot_cancel(mock_do_snapshot_request, mock_get, mock_cosmos):
"""
Test _snapshot_manage(), should not fail
"""
mock_cosmos().enabled.return_value = True
m = mock.MagicMock()
m.json.return_value = {
'capabilities': [{'name': 'SUPPORT_CLUSTER_REPORT'}]
}
mock_get.return_value = m
mock_do_snapshot_request.return_value = {
'status': 'success'
}
# _snapshot_manage(list_snapshots, status, cancel, json)
main._snapshot_manage(False, False, True, False)
mock_do_snapshot_request.assert_called_once_with(
'/system/health/v1/report/snapshot/cancel',
'POST'
)
@mock.patch('dcos.cosmospackage.Cosmos')
@mock.patch('dcoscli.node.main._do_request')
@mock.patch('dcoscli.node.main._get_snapshots_list')
def test_node_snapshot_download(mock_get_snapshot_list, mock_do_request,
mock_cosmos):
mock_cosmos().enabled.return_value = True
mock_get_snapshot_list.return_value = [('snap.zip', 123)]
main._snapshot_download('snap.zip', None)
mock_do_request.assert_called_with(
'/system/health/v1/report/snapshot/serve/snap.zip', 'GET',
stream=True)

31
dcos/cosmospackage.py
View File

@@ -20,8 +20,37 @@ class Cosmos():
def __init__(self, cosmos_url):
self.cosmos_url = cosmos_url
def has_capability(self, capability):
"""Check if cluster has a capability.
:param capability: capability name
:type capability: string
:return: does the cluster has capability
:rtype: bool
"""
if not self.enabled():
return False
try:
url = urllib.parse.urljoin(self.cosmos_url, 'capabilities')
response = http.get(url,
headers=_get_capabilities_header()).json()
except Exception as e:
logger.exception(e)
return False
if 'capabilities' not in response:
logger.error(
'Request to get cluster capabilities: {} '
'returned unexpected response: {}. '
'Missing "capabilities" field'.format(url, response))
return False
return {'name': capability} in response['capabilities']
def enabled(self):
"""Returns whether or not cosmos is enabled on specified dcos cluter
"""Returns whether or not cosmos is enabled on specified dcos cluster
:rtype: bool
"""