openstack/deb-python-dcos
Code Issues Proposed changes

dcos node ssh

Browse Source
This commit is contained in:
Michael Gummelt
2015-07-01 10:46:36 -07:00
parent 0b88bc6830
commit ac41782ef1
9 changed files with 253 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
cli/dcoscli/node/main.py
View File

@@ -4,18 +4,28 @@ Usage:
dcos node --info
dcos node [--json]
dcos node log [--follow --lines=N --master --slave=<slave-id>]
dcos node ssh [--option SSHOPT=VAL ...]
[--config-file=<path>]
[--user=<user>]
(--master | --slave=<slave-id>)
Options:
-h, --help Show this screen
--info Show a short description of this subcommand
--json Print json-formatted nodes
--follow Output data as the file grows
--lines=N Output the last N lines [default: 10]
--master Output the leading master's Mesos log
--slave=<slave-id> Output this slave's Mesos log
--version Show version
-h, --help Show this screen
--info Show a short description of this subcommand
--json Print json-formatted nodes
--follow Output data as the file grows
--lines=N Output the last N lines [default: 10]
--master Access the leading master
--slave=<slave-id> Access the slave with the provided ID
--option SSHOPT=VAL SSH option (see `man ssh_config`)
--config-file=<path> Path to ssh config file
--user=<user> SSH user [default: core]
--version Show version
"""
import os
import subprocess
import dcoscli
import docopt
from dcos import cmds, emitting, errors, mesos, util
@@ -61,6 +71,12 @@ def _cmds():
arg_keys=['--follow', '--lines', '--master', '--slave'],
function=_log),
cmds.Command(
hierarchy=['node', 'ssh'],
arg_keys=['--master', '--slave', '--option', '--config-file',
'--user'],
function=_ssh),
cmds.Command(
hierarchy=['node'],
arg_keys=['--json'],
@@ -80,7 +96,7 @@ def _info():
def _list(json_):
"""List dcos nodes
"""List DCOS nodes
:param json_: If true, output json.
Otherwise, output a human readable table.
@@ -89,7 +105,7 @@ def _list(json_):
:rtype: int
"""
client = mesos.MesosClient()
client = mesos.DCOSClient()
slaves = client.get_state_summary()['slaves']
if json_:
emitter.publish(slaves)
@@ -148,3 +164,61 @@ def _mesos_files(master, slave_id):
slave = mesos.get_master().slave(slave_id)
files.append(mesos.MesosFile('/slave/log', slave=slave))
return files
def _ssh(master, slave, option, config_file, user):
"""SSH into a DCOS node. Since only the masters are definitely
publicly available, we first ssh into an arbitrary master, then
hop to the desired node.
:param master: True if the user has opted to SSH into the leading
master
:type master: bool | None
:param slave: The slave ID if the user has opted to SSH into a slave
:type slave: str | None
:param option: SSH option
:type option: [str]
:param config_file: SSH config file
:type config_file: str | None
:param user: SSH user
:type user: str | None
:rtype: int
:returns: process return code
"""
if not os.environ.get('SSH_AUTH_SOCK'):
raise DCOSException(
"There is no SSH_AUTH_SOCK env variable, which likely means you " +
"aren't running `ssh-agent`. `dcos node ssh` depends on " +
"`ssh-agent` so we can safely use your private key to hop " +
"between nodes in your cluster. Please run `ssh-agent`, " +
"then add your private key with `ssh-add`.")
master_public_ip = mesos.DCOSClient().metadata()['PUBLIC_IPV4']
ssh_options = ' '.join('-o {}'.format(opt) for opt in option)
if config_file:
ssh_config = '-F {}'.format(config_file)
else:
ssh_config = ''
if master:
host = 'leader.mesos'
else:
summary = mesos.DCOSClient().get_state_summary()
slave_obj = next((slave_ for slave_ in summary['slaves']
if slave_['id'] == slave),
None)
if slave_obj:
host = mesos.parse_pid(slave_obj['pid'])[1]
else:
raise DCOSException('No slave found with ID [{}]'.format(slave))
cmd = "ssh -A -t {0} {1} {2}@{3} ssh -A -t {2}@{4}".format(
ssh_options,
ssh_config,
user,
master_public_ip,
host)
return subprocess.call(cmd, shell=True)

2
cli/dcoscli/service/main.py
View File

@@ -121,5 +121,5 @@ def _shutdown(service_id):
:rtype: int
"""
mesos.MesosClient().shutdown_framework(service_id)
mesos.DCOSClient().shutdown_framework(service_id)
return 0

2
cli/dcoscli/task/main.py
View File

@@ -170,7 +170,7 @@ def _mesos_files(completed, fltr, path):
"""
# get tasks
client = mesos.MesosClient()
client = mesos.DCOSClient()
master = mesos.Master(client.get_master_state())
tasks = master.tasks(completed=completed, fltr=fltr)

2
cli/tests/data/node/ssh_config Normal file
View File

@@ -0,0 +1,2 @@
Host *
Protocol 0

119
cli/tests/integrations/test_node.py
View File

@@ -1,7 +1,12 @@
import json
import os
import pty
import re
import subprocess
import time
import dcos.util as util
from dcos import mesos
from dcos.util import create_schema
from ..fixtures.node import slave_fixture
@@ -15,16 +20,23 @@ Usage:
dcos node --info
dcos node [--json]
dcos node log [--follow --lines=N --master --slave=<slave-id>]
dcos node ssh [--option SSHOPT=VAL ...]
[--config-file=<path>]
[--user=<user>]
(--master | --slave=<slave-id>)
Options:
-h, --help Show this screen
--info Show a short description of this subcommand
--json Print json-formatted nodes
--follow Output data as the file grows
--lines=N Output the last N lines [default: 10]
--master Output the leading master's Mesos log
--slave=<slave-id> Output this slave's Mesos log
--version Show version
-h, --help Show this screen
--info Show a short description of this subcommand
--json Print json-formatted nodes
--follow Output data as the file grows
--lines=N Output the last N lines [default: 10]
--master Access the leading master
--slave=<slave-id> Access the slave with the provided ID
--option SSHOPT=VAL SSH option (see `man ssh_config`)
--config-file=<path> Path to ssh config file
--user=<user> SSH user [default: core]
--version Show version
"""
assert_command(['dcos', 'node', '--help'], stdout=stdout)
@@ -103,13 +115,85 @@ def test_node_log_invalid_lines():
returncode=1)
def _node():
returncode, stdout, stderr = exec_command(['dcos', 'node', '--json'])
def test_node_ssh_master():
_node_ssh(['--master'])
assert returncode == 0
assert stderr == b''
return json.loads(stdout.decode('utf-8'))
def test_node_ssh_slave():
slave_id = mesos.DCOSClient().get_state_summary()['slaves'][0]['id']
_node_ssh(['--slave={}'.format(slave_id)])
def test_node_ssh_option():
stdout, stderr = _node_ssh_output(
['--master', '--option', 'Protocol=0'])
assert stdout == b''
assert stderr.startswith(b'ignoring bad proto spec')
def test_node_ssh_config_file():
stdout, stderr = _node_ssh_output(
['--master', '--config-file', 'tests/data/node/ssh_config'])
assert stdout == b''
assert stderr.startswith(b'ignoring bad proto spec')
def test_node_ssh_user():
stdout, stderr = _node_ssh_output(
['--master', '--user=bogus', '--option', 'PasswordAuthentication=no'])
assert stdout == b''
assert stderr.startswith(b'Permission denied')
def test_node_ssh_no_agent():
stderr = (b"There is no SSH_AUTH_SOCK env variable, which likely means "
b"you aren't running `ssh-agent`. `dcos node ssh` depends on"
b" `ssh-agent` so we can safely use your private key to hop "
b"between nodes in your cluster. Please run `ssh-agent`, then "
b"add your private key with `ssh-add`.\n")
assert_command(['dcos', 'node', 'ssh', '--master'],
stdout=b'',
stderr=stderr,
returncode=1)
def _node_ssh_output(args):
# ssh must run with stdin attached to a tty
master, slave = pty.openpty()
cmd = ('ssh-agent /bin/bash -c ' +
'"ssh-add /host-home/.vagrant.d/insecure_private_key ' +
'2> /dev/null && dcos node ssh --option StrictHostKeyChecking=no' +
' {}"').format(' '.join(args))
proc = subprocess.Popen(cmd,
stdin=slave,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=os.setsid,
close_fds=True,
shell=True)
os.close(slave)
# wait for the ssh connection
time.sleep(8)
# kill the whole process group
os.killpg(os.getpgid(proc.pid), 15)
os.close(master)
return proc.communicate()
def _node_ssh(args):
stdout, stderr = _node_ssh_output(args)
print('SSH STDOUT: {}'.format(stdout.decode('utf-8')))
print('SSH STDERR: {}'.format(stderr.decode('utf-8')))
assert stdout
assert ((stderr == b'') or
(len(stderr.split('\n')) == 2 and
stderr.startswith('Warning: Permanently added')))
def _get_schema(slave):
@@ -120,3 +204,12 @@ def _get_schema(slave):
schema['properties']['attributes']['additionalProperties'] = True
return schema
def _node():
returncode, stdout, stderr = exec_command(['dcos', 'node', '--json'])
assert returncode == 0
assert stderr == b''
return json.loads(stdout.decode('utf-8'))

8
cli/tests/integrations/test_task.py
View File

@@ -252,10 +252,10 @@ def test_log_completed():
def test_log_master_unavailable():
""" Test master's state.json being unavailable """
client = mesos.MesosClient()
client = mesos.DCOSClient()
client.get_master_state = _mock_exception()
with patch('dcos.mesos.MesosClient', return_value=client):
with patch('dcos.mesos.DCOSClient', return_value=client):
args = ['task', 'log', '_']
assert_mock(main, args, returncode=1, stderr=(b"exception\n"))
@@ -263,10 +263,10 @@ def test_log_master_unavailable():
def test_log_slave_unavailable():
""" Test slave's state.json being unavailable """
with app(SLEEP1, 'test-app', True):
client = mesos.MesosClient()
client = mesos.DCOSClient()
client.get_slave_state = _mock_exception()
with patch('dcos.mesos.MesosClient', return_value=client):
with patch('dcos.mesos.DCOSClient', return_value=client):
args = ['task', 'log', 'test-app']
stderr = (b"""Error accessing slave: exception\n"""
b"""No matching tasks. Exiting.\n""")

40
dcos/mesos.py
View File

@@ -23,11 +23,11 @@ def get_master():
:rtype: Master
"""
return Master(MesosClient().get_master_state())
return Master(DCOSClient().get_master_state())
class MesosClient(object):
"""Client for communicating with the Mesos master"""
class DCOSClient(object):
"""Client for communicating with DCOS"""
def __init__(self):
config = util.get_config()
@@ -40,8 +40,21 @@ class MesosClient(object):
else:
self._mesos_master_url = mesos_master_url
def get_dcos_url(self, path):
""" Create a DCOS URL
:param path: the path suffix of the URL
:type path: str
:returns: DCOS URL
:rtype: str
"""
if self._dcos_url:
return urllib.parse.urljoin(self._dcos_url, path)
else:
raise util.missing_config_exception('core.dcos_url')
def master_url(self, path):
""" Create a URL that hits the master
""" Create a master URL
:param path: the path suffix of the desired URL
:type path: str
@@ -55,7 +68,7 @@ class MesosClient(object):
# TODO (mgummelt): this doesn't work with self._mesos_master_url
def slave_url(self, slave_id, path):
""" Create a URL that hits the slave
""" Create a slave URL
:param slave_id: slave ID
:type slave_id: str
@@ -180,11 +193,20 @@ class MesosClient(object):
url = self.master_url('master/shutdown')
http.post(url, data=data)
def metadata(self):
""" Get /metadata
:returns: /metadata content
:rtype: dict
"""
url = self.get_dcos_url('metadata')
return http.get(url).json()
class Master(object):
"""Mesos Master Model
:param state: Mesos master state json
:param state: Mesos master's state.json
:type state: dict
"""
@@ -417,7 +439,7 @@ class Slave(object):
"""
if not self._state:
self._state = MesosClient().get_slave_state(self['id'])
self._state = DCOSClient().get_slave_state(self['id'])
return self._state
def _framework_dicts(self):
@@ -611,7 +633,7 @@ class MesosFile(object):
:param slave: slave where the file lives
:type slave: Slave | None
:param mesos_client: client to use for network requests
:type mesos_client: MesosClient | None
:type mesos_client: DCOSClient | None
"""
@@ -630,7 +652,7 @@ class MesosFile(object):
self._task = task
self._path = path
self._mesos_client = mesos_client or MesosClient()
self._mesos_client = mesos_client or DCOSClient()
self._cursor = 0
def size(self):

12
dcos/package.py
View File

@@ -165,7 +165,7 @@ def uninstall(package_name, remove_all, app_id, cli, app):
remove_all,
app_id,
marathon.create_client(),
mesos.MesosClient())
mesos.DCOSClient())
if num_apps > 0:
uninstalled = True
@@ -192,7 +192,7 @@ def uninstall_subcommand(distribution_name):
return subcommand.uninstall(distribution_name)
def uninstall_app(app_name, remove_all, app_id, init_client, mesos_client):
def uninstall_app(app_name, remove_all, app_id, init_client, dcos_client):
"""Uninstalls an app.
:param app_name: The app to uninstall
@@ -203,8 +203,8 @@ def uninstall_app(app_name, remove_all, app_id, init_client, mesos_client):
:type app_id: str
:param init_client: The program to use to run the app
:type init_client: object
:param mesos_client: the mesos client
:type mesos_client: dcos.mesos.MesosClient
:param dcos_client: the DCOS client
:type dcos_client: dcos.mesos.DCOSClient
:returns: number of apps uninstalled
:rtype: int
"""
@@ -245,7 +245,7 @@ def uninstall_app(app_name, remove_all, app_id, init_client, mesos_client):
if framework_name is not None:
logger.info(
'Trying to shutdown framework {}'.format(framework_name))
frameworks = mesos.Master(mesos_client.get_master_state()) \
frameworks = mesos.Master(dcos_client.get_master_state()) \
.frameworks(inactive=True)
# Look up all the framework names
@@ -259,7 +259,7 @@ def uninstall_app(app_name, remove_all, app_id, init_client, mesos_client):
'Found the following frameworks: {}'.format(framework_ids))
if len(framework_ids) == 1:
mesos_client.shutdown_framework(framework_ids[0])
dcos_client.shutdown_framework(framework_ids[0])
elif len(framework_ids) > 1:
raise DCOSException(
"Unable to shutdown the framework for [{}] because there "

23
dcos/util.py
View File

@@ -108,6 +108,7 @@ def get_config():
:rtype: Toml
"""
# avoid circular import
from dcos import config
return config.load_from_path(
@@ -128,15 +129,27 @@ def get_config_vals(config, keys):
missing = [key for key in keys if key not in config]
if missing:
msg = '\n'.join(
'Missing required config parameter: "{0}".'.format(key) +
' Please run `dcos config set {0} <value>`.'.format(key)
for key in keys)
raise DCOSException(msg)
raise missing_config_exception(keys)
return [config[key] for key in keys]
def missing_config_exception(keys):
""" DCOSException for a missing config value
:param keys: keys in the config dict
:type keys: [str]
:returns: DCOSException
:rtype: DCOSException
"""
msg = '\n'.join(
'Missing required config parameter: "{0}".'.format(key) +
' Please run `dcos config set {0} <value>`.'.format(key)
for key in keys)
return DCOSException(msg)
def which(program):
"""Returns the path to the named executable program.