openstack/deb-python-dcos
Code Issues Proposed changes

Merge pull request #134 from mesosphere/list_commands

Browse Source 
list command in 'dcos package list-installed' -- DCOS-1142
This commit is contained in:
mgummelt
2015-04-28 17:40:33 -07:00
parent e3f31e8791 d4953b3e93
commit c36e45f61a
8 changed files with 416 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cli/dcoscli/analytics.py
View File

@@ -71,7 +71,7 @@ def _send_segment_event(event, properties):
requests.post(SEGMENT_URL,
json=data,
auth=HTTPBasicAuth(key, ''),
timeout=3)
timeout=1)
except Exception as e:
logger.exception(e)

27
cli/dcoscli/package/main.py
View File

@@ -376,26 +376,23 @@ def _list(endpoints, app_id, package_name):
init_client = marathon.create_client(config)
def keep(pkg):
if package_name and pkg.get('name', '') != package_name:
return False
if app_id and pkg.get('appId', '') != app_id:
return False
return True
installed, error = package.list_installed_packages(init_client, keep)
installed, error = package.installed_packages(init_client, endpoints)
if error is not None:
emitter.publish(error)
return 1
if endpoints:
installed, error = package.get_tasks_multiple(init_client, installed)
if error is not None:
emitter.publish(error)
return 1
results = []
for pkg in installed:
if not ((package_name and pkg.name() != package_name) or
(app_id and pkg.app and pkg.app['appId'] != app_id)):
result, err = pkg.dict()
if err is not None:
emitter.publish(err)
return 1
emitter.publish(installed)
results.append(result)
emitter.publish(results)
return 0

30
cli/tests/integrations/cli/common.py
View File

@@ -30,3 +30,33 @@ def exec_command(cmd, env=None, stdin=None):
print('STDERR: {}'.format(stderr.decode('utf-8')))
return (process.returncode, stdout, stderr)
def assert_command(cmd,
returncode=0,
stdout=b'',
stderr=b'',
env=None,
stdin=None):
"""Execute CLI command and assert expected behavior.
:param cmd: Program and arguments
:type cmd: list of str
:param returncode: Expected return code
:type returncode: int
:param stdout: Expected stdout
:type stdout: str
:param stderr: Expected stderr
:type stderr: str
:param env: Environment variables
:type env: dict of str to str
:param stdin: File to use for stdin
:type stdin: file
:rtype: None
"""
returncode_, stdout_, stderr_ = exec_command(cmd, env, stdin)
assert returncode_ == returncode
assert stdout_ == stdout
assert stderr_ == stderr

2
cli/tests/integrations/cli/test_analytics.py
View File

@@ -53,7 +53,7 @@ def test_no_exc():
assert kwargs['json'] == {'anonymousId': ANON_ID,
'event': SEGMENT_IO_CLI_EVENT,
'properties': props}
assert kwargs['timeout'] == 3
assert kwargs['timeout'] == 1
# rollbar
assert rollbar.report_message.call_count == 0

260
cli/tests/integrations/cli/test_package.py
View File

@@ -4,7 +4,7 @@ import os
import six
from dcos.api import subcommand
from common import exec_command
from common import assert_command, exec_command
def test_package():
@@ -143,17 +143,8 @@ tutorial-gce.html",
def test_bad_install():
returncode, stdout, stderr = exec_command(
['dcos',
'package',
'install',
'mesos-dns',
'--options=tests/data/package/mesos-dns-config-bad.json'])
assert returncode == 1
assert stdout == b''
assert stderr == b"""\
args = ['--options=tests/data/package/mesos-dns-config-bad.json']
stderr = b"""\
Error: 'mesos-dns/config-url' is a required property
Value: {"mesos-dns/host": false}
@@ -161,19 +152,14 @@ Error: False is not of type 'string'
Path: mesos-dns/host
Value: false
"""
_install_mesos_dns(args=args,
returncode=1,
stdout=b'',
stderr=stderr)
def test_install():
returncode, stdout, stderr = exec_command(
['dcos',
'package',
'install',
'mesos-dns',
'--options=tests/data/package/mesos-dns-config.json'])
assert returncode == 0
assert stdout == b'Installing package [mesos-dns] version [alpha]\n'
assert stderr == b''
_install_mesos_dns()
def test_package_metadata():
@@ -254,32 +240,17 @@ wLjEuMCJdfQ=="""
def test_install_with_id():
returncode, stdout, stderr = exec_command(
['dcos',
'package',
'install',
'mesos-dns',
'--options=tests/data/package/mesos-dns-config.json',
'--app-id=dns-1'])
args = ['--options=tests/data/package/mesos-dns-config.json',
'--app-id=dns-1']
stdout = b"""Installing package [mesos-dns] version [alpha] \
with app id [dns-1]\n"""
_install_mesos_dns(args=args, stdout=stdout)
assert returncode == 0
assert stdout == b"""Installing package [mesos-dns] version [alpha] \
with app id [dns-1]
"""
assert stderr == b''
returncode, stdout, stderr = exec_command(
['dcos',
'package',
'install',
'mesos-dns',
'--options=tests/data/package/mesos-dns-config.json',
'--app-id=dns-2'])
assert returncode == 0
assert stdout == b"""Installing package [mesos-dns] version [alpha] \
args = ['--options=tests/data/package/mesos-dns-config.json',
'--app-id=dns-2']
stdout = b"""Installing package [mesos-dns] version [alpha] \
with app id [dns-2]\n"""
assert stderr == b''
_install_mesos_dns(args=args, stdout=stdout)
def test_install_missing_package():
@@ -294,38 +265,19 @@ You may need to run 'dcos package update' to update your repositories
def test_uninstall_with_id():
returncode, stdout, stderr = exec_command(
['dcos', 'package', 'uninstall', 'mesos-dns', '--app-id=dns-1'])
assert returncode == 0
assert stdout == b''
assert stderr == b''
_uninstall_mesos_dns(args=['--app-id=dns-1'])
def test_uninstall_all():
returncode, stdout, stderr = exec_command(
['dcos', 'package', 'uninstall', 'mesos-dns', '--all'])
assert returncode == 0
assert stdout == b''
assert stderr == b''
_uninstall_mesos_dns(args=['--all'])
def test_uninstall_missing():
returncode, stdout, stderr = exec_command(
['dcos', 'package', 'uninstall', 'mesos-dns'])
stderr = b'Package [mesos-dns] is not installed.\n'
_uninstall_mesos_dns(returncode=1, stderr=stderr)
assert returncode == 1
assert stdout == b''
assert stderr == b'Package [mesos-dns] is not installed.\n'
returncode, stdout, stderr = exec_command(
['dcos', 'package', 'uninstall', 'mesos-dns', '--app-id=dns-1'])
assert returncode == 1
assert stdout == b''
assert stderr == b"""Package [mesos-dns] with id [dns-1] is not \
installed.\n"""
stderr = b'Package [mesos-dns] with id [dns-1] is not installed.\n'
_uninstall_mesos_dns(args=['--app-id=dns-1'], returncode=1, stderr=stderr)
def test_uninstall_subcommand():
@@ -352,43 +304,23 @@ Installing CLI subcommand for package [helloworld]
def test_list_installed():
returncode, stdout, stderr = exec_command(['dcos',
'package',
'list-installed'])
assert_command(['dcos', 'package', 'list-installed'],
stdout=b'[]\n')
assert returncode == 0
assert stdout == b'[]\n'
assert stderr == b''
assert_command(['dcos', 'package', 'list-installed', 'xyzzy'],
stdout=b'[]\n')
returncode, stdout, stderr = exec_command(
['dcos', 'package', 'list-installed', 'xyzzy'])
assert_command(['dcos', 'package', 'list-installed', '--app-id=/xyzzy'],
stdout=b'[]\n')
assert returncode == 0
assert stdout == b'[]\n'
assert stderr == b''
returncode, stdout, stderr = exec_command(
['dcos', 'package', 'list-installed', '--app-id=/xyzzy'])
assert returncode == 0
assert stdout == b'[]\n'
assert stderr == b''
returncode, stdout, stderr = exec_command(
['dcos',
'package',
'install',
'mesos-dns',
'--options=tests/data/package/mesos-dns-config.json'])
assert returncode == 0
assert stdout == b'Installing package [mesos-dns] version [alpha]\n'
assert stderr == b''
_install_mesos_dns()
expected_output = b"""\
[
{
"appId": "/mesos-dns",
"app": {
"appId": "/mesos-dns"
},
"description": "DNS-based service discovery for Mesos.",
"maintainer": "support@mesosphere.io",
"name": "mesos-dns",
@@ -396,7 +328,7 @@ def test_list_installed():
"postInstallNotes": "Please refer to the tutorial instructions for \
further setup requirements: http://mesosphere.github.io/mesos-dns/docs\
/tutorial-gce.html",
"registryVersion": "0.1.0-alpha",
"releaseVersion": "0",
"scm": "https://github.com/mesosphere/mesos-dns.git",
"tags": [
"mesosphere"
@@ -406,26 +338,93 @@ further setup requirements: http://mesosphere.github.io/mesos-dns/docs\
}
]
"""
returncode, stdout, stderr = exec_command(
['dcos', 'package', 'list-installed'])
assert_command(['dcos', 'package', 'list-installed'],
stdout=expected_output)
assert returncode == 0
assert stderr == b''
assert stdout == expected_output
assert_command(['dcos', 'package', 'list-installed', 'mesos-dns'],
stdout=expected_output)
returncode, stdout, stderr = exec_command(
['dcos', 'package', 'list-installed', 'mesos-dns'])
assert_command(
['dcos', 'package', 'list-installed', '--app-id=/mesos-dns'],
stdout=expected_output)
assert returncode == 0
assert stderr == b''
assert stdout == expected_output
assert_command(
['dcos', 'package', 'list-installed', 'ceci-nest-pas-une-package'],
stdout=b'[]\n')
returncode, stdout, stderr = exec_command(
['dcos', 'package', 'list-installed', '--app-id=/mesos-dns'])
assert_command(
['dcos', 'package', 'list-installed',
'--app-id=/ceci-nest-pas-une-package'],
stdout=b'[]\n')
assert returncode == 0
assert stderr == b''
assert stdout == expected_output
_uninstall_mesos_dns()
def test_list_installed_cli():
stdout = b"""Installing package [helloworld] version [0.1.0]
Installing CLI subcommand for package [helloworld]
"""
assert_command(['dcos', 'package', 'install', 'helloworld'],
stdout=stdout)
stdout = b"""\
[
{
"app": {
"appId": "/helloworld"
},
"command": {
"name": "helloworld"
},
"description": "Example DCOS application package",
"maintainer": "support@mesosphere.io",
"name": "helloworld",
"packageSource": "git://github.com/mesosphere/universe.git",
"releaseVersion": "0",
"tags": [
"mesosphere",
"example",
"subcommand"
],
"version": "0.1.0",
"website": "https://github.com/mesosphere/dcos-helloworld"
}
]
"""
assert_command(['dcos', 'package', 'list-installed'],
stdout=stdout)
assert_command(['dcos', 'package', 'uninstall', 'helloworld'])
stdout = b"Installing CLI subcommand for package [helloworld]\n"
assert_command(['dcos', 'package', 'install', 'helloworld', '--cli'],
stdout=stdout)
stdout = b"""\
[
{
"command": {
"name": "helloworld"
},
"description": "Example DCOS application package",
"maintainer": "support@mesosphere.io",
"name": "helloworld",
"packageSource": "git://github.com/mesosphere/universe.git",
"releaseVersion": "0",
"tags": [
"mesosphere",
"example",
"subcommand"
],
"version": "0.1.0",
"website": "https://github.com/mesosphere/dcos-helloworld"
}
]
"""
assert_command(['dcos', 'package', 'list-installed'],
stdout=stdout)
assert_command(['dcos', 'package', 'uninstall', 'helloworld'])
def test_search():
@@ -451,15 +450,6 @@ def test_search():
assert stderr == b''
def test_cleanup():
returncode, stdout, stderr = exec_command(
['dcos', 'package', 'uninstall', 'mesos-dns'])
assert returncode == 0
assert stdout == b''
assert stderr == b''
def get_app_labels(app_id):
returncode, stdout, stderr = exec_command(
['dcos', 'marathon', 'app', 'show', app_id])
@@ -469,3 +459,21 @@ def get_app_labels(app_id):
app_json = json.loads(stdout.decode('utf-8'))
return app_json.get('labels')
def _uninstall_mesos_dns(args=[],
returncode=0,
stdout=b'',
stderr=b''):
cmd = ['dcos', 'package', 'uninstall', 'mesos-dns'] + args
assert_command(cmd, returncode, stdout, stderr)
def _install_mesos_dns(
args=['--options=tests/data/package/mesos-dns-config.json'],
returncode=0,
stdout=b'Installing package [mesos-dns] version [alpha]\n',
stderr=b''):
cmd = ['dcos', 'package', 'install', 'mesos-dns'] + args
assert_command(cmd, returncode, stdout, stderr)

215
dcos/api/package.py
View File

@@ -23,13 +23,13 @@ emitter = emitting.FlatEmitter()
PACKAGE_METADATA_KEY = 'DCOS_PACKAGE_METADATA'
PACKAGE_REGISTRY_VERSION_KEY = 'DCOS_PACKAGE_REGISTRY_VERSION'
PACKAGE_NAME_KEY = 'DCOS_PACKAGE_NAME'
PACKAGE_VERSION_KEY = 'DCOS_PACKAGE_VERSION'
PACKAGE_SOURCE_KEY = 'DCOS_PACKAGE_SOURCE'
PACKAGE_FRAMEWORK_KEY = 'DCOS_PACKAGE_IS_FRAMEWORK'
PACKAGE_RELEASE_KEY = 'DCOS_PACKAGE_RELEASE'
PACKAGE_COMMAND_KEY = 'DCOS_PACKAGE_COMMAND'
PACKAGE_REGISTRY_VERSION_KEY = 'DCOS_PACKAGE_REGISTRY_VERSION'
def install_app(pkg, version, init_client, options, app_id):
@@ -229,12 +229,162 @@ The app ids of the installed package instances are: [{}].""".format(
return (len(matching_apps), None)
def list_installed_packages(init_client, result_predicate=lambda x: True):
class InstalledPackage(object):
"""Represents an intalled DCOS package. One of `app` and
`subcommand` must be supplied.
:param app: A dictionary representing a marathon app. Of the
format returned by `installed_apps()`
:type app: dict
:param subcommand: Installed subcommand
:type subcommand: subcommand.InstalledSubcommand
"""
def __init__(self, app=None, subcommand=None):
assert app or subcommand
self.app = app
self.subcommand = subcommand
def name(self):
"""
:returns: The name of the package
:rtype: str
"""
if self.subcommand:
return self.subcommand.name
else:
return self.app['name']
def dict(self):
""" A dictionary representation of the package. Used by `dcos package
list-installed`.
:returns: A dictionary representation of the package.
:rtype: (dict, None)
"""
ret = {'name': self.name}
if self.subcommand:
ret['command'] = {'name': self.subcommand.name}
if self.app:
ret['app'] = {'appId': self.app['appId']}
if self.subcommand:
package_json, err = self.subcommand.package_json()
if err is not None:
return (None, err)
ret.update(package_json)
package_source, err = self.subcommand.package_source()
if err is not None:
return (None, err)
ret['packageSource'] = package_source
package_version, err = self.subcommand.package_version()
if err is not None:
return (None, err)
ret['releaseVersion'] = package_version
else:
ret.update(self.app)
ret.pop('appId')
return (ret, None)
def installed_packages(init_client, endpoints):
"""Returns all installed packages in the format:
[{
'app': {
'id': <id>
},
'command': {
'name': <name>
}
...<metadata>...
}]
:param init_client: The program to use to list packages
:type init_client: object
:param result_predicate: The predicate to use to filter results
:type result_predicate: function(dict): bool
:param endpoints: Whether to include a list of
endpoints as port-host pairs
:type endpoints: boolean
:returns: A list of installed packages
:rtype: ([InstalledPackage], Error)
"""
apps, error = installed_apps(init_client, endpoints)
if error is not None:
return (None, error)
subcommands, error = installed_subcommands()
if error is not None:
return (None, error)
dicts = collections.defaultdict(lambda: {'app': None, 'command': None})
for app in apps:
key = (app['name'], app['releaseVersion'], app['packageSource'])
dicts[key]['app'] = app
for subcmd in subcommands:
package_version, err = subcmd.package_version()
if err is not None:
return (None, err)
package_source, err = subcmd.package_source()
if err is not None:
return (None, err)
key = (subcmd.name, package_version, package_source)
dicts[key]['command'] = subcmd
pkgs = []
for key, pkg in dicts.items():
pkgs.append(InstalledPackage(pkg['app'], pkg['command']))
return (pkgs, None)
def installed_subcommands():
"""Returns all installed subcommands.
:returns: all installed subcommands
:rtype: ([InstalledSubcommand], Error)
"""
ret = [subcommand.InstalledSubcommand(name) for name in
subcommand.distributions(util.dcos_path())]
return (ret, None)
def installed_apps(init_client, endpoints=False):
"""
Returns all installed apps. An app is of the format:
{
'appId': <appId>,
'packageSource': <source>,
'registryVersion': <app_version>,
'releaseVersion': <release_version>
'endpoints' (optional): [{
'host': <host>,
'ports': <ports>,
}]
..<package.json properties>..
}
:param init_client: The program to use to list packages
:type init_client: object
:param endpoints: Whether to include a list of
endpoints as port-host pairs
:type endpoints: boolean
:returns: all installed apps
:rtype: (list of dict, Error)
"""
@@ -242,7 +392,7 @@ def list_installed_packages(init_client, result_predicate=lambda x: True):
if error is not None:
return (None, error)
encoded_pkgs = [(a['id'], a['labels'])
encoded_apps = [(a['id'], a['labels'])
for a in apps
if a.get('labels', {}).get(PACKAGE_METADATA_KEY)]
@@ -250,44 +400,33 @@ def list_installed_packages(init_client, result_predicate=lambda x: True):
app_id, labels = pair
encoded = labels.get(PACKAGE_METADATA_KEY, {})
source = labels.get(PACKAGE_SOURCE_KEY)
registry_version = labels.get(PACKAGE_REGISTRY_VERSION_KEY)
release_version = labels.get(PACKAGE_RELEASE_KEY)
decoded = base64.b64decode(six.b(encoded)).decode()
decoded_json, error = util.load_jsons(decoded)
if error is None:
decoded_json['appId'] = app_id
decoded_json['packageSource'] = source
decoded_json['registryVersion'] = registry_version
decoded_json['releaseVersion'] = release_version
return (decoded_json, error)
decoded_pkgs = [decode_and_add_context(encoded)
for encoded in encoded_pkgs]
decoded_apps = [decode_and_add_context(encoded)
for encoded in encoded_apps]
# Filter elements that failed to parse correctly as JSON,
# or do not match the supplied predicate
pkgs = [pair[0] for pair in decoded_pkgs
if pair[1] is None and result_predicate(pair[0])]
valid_apps = [pair[0] for pair in decoded_apps if pair[1] is None]
return (pkgs, None)
if endpoints:
for app in valid_apps:
tasks, err = init_client.get_tasks(app["appId"])
if err is not None:
return (None, err)
app['endpoints'] = [{"host": t["host"], "ports": t["ports"]}
for t in tasks]
def get_tasks_multiple(init_client, apps):
"""Adds tasks to app dictionary
:param init_client: The program to use to list packages
:type init_client: object
:param apps: list of dict
:type apps: object
:rtype: (list, Error)
"""
for app in apps:
tasks, err = init_client.get_tasks(app["appId"])
if err is not None:
return (None, err)
app["endpoints"] = [{"host": t["host"], "ports": t["ports"]}
for t in tasks]
return (apps, None)
return (valid_apps, None)
def search(query, cfg):
@@ -1075,16 +1214,10 @@ class Package():
"""
data, error = self._data(path)
if error is not None:
return (None, error)
try:
result = json.loads(data)
except ValueError:
return (None, Error(''))
return (result, None)
return util.load_jsons(data)
def _data(self, path):
"""Returns the content of the supplied file, relative to the base path.
@@ -1096,15 +1229,7 @@ class Package():
"""
full_path = os.path.join(self.path, path)
if not os.path.isfile(full_path):
return (None, Error('Path [{}] is not a file'.format(full_path)))
try:
with open(full_path) as fd:
content = fd.read()
return (content, None)
except IOError:
return (None, Error('Unable to open file [{}]'.format(full_path)))
return util.read_file(full_path)
def package_versions(self):
"""Returns all of the available package versions, most recent first.

47
dcos/api/subcommand.py
View File

@@ -415,3 +415,50 @@ def _generic_error(package_name):
return errors.DefaultError(
'Error installing {!r} package'.format(package_name))
class InstalledSubcommand(object):
""" Represents an installed subcommand.
:param name: The name of the subcommand
:type name: str
"""
def __init__(self, name):
self.name = name
def _dir(self):
"""
:returns: path to this subcommand's directory.
:rtype: (str, Error)
"""
return package_dir(self.name)
def package_version(self):
"""
:returns: this subcommand's version.
:rtype: (str, Error)
"""
version_path = os.path.join(self._dir(), 'version')
return util.read_file(version_path)
def package_source(self):
"""
:returns: this subcommand's source.
:rtype: (str, Error)
"""
source_path = os.path.join(self._dir(), 'source')
return util.read_file(source_path)
def package_json(self):
"""
:returns: contents of this subcommand's package.json file.
:rtype: (dict, Error)
"""
package_json_path = os.path.join(self._dir(), 'package.json')
with open(package_json_path) as package_json_file:
return util.load_json(package_json_file)

23
dcos/api/util.py
View File

@@ -70,6 +70,26 @@ def ensure_dir(directory):
os.makedirs(directory, 0o775)
def read_file(path):
"""
:param path: path to file
:type path: str
:returns: contents of file
:rtype: (str, Error)
"""
if not os.path.isfile(path):
return (None, errors.DefaultError(
'Path [{}] is not a file'.format(path)))
try:
with open(path) as fd:
content = fd.read()
return (content, None)
except IOError:
return (None, errors.DefaultError(
'Unable to open file [{}]'.format(path)))
def which(program):
"""Returns the path to the named executable program.
@@ -103,8 +123,7 @@ def dcos_path():
"""
dcos_bin_dir = os.path.realpath(sys.argv[0])
dcos_dir = os.path.dirname(os.path.dirname(dcos_bin_dir))
return dcos_dir
return os.path.dirname(os.path.dirname(dcos_bin_dir))
def configure_logger_from_environ():