From d4953b3e9378d4add445cfb7174193a364bbff17 Mon Sep 17 00:00:00 2001 From: Michael Gummelt Date: Thu, 23 Apr 2015 10:36:20 -0700 Subject: [PATCH] list command in 'dcos package list-installed' --- cli/dcoscli/analytics.py | 2 +- cli/dcoscli/package/main.py | 27 +- cli/tests/integrations/cli/common.py | 30 +++ cli/tests/integrations/cli/test_analytics.py | 2 +- cli/tests/integrations/cli/test_package.py | 260 ++++++++++--------- dcos/api/package.py | 215 +++++++++++---- dcos/api/subcommand.py | 47 ++++ dcos/api/util.py | 23 +- 8 files changed, 416 insertions(+), 190 deletions(-) diff --git a/cli/dcoscli/analytics.py b/cli/dcoscli/analytics.py index d22c558..46d792f 100644 --- a/cli/dcoscli/analytics.py +++ b/cli/dcoscli/analytics.py @@ -71,7 +71,7 @@ def _send_segment_event(event, properties): requests.post(SEGMENT_URL, json=data, auth=HTTPBasicAuth(key, ''), - timeout=3) + timeout=1) except Exception as e: logger.exception(e) diff --git a/cli/dcoscli/package/main.py b/cli/dcoscli/package/main.py index 0f9de96..53a2494 100644 --- a/cli/dcoscli/package/main.py +++ b/cli/dcoscli/package/main.py @@ -376,26 +376,23 @@ def _list(endpoints, app_id, package_name): init_client = marathon.create_client(config) - def keep(pkg): - if package_name and pkg.get('name', '') != package_name: - return False - if app_id and pkg.get('appId', '') != app_id: - return False - return True - - installed, error = package.list_installed_packages(init_client, keep) - + installed, error = package.installed_packages(init_client, endpoints) if error is not None: emitter.publish(error) return 1 - if endpoints: - installed, error = package.get_tasks_multiple(init_client, installed) - if error is not None: - emitter.publish(error) - return 1 + results = [] + for pkg in installed: + if not ((package_name and pkg.name() != package_name) or + (app_id and pkg.app and pkg.app['appId'] != app_id)): + result, err = pkg.dict() + if err is not None: + emitter.publish(err) + return 1 - emitter.publish(installed) + results.append(result) + + emitter.publish(results) return 0 diff --git a/cli/tests/integrations/cli/common.py b/cli/tests/integrations/cli/common.py index 638dda7..2d8d50f 100644 --- a/cli/tests/integrations/cli/common.py +++ b/cli/tests/integrations/cli/common.py @@ -30,3 +30,33 @@ def exec_command(cmd, env=None, stdin=None): print('STDERR: {}'.format(stderr.decode('utf-8'))) return (process.returncode, stdout, stderr) + + +def assert_command(cmd, + returncode=0, + stdout=b'', + stderr=b'', + env=None, + stdin=None): + """Execute CLI command and assert expected behavior. + + :param cmd: Program and arguments + :type cmd: list of str + :param returncode: Expected return code + :type returncode: int + :param stdout: Expected stdout + :type stdout: str + :param stderr: Expected stderr + :type stderr: str + :param env: Environment variables + :type env: dict of str to str + :param stdin: File to use for stdin + :type stdin: file + :rtype: None + """ + + returncode_, stdout_, stderr_ = exec_command(cmd, env, stdin) + + assert returncode_ == returncode + assert stdout_ == stdout + assert stderr_ == stderr diff --git a/cli/tests/integrations/cli/test_analytics.py b/cli/tests/integrations/cli/test_analytics.py index 7b01dec..339ed9a 100644 --- a/cli/tests/integrations/cli/test_analytics.py +++ b/cli/tests/integrations/cli/test_analytics.py @@ -53,7 +53,7 @@ def test_no_exc(): assert kwargs['json'] == {'anonymousId': ANON_ID, 'event': SEGMENT_IO_CLI_EVENT, 'properties': props} - assert kwargs['timeout'] == 3 + assert kwargs['timeout'] == 1 # rollbar assert rollbar.report_message.call_count == 0 diff --git a/cli/tests/integrations/cli/test_package.py b/cli/tests/integrations/cli/test_package.py index 80e8da3..2deba63 100644 --- a/cli/tests/integrations/cli/test_package.py +++ b/cli/tests/integrations/cli/test_package.py @@ -4,7 +4,7 @@ import os import six from dcos.api import subcommand -from common import exec_command +from common import assert_command, exec_command def test_package(): @@ -143,17 +143,8 @@ tutorial-gce.html", def test_bad_install(): - returncode, stdout, stderr = exec_command( - ['dcos', - 'package', - 'install', - 'mesos-dns', - '--options=tests/data/package/mesos-dns-config-bad.json']) - - assert returncode == 1 - assert stdout == b'' - - assert stderr == b"""\ + args = ['--options=tests/data/package/mesos-dns-config-bad.json'] + stderr = b"""\ Error: 'mesos-dns/config-url' is a required property Value: {"mesos-dns/host": false} @@ -161,19 +152,14 @@ Error: False is not of type 'string' Path: mesos-dns/host Value: false """ + _install_mesos_dns(args=args, + returncode=1, + stdout=b'', + stderr=stderr) def test_install(): - returncode, stdout, stderr = exec_command( - ['dcos', - 'package', - 'install', - 'mesos-dns', - '--options=tests/data/package/mesos-dns-config.json']) - - assert returncode == 0 - assert stdout == b'Installing package [mesos-dns] version [alpha]\n' - assert stderr == b'' + _install_mesos_dns() def test_package_metadata(): @@ -254,32 +240,17 @@ wLjEuMCJdfQ==""" def test_install_with_id(): - returncode, stdout, stderr = exec_command( - ['dcos', - 'package', - 'install', - 'mesos-dns', - '--options=tests/data/package/mesos-dns-config.json', - '--app-id=dns-1']) + args = ['--options=tests/data/package/mesos-dns-config.json', + '--app-id=dns-1'] + stdout = b"""Installing package [mesos-dns] version [alpha] \ +with app id [dns-1]\n""" + _install_mesos_dns(args=args, stdout=stdout) - assert returncode == 0 - assert stdout == b"""Installing package [mesos-dns] version [alpha] \ -with app id [dns-1] -""" - assert stderr == b'' - - returncode, stdout, stderr = exec_command( - ['dcos', - 'package', - 'install', - 'mesos-dns', - '--options=tests/data/package/mesos-dns-config.json', - '--app-id=dns-2']) - - assert returncode == 0 - assert stdout == b"""Installing package [mesos-dns] version [alpha] \ + args = ['--options=tests/data/package/mesos-dns-config.json', + '--app-id=dns-2'] + stdout = b"""Installing package [mesos-dns] version [alpha] \ with app id [dns-2]\n""" - assert stderr == b'' + _install_mesos_dns(args=args, stdout=stdout) def test_install_missing_package(): @@ -294,38 +265,19 @@ You may need to run 'dcos package update' to update your repositories def test_uninstall_with_id(): - returncode, stdout, stderr = exec_command( - ['dcos', 'package', 'uninstall', 'mesos-dns', '--app-id=dns-1']) - - assert returncode == 0 - assert stdout == b'' - assert stderr == b'' + _uninstall_mesos_dns(args=['--app-id=dns-1']) def test_uninstall_all(): - returncode, stdout, stderr = exec_command( - ['dcos', 'package', 'uninstall', 'mesos-dns', '--all']) - - assert returncode == 0 - assert stdout == b'' - assert stderr == b'' + _uninstall_mesos_dns(args=['--all']) def test_uninstall_missing(): - returncode, stdout, stderr = exec_command( - ['dcos', 'package', 'uninstall', 'mesos-dns']) + stderr = b'Package [mesos-dns] is not installed.\n' + _uninstall_mesos_dns(returncode=1, stderr=stderr) - assert returncode == 1 - assert stdout == b'' - assert stderr == b'Package [mesos-dns] is not installed.\n' - - returncode, stdout, stderr = exec_command( - ['dcos', 'package', 'uninstall', 'mesos-dns', '--app-id=dns-1']) - - assert returncode == 1 - assert stdout == b'' - assert stderr == b"""Package [mesos-dns] with id [dns-1] is not \ -installed.\n""" + stderr = b'Package [mesos-dns] with id [dns-1] is not installed.\n' + _uninstall_mesos_dns(args=['--app-id=dns-1'], returncode=1, stderr=stderr) def test_uninstall_subcommand(): @@ -352,43 +304,23 @@ Installing CLI subcommand for package [helloworld] def test_list_installed(): - returncode, stdout, stderr = exec_command(['dcos', - 'package', - 'list-installed']) + assert_command(['dcos', 'package', 'list-installed'], + stdout=b'[]\n') - assert returncode == 0 - assert stdout == b'[]\n' - assert stderr == b'' + assert_command(['dcos', 'package', 'list-installed', 'xyzzy'], + stdout=b'[]\n') - returncode, stdout, stderr = exec_command( - ['dcos', 'package', 'list-installed', 'xyzzy']) + assert_command(['dcos', 'package', 'list-installed', '--app-id=/xyzzy'], + stdout=b'[]\n') - assert returncode == 0 - assert stdout == b'[]\n' - assert stderr == b'' - - returncode, stdout, stderr = exec_command( - ['dcos', 'package', 'list-installed', '--app-id=/xyzzy']) - - assert returncode == 0 - assert stdout == b'[]\n' - assert stderr == b'' - - returncode, stdout, stderr = exec_command( - ['dcos', - 'package', - 'install', - 'mesos-dns', - '--options=tests/data/package/mesos-dns-config.json']) - - assert returncode == 0 - assert stdout == b'Installing package [mesos-dns] version [alpha]\n' - assert stderr == b'' + _install_mesos_dns() expected_output = b"""\ [ { - "appId": "/mesos-dns", + "app": { + "appId": "/mesos-dns" + }, "description": "DNS-based service discovery for Mesos.", "maintainer": "support@mesosphere.io", "name": "mesos-dns", @@ -396,7 +328,7 @@ def test_list_installed(): "postInstallNotes": "Please refer to the tutorial instructions for \ further setup requirements: http://mesosphere.github.io/mesos-dns/docs\ /tutorial-gce.html", - "registryVersion": "0.1.0-alpha", + "releaseVersion": "0", "scm": "https://github.com/mesosphere/mesos-dns.git", "tags": [ "mesosphere" @@ -406,26 +338,93 @@ further setup requirements: http://mesosphere.github.io/mesos-dns/docs\ } ] """ - returncode, stdout, stderr = exec_command( - ['dcos', 'package', 'list-installed']) + assert_command(['dcos', 'package', 'list-installed'], + stdout=expected_output) - assert returncode == 0 - assert stderr == b'' - assert stdout == expected_output + assert_command(['dcos', 'package', 'list-installed', 'mesos-dns'], + stdout=expected_output) - returncode, stdout, stderr = exec_command( - ['dcos', 'package', 'list-installed', 'mesos-dns']) + assert_command( + ['dcos', 'package', 'list-installed', '--app-id=/mesos-dns'], + stdout=expected_output) - assert returncode == 0 - assert stderr == b'' - assert stdout == expected_output + assert_command( + ['dcos', 'package', 'list-installed', 'ceci-nest-pas-une-package'], + stdout=b'[]\n') - returncode, stdout, stderr = exec_command( - ['dcos', 'package', 'list-installed', '--app-id=/mesos-dns']) + assert_command( + ['dcos', 'package', 'list-installed', + '--app-id=/ceci-nest-pas-une-package'], + stdout=b'[]\n') - assert returncode == 0 - assert stderr == b'' - assert stdout == expected_output + _uninstall_mesos_dns() + + +def test_list_installed_cli(): + stdout = b"""Installing package [helloworld] version [0.1.0] +Installing CLI subcommand for package [helloworld] +""" + assert_command(['dcos', 'package', 'install', 'helloworld'], + stdout=stdout) + + stdout = b"""\ +[ + { + "app": { + "appId": "/helloworld" + }, + "command": { + "name": "helloworld" + }, + "description": "Example DCOS application package", + "maintainer": "support@mesosphere.io", + "name": "helloworld", + "packageSource": "git://github.com/mesosphere/universe.git", + "releaseVersion": "0", + "tags": [ + "mesosphere", + "example", + "subcommand" + ], + "version": "0.1.0", + "website": "https://github.com/mesosphere/dcos-helloworld" + } +] +""" + assert_command(['dcos', 'package', 'list-installed'], + stdout=stdout) + + assert_command(['dcos', 'package', 'uninstall', 'helloworld']) + + stdout = b"Installing CLI subcommand for package [helloworld]\n" + assert_command(['dcos', 'package', 'install', 'helloworld', '--cli'], + stdout=stdout) + + stdout = b"""\ +[ + { + "command": { + "name": "helloworld" + }, + "description": "Example DCOS application package", + "maintainer": "support@mesosphere.io", + "name": "helloworld", + "packageSource": "git://github.com/mesosphere/universe.git", + "releaseVersion": "0", + "tags": [ + "mesosphere", + "example", + "subcommand" + ], + "version": "0.1.0", + "website": "https://github.com/mesosphere/dcos-helloworld" + } +] +""" + assert_command(['dcos', 'package', 'list-installed'], + stdout=stdout) + + assert_command(['dcos', 'package', 'uninstall', 'helloworld']) def test_search(): @@ -451,15 +450,6 @@ def test_search(): assert stderr == b'' -def test_cleanup(): - returncode, stdout, stderr = exec_command( - ['dcos', 'package', 'uninstall', 'mesos-dns']) - - assert returncode == 0 - assert stdout == b'' - assert stderr == b'' - - def get_app_labels(app_id): returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'show', app_id]) @@ -469,3 +459,21 @@ def get_app_labels(app_id): app_json = json.loads(stdout.decode('utf-8')) return app_json.get('labels') + + +def _uninstall_mesos_dns(args=[], + returncode=0, + stdout=b'', + stderr=b''): + cmd = ['dcos', 'package', 'uninstall', 'mesos-dns'] + args + assert_command(cmd, returncode, stdout, stderr) + + +def _install_mesos_dns( + args=['--options=tests/data/package/mesos-dns-config.json'], + returncode=0, + stdout=b'Installing package [mesos-dns] version [alpha]\n', + stderr=b''): + + cmd = ['dcos', 'package', 'install', 'mesos-dns'] + args + assert_command(cmd, returncode, stdout, stderr) diff --git a/dcos/api/package.py b/dcos/api/package.py index e73ed2f..f65f158 100644 --- a/dcos/api/package.py +++ b/dcos/api/package.py @@ -23,13 +23,13 @@ emitter = emitting.FlatEmitter() PACKAGE_METADATA_KEY = 'DCOS_PACKAGE_METADATA' -PACKAGE_REGISTRY_VERSION_KEY = 'DCOS_PACKAGE_REGISTRY_VERSION' PACKAGE_NAME_KEY = 'DCOS_PACKAGE_NAME' PACKAGE_VERSION_KEY = 'DCOS_PACKAGE_VERSION' PACKAGE_SOURCE_KEY = 'DCOS_PACKAGE_SOURCE' PACKAGE_FRAMEWORK_KEY = 'DCOS_PACKAGE_IS_FRAMEWORK' PACKAGE_RELEASE_KEY = 'DCOS_PACKAGE_RELEASE' PACKAGE_COMMAND_KEY = 'DCOS_PACKAGE_COMMAND' +PACKAGE_REGISTRY_VERSION_KEY = 'DCOS_PACKAGE_REGISTRY_VERSION' def install_app(pkg, version, init_client, options, app_id): @@ -229,12 +229,162 @@ The app ids of the installed package instances are: [{}].""".format( return (len(matching_apps), None) -def list_installed_packages(init_client, result_predicate=lambda x: True): +class InstalledPackage(object): + """Represents an intalled DCOS package. One of `app` and + `subcommand` must be supplied. + + :param app: A dictionary representing a marathon app. Of the + format returned by `installed_apps()` + :type app: dict + :param subcommand: Installed subcommand + :type subcommand: subcommand.InstalledSubcommand """ + + def __init__(self, app=None, subcommand=None): + assert app or subcommand + self.app = app + self.subcommand = subcommand + + def name(self): + """ + :returns: The name of the package + :rtype: str + """ + if self.subcommand: + return self.subcommand.name + else: + return self.app['name'] + + def dict(self): + """ A dictionary representation of the package. Used by `dcos package + list-installed`. + + :returns: A dictionary representation of the package. + :rtype: (dict, None) + """ + ret = {'name': self.name} + + if self.subcommand: + ret['command'] = {'name': self.subcommand.name} + + if self.app: + ret['app'] = {'appId': self.app['appId']} + + if self.subcommand: + package_json, err = self.subcommand.package_json() + if err is not None: + return (None, err) + + ret.update(package_json) + + package_source, err = self.subcommand.package_source() + if err is not None: + return (None, err) + + ret['packageSource'] = package_source + + package_version, err = self.subcommand.package_version() + if err is not None: + return (None, err) + + ret['releaseVersion'] = package_version + else: + ret.update(self.app) + ret.pop('appId') + + return (ret, None) + + +def installed_packages(init_client, endpoints): + """Returns all installed packages in the format: + + [{ + 'app': { + 'id': + }, + 'command': { + 'name': + } + ...... + }] + :param init_client: The program to use to list packages :type init_client: object - :param result_predicate: The predicate to use to filter results - :type result_predicate: function(dict): bool + :param endpoints: Whether to include a list of + endpoints as port-host pairs + :type endpoints: boolean + :returns: A list of installed packages + :rtype: ([InstalledPackage], Error) + """ + + apps, error = installed_apps(init_client, endpoints) + if error is not None: + return (None, error) + + subcommands, error = installed_subcommands() + if error is not None: + return (None, error) + + dicts = collections.defaultdict(lambda: {'app': None, 'command': None}) + + for app in apps: + key = (app['name'], app['releaseVersion'], app['packageSource']) + dicts[key]['app'] = app + + for subcmd in subcommands: + package_version, err = subcmd.package_version() + if err is not None: + return (None, err) + + package_source, err = subcmd.package_source() + if err is not None: + return (None, err) + + key = (subcmd.name, package_version, package_source) + dicts[key]['command'] = subcmd + + pkgs = [] + + for key, pkg in dicts.items(): + pkgs.append(InstalledPackage(pkg['app'], pkg['command'])) + + return (pkgs, None) + + +def installed_subcommands(): + """Returns all installed subcommands. + + :returns: all installed subcommands + :rtype: ([InstalledSubcommand], Error) + """ + + ret = [subcommand.InstalledSubcommand(name) for name in + subcommand.distributions(util.dcos_path())] + return (ret, None) + + +def installed_apps(init_client, endpoints=False): + """ + Returns all installed apps. An app is of the format: + + { + 'appId': , + 'packageSource': , + 'registryVersion': , + 'releaseVersion': + 'endpoints' (optional): [{ + 'host': , + 'ports': , + }] + .... + } + + :param init_client: The program to use to list packages + :type init_client: object + :param endpoints: Whether to include a list of + endpoints as port-host pairs + :type endpoints: boolean + :returns: all installed apps :rtype: (list of dict, Error) """ @@ -242,7 +392,7 @@ def list_installed_packages(init_client, result_predicate=lambda x: True): if error is not None: return (None, error) - encoded_pkgs = [(a['id'], a['labels']) + encoded_apps = [(a['id'], a['labels']) for a in apps if a.get('labels', {}).get(PACKAGE_METADATA_KEY)] @@ -250,44 +400,33 @@ def list_installed_packages(init_client, result_predicate=lambda x: True): app_id, labels = pair encoded = labels.get(PACKAGE_METADATA_KEY, {}) source = labels.get(PACKAGE_SOURCE_KEY) - registry_version = labels.get(PACKAGE_REGISTRY_VERSION_KEY) + release_version = labels.get(PACKAGE_RELEASE_KEY) decoded = base64.b64decode(six.b(encoded)).decode() decoded_json, error = util.load_jsons(decoded) if error is None: decoded_json['appId'] = app_id decoded_json['packageSource'] = source - decoded_json['registryVersion'] = registry_version + decoded_json['releaseVersion'] = release_version return (decoded_json, error) - decoded_pkgs = [decode_and_add_context(encoded) - for encoded in encoded_pkgs] + decoded_apps = [decode_and_add_context(encoded) + for encoded in encoded_apps] # Filter elements that failed to parse correctly as JSON, # or do not match the supplied predicate - pkgs = [pair[0] for pair in decoded_pkgs - if pair[1] is None and result_predicate(pair[0])] + valid_apps = [pair[0] for pair in decoded_apps if pair[1] is None] - return (pkgs, None) + if endpoints: + for app in valid_apps: + tasks, err = init_client.get_tasks(app["appId"]) + if err is not None: + return (None, err) + app['endpoints'] = [{"host": t["host"], "ports": t["ports"]} + for t in tasks] -def get_tasks_multiple(init_client, apps): - """Adds tasks to app dictionary - :param init_client: The program to use to list packages - :type init_client: object - :param apps: list of dict - :type apps: object - :rtype: (list, Error) - """ - - for app in apps: - tasks, err = init_client.get_tasks(app["appId"]) - if err is not None: - return (None, err) - app["endpoints"] = [{"host": t["host"], "ports": t["ports"]} - for t in tasks] - - return (apps, None) + return (valid_apps, None) def search(query, cfg): @@ -1075,16 +1214,10 @@ class Package(): """ data, error = self._data(path) - if error is not None: return (None, error) - try: - result = json.loads(data) - except ValueError: - return (None, Error('')) - - return (result, None) + return util.load_jsons(data) def _data(self, path): """Returns the content of the supplied file, relative to the base path. @@ -1096,15 +1229,7 @@ class Package(): """ full_path = os.path.join(self.path, path) - if not os.path.isfile(full_path): - return (None, Error('Path [{}] is not a file'.format(full_path))) - - try: - with open(full_path) as fd: - content = fd.read() - return (content, None) - except IOError: - return (None, Error('Unable to open file [{}]'.format(full_path))) + return util.read_file(full_path) def package_versions(self): """Returns all of the available package versions, most recent first. diff --git a/dcos/api/subcommand.py b/dcos/api/subcommand.py index fa26069..66756ad 100644 --- a/dcos/api/subcommand.py +++ b/dcos/api/subcommand.py @@ -415,3 +415,50 @@ def _generic_error(package_name): return errors.DefaultError( 'Error installing {!r} package'.format(package_name)) + + +class InstalledSubcommand(object): + """ Represents an installed subcommand. + + :param name: The name of the subcommand + :type name: str + """ + + def __init__(self, name): + self.name = name + + def _dir(self): + """ + :returns: path to this subcommand's directory. + :rtype: (str, Error) + """ + + return package_dir(self.name) + + def package_version(self): + """ + :returns: this subcommand's version. + :rtype: (str, Error) + """ + + version_path = os.path.join(self._dir(), 'version') + return util.read_file(version_path) + + def package_source(self): + """ + :returns: this subcommand's source. + :rtype: (str, Error) + """ + + source_path = os.path.join(self._dir(), 'source') + return util.read_file(source_path) + + def package_json(self): + """ + :returns: contents of this subcommand's package.json file. + :rtype: (dict, Error) + """ + + package_json_path = os.path.join(self._dir(), 'package.json') + with open(package_json_path) as package_json_file: + return util.load_json(package_json_file) diff --git a/dcos/api/util.py b/dcos/api/util.py index 3f8dfe4..ab782c4 100644 --- a/dcos/api/util.py +++ b/dcos/api/util.py @@ -70,6 +70,26 @@ def ensure_dir(directory): os.makedirs(directory, 0o775) +def read_file(path): + """ + :param path: path to file + :type path: str + :returns: contents of file + :rtype: (str, Error) + """ + if not os.path.isfile(path): + return (None, errors.DefaultError( + 'Path [{}] is not a file'.format(path))) + + try: + with open(path) as fd: + content = fd.read() + return (content, None) + except IOError: + return (None, errors.DefaultError( + 'Unable to open file [{}]'.format(path))) + + def which(program): """Returns the path to the named executable program. @@ -103,8 +123,7 @@ def dcos_path(): """ dcos_bin_dir = os.path.realpath(sys.argv[0]) - dcos_dir = os.path.dirname(os.path.dirname(dcos_bin_dir)) - return dcos_dir + return os.path.dirname(os.path.dirname(dcos_bin_dir)) def configure_logger_from_environ():