diff --git a/cli/dcoscli/config/main.py b/cli/dcoscli/config/main.py index 92a7a14..5da7ea8 100644 --- a/cli/dcoscli/config/main.py +++ b/cli/dcoscli/config/main.py @@ -16,8 +16,8 @@ Options: index of zero Positional Arguments: - The name of the property - The value of the property + The name of the property + The value of the property """ import collections diff --git a/cli/dcoscli/package/main.py b/cli/dcoscli/package/main.py index 41863b7..e20a8ee 100644 --- a/cli/dcoscli/package/main.py +++ b/cli/dcoscli/package/main.py @@ -4,8 +4,8 @@ Usage: dcos package --config-schema dcos package describe dcos package info - dcos package install [--options= --app-id=] - + dcos package install [--options= --app-id= --cli --app] + dcos package list-installed [--endpoints --app-id= ] dcos package search dcos package sources @@ -13,8 +13,14 @@ Usage: dcos package update Options: - -h, --help Show this screen - --version Show version + --all Apply the operation to all matching packages + --app-id= The application id + --cli Apply the operation only to the package's CLI + --help Show this screen + --options= Path to a JSON file containing package installation + options + --app Apply the operation only to the package's application + --version Show version Configuration: [package] @@ -94,7 +100,8 @@ def _cmds(): cmds.Command( hierarchy=['package', 'install'], - arg_keys=['', '--options', '--app-id'], + arg_keys=['', '--options', '--app-id', '--cli', + '--app'], function=_install), cmds.Command( @@ -243,19 +250,28 @@ def _describe(package_name): return 0 -def _install(package_name, options_file, app_id): +def _install(package_name, options_file, app_id, cli, app): """Install the specified package. - :param package_name: The package to install + :param package_name: the package to install :type package_name: str - :param options_file: Path to file containing option values + :param options_file: path to file containing option values :type options_file: str - :param app_id: App ID for installation of this package + :param app_id: app ID for installation of this package :type app_id: str - :returns: Process status + :param cli: indicates if the cli should be installed + :type cli: bool + :param app: indicate if the application should be installed + :type app: bool + :returns: process status :rtype: int """ + if cli is False and app is False: + # Install both if neither flag is specified + cli = True + app = True + config = _load_config() pkg = package.resolve_package(package_name, config) @@ -274,8 +290,6 @@ def _install(package_name, options_file, app_id): emitter.publish(e.message) return 1 - init_client = marathon.create_client(config) - # TODO(CD): Make package version to install configurable pkg_version, version_error = pkg.latest_version() @@ -283,17 +297,26 @@ def _install(package_name, options_file, app_id): emitter.publish(version_error) return 1 - install_error = package.install( - pkg, - pkg_version, - init_client, - options_json, - app_id, - config) + if app: + # Install in Marathon + init_client = marathon.create_client(config) - if install_error is not None: - emitter.publish(install_error) - return 1 + install_error = package.install_app( + pkg, + pkg_version, + init_client, + options_json, + app_id) + if install_error is not None: + emitter.publish(install_error) + return 1 + + if cli and pkg.is_command_defined(pkg_version): + # Install subcommand + err = package.install_subcommand(pkg, pkg_version, options_json) + if err is not None: + emitter.publish(err) + return 1 return 0 @@ -383,8 +406,7 @@ def _uninstall(package_name, remove_all, app_id): package_name, remove_all, app_id, - init_client, - config) + init_client) if uninstall_error is not None: emitter.publish(uninstall_error) diff --git a/cli/dcoscli/subcommand/main.py b/cli/dcoscli/subcommand/main.py index 1580958..852b38a 100644 --- a/cli/dcoscli/subcommand/main.py +++ b/cli/dcoscli/subcommand/main.py @@ -17,8 +17,6 @@ Positional arguments: """ import json import os -import shutil -import subprocess import dcoscli import docopt @@ -133,27 +131,22 @@ def _install(package): dcos_config = config.load_from_path(os.environ[constants.DCOS_CONFIG_ENV]) - bin_directory = os.path.dirname(util.process_executable_path()) - - subcommand_directory = os.path.join( - os.path.dirname(bin_directory), - constants.DCOS_SUBCOMMAND_SUBDIR) - if not os.path.exists(subcommand_directory): - logger.info('Creating directory: %r', subcommand_directory) - os.mkdir(subcommand_directory, 0o775) + install_operation = { + 'pip': [package] + } + if 'subcommand.pip_find_links' in dcos_config: + install_operation['pip'].append( + '--find-links {}'.format(dcos_config['subcommand.pip_find_links'])) distribution_name, err = _distribution_name(package) if err is not None: emitter.publish(err) return 1 - package_directory = os.path.join(subcommand_directory, distribution_name) - - err = _install_subcommand( - bin_directory, - package_directory, - package, - dcos_config.get('subcommand.pip_find_links')) + err = subcommand.install( + distribution_name, + install_operation, + util.dcos_path()) if err is not None: emitter.publish(err) return 1 @@ -167,13 +160,7 @@ def _uninstall(package_name): :rtype: int """ - subcommand_directory = os.path.join( - util.dcos_path(), - constants.DCOS_SUBCOMMAND_SUBDIR, - package_name) - - if os.path.isdir(subcommand_directory): - shutil.rmtree(subcommand_directory) + subcommand.uninstall(package_name, util.dcos_path()) return 0 @@ -193,93 +180,3 @@ def _distribution_name(package_path): errors.DefaultError( 'Failed to read file: {}'.format(error)) ) - - -def _install_subcommand( - bin_directory, - package_directory, - package, - wheel_cache): - """ - :param: bin_directory: the path to the directory containing the - executables (virtualenv, etc). - :type: str - :param package_directory: the path to the directory for the package - :type: str - :param package: the path to Python wheel package - :type: str - :returns: an Error if it failed to install the package; None otherwise - :rtype: dcos.api.errors.Error - """ - - new_package_dir = not os.path.exists(package_directory) - - if not os.path.exists(os.path.join(package_directory, 'bin', 'pip')): - cmd = [os.path.join(bin_directory, 'virtualenv'), package_directory] - - if _execute_command(cmd) != 0: - return _generic_error(package) - - cmd = [ - os.path.join(package_directory, 'bin', 'pip'), - 'install', - '--upgrade', - '--force-reinstall', - ] - - if wheel_cache is not None: - cmd.append('--find-links') - cmd.append(wheel_cache) - - cmd.append(package) - - if _execute_command(cmd) != 0: - # We should remove the diretory that we just created - if new_package_dir: - shutil.rmtree(package_directory) - - return _generic_error(package) - - return None - - -def _execute_command(command): - """ - :param command: the command to execute - :type command: list of str - :returns: the process return code - :rtype: int - """ - - logger.info('Calling: %r', command) - - process = subprocess.Popen( - command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - stdout, stderr = process.communicate() - - if process.returncode != 0: - logger.error("Install script's stdout: %s", stdout) - logger.error("Install script's stderr: %s", stderr) - else: - logger.info("Install script's stdout: %s", stdout) - logger.info("Install script's stderr: %s", stderr) - - return process.returncode - - -def _generic_error(package): - """ - :param package: path the subcommand package - :type: str - :returns: generic error when installing package - :rtype: dcos.api.errors.Error - """ - distribution_name, err = _distribution_name(package) - if err is not None: - return err - - return errors.DefaultError( - 'Error installing {!r} package'.format(distribution_name)) diff --git a/cli/tests/integrations/cli/test_config.py b/cli/tests/integrations/cli/test_config.py index 3155c52..4bf1206 100644 --- a/cli/tests/integrations/cli/test_config.py +++ b/cli/tests/integrations/cli/test_config.py @@ -38,8 +38,8 @@ Options: index of zero Positional Arguments: - The name of the property - The value of the property + The name of the property + The value of the property """ assert stderr == b'' diff --git a/cli/tests/integrations/cli/test_marathon.py b/cli/tests/integrations/cli/test_marathon.py index 7c585aa..772d183 100644 --- a/cli/tests/integrations/cli/test_marathon.py +++ b/cli/tests/integrations/cli/test_marathon.py @@ -118,7 +118,7 @@ def test_add_bad_json_app(): assert returncode == 1 assert stdout == b'' - assert stderr == b'Error loading JSON.\n' + assert stderr.decode('utf-8').startswith('Error loading JSON: ') def test_add_existing_app(): diff --git a/cli/tests/integrations/cli/test_package.py b/cli/tests/integrations/cli/test_package.py index ac3a5e5..2472957 100644 --- a/cli/tests/integrations/cli/test_package.py +++ b/cli/tests/integrations/cli/test_package.py @@ -15,8 +15,8 @@ Usage: dcos package --config-schema dcos package describe dcos package info - dcos package install [--options= --app-id=] - + dcos package install [--options= --app-id= --cli --app] + dcos package list-installed [--endpoints --app-id= ] dcos package search dcos package sources @@ -24,8 +24,14 @@ Usage: dcos package update Options: - -h, --help Show this screen - --version Show version + --all Apply the operation to all matching packages + --app-id= The application id + --cli Apply the operation only to the package's CLI + --help Show this screen + --options= Path to a JSON file containing package installation + options + --app Apply the operation only to the package's application + --version Show version Configuration: [package] diff --git a/dcos/api/package.py b/dcos/api/package.py index 9922b02..2412821 100644 --- a/dcos/api/package.py +++ b/dcos/api/package.py @@ -11,17 +11,11 @@ import zipfile import git import portalocker +import pystache import six -from dcos.api import constants, emitting, errors, util +from dcos.api import constants, emitting, errors, subcommand, util -try: - # Python 2 - from urlparse import urlparse - from urllib import urlretrieve -except ImportError: - # Python 3 - from urllib.parse import urlparse - from urllib.request import urlretrieve +from six.moves import urllib logger = util.get_logger(__name__) @@ -36,62 +30,73 @@ PACKAGE_SOURCE_KEY = 'DCOS_PACKAGE_SOURCE' PACKAGE_FRAMEWORK_KEY = 'DCOS_PACKAGE_IS_FRAMEWORK' -def install(pkg, version, init_client, user_options, app_id, cfg): - """Installs a package. - - :param pkg: The package to install - :type pkg: Package - :param version: The package version to install - :type version: str - :param init_client: The program to use to run the package - :type init_client: object - :param user_options: Package parameters - :type user_options: dict - :param app_id: App ID for installation of this package - :type app_id: str - :param cfg: Configuration dictionary - :type cfg: dcos.api.config.Toml - :rtype: Error +def _merge_options(pkg, version, user_options): + """ + :param pkg: the package to install + :type pkg: Package + :param version: the package version to install + :type version: str + :param user_options: package parameters + :type user_options: dict + :returns: a dictionary with the user supplied options + :rtype: (dict, dcos.api.errors.Error) """ - if user_options is None: user_options = {} - config_schema, schema_error = pkg.config_json(version) + config_schema, err = pkg.config_json(version) + if err is not None: + return (None, err) - if schema_error is not None: - return schema_error - - default_options, default_error = _extract_default_values(config_schema) - - if default_error is not None: - return default_error + default_options, err = _extract_default_values(config_schema) + if err is not None: + return (None, err) # Merge option overrides options = dict(list(default_options.items()) + list(user_options.items())) # Validate options with the config schema err = util.validate_json(options, config_schema) + if err is not None: + return (None, err) + + return (options, None) + + +def install_app(pkg, version, init_client, user_options, app_id): + """Installs a package's application + + :param pkg: the package to install + :type pkg: Package + :param version: the package version to install + :type version: str + :param init_client: the program to use to run the package + :type init_client: object + :param user_options: package parameters + :type user_options: dict + :param app_id: app ID for installation of this package + :type app_id: str + :rtype: Error + """ + + options, err = _merge_options(pkg, version, user_options) if err is not None: return err # Insert option parameters into the init template - template, tmpl_error = pkg.marathon_template(version) - - if tmpl_error is not None: - return tmpl_error + template, err = pkg.marathon_template(version) + if err is not None: + return err # Render the init template with the marshaled options - init_desc, render_error = util.render_mustache_json(template, options) - - if render_error is not None: - return render_error + init_desc, err = util.render_mustache_json(template, options) + if err is not None: + return err # Add package metadata - package_labels, label_error = _make_package_labels(pkg, version) - - if label_error is not None: - return label_error + package_labels, err = _make_package_labels(pkg, version) + if err is not None: + return err # Preserve existing labels labels = init_desc.get('labels', {}) @@ -149,7 +154,40 @@ def _make_package_labels(pkg, version): return (package_labels, None) -def uninstall(package_name, remove_all, app_id, init_client, config): +def install_subcommand(pkg, version, user_options): + """Installs a package's command line interface + + :param pkg: the package to install + :type pkg: Package + :param version: the package version to install + :type version: str + :param user_options: package parameters + :type user_options: dict + :rtype: dcos.api.errors.Error + """ + + options, err = _merge_options(pkg, version, user_options) + if err is not None: + return err + + # Insert option parameters into the init template + init_template, err = pkg.command_template(version) + if err is not None: + return err + + rendered_template = pystache.render(init_template, options) + + install_operation, err = util.load_jsons(rendered_template) + if err is not None: + return err + + return subcommand.install( + pkg.name(), + install_operation, + util.dcos_path()) + + +def uninstall(package_name, remove_all, app_id, init_client): """Uninstalls a package. :param package_name: The package to uninstall @@ -160,8 +198,6 @@ def uninstall(package_name, remove_all, app_id, init_client, config): :type app_id: str :param init_client: The program to use to run the package :type init_client: object - :param cfg: Configuration dictionary - :type cfg: dcos.api.config.Toml :rtype: Error """ @@ -410,7 +446,7 @@ def url_to_source(url): :rtype: (Source, Error) """ - parse_result = urlparse(url) + parse_result = urllib.parse.urlparse(url) scheme = parse_result.scheme if scheme == 'file': @@ -612,7 +648,7 @@ class FileSource(Source): """ # copy the source to the target_directory - parse_result = urlparse(self._url) + parse_result = urllib.parse.urlparse(self._url) source_dir = parse_result.path try: shutil.copytree(source_dir, target_dir) @@ -655,7 +691,7 @@ class HttpSource(Source): tmp_file = os.path.join(tmp_dir, 'packages.zip') # Download the zip file. - urlretrieve(self.url, tmp_file) + urllib.request.urlretrieve(self.url, tmp_file) # Unzip the downloaded file. packages_zip = zipfile.ZipFile(tmp_file, 'r') @@ -919,14 +955,27 @@ class Package(): return self._registry - def command_json(self, version): + def is_command_defined(self, version): + """Returns true if the package defines a command; false otherwise. + + :param version: package version + :type version: str + :rtype: bool + """ + + return os.path.isfile( + os.path.join( + self.path, + os.path.join(version, 'command.json'))) + + def command_template(self, version): """Returns the JSON content of the command.json file. :returns: Package command data - :rtype: (dict, Error) + :rtype: (str, Error) """ - return self._json(os.path.join(version, 'command.json')) + return self._data(os.path.join(version, 'command.json')) def config_json(self, version): """Returns the JSON content of the config.json file. @@ -950,7 +999,7 @@ class Package(): """Returns the JSON content of the marathon.json file. :returns: Package marathon data - :rtype: str or Error + :rtype: (str, Error) """ return self._data(os.path.join(version, 'marathon.json')) diff --git a/dcos/api/subcommand.py b/dcos/api/subcommand.py index 44b89fd..753ddd7 100644 --- a/dcos/api/subcommand.py +++ b/dcos/api/subcommand.py @@ -1,8 +1,13 @@ +from __future__ import print_function + import json import os +import shutil import subprocess -from dcos.api import constants, errors +from dcos.api import constants, errors, util + +logger = util.get_logger(__name__) def command_executables(subcommand, dcos_path): @@ -150,3 +155,146 @@ def noun(executable_path): basename = os.path.basename(executable_path) return basename[len(constants.DCOS_COMMAND_PREFIX):] + + +def install(distribution_name, install_operation, dcos_path): + """Installs the dcos cli subcommand + + :param distribution_name: the name of the package + :type distribution_name: str + :param install_operation: operation to use to install subcommand + :type install_operation: dict + :param dcos_path: path to the dcos cli directory + :type dcos_path: str + :returns: an error if the subcommand failed; None otherwise + :rtype: dcos.api.errors.Error + """ + + subcommand_directory = os.path.join( + dcos_path, + constants.DCOS_SUBCOMMAND_SUBDIR) + if not os.path.exists(subcommand_directory): + logger.info('Creating directory: %r', subcommand_directory) + os.mkdir(subcommand_directory, 0o775) + + package_directory = os.path.join(subcommand_directory, distribution_name) + + if 'pip' in install_operation: + return _install_with_pip( + distribution_name, + os.path.join(dcos_path, 'bin'), + package_directory, + install_operation['pip']) + else: + return errors.DefaultError( + "Installation methods '{}' not supported".format( + install_operation.keys())) + + +def uninstall(distribution_name, dcos_path): + """Uninstall the dcos cli subcommand + + :param distribution_name: the name of the package + :type distribution_name: str + :param dcos_path: the path to the dcos cli directory + :type dcos_path: str + """ + + subcommand_directory = os.path.join( + dcos_path, + constants.DCOS_SUBCOMMAND_SUBDIR, + distribution_name) + + if os.path.isdir(subcommand_directory): + shutil.rmtree(subcommand_directory) + + +def _install_with_pip( + distribution_name, + bin_directory, + package_directory, + requirements): + """ + :param distribution_name: the name of the package + :type distribution_name: str + :param bin_directory: the path to the directory containing the + executables (virtualenv, etc). + :type bin_directory: str + :param package_directory: the path to the directory for the package + :type package_directory: str + :param requirements: the list of pip requirements + :type requirements: list of str + :returns: an Error if it failed to install the package; None otherwise + :rtype: dcos.api.errors.Error + """ + + new_package_dir = not os.path.exists(package_directory) + + if not os.path.exists(os.path.join(package_directory, 'bin', 'pip')): + cmd = [os.path.join(bin_directory, 'virtualenv'), package_directory] + + if _execute_command(cmd) != 0: + return _generic_error(distribution_name) + + with util.temptext() as text_file: + fd, requirement_path = text_file + + # Write the requirements to the file + with os.fdopen(fd, 'w') as requirements_file: + for line in requirements: + print(line, file=requirements_file) + + cmd = [ + os.path.join(package_directory, 'bin', 'pip'), + 'install', + '--requirement', + requirement_path, + ] + + if _execute_command(cmd) != 0: + # We should remove the diretory that we just created + if new_package_dir: + shutil.rmtree(package_directory) + + return _generic_error(distribution_name) + + return None + + +def _execute_command(command): + """ + :param command: the command to execute + :type command: list of str + :returns: the process return code + :rtype: int + """ + + logger.info('Calling: %r', command) + + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + stdout, stderr = process.communicate() + + if process.returncode != 0: + logger.error("Install script's stdout: %s", stdout) + logger.error("Install script's stderr: %s", stderr) + else: + logger.info("Install script's stdout: %s", stdout) + logger.info("Install script's stderr: %s", stderr) + + return process.returncode + + +def _generic_error(distribution_name): + """ + :param package: package name + :type: str + :returns: generic error when installing package + :rtype: dcos.api.errors.Error + """ + + return errors.DefaultError( + 'Error installing {!r} package'.format(distribution_name)) diff --git a/dcos/api/util.py b/dcos/api/util.py index af24424..85839ce 100644 --- a/dcos/api/util.py +++ b/dcos/api/util.py @@ -22,7 +22,7 @@ def tempdir(): lexical scope of the returned file descriptor. :return: Reference to a temporary directory - :rtype: file descriptor + :rtype: str """ tmpdir = tempfile.mkdtemp() @@ -32,6 +32,31 @@ def tempdir(): shutil.rmtree(tmpdir, ignore_errors=True) +@contextlib.contextmanager +def temptext(): + """A context manager for temporary files. + + The lifetime of the returned temporary file corresponds to the + lexical scope of the returned file descriptor. + + :return: reference to a temporary file + :rtype: (fd, str) + """ + + fd, path = tempfile.mkstemp() + try: + yield (fd, path) + finally: + # Close the file descriptor and ignore errors + try: + os.close(fd) + except OSError: + pass + + # delete the path + shutil.rmtree(path, ignore_errors=True) + + def which(program): """Returns the path to the named executable program. @@ -135,13 +160,15 @@ def load_json(reader): try: return (json.load(reader), None) - except: - error = sys.exc_info()[0] + except Exception as error: logger = get_logger(__name__) logger.error( 'Unhandled exception while loading JSON: %r', error) - return (None, errors.DefaultError('Error loading JSON.')) + return ( + None, + errors.DefaultError('Error loading JSON: {}'.format(error)) + ) def load_jsons(value):