diff --git a/cli/dcoscli/package/main.py b/cli/dcoscli/package/main.py index b7f2b98..4dc3be8 100644 --- a/cli/dcoscli/package/main.py +++ b/cli/dcoscli/package/main.py @@ -386,13 +386,13 @@ def _install(package_name, package_version, options_path, app_id, cli, app, options, app_id) - if cli and pkg.has_command_definition(): + if cli and pkg.has_cli_definition(): # Install subcommand msg = 'Installing CLI subcommand for package [{}] version [{}]'.format( pkg.name(), pkg.version()) emitter.publish(msg) - subcommand.install(pkg, pkg.options(user_options)) + subcommand.install(pkg) subcommand_paths = subcommand.get_package_commands(package_name) new_commands = [os.path.basename(p).replace('-', ' ', 1) diff --git a/cli/tests/unit/test_http_auth.py b/cli/tests/unit/test_http_auth.py index ebe3e34..9c1a690 100644 --- a/cli/tests/unit/test_http_auth.py +++ b/cli/tests/unit/test_http_auth.py @@ -37,7 +37,7 @@ def test_get_auth_scheme_bad_request(): with patch('requests.Response') as mock: mock.headers = {'www-authenticate': ''} res = http.get_auth_scheme(mock) - assert res is None + assert res == (None, None) @patch('requests.Response') diff --git a/dcos/constants.py b/dcos/constants.py index fe69461..42ba248 100644 --- a/dcos/constants.py +++ b/dcos/constants.py @@ -1,8 +1,8 @@ DCOS_DIR = ".dcos" """DCOS data directory. Can store subcommands and the config file.""" -DCOS_SUBCOMMAND_VIRTUALENV_SUBDIR = 'env' -"""In a package's directory, this is the virtualenv subdirectory.""" +DCOS_SUBCOMMAND_ENV_SUBDIR = 'env' +"""In a package's directory, this is the cli contents subdirectory.""" DCOS_SUBCOMMAND_SUBDIR = 'subcommands' """Name of the subdirectory that contains all of the subcommands. This is diff --git a/dcos/cosmospackage.py b/dcos/cosmospackage.py index 0f2f75e..4fc4ef4 100644 --- a/dcos/cosmospackage.py +++ b/dcos/cosmospackage.py @@ -354,7 +354,7 @@ class CosmosPackageVersion(): return self._config_json - def _resource_json(self): + def resource_json(self): """Returns the JSON content of the resource.json file. :returns: Package resources @@ -406,13 +406,14 @@ class CosmosPackageVersion(): return user_options - def has_command_definition(self): + def has_cli_definition(self): """Returns true if the package defines a command; false otherwise. :rtype: bool """ - return self._command_json is not None + return self._command_json is not None or ( + self._resource_json and self._resource_json.get("cli")) def command_json(self): """Returns the JSON content of the command.json file. diff --git a/dcos/http.py b/dcos/http.py index 583b00c..47531fd 100644 --- a/dcos/http.py +++ b/dcos/http.py @@ -354,12 +354,12 @@ def _get_auth_credentials(username, hostname): def get_auth_scheme(response): """Return authentication scheme and realm requested by server for 'Basic' - or 'acsjwt' (DCOS acs auth) or 'oauthjwt' (DCOS acs oauth) type or None + or 'acsjwt' (DCOS acs auth) or 'oauthjwt' (DCOS acs oauth) type :param response: requests.response :type response: requests.Response :returns: auth_scheme, realm - :rtype: (str, str) | None + :rtype: (str, str) """ if 'www-authenticate' in response.headers: @@ -375,9 +375,9 @@ def get_auth_scheme(response): realm = scheme_info[-1].strip(' \'\"').lower() return auth_scheme, realm else: - return None + return None, None else: - return None + return None, None def _get_http_auth(response, url, auth_scheme): diff --git a/dcos/subcommand.py b/dcos/subcommand.py index e8900d4..be82afe 100644 --- a/dcos/subcommand.py +++ b/dcos/subcommand.py @@ -1,12 +1,17 @@ from __future__ import print_function +import hashlib import json import os +import platform import shutil +import stat import subprocess import sys +import zipfile from subprocess import PIPE, Popen +import requests from dcos import constants, emitting, util from dcos.errors import DCOSException @@ -52,8 +57,9 @@ def get_package_commands(package_name): :returns: list of all the dcos program paths in package :rtype: [str] """ + bin_dir = os.path.join(_package_dir(package_name), - constants.DCOS_SUBCOMMAND_VIRTUALENV_SUBDIR, + constants.DCOS_SUBCOMMAND_ENV_SUBDIR, BIN_DIRECTORY) executables = [] @@ -122,7 +128,7 @@ def distributions(): os.path.join( subcommand_dir, subdir, - constants.DCOS_SUBCOMMAND_VIRTUALENV_SUBDIR)) + constants.DCOS_SUBCOMMAND_ENV_SUBDIR)) ] else: return [] @@ -219,40 +225,124 @@ def _write_package_json(pkg): json.dump(package_json, package_file) -def _install_env(pkg, options): - """ Install subcommand virtual env. +def _hashfile(filename): + """Calculates the sha256 of a file - :param pkg: the package to install - :type pkg: PackageVersion - :param options: package parameters - :type options: dict + :param filename: path to the file to sum + :type filename: str + :returns: digest in hexadecimal + :rtype: str + """ + + hasher = hashlib.sha256() + with open(filename, 'rb') as f: + for chunk in iter(lambda: f.read(4096), b''): + hasher.update(chunk) + return hasher.hexdigest() + + +def _check_hash(filename, content_hashes): + """Validates whether downloaded binary matches expected hash + + :param filename: path to binary + :type filename: str + :param content_hashes: list of hash algorithms/value + :type content_hashes: [{"algo": , "value": }] + :returns: None if valid hash, else throws exception :rtype: None """ - pkg_dir = _package_dir(pkg.name()) - - install_operation = pkg.command_json() - - env_dir = os.path.join(pkg_dir, - constants.DCOS_SUBCOMMAND_VIRTUALENV_SUBDIR) - - if 'pip' in install_operation: - _install_with_pip( - pkg.name(), - env_dir, - install_operation['pip']) + content_hash = next((contents for contents in content_hashes + if contents.get("algo") == "sha256"), + None) + if content_hash: + expected_value = content_hash.get("value") + actual_value = _hashfile(filename) + if expected_value != actual_value: + raise DCOSException( + "The hash for the downloaded subcommand [{}] " + "does not match the expected value [{}]. Aborting...".format( + actual_value, expected_value)) + else: + return else: - raise DCOSException("Installation methods '{}' not supported".format( - install_operation.keys())) + raise DCOSException( + "Hash algorithm specified is unsupported. " + "Please contact the package maintainer. Aborting...") -def install(pkg, options): +def _get_cli_binary_info(cli_resources): + """Find compatible cli binary, if one exists + + :param cli_resources: cli property of resource.json + :type resources: {} + :returns: {"url": , "kind": , "contentHash": [{}]} + :rtype: {} | None + """ + + if "binaries" in cli_resources: + binaries = cli_resources["binaries"] + arch = platform.architecture()[0] + if arch != "64bit": + raise DCOSException( + "There is no compatible subcommand for your architecture [{}] " + "We only support x86-64. Aborting...".format(arch)) + system = platform.system().lower() + binary = binaries.get(system) + if binary is None: + raise DCOSException( + "There is not compatible subcommand for your system [{}] " + "Aborting...".format(system)) + elif "x86-64" in binary: + return binary["x86-64"] + + raise DCOSException( + "The CLI subcommand has unexpected format [{}]. " + "Please contact the package maintainer. Aborting...".format( + cli_resources)) + + +def _install_cli(pkg): + """Install subcommand cli + + :param pkg: the package to install + :type pkg: PackageVersion + :rtype: None + """ + + with util.remove_path_on_error(_package_dir(pkg.name())) as pkg_dir: + env_dir = os.path.join(pkg_dir, constants.DCOS_SUBCOMMAND_ENV_SUBDIR) + + resources = pkg.resource_json() + + if resources and resources.get("cli") is not None: + binary = resources["cli"] + binary_cli = _get_cli_binary_info(binary) + _install_with_binary( + pkg.name(), + env_dir, + binary_cli) + elif pkg.command_json() is not None: + install_operation = pkg.command_json() + if 'pip' in install_operation: + _install_with_pip( + pkg.name(), + env_dir, + install_operation['pip']) + else: + raise DCOSException( + "Installation methods '{}' not supported".format( + install_operation.keys())) + else: + raise DCOSException( + "Could not find a CLI subcommand for your platform") + + +def install(pkg): """Installs the dcos cli subcommand :param pkg: the package to install :type pkg: Package - :param options: package parameters - :type options: dict :rtype: None """ @@ -261,7 +351,7 @@ def install(pkg, options): _write_package_json(pkg) - _install_env(pkg, options) + _install_cli(pkg) def _subcommand_dir(): @@ -324,6 +414,86 @@ def _find_virtualenv(bin_directory): return virtualenv_path +def _download_and_store(url, location): + """Download given url and store in location on disk + + :param url: url to download + :type url: str + :param location: path to file to store url + :type location: str + :rtype: None + """ + + with open(location, 'wb') as f: + r = requests.get(url, stream=True) + for chunk in r.iter_content(1024): + f.write(chunk) + + +def _install_with_binary( + package_name, + env_directory, + binary_cli): + """ + :param package_name: the name of the package + :type package_name: str + :param env_directory: the path to the directory in which to install the + package's binary_cli + :type env_directory: str + :param binary_cli: binary cli to install + :type binary_cli: str + :rtype: None + """ + + binary_url, kind = binary_cli.get("url"), binary_cli.get("kind") + + try: + env_bin_dir = os.path.join(env_directory, BIN_DIRECTORY) + + if kind in ["executable", "zip"]: + with util.temptext() as file_tmp: + _, binary_tmp = file_tmp + _download_and_store(binary_url, binary_tmp) + _check_hash(binary_tmp, binary_cli.get("contentHash")) + + if kind == "executable": + util.ensure_dir_exists(env_bin_dir) + binary_name = "dcos-{}".format(package_name) + binary_file = os.path.join(env_bin_dir, binary_name) + shutil.move(binary_tmp, binary_file) + else: + # kind == "zip" + with zipfile.ZipFile(binary_tmp) as zf: + zf.extractall(env_directory) + + # check contents for package_name/env/bin folder structure + if not os.path.exists(env_bin_dir): + msg = ( + "CLI subcommand for [{}] has an unexpected format. " + "Please contact the package maintainer".format( + package_name)) + raise DCOSException(msg) + else: + msg = ("CLI subcommand for [{}] is an unsupported type: {}" + "Please contact the package maintainer".format( + package_name, kind)) + raise DCOSException(msg) + + # make binar(ies) executable + for f in os.listdir(env_bin_dir): + binary = os.path.join(env_bin_dir, f) + if (f.startswith(constants.DCOS_COMMAND_PREFIX)): + st = os.stat(binary) + os.chmod(binary, st.st_mode | stat.S_IEXEC) + except DCOSException: + raise + except Exception as e: + logger.exception(e) + raise _generic_error(package_name) + + return None + + def _install_with_pip( package_name, env_directory, diff --git a/dcos/util.py b/dcos/util.py index 2659faa..dde7f31 100644 --- a/dcos/util.py +++ b/dcos/util.py @@ -77,6 +77,21 @@ def temptext(): shutil.rmtree(path, ignore_errors=True) +@contextlib.contextmanager +def remove_path_on_error(path): + """A context manager for modifying a specific path + `path` and all subpaths will be removed on error + + :rtype: None + """ + + try: + yield path + except: + shutil.rmtree(path, ignore_errors=True) + raise + + def sh_copy(src, dst): """Copy file src to the file or directory dst.