1e2627ba3c
Builtin, third party, and first party are all supposed to have a newline between them. Most of the codebase was doing this, but some had snuck through. This updates those ones to follow the coding style. These are caught by recent changes in the code style checking plugin making it more strict / accurate.
702 lines
20 KiB
Python
702 lines
20 KiB
Python
from __future__ import print_function
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import platform
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import zipfile
|
|
from distutils.version import LooseVersion
|
|
|
|
import requests
|
|
|
|
from dcos import constants, util
|
|
from dcos.errors import DCOSException
|
|
from dcos.subprocess import Subproc
|
|
|
|
logger = util.get_logger(__name__)
|
|
|
|
|
|
def command_executables(subcommand):
|
|
"""List the real path to executable dcos program for specified subcommand.
|
|
|
|
:param subcommand: name of subcommand. E.g. marathon
|
|
:type subcommand: str
|
|
:returns: the dcos program path
|
|
:rtype: str
|
|
"""
|
|
|
|
executables = []
|
|
if subcommand in default_subcommands():
|
|
executables += [default_list_paths()]
|
|
|
|
executables += [
|
|
command_path
|
|
for command_path in list_paths()
|
|
if noun(command_path) == subcommand
|
|
]
|
|
|
|
if len(executables) > 1:
|
|
msg = 'Found more than one executable for command {!r}.'
|
|
raise DCOSException(msg.format(subcommand))
|
|
|
|
if len(executables) == 0:
|
|
msg = "{!r} is not a dcos command."
|
|
raise DCOSException(msg.format(subcommand))
|
|
|
|
return executables[0]
|
|
|
|
|
|
def get_package_commands(package_name):
|
|
"""List the real path(s) to executables for a specific dcos subcommand
|
|
|
|
:param package_name: package name
|
|
:type package_name: str
|
|
:returns: list of all the dcos program paths in package
|
|
:rtype: [str]
|
|
"""
|
|
|
|
bin_dir = os.path.join(_package_dir(package_name),
|
|
constants.DCOS_SUBCOMMAND_ENV_SUBDIR,
|
|
BIN_DIRECTORY)
|
|
|
|
executables = []
|
|
for filename in os.listdir(bin_dir):
|
|
path = os.path.join(bin_dir, filename)
|
|
|
|
if (filename.startswith(constants.DCOS_COMMAND_PREFIX) and
|
|
_is_executable(path)):
|
|
|
|
executables.append(path)
|
|
|
|
return executables
|
|
|
|
|
|
def default_list_paths():
|
|
"""List the real path to dcos executable
|
|
|
|
:returns: list dcos program path
|
|
:rtype: str
|
|
"""
|
|
|
|
# Let's get all the default subcommands
|
|
binpath = util.dcos_bin_path()
|
|
return os.path.join(binpath, "dcos")
|
|
|
|
|
|
def list_paths():
|
|
"""List the real path to executable dcos subcommand programs.
|
|
|
|
:returns: list of all the dcos program paths
|
|
:rtype: [str]
|
|
"""
|
|
|
|
subcommands = []
|
|
for package in distributions():
|
|
subcommands += get_package_commands(package)
|
|
|
|
return subcommands
|
|
|
|
|
|
def _is_executable(path):
|
|
"""
|
|
:param path: the path to a program
|
|
:type path: str
|
|
:returns: True if the path is an executable; False otherwise
|
|
:rtype: bool
|
|
"""
|
|
|
|
return os.access(path, os.X_OK) and (
|
|
not util.is_windows_platform() or path.endswith('.exe'))
|
|
|
|
|
|
def distributions():
|
|
"""List all of the installed subcommand packages
|
|
|
|
:returns: a list of packages
|
|
:rtype: list of str
|
|
"""
|
|
|
|
subcommand_dir = _subcommand_dir()
|
|
|
|
if os.path.isdir(subcommand_dir):
|
|
return [
|
|
subdir for subdir in os.listdir(subcommand_dir)
|
|
if os.path.isdir(
|
|
os.path.join(
|
|
subcommand_dir,
|
|
subdir,
|
|
constants.DCOS_SUBCOMMAND_ENV_SUBDIR))
|
|
]
|
|
else:
|
|
return []
|
|
|
|
|
|
# must also add subcommand name to dcoscli.subcommand._default_modules
|
|
def default_subcommands():
|
|
"""List the default dcos cli subcommands
|
|
|
|
:returns: list of all the default dcos cli subcommands
|
|
:rtype: [str]
|
|
"""
|
|
return ["auth", "config", "help", "job", "marathon",
|
|
"node", "package", "service", "task"]
|
|
|
|
|
|
def documentation(executable_path):
|
|
"""Gather subcommand summary
|
|
|
|
:param executable_path: real path to the dcos subcommands
|
|
:type executable_path: str
|
|
:returns: subcommand and its summary
|
|
:rtype: (str, str)
|
|
"""
|
|
|
|
path_noun = noun(executable_path)
|
|
return (path_noun, info(executable_path, path_noun))
|
|
|
|
|
|
def info(executable_path, path_noun):
|
|
"""Collects subcommand information
|
|
|
|
:param executable_path: real path to the dcos subcommand
|
|
:type executable_path: str
|
|
:param path_noun: subcommand
|
|
:type path_noun: str
|
|
:returns: the subcommand information
|
|
:rtype: str
|
|
"""
|
|
|
|
out = Subproc().check_output(
|
|
[executable_path, path_noun, '--info'])
|
|
|
|
return out.decode('utf-8').strip()
|
|
|
|
|
|
def config_schema(executable_path, noun=None):
|
|
"""Collects subcommand config schema
|
|
|
|
:param executable_path: real path to the dcos subcommand
|
|
:type executable_path: str
|
|
:param noun: name of subcommand
|
|
:type noun: str
|
|
:returns: the subcommand config schema
|
|
:rtype: dict
|
|
"""
|
|
if noun is None:
|
|
noun = noun(executable_path)
|
|
|
|
out = Subproc().check_output(
|
|
[executable_path, noun, '--config-schema'])
|
|
|
|
return json.loads(out.decode('utf-8'))
|
|
|
|
|
|
def noun(executable_path):
|
|
"""Extracts the subcommand single noun from the path to the executable.
|
|
E.g for :code:`bin/dcos-subcommand` this method returns :code:`subcommand`.
|
|
|
|
:param executable_path: real pth to the dcos subcommand
|
|
:type executable_path: str
|
|
:returns: the subcommand
|
|
:rtype: str
|
|
"""
|
|
|
|
basename = os.path.basename(executable_path)
|
|
noun = basename[len(constants.DCOS_COMMAND_PREFIX):].replace('.exe', '')
|
|
return noun
|
|
|
|
|
|
def _write_package_json(pkg):
|
|
""" Write package.json locally.
|
|
|
|
:param pkg: the package being installed
|
|
:type pkg: PackageVersion
|
|
:rtype: None
|
|
"""
|
|
|
|
pkg_dir = _package_dir(pkg.name())
|
|
|
|
package_path = os.path.join(pkg_dir, 'package.json')
|
|
|
|
package_json = pkg.package_json()
|
|
|
|
with util.open_file(package_path, 'w') as package_file:
|
|
json.dump(package_json, package_file)
|
|
|
|
|
|
def _hashfile(filename):
|
|
"""Calculates the sha256 of a file
|
|
|
|
:param filename: path to the file to sum
|
|
:type filename: str
|
|
:returns: digest in hexadecimal
|
|
:rtype: str
|
|
"""
|
|
|
|
hasher = hashlib.sha256()
|
|
with open(filename, 'rb') as f:
|
|
for chunk in iter(lambda: f.read(4096), b''):
|
|
hasher.update(chunk)
|
|
return hasher.hexdigest()
|
|
|
|
|
|
def _check_hash(filename, content_hashes):
|
|
"""Validates whether downloaded binary matches expected hash
|
|
|
|
:param filename: path to binary
|
|
:type filename: str
|
|
:param content_hashes: list of hash algorithms/value
|
|
:type content_hashes: [{"algo": <str>, "value": <str>}]
|
|
:returns: None if valid hash, else throws exception
|
|
:rtype: None
|
|
"""
|
|
|
|
content_hash = next((contents for contents in content_hashes
|
|
if contents.get("algo") == "sha256"),
|
|
None)
|
|
if content_hash:
|
|
expected_value = content_hash.get("value")
|
|
actual_value = _hashfile(filename)
|
|
if expected_value != actual_value:
|
|
raise DCOSException(
|
|
"The hash for the downloaded subcommand [{}] "
|
|
"does not match the expected value [{}]. Aborting...".format(
|
|
actual_value, expected_value))
|
|
else:
|
|
return
|
|
else:
|
|
raise DCOSException(
|
|
"Hash algorithm specified is unsupported. "
|
|
"Please contact the package maintainer. Aborting...")
|
|
|
|
|
|
def _get_cli_binary_info(cli_resources):
|
|
"""Find compatible cli binary, if one exists
|
|
|
|
:param cli_resources: cli property of resource.json
|
|
:type resources: {}
|
|
:returns: {"url": <str>, "kind": <str>, "contentHash": [{}]}
|
|
:rtype: {} | None
|
|
"""
|
|
|
|
if "binaries" in cli_resources:
|
|
binaries = cli_resources["binaries"]
|
|
arch = platform.architecture()[0]
|
|
if arch != "64bit":
|
|
raise DCOSException(
|
|
"There is no compatible subcommand for your architecture [{}] "
|
|
"We only support x86-64. Aborting...".format(arch))
|
|
system = platform.system().lower()
|
|
binary = binaries.get(system)
|
|
if binary is None:
|
|
raise DCOSException(
|
|
"There is not compatible subcommand for your system [{}] "
|
|
"Aborting...".format(system))
|
|
elif "x86-64" in binary:
|
|
return binary["x86-64"]
|
|
|
|
raise DCOSException(
|
|
"The CLI subcommand has unexpected format [{}]. "
|
|
"Please contact the package maintainer. Aborting...".format(
|
|
cli_resources))
|
|
|
|
|
|
def _install_cli(pkg):
|
|
"""Install subcommand cli
|
|
|
|
:param pkg: the package to install
|
|
:type pkg: PackageVersion
|
|
:rtype: None
|
|
"""
|
|
|
|
with util.remove_path_on_error(_package_dir(pkg.name())) as pkg_dir:
|
|
env_dir = os.path.join(pkg_dir, constants.DCOS_SUBCOMMAND_ENV_SUBDIR)
|
|
|
|
resources = pkg.resource_json()
|
|
|
|
if resources and resources.get("cli") is not None:
|
|
binary = resources["cli"]
|
|
binary_cli = _get_cli_binary_info(binary)
|
|
_install_with_binary(
|
|
pkg.name(),
|
|
env_dir,
|
|
binary_cli)
|
|
elif pkg.command_json() is not None:
|
|
install_operation = pkg.command_json()
|
|
if 'pip' in install_operation:
|
|
_install_with_pip(
|
|
pkg.name(),
|
|
env_dir,
|
|
install_operation['pip'])
|
|
else:
|
|
raise DCOSException(
|
|
"Installation methods '{}' not supported".format(
|
|
install_operation.keys()))
|
|
else:
|
|
raise DCOSException(
|
|
"Could not find a CLI subcommand for your platform")
|
|
|
|
|
|
def install(pkg):
|
|
"""Installs the dcos cli subcommand
|
|
|
|
:param pkg: the package to install
|
|
:type pkg: Package
|
|
:rtype: None
|
|
"""
|
|
|
|
pkg_dir = _package_dir(pkg.name())
|
|
util.ensure_dir_exists(pkg_dir)
|
|
|
|
_write_package_json(pkg)
|
|
|
|
_install_cli(pkg)
|
|
|
|
|
|
def _subcommand_dir():
|
|
""" Returns ~/.dcos/subcommands """
|
|
return os.path.expanduser(os.path.join("~",
|
|
constants.DCOS_DIR,
|
|
constants.DCOS_SUBCOMMAND_SUBDIR))
|
|
|
|
|
|
def _package_dir(name):
|
|
""" Returns ~/.dcos/subcommands/<name>
|
|
|
|
:param name: package name
|
|
:type name: str
|
|
:rtype: str
|
|
"""
|
|
return os.path.join(_subcommand_dir(),
|
|
name)
|
|
|
|
|
|
def uninstall(package_name):
|
|
"""Uninstall the dcos cli subcommand
|
|
|
|
:param package_name: the name of the package
|
|
:type package_name: str
|
|
:returns: True if the subcommand was uninstalled
|
|
:rtype: bool
|
|
"""
|
|
|
|
pkg_dir = _package_dir(package_name)
|
|
|
|
if os.path.isdir(pkg_dir):
|
|
shutil.rmtree(pkg_dir)
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
BIN_DIRECTORY = 'Scripts' if util.is_windows_platform() else 'bin'
|
|
|
|
|
|
def _find_virtualenv(bin_directory):
|
|
"""
|
|
:param bin_directory: directory to first use to find virtualenv
|
|
:type bin_directory: str
|
|
:returns: Absolute path to virutalenv program
|
|
:rtype: str
|
|
"""
|
|
|
|
virtualenv_path = os.path.join(bin_directory, 'virtualenv')
|
|
if not os.path.exists(virtualenv_path):
|
|
virtualenv_path = util.which('virtualenv')
|
|
|
|
if virtualenv_path is None:
|
|
msg = ("Unable to install CLI subcommand. "
|
|
"Missing required program 'virtualenv'.\n"
|
|
"Please see installation instructions: "
|
|
"https://virtualenv.pypa.io/en/latest/installation.html")
|
|
raise DCOSException(msg)
|
|
|
|
return virtualenv_path
|
|
|
|
|
|
def _download_and_store(url, location):
|
|
"""Download given url and store in location on disk
|
|
|
|
:param url: url to download
|
|
:type url: str
|
|
:param location: path to file to store url
|
|
:type location: str
|
|
:rtype: None
|
|
"""
|
|
|
|
with open(location, 'wb') as f:
|
|
r = requests.get(url, stream=True)
|
|
for chunk in r.iter_content(1024):
|
|
f.write(chunk)
|
|
|
|
|
|
def _install_with_binary(
|
|
package_name,
|
|
env_directory,
|
|
binary_cli):
|
|
"""
|
|
:param package_name: the name of the package
|
|
:type package_name: str
|
|
:param env_directory: the path to the directory in which to install the
|
|
package's binary_cli
|
|
:type env_directory: str
|
|
:param binary_cli: binary cli to install
|
|
:type binary_cli: str
|
|
:rtype: None
|
|
"""
|
|
|
|
binary_url, kind = binary_cli.get("url"), binary_cli.get("kind")
|
|
|
|
try:
|
|
env_bin_dir = os.path.join(env_directory, BIN_DIRECTORY)
|
|
|
|
if kind in ["executable", "zip"]:
|
|
with util.temptext() as file_tmp:
|
|
_, binary_tmp = file_tmp
|
|
_download_and_store(binary_url, binary_tmp)
|
|
_check_hash(binary_tmp, binary_cli.get("contentHash"))
|
|
|
|
if kind == "executable":
|
|
util.ensure_dir_exists(env_bin_dir)
|
|
binary_name = "dcos-{}".format(package_name)
|
|
if util.is_windows_platform():
|
|
binary_name += '.exe'
|
|
binary_file = os.path.join(env_bin_dir, binary_name)
|
|
|
|
# copy to avoid windows error of moving open file
|
|
# binary_tmp will be removed by context manager
|
|
shutil.copy(binary_tmp, binary_file)
|
|
else:
|
|
# kind == "zip"
|
|
with zipfile.ZipFile(binary_tmp) as zf:
|
|
zf.extractall(env_directory)
|
|
|
|
# check contents for package_name/env/bin folder structure
|
|
if not os.path.exists(env_bin_dir):
|
|
msg = (
|
|
"CLI subcommand for [{}] has an unexpected format. "
|
|
"Please contact the package maintainer".format(
|
|
package_name))
|
|
raise DCOSException(msg)
|
|
else:
|
|
msg = ("CLI subcommand for [{}] is an unsupported type: {}"
|
|
"Please contact the package maintainer".format(
|
|
package_name, kind))
|
|
raise DCOSException(msg)
|
|
|
|
# make binar(ies) executable
|
|
for f in os.listdir(env_bin_dir):
|
|
binary = os.path.join(env_bin_dir, f)
|
|
if (f.startswith(constants.DCOS_COMMAND_PREFIX)):
|
|
st = os.stat(binary)
|
|
os.chmod(binary, st.st_mode | stat.S_IEXEC)
|
|
except DCOSException:
|
|
raise
|
|
except Exception as e:
|
|
logger.exception(e)
|
|
raise _generic_error(package_name)
|
|
|
|
return None
|
|
|
|
|
|
def _install_with_pip(
|
|
package_name,
|
|
env_directory,
|
|
requirements):
|
|
"""
|
|
:param package_name: the name of the package
|
|
:type package_name: str
|
|
:param env_directory: the path to the directory in which to install the
|
|
package's virtual env
|
|
:type env_directory: str
|
|
:param requirements: the list of pip requirements
|
|
:type requirements: list of str
|
|
:rtype: None
|
|
"""
|
|
|
|
bin_directory = util.dcos_bin_path()
|
|
new_package_dir = not os.path.exists(env_directory)
|
|
|
|
pip_path = os.path.join(env_directory, BIN_DIRECTORY, 'pip')
|
|
if not os.path.exists(pip_path):
|
|
virtualenv_path = _find_virtualenv(bin_directory)
|
|
|
|
virtualenv_version = _execute_command(
|
|
[virtualenv_path, '--version'])[0].strip().decode('utf-8')
|
|
if LooseVersion("12") > LooseVersion(virtualenv_version):
|
|
msg = ("Unable to install CLI subcommand. "
|
|
"Required program 'virtualenv' must be version 12+, "
|
|
"currently version {}\n"
|
|
"Please see installation instructions: "
|
|
"https://virtualenv.pypa.io/en/latest/installation.html"
|
|
"".format(virtualenv_version))
|
|
raise DCOSException(msg)
|
|
|
|
cmd = [_find_virtualenv(bin_directory), env_directory]
|
|
|
|
if _execute_command(cmd)[2] != 0:
|
|
raise _generic_error(package_name)
|
|
|
|
# Do not replace util.temptext NamedTemporaryFile
|
|
# otherwise bad things will happen on Windows
|
|
with util.temptext() as text_file:
|
|
fd, requirement_path = text_file
|
|
|
|
# Write the requirements to the file
|
|
with os.fdopen(fd, 'w') as requirements_file:
|
|
for line in requirements:
|
|
print(line, file=requirements_file)
|
|
|
|
cmd = [
|
|
os.path.join(env_directory, BIN_DIRECTORY, 'pip'),
|
|
'install',
|
|
'--requirement',
|
|
requirement_path,
|
|
]
|
|
|
|
if _execute_command(cmd)[2] != 0:
|
|
# We should remove the directory that we just created
|
|
if new_package_dir:
|
|
shutil.rmtree(env_directory)
|
|
|
|
raise _generic_error(package_name)
|
|
return None
|
|
|
|
|
|
def _execute_command(command):
|
|
"""
|
|
:param command: a command to execute
|
|
:type command: list of str
|
|
:returns: stdout, stderr, the process return code
|
|
:rtype: str, str, int
|
|
"""
|
|
|
|
logger.info('Calling: %r', command)
|
|
|
|
process = Subproc().popen(
|
|
command,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE)
|
|
|
|
stdout, stderr = process.communicate()
|
|
|
|
if process.returncode != 0:
|
|
logger.error("Command script's stdout: %s", stdout)
|
|
logger.error("Command script's stderr: %s", stderr)
|
|
else:
|
|
logger.info("Command script's stdout: %s", stdout)
|
|
logger.info("Command script's stderr: %s", stderr)
|
|
|
|
return stdout, stderr, process.returncode
|
|
|
|
|
|
def _generic_error(package_name):
|
|
"""
|
|
:param package: package name
|
|
:type: str
|
|
:returns: generic error when installing package
|
|
:rtype: DCOSException
|
|
"""
|
|
|
|
return DCOSException(
|
|
('Error installing {!r} package.\n'
|
|
'Run with `dcos --log-level=ERROR` to see the full output.').format(
|
|
package_name))
|
|
|
|
|
|
class InstalledSubcommand(object):
|
|
""" Represents an installed subcommand.
|
|
|
|
:param name: The name of the subcommand
|
|
:type name: str
|
|
"""
|
|
|
|
def __init__(self, name):
|
|
self.name = name
|
|
|
|
def _dir(self):
|
|
"""
|
|
:returns: path to this subcommand's directory.
|
|
:rtype: str
|
|
"""
|
|
|
|
return _package_dir(self.name)
|
|
|
|
def package_revision(self):
|
|
"""
|
|
:returns: this subcommand's version.
|
|
:rtype: str
|
|
"""
|
|
|
|
version_path = os.path.join(self._dir(), 'version')
|
|
return util.read_file(version_path)
|
|
|
|
def package_source(self):
|
|
"""
|
|
:returns: this subcommand's source.
|
|
:rtype: str
|
|
"""
|
|
|
|
source_path = os.path.join(self._dir(), 'source')
|
|
return util.read_file(source_path)
|
|
|
|
def package_json(self):
|
|
"""
|
|
:returns: contents of this subcommand's package.json file.
|
|
:rtype: dict
|
|
"""
|
|
|
|
package_json_path = os.path.join(self._dir(), 'package.json')
|
|
with util.open_file(package_json_path) as package_json_file:
|
|
return util.load_json(package_json_file)
|
|
|
|
|
|
class SubcommandProcess():
|
|
|
|
def __init__(self, executable, command, args):
|
|
"""Representes a subcommand running by a forked process
|
|
|
|
:param executable: executable to run
|
|
:type executable: executable
|
|
:param command: command to run by executable
|
|
:type command: str
|
|
:param args: arguments for command
|
|
:type args: [str]
|
|
"""
|
|
|
|
self._executable = executable
|
|
self._command = command
|
|
self._args = args
|
|
|
|
def run_and_capture(self):
|
|
"""
|
|
Run a command and capture exceptions. This is a blocking call
|
|
:returns: tuple of exitcode, error (or None)
|
|
:rtype: int, str | None
|
|
"""
|
|
|
|
subproc = Subproc().popen(
|
|
[self._executable, self._command] + self._args,
|
|
stderr=subprocess.PIPE)
|
|
|
|
err = ''
|
|
while subproc.poll() is None:
|
|
line = subproc.stderr.readline().decode('utf-8')
|
|
err += line
|
|
sys.stderr.write(line)
|
|
sys.stderr.flush()
|
|
|
|
exitcode = subproc.poll()
|
|
# We only want to catch exceptions, not other stderr messages
|
|
# (such as "task does not exist", so we look for the 'Traceback'
|
|
# string. This only works for python, so we'll need to revisit
|
|
# this in the future when we support subcommands written in other
|
|
# languages.
|
|
err = ('Traceback' in err and err) or None
|
|
|
|
return exitcode, err
|