From 0af60ffa4d37ea1c5f31f8fbd80354af26fab984 Mon Sep 17 00:00:00 2001 From: Tamar Ben-Shachar Date: Mon, 22 Feb 2016 16:35:44 -0800 Subject: [PATCH] support for cosmos + remove support for legacy --- README.rst | 3 - bin/install/install-dcos-cli.sh | 20 +- bin/install/install-optout-dcos-cli.sh | 20 +- bin/install/legacy/install-legacy-dcos-cli.sh | 154 ++ .../legacy/install-legacy-optout-dcos-cli.sh | 155 ++ bin/install/upload_to_s3.sh | 38 + cli/dcoscli/config/main.py | 95 +- cli/dcoscli/data/config-schema/package.json | 24 +- cli/dcoscli/data/help/config.txt | 6 +- cli/dcoscli/data/help/package.txt | 31 +- cli/dcoscli/package/main.py | 226 ++- cli/dcoscli/service/main.py | 4 +- cli/dcoscli/tables.py | 3 - .../data/config/missing_params_dcos.toml | 3 +- cli/tests/data/dcos.toml | 5 +- cli/tests/data/help/config.txt | 6 +- cli/tests/data/help/package.txt | 31 +- .../data/marathon/groups/complicated.json | 2 +- .../marathon/missing_marathon_params.toml | 3 +- .../package/json/cassandra_single_node.json | 12 + .../package/json/test_describe_app_cli.json | 4 +- .../json/test_describe_app_marathon.json | 2 +- .../json/test_describe_app_options.json | 10 +- .../package/json/test_describe_marathon.json | 8 +- ...son => test_describe_marathon_0.11.1.json} | 12 +- .../test_describe_marathon_app_render.json | 10 +- ...est_describe_marathon_package_version.json | 23 +- ...st_describe_marathon_package_versions.json | 10 +- .../data/package/json/test_list_chronos.json | 26 + .../json/test_list_chronos_two_users.json | 28 + .../json/test_list_chronos_user_1.json | 26 + .../json/test_list_chronos_user_2.json | 27 + cli/tests/data/ssl/ssl.toml | 3 +- cli/tests/fixtures/package.py | 211 ++- cli/tests/integrations/common.py | 8 +- cli/tests/integrations/test_config.py | 289 +-- cli/tests/integrations/test_marathon.py | 2 + cli/tests/integrations/test_package.py | 543 +++--- cli/tests/integrations/test_ssl.py | 8 +- cli/tests/integrations/test_task.py | 5 +- cli/tests/unit/data/package_search.txt | 16 +- dcos/config.py | 94 +- dcos/cosmospackage.py | 493 +++++- dcos/marathon.py | 12 + dcos/package.py | 1549 +---------------- dcos/subcommand.py | 81 +- dcos/util.py | 14 + tests/test_package.py | 117 -- win_bin/install/install-dcos-cli.ps1 | 4 - .../legacy/install-legacy-dcos-cli.ps1 | 147 ++ 50 files changed, 1939 insertions(+), 2684 deletions(-) create mode 100755 bin/install/legacy/install-legacy-dcos-cli.sh create mode 100755 bin/install/legacy/install-legacy-optout-dcos-cli.sh create mode 100755 bin/install/upload_to_s3.sh create mode 100644 cli/tests/data/package/json/cassandra_single_node.json rename cli/tests/data/package/json/{test_describe_marathon_0.8.1.json => test_describe_marathon_0.11.1.json} (71%) create mode 100644 cli/tests/data/package/json/test_list_chronos.json create mode 100644 cli/tests/data/package/json/test_list_chronos_two_users.json create mode 100644 cli/tests/data/package/json/test_list_chronos_user_1.json create mode 100644 cli/tests/data/package/json/test_list_chronos_user_2.json delete mode 100644 tests/test_package.py create mode 100644 win_bin/install/legacy/install-legacy-dcos-cli.ps1 diff --git a/README.rst b/README.rst index d5ebd7e..b8cbe8f 100644 --- a/README.rst +++ b/README.rst @@ -96,9 +96,6 @@ Configure Environment and Run installation of DCOS:: dcos config set core.dcos_url http://dcos-ea-1234.us-west-2.elb.amazonaws.com - dcos config append package.sources https://universe.mesosphere.com/repo - dcos config set package.cache /tmp/dcos - dcos package update #. Get started by calling the DCOS CLI help:: diff --git a/bin/install/install-dcos-cli.sh b/bin/install/install-dcos-cli.sh index 3a30174..b6b0396 100755 --- a/bin/install/install-dcos-cli.sh +++ b/bin/install/install-dcos-cli.sh @@ -66,12 +66,29 @@ check_pip_version() fi } +check_dcoscli_version() +{ + if [ ! -z "$DCOS_CLI_VERSION" ]; then + # result is the larger of the two versions + COSMOS_VERSION="0.4.0" + # convert the str to numbers, sort, and return the larger + result=$(echo -e "$COSMOS_VERSION\n$DCOS_CLI_VERSION" | sed '/^$/d' | sort -nr | head -1) + # if DCOS_CLI_VERSION < COSMOS_VERSION, exit + if [ "$result" != "$DCOS_CLI_VERSION" ]; then + echo "Please use legacy installer for dcoscli versions <0.4.0. Aborting."; + exit 1; + fi + fi + exit 1; +} + if [ "$#" -lt 2 ]; then usage; exit 1; fi check_pip_version; +check_dcoscli_version; ARGS=( "$@" ); @@ -113,9 +130,6 @@ dcos config set core.reporting true dcos config set core.dcos_url $DCOS_URL dcos config set core.ssl_verify false dcos config set core.timeout 5 -dcos config set package.cache ~/.dcos/cache -dcos config set package.sources '["https://universe.mesosphere.com/repo"]' -dcos package update ADD_PATH="" while [ $# -gt 0 ]; do diff --git a/bin/install/install-optout-dcos-cli.sh b/bin/install/install-optout-dcos-cli.sh index 41ba7b2..8b739df 100755 --- a/bin/install/install-optout-dcos-cli.sh +++ b/bin/install/install-optout-dcos-cli.sh @@ -66,12 +66,29 @@ check_pip_version() fi } +check_dcoscli_version() +{ + if [ ! -z "$DCOS_CLI_VERSION" ]; then + # result is the larger of the two versions + COSMOS_VERSION="0.4.0" + # convert the str to numbers, sort, and return the larger + result=$(echo -e "$COSMOS_VERSION\n$DCOS_CLI_VERSION" | sed '/^$/d' | sort -nr | head -1) + # if DCOS_CLI_VERSION < COSMOS_VERSION, exit + if [ "$result" != "$DCOS_CLI_VERSION" ]; then + echo "Please use legacy installer for dcoscli versions <0.4.0. Aborting."; + exit 1; + fi + fi + exit 1; +} + if [ "$#" -lt 2 ]; then usage; exit 1; fi check_pip_version; +check_dcoscli_version; ARGS=( "$@" ); @@ -114,9 +131,6 @@ dcos config set core.reporting false dcos config set core.dcos_url $DCOS_URL dcos config set core.ssl_verify false dcos config set core.timeout 5 -dcos config set package.cache ~/.dcos/cache -dcos config set package.sources '["https://universe.mesosphere.com/repo"]' -dcos package update ADD_PATH="" while [ $# -gt 0 ]; do diff --git a/bin/install/legacy/install-legacy-dcos-cli.sh b/bin/install/legacy/install-legacy-dcos-cli.sh new file mode 100755 index 0000000..4badf4c --- /dev/null +++ b/bin/install/legacy/install-legacy-dcos-cli.sh @@ -0,0 +1,154 @@ +#!/bin/bash + +set -o errexit -o pipefail + +usage() +{ # Show usage information. + echo "$(basename "$(test -L "$0" && readlink "$0" || echo "$0")") [--add-path yes/no]" +} + +post_install_message() +{ + echo 'Finished installing and configuring DCOS CLI.' + echo '' + echo 'Run this command to set up your environment and to get started:' + echo "source $1 && dcos help" +} + +RC_NAME="" + +write_to_profile() +{ + echo "" >> ~/"$2"; + echo "# path to the DCOS CLI binary" >> ~/"$2"; + echo "if [[ \"\$PATH\" != *\"$1\"* ]];" >> ~/"$2"; + echo " then export PATH=\$PATH:$1;" >> ~/"$2"; + echo "fi" >> ~/"$2"; +} + +add_dcos_path_to_profile() +{ + UNAME=`uname` + case "$UNAME" in + Linux ) RC_NAME=".bashrc";; + Darwin ) RC_NAME=".bash_profile";; + CYGWIN* ) RC_NAME=".bashrc";; + MINGW* ) RC_NAME=".profile";; + * ) RC_NAME=".bashrc";; + esac + write_to_profile "$1" "$RC_NAME" +} + +prompt_add_dcos_path_to_profile() +{ + while true; do + echo "" + read -p "Modify your bash profile to add DCOS to your PATH? [yes/no] " ANSWER + echo "" + case "$ANSWER" in + [Yy]* ) add_dcos_path_to_profile "$1"; break;; + [Nn]* ) break;; + * ) echo "Please answer yes or no.";; + esac + done +} + +check_pip_version() +{ + PIP_INFO=$(pip -V); + REGEX="([0-9]+)\.([0-9]+)"; + [[ $PIP_INFO =~ $REGEX ]]; + MAJOR_PIP_VERSION="${BASH_REMATCH[1]}"; + MINOR_PIP_VERSION="${BASH_REMATCH[2]}"; + if [ "$MAJOR_PIP_VERSION" -lt 1 ] || ([ "$MAJOR_PIP_VERSION" -eq 1 ] && [ "$MINOR_PIP_VERSION" -le 4 ]); + then echo "Pip version must be greater than 1.4. Aborting."; + exit 1; + fi +} + +check_dcoscli_version() +{ + if [ ! -z "$DCOS_CLI_VERSION" ]; then + # result is the larger of the two versions + COSMOS_VERSION="0.4.0" + # convert the str to numbers, sort, and return the larger + result=$(echo -e "$COSMOS_VERSION\n$DCOS_CLI_VERSION" | sed '/^$/d' | sort -nr | head -1) + # if DCOS_CLI_VERSION >= COSMOS_VERSION, exit + if [ "$result" = "$DCOS_CLI_VERSION" ]; then + echo "Legacy mode is only supported in dcoscli version <0.4.0. Aborting."; + exit 1; + fi + fi +} + +if [ "$#" -lt 2 ]; then + usage; + exit 1; +fi + +check_pip_version; +check_dcoscli_version; + +ARGS=( "$@" ); + +VIRTUAL_ENV_PATH=$(python -c "import os; print(os.path.realpath('"${ARGS[0]}"'))") +if [[ $VIRTUAL_ENV_PATH =~ \ ]]; + then echo "Spaces are not permitted in the installation path. Please try again with another path."; + exit 1; +fi +DCOS_URL=${ARGS[1]} + +command -v virtualenv >/dev/null 2>&1 || { echo "Cannot find virtualenv. You may need to install it by following the documentation at https://docs.mesosphere.com/install/cli/#linux. Aborting."; exit 1; } + +VIRTUALENV_VERSION=$(virtualenv --version) +VERSION_REGEX="s#[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)#\1#" + +eval MAJOR=`echo $VIRTUALENV_VERSION | sed -e $VERSION_REGEX` +if [ $MAJOR -lt 12 ]; + then echo "Virtualenv version must be 12 or greater. Aborting."; + exit 1; +fi + +echo "Installing DCOS CLI from PyPI..."; +echo ""; + +# Let's first setup a virtualenv: we are assuming that the path is absolute +mkdir -p "$VIRTUAL_ENV_PATH" +virtualenv "$VIRTUAL_ENV_PATH" + +# Install the DCOS CLI package, using version if set +if [ -z "$DCOS_CLI_VERSION" ]; then + "$VIRTUAL_ENV_PATH/bin/pip" install --quiet "dcoscli<0.4.0" +else + "$VIRTUAL_ENV_PATH/bin/pip" install --quiet "dcoscli==$DCOS_CLI_VERSION" +fi + +ENV_SETUP="$VIRTUAL_ENV_PATH/bin/env-setup" +source "$ENV_SETUP" +dcos config set core.reporting true +dcos config set core.dcos_url $DCOS_URL +dcos config set core.ssl_verify false +dcos config set core.timeout 5 +dcos config set package.cache ~/.dcos/cache +dcos config set package.sources '["https://github.com/mesosphere/universe/archive/version-1.x.zip"]' +dcos package update + +ADD_PATH="" +while [ $# -gt 0 ]; do + case "$1" in + --add-path ) ADD_PATH="$2"; break;; + * ) shift;; + esac +done + +case "$ADD_PATH" in + [Yy]* ) add_dcos_path_to_profile "$VIRTUAL_ENV_PATH/bin";; + [Nn]* ) ;; + * ) prompt_add_dcos_path_to_profile "$VIRTUAL_ENV_PATH/bin";; +esac + +if [ -z "$RC_NAME" ]; then + post_install_message "$ENV_SETUP" +else + post_install_message "~/$RC_NAME" +fi diff --git a/bin/install/legacy/install-legacy-optout-dcos-cli.sh b/bin/install/legacy/install-legacy-optout-dcos-cli.sh new file mode 100755 index 0000000..1413f6e --- /dev/null +++ b/bin/install/legacy/install-legacy-optout-dcos-cli.sh @@ -0,0 +1,155 @@ +#!/bin/bash + +set -o errexit -o pipefail + +usage() +{ # Show usage information. + echo "$(basename "$(test -L "$0" && readlink "$0" || echo "$0")") [--add-path yes/no]" +} + +post_install_message() +{ + echo 'Finished installing and configuring DCOS CLI.' + echo '' + echo 'Run this command to set up your environment and to get started:' + echo "source $1 && dcos help" +} + +RC_NAME="" + +write_to_profile() +{ + echo "" >> ~/"$2"; + echo "# path to the DCOS CLI binary" >> ~/"$2"; + echo "if [[ \"\$PATH\" != *\"$1\"* ]];" >> ~/"$2"; + echo " then export PATH=\$PATH:$1;" >> ~/"$2"; + echo "fi" >> ~/"$2"; +} + +add_dcos_path_to_profile() +{ + UNAME=`uname` + case "$UNAME" in + Linux ) RC_NAME=".bashrc";; + Darwin ) RC_NAME=".bash_profile";; + CYGWIN* ) RC_NAME=".bashrc";; + MINGW* ) RC_NAME=".profile";; + * ) RC_NAME=".bashrc";; + esac + write_to_profile "$1" "$RC_NAME" +} + +prompt_add_dcos_path_to_profile() +{ + while true; do + echo "" + read -p "Modify your bash profile to add DCOS to your PATH? [yes/no] " ANSWER + echo "" + case "$ANSWER" in + [Yy]* ) add_dcos_path_to_profile "$1"; break;; + [Nn]* ) break;; + * ) echo "Please answer yes or no.";; + esac + done +} + +check_pip_version() +{ + PIP_INFO=$(pip -V); + REGEX="([0-9]+)\.([0-9]+)"; + [[ $PIP_INFO =~ $REGEX ]]; + MAJOR_PIP_VERSION="${BASH_REMATCH[1]}"; + MINOR_PIP_VERSION="${BASH_REMATCH[2]}"; + if [ "$MAJOR_PIP_VERSION" -lt 1 ] || ([ "$MAJOR_PIP_VERSION" -eq 1 ] && [ "$MINOR_PIP_VERSION" -le 4 ]); + then echo "Pip version must be greater than 1.4. Aborting."; + exit 1; + fi +} + +check_dcoscli_version() +{ + if [ ! -z "$DCOS_CLI_VERSION" ]; then + COSMOS_VERSION="0.4.0" + # result is the larger of the two versions + # convert the str to numbers, sort, and return the larger + result=$(echo -e "$COSMOS_VERSION\n$DCOS_CLI_VERSION" | sed '/^$/d' | sort -nr | head -1) + # if DCOS_CLI_VERSION >= COSMOS_VERSION, exit + if [ "$result" = "$DCOS_CLI_VERSION" ]; then + echo "Legacy mode is only supported in dcoscli version <0.4.0. Aborting."; + exit 1; + fi + fi +} + +if [ "$#" -lt 2 ]; then + usage; + exit 1; +fi + +check_pip_version; +check_dcoscli_version; + +ARGS=( "$@" ); + +VIRTUAL_ENV_PATH=$(python -c "import os; print(os.path.realpath('"${ARGS[0]}"'))") +if [[ $VIRTUAL_ENV_PATH =~ \ ]]; + then echo "Spaces are not permitted in the installation path. Please try again with another path."; + exit 1; +fi +DCOS_URL=${ARGS[1]} + +command -v virtualenv >/dev/null 2>&1 || { echo "Cannot find virtualenv. You may need to install it by following the documentation at https://docs.mesosphere.com/install/cli/#linux. Aborting."; exit 1; } + +VIRTUALENV_VERSION=$(virtualenv --version) +VERSION_REGEX="s#[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)#\1#" + +eval MAJOR=`echo $VIRTUALENV_VERSION | sed -e $VERSION_REGEX` +if [ $MAJOR -lt 12 ]; + then echo "Virtualenv version must be 12 or greater. Aborting."; + exit 1; +fi + +echo "Installing DCOS CLI from PyPI..."; +echo ""; + +# Let's first setup a virtualenv: we are assuming that the path is absolute +mkdir -p "$VIRTUAL_ENV_PATH" +virtualenv "$VIRTUAL_ENV_PATH" + +# Install the DCOS CLI package, using version if set +if [ -z "$DCOS_CLI_VERSION" ]; then + "$VIRTUAL_ENV_PATH/bin/pip" install --quiet "dcoscli<0.4.0" +else + "$VIRTUAL_ENV_PATH/bin/pip" install --quiet "dcoscli==$DCOS_CLI_VERSION" +fi + +ENV_SETUP="$VIRTUAL_ENV_PATH/bin/env-setup" +source "$ENV_SETUP" +dcos config set core.email anonymous-optout +dcos config set core.reporting false +dcos config set core.dcos_url $DCOS_URL +dcos config set core.ssl_verify false +dcos config set core.timeout 5 +dcos config set package.cache ~/.dcos/cache +dcos config set package.sources '["https://github.com/mesosphere/universe/archive/version-1.x.zip"]' +dcos package update + +ADD_PATH="" +while [ $# -gt 0 ]; do + case "$1" in + --add-path ) ADD_PATH="$2"; break;; + * ) shift;; + esac +done + +case "$ADD_PATH" in + [Yy]* ) add_dcos_path_to_profile "$VIRTUAL_ENV_PATH/bin";; + [Nn]* ) ;; + * ) prompt_add_dcos_path_to_profile "$VIRTUAL_ENV_PATH/bin";; +esac + +if [ -z "$RC_NAME" ]; then + post_install_message "$ENV_SETUP" +else + post_install_message "~/$RC_NAME" +fi diff --git a/bin/install/upload_to_s3.sh b/bin/install/upload_to_s3.sh new file mode 100755 index 0000000..d3c8ebe --- /dev/null +++ b/bin/install/upload_to_s3.sh @@ -0,0 +1,38 @@ +#!/bin/bash -x + +COSMOS_VERSION="0.4.0" + +cosmos_cli() +{ + # result is the larger of the two versions + # convert the str to numbers, sort, and return the larger + result=$(echo -e "$COSMOS_VERSION\n$TAG_VERSION" | sed '/^$/d' | sort -nr | head -1) + # if TAGGED_VERSION >= COSMOS_VERSION we want to use cosmos cli + [[ "$result" = "$TAG_VERSION" ]] +} + +if cosmos_cli ; then + aws s3 --region=us-east-1 cp \ + dcos-cli/bin/install/install-dcos-cli.sh \ + %aws.bash_destination_url% + + aws s3 --region=us-east-1 cp \ + dcos-cli/bin/install/install-optout-dcos-cli.sh \ + %aws.bash_optout_destination_url% + + aws s3 --region=us-east-1 cp \ + dcos-cli/win_bin/install/install-dcos-cli.ps1 \ + %aws.powershell_destination_url% +else + aws s3 --region=us-east-1 cp \ + dcos-cli/bin/install/legacy/install-legacy-dcos-cli.sh \ + %aws.legacy_bash_destination_url% + + aws s3 --region=us-east-1 cp \ + dcos-cli/bin/install/install-legacy_optout-dcos-cli.sh \ + %aws.bash_legacy_optout_destination_url% + + aws s3 --region=us-east-1 cp \ + dcos-cli/win_bin/install/legacy/install-legacy-dcos-cli.ps1 \ + %aws.legacy_powershell_destination_legacy_url% +fi diff --git a/cli/dcoscli/config/main.py b/cli/dcoscli/config/main.py index b0bf3d9..dc72cc6 100644 --- a/cli/dcoscli/config/main.py +++ b/cli/dcoscli/config/main.py @@ -1,10 +1,9 @@ import collections -import copy import dcoscli import docopt import pkg_resources -from dcos import cmds, config, emitting, http, jsonitem, util +from dcos import cmds, config, emitting, http, util from dcos.errors import DCOSException from dcoscli import analytics from dcoscli.main import decorate_docopt_usage @@ -55,19 +54,9 @@ def _cmds(): arg_keys=['', ''], function=_set), - cmds.Command( - hierarchy=['config', 'append'], - arg_keys=['', ''], - function=_append), - - cmds.Command( - hierarchy=['config', 'prepend'], - arg_keys=['', ''], - function=_prepend), - cmds.Command( hierarchy=['config', 'unset'], - arg_keys=['', '--index'], + arg_keys=[''], function=_unset), cmds.Command( @@ -112,56 +101,13 @@ def _set(name, value): return 0 -def _append(name, value): +def _unset(name): """ :returns: process status :rtype: int """ - toml_config = util.get_config(True) - - python_value = _parse_array_item(name, value) - toml_config_pre = copy.deepcopy(toml_config) - section = name.split(".", 1)[0] - if section not in toml_config_pre._dictionary: - toml_config_pre._dictionary[section] = {} - - toml_config[name] = toml_config.get(name, []) + python_value - - config.check_config(toml_config_pre, toml_config) - - config.save(toml_config) - return 0 - - -def _prepend(name, value): - """ - :returns: process status - :rtype: int - """ - - toml_config = util.get_config(True) - - python_value = _parse_array_item(name, value) - - toml_config_pre = copy.deepcopy(toml_config) - section = name.split(".", 1)[0] - if section not in toml_config_pre._dictionary: - toml_config_pre._dictionary[section] = {} - toml_config[name] = python_value + toml_config.get(name, []) - config.check_config(toml_config_pre, toml_config) - - config.save(toml_config) - return 0 - - -def _unset(name, index): - """ - :returns: process status - :rtype: int - """ - - config.unset(name, index) + config.unset(name) return 0 @@ -205,36 +151,3 @@ def _validate(): emitter.publish("Congratulations, your configuration is valid!") return 0 - - -def _parse_array_item(name, value): - """ - :param name: the name of the property - :type name: str - :param value: the value to parse - :type value: str - :returns: the parsed value as an array with one element - :rtype: (list of any, dcos.errors.Error) where any is string, int, - float, bool, array or dict - """ - - section, subkey = config.split_key(name) - - config_schema = config.get_config_schema(section) - - parser = jsonitem.find_parser(subkey, config_schema) - - if parser.schema['type'] != 'array': - raise DCOSException( - "Append/Prepend not supported on '{0}' properties - use 'dcos " - "config set {0} {1}'".format(name, value)) - - if ('items' in parser.schema and - parser.schema['items']['type'] == 'string'): - - value = '["' + value + '"]' - else: - # We are going to assume that wrapping it in an array is enough - value = '[' + value + ']' - - return parser(value) diff --git a/cli/dcoscli/data/config-schema/package.json b/cli/dcoscli/data/config-schema/package.json index ac78663..8cc8d1e 100644 --- a/cli/dcoscli/data/config-schema/package.json +++ b/cli/dcoscli/data/config-schema/package.json @@ -2,25 +2,13 @@ "$schema": "http://json-schema.org/schema#", "type": "object", "properties": { - "sources": { - "type": "array", - "items": { - "type": "string", - "pattern": "^((?:(?:(https?|file))://)(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\\.?)+(?:[a-zA-Z]{2,6}\\.?|[a-zA-Z0-9-]{2,}\\.?)?|\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})?(?::\\d+)?(?:/?|[/?]\\S+)|((git|ssh|https?)|(git@[\\w\\.]+))(:(//)?)([\\w\\.@\\:/\\-~]+)(\\.git)(/)?)$" - }, - "title": "Package sources", - "description": "The list of package source in search order", - "default": [ "git://github.com/mesosphere/universe.git" ], - "additionalItems": false, - "uniqueItems": true - }, - "cache": { + "cosmos_url": { "type": "string", - "title": "Package cache directory", - "description": "Path to the local package cache directory", - "default": "/tmp/cache" + "format": "uri", + "title": "Cosmos base URL", + "description": "Base URL for talking to COSMOS. It overwrites the value specified in core.dcos_url", + "default": "http://localhost:7070" } }, - "additionalProperties": false, - "required": ["sources", "cache"] + "additionalProperties": false } diff --git a/cli/dcoscli/data/help/config.txt b/cli/dcoscli/data/help/config.txt index b93d109..63b5ee7 100644 --- a/cli/dcoscli/data/help/config.txt +++ b/cli/dcoscli/data/help/config.txt @@ -2,19 +2,15 @@ Get and set DCOS CLI configuration properties Usage: dcos config --info - dcos config append - dcos config prepend dcos config set dcos config show [] - dcos config unset [--index=] + dcos config unset dcos config validate Options: -h, --help Show this screen --info Show a short description of this subcommand --version Show version - --index= Index into the list. The first element in the list has an - index of zero Positional Arguments: The name of the property diff --git a/cli/dcoscli/data/help/package.txt b/cli/dcoscli/data/help/package.txt index a15a333..50a1598 100644 --- a/cli/dcoscli/data/help/package.txt +++ b/cli/dcoscli/data/help/package.txt @@ -7,19 +7,20 @@ Usage: [--render] [--package-versions] [--options=] - [--package-version=] + [--package-version=] - dcos package install [--cli | [--app --app-id=]] - [--package-version=] + dcos package install [--cli | [--app --app-id=]] + [--package-version=] [--options=] [--yes] - dcos package list [--json --endpoints --app-id= ] + dcos package list [--json --app-id= ] dcos package search [--json ] - dcos package sources + dcos package repo add [--index=] + dcos package repo remove (--repo-name= | --repo-url=) + dcos package repo list dcos package uninstall [--cli | [--app --app-id= --all]] - dcos package update [--validate] Options: --all @@ -41,13 +42,16 @@ Options: -h, --help Show this screen + --index= + Index into the list. The first element in the list has an index of zero + --info Show a short description of this subcommand --options= Path to a JSON file containing package installation options - --package-version= + --package-version= Package version to install --package-versions @@ -58,8 +62,11 @@ Options: values from config.json and --options. If not provided, print the raw templates. - --validate - Validate package content when updating sources + --repo-name= + Name for repository + + --repo-url= + URL of repository of DCOS packages. E.g. https://universe.mesosphere.com/repo --version Show version @@ -73,3 +80,9 @@ Positional Arguments: Pattern to use for searching for package + + + Name for repository + + + URL of repository of DCOS packages. E.g. https://universe.mesosphere.com/repo diff --git a/cli/dcoscli/package/main.py b/cli/dcoscli/package/main.py index aa36652..a1ffc6b 100644 --- a/cli/dcoscli/package/main.py +++ b/cli/dcoscli/package/main.py @@ -9,8 +9,8 @@ from collections import defaultdict import dcoscli import docopt import pkg_resources -from dcos import (cmds, cosmospackage, emitting, errors, http, marathon, - options, package, subcommand, util) +from dcos import (cmds, cosmospackage, emitting, errors, http, options, + package, subcommand, util) from dcos.errors import DCOSException from dcoscli import tables from dcoscli.main import decorate_docopt_usage @@ -54,14 +54,19 @@ def _cmds(): return [ cmds.Command( - hierarchy=['package', 'sources'], + hierarchy=['package', 'repo', 'list'], arg_keys=[], - function=_list_sources), + function=_list_response), cmds.Command( - hierarchy=['package', 'update'], - arg_keys=['--validate'], - function=_update), + hierarchy=['package', 'repo', 'add'], + arg_keys=['', '', '--index'], + function=_add_repo), + + cmds.Command( + hierarchy=['package', 'repo', 'remove'], + arg_keys=['--repo-name', '--repo-url'], + function=_remove_repo), cmds.Command( hierarchy=['package', 'describe'], @@ -78,7 +83,7 @@ def _cmds(): cmds.Command( hierarchy=['package', 'list'], - arg_keys=['--json', '--endpoints', '--app-id', ''], + arg_keys=['--json', '--app-id', ''], function=_list), cmds.Command( @@ -107,7 +112,6 @@ def _package(config_schema, info): :returns: Process status :rtype: int """ - if config_schema: schema = json.loads( pkg_resources.resource_string( @@ -134,37 +138,57 @@ def _info(): return 0 -def _list_sources(): - """List configured package sources. +def _list_response(): + """List configured package repositories. :returns: Process status :rtype: int """ - _check_cluster_capabilities() - config = util.get_config() + package_manager = _get_package_manager() + repos = package_manager.get_repos() - sources = package.list_sources(config) - - for source in sources: - emitter.publish("{} {}".format(source.hash(), source.url)) + if repos: + emitter.publish(repos) + else: + msg = ("There are currently no repos configured. " + "Please use `dcos package repo add` to add a repo") + raise DCOSException(msg) return 0 -def _update(validate): - """Update local package definitions from sources. +def _add_repo(repo_name, repo_url, index): + """Add package repo and update repo with new repo - :param validate: Whether to validate package content when updating sources. - :type validate: bool + :param repo_name: name to call repo + :type repo_name: str + :param repo_url: location of repo to add + :type repo_url: str + :param index: index to add this repo + :type index: int + :rtype: None + """ + + package_manager = _get_package_manager() + package_manager.add_repo(repo_name, repo_url, index) + + return 0 + + +def _remove_repo(repo_name, repo_url): + """Remove package repo and update repo with new repo + + :param repo_name: name to call repo + :type repo_name: str + :param repo_url: location of repo to add + :type repo_url: str :returns: Process status :rtype: int """ - _check_cluster_capabilities() - config = util.get_config() - - package.update_sources(config, validate) + package_manager = _get_package_manager() + package_manager.remove_repo(repo_name, repo_url) return 0 @@ -202,7 +226,6 @@ def _describe(package_name, :rtype: int """ - _check_cluster_capabilities() # If the user supplied template options, they definitely want to # render the template if options_path: @@ -216,51 +239,40 @@ def _describe(package_name, 'If --package-versions is provided, no other option can be ' 'provided') - pkg = package.resolve_package(package_name) - if pkg is None: - raise DCOSException("Package [{}] not found".format(package_name)) + package_manager = _get_package_manager() + pkg = package_manager.get_package_version(package_name, package_version) - pkg_revision = pkg.latest_package_revision(package_version) - - if pkg_revision is None: - raise DCOSException("Version {} of package [{}] is not available". - format(package_version, package_name)) - - pkg_json = pkg.package_json(pkg_revision) + pkg_json = pkg.package_json() if package_version is None: - revision_map = pkg.package_revisions_map() - pkg_versions = list(revision_map.values()) + pkg_versions = pkg.package_versions() del pkg_json['version'] pkg_json['versions'] = pkg_versions if package_versions: - emitter.publish('\n'.join(pkg_json['versions'])) + emitter.publish(pkg.package_versions()) elif cli or app or config: user_options = _user_options(options_path) - options = pkg.options(pkg_revision, user_options) + options = pkg.options(user_options) if cli: if render: - cli_output = pkg.command_json(pkg_revision, options) + cli_output = pkg.command_json(options) else: - cli_output = pkg.command_template(pkg_revision) - if cli_output and cli_output[-1] == '\n': - cli_output = cli_output[:-1] + cli_output = pkg.command_template() emitter.publish(cli_output) if app: if render: - app_output = pkg.marathon_json(pkg_revision, options) + app_output = pkg.marathon_json(options) else: - app_output = pkg.marathon_template(pkg_revision) + app_output = pkg.marathon_template() if app_output and app_output[-1] == '\n': app_output = app_output[:-1] emitter.publish(app_output) if config: - config_output = pkg.config_json(pkg_revision) + config_output = pkg.config_json() emitter.publish(config_output) else: - pkg_json = pkg.package_json(pkg_revision) emitter.publish(pkg_json) return 0 @@ -329,36 +341,19 @@ def _install(package_name, package_version, options_path, app_id, cli, app, :rtype: int """ - _check_cluster_capabilities() if cli is False and app is False: # Install both if neither flag is specified cli = app = True - config = util.get_config() - - pkg = package.resolve_package(package_name, config) - if pkg is None: - msg = "Package [{}] not found\n".format(package_name) + \ - "You may need to run 'dcos package update' to update your " + \ - "repositories" - raise DCOSException(msg) - - pkg_revision = pkg.latest_package_revision(package_version) - if pkg_revision is None: - if package_version is not None: - msg = "Version {} of package [{}] is not available".format( - package_version, package_name) - else: - msg = "Package [{}] not available".format(package_name) - raise DCOSException(msg) - # Expand ~ in the options file path if options_path: options_path = os.path.expanduser(options_path) - user_options = _user_options(options_path) - pkg_json = pkg.package_json(pkg_revision) + package_manager = _get_package_manager() + pkg = package_manager.get_package_version(package_name, package_version) + + pkg_json = pkg.package_json() pre_install_notes = pkg_json.get('preInstallNotes') if pre_install_notes: emitter.publish(pre_install_notes) @@ -366,37 +361,31 @@ def _install(package_name, package_version, options_path, app_id, cli, app, emitter.publish('Exiting installation.') return 0 - options = pkg.options(pkg_revision, user_options) + # render options before start installation + options = pkg.options(user_options) - revision_map = pkg.package_revisions_map() - package_version = revision_map.get(pkg_revision) + if app and pkg.has_mustache_definition(): - if app and (pkg.has_marathon_definition(pkg_revision) or - pkg.has_marathon_mustache_definition(pkg_revision)): # Install in Marathon msg = 'Installing Marathon app for package [{}] version [{}]'.format( - pkg.name(), package_version) + pkg.name(), pkg.version()) if app_id is not None: msg += ' with app id [{}]'.format(app_id) emitter.publish(msg) - init_client = marathon.create_client(config) - - package.install_app( + package_manager.install_app( pkg, - pkg_revision, - init_client, options, app_id) - if cli and pkg.has_command_definition(pkg_revision): + if cli and pkg.has_command_definition(): # Install subcommand msg = 'Installing CLI subcommand for package [{}] version [{}]'.format( - pkg.name(), package_version) + pkg.name(), pkg.version()) emitter.publish(msg) - subcommand.install(pkg, pkg_revision, options) + subcommand.install(pkg, pkg.options(user_options)) subcommand_paths = subcommand.get_package_commands(package_name) new_commands = [os.path.basename(p).replace('-', ' ', 1) @@ -415,14 +404,11 @@ def _install(package_name, package_version, options_path, app_id, cli, app, return 0 -def _list(json_, endpoints, app_id, package_name): +def _list(json_, app_id, package_name): """List installed apps :param json_: output json if True :type json_: bool - :param endpoints: Whether to include a list of - endpoints as port-host pairs - :type endpoints: boolean :param app_id: App ID of app to show :type app_id: str :param package_name: The package to show @@ -431,26 +417,13 @@ def _list(json_, endpoints, app_id, package_name): :rtype: int """ - _check_cluster_capabilities() - config = util.get_config() - init_client = marathon.create_client(config) - installed = package.installed_packages(init_client, endpoints) + package_manager = _get_package_manager() + if app_id is not None: + app_id = util.normalize_app_id(app_id) + results = package.installed_packages( + package_manager, app_id, package_name) # only emit those packages that match the provided package_name and app_id - results = [] - for pkg in installed: - pkg_info = pkg.dict() - if (_matches_package_name(package_name, pkg_info) and - _matches_app_id(app_id, pkg_info)): - if app_id: - # if the user is asking a specific id then only show that id - pkg_info['apps'] = [ - app for app in pkg_info['apps'] - if app == app_id - ] - - results.append(pkg_info) - if results or json_: emitting.publish_table(emitter, results, tables.package_table, json_) else: @@ -499,13 +472,11 @@ def _search(json_, query): :rtype: int """ - _check_cluster_capabilities() if not query: query = '' - config = util.get_config() - results = [index_entry.as_dict() - for index_entry in package.search(query, config)] + package_manager = _get_package_manager() + results = package_manager.search_sources(query) if any(result['packages'] for result in results) or json_: emitting.publish_table(emitter, @@ -530,8 +501,9 @@ def _uninstall(package_name, remove_all, app_id, cli, app): :rtype: int """ - _check_cluster_capabilities() - err = package.uninstall(package_name, remove_all, app_id, cli, app) + package_manager = _get_package_manager() + err = package.uninstall( + package_manager, package_name, remove_all, app_id, cli, app) if err is not None: emitter.publish(err) return 1 @@ -763,19 +735,31 @@ def _bundle_screenshots(screenshot_directory, zip_file): arcname='images/screenshots/{}'.format(filename)) -def _check_cluster_capabilities(): - """Make sure this version the cli is compatible with version of DCOS +def _get_cosmos_url(): + """ + :returns: cosmos base url + :rtype: str + """ + config = util.get_config() + cosmos_url = config.get("package.cosmos_url") + if cosmos_url is None: + cosmos_url = util.get_config_vals(['core.dcos_url'], config)[0] + return cosmos_url + + +def _get_package_manager(): + """Returns type of package manager to use :returns: PackageManager instance - :rtype: None + :rtype: PackageManager """ - dcos_url = util.get_config().get("core.dcos_url") - cosmos_manager = cosmospackage.Cosmos(dcos_url) + cosmos_url = _get_cosmos_url() + cosmos_manager = cosmospackage.Cosmos(cosmos_url) if cosmos_manager.enabled(): - msg = ("This version of the DCOS CLI is not supported for your " - "cluster. Please upgrade the CLI to the latest version: " - "https://docs.mesosphere.com/administration/introcli/updatecli/" - ) - + return cosmos_manager + else: + msg = ("This version of the dcos-cli is unsupported for your DCOS " + "cluster. Please use a dcos-cli version < 0.4.0 or upgrade your" + " cluster to DCOS version >= 1.6.1") raise DCOSException(msg) diff --git a/cli/dcoscli/service/main.py b/cli/dcoscli/service/main.py index 8953fc0..7e5d354 100644 --- a/cli/dcoscli/service/main.py +++ b/cli/dcoscli/service/main.py @@ -3,7 +3,7 @@ import subprocess import dcoscli import docopt import pkg_resources -from dcos import cmds, emitting, marathon, mesos, package, util +from dcos import cmds, emitting, marathon, mesos, util from dcos.errors import DCOSException, DefaultError from dcoscli import log, tables from dcoscli.main import decorate_docopt_usage @@ -234,7 +234,7 @@ def _get_service_app(marathon_client, service_name): :rtype: dict """ - apps = package.get_apps_for_framework(service_name, marathon_client) + apps = marathon_client.get_apps_for_framework(service_name) if len(apps) > 1: raise DCOSException( diff --git a/cli/dcoscli/tables.py b/cli/dcoscli/tables.py index a608f93..b518b67 100644 --- a/cli/dcoscli/tables.py +++ b/cli/dcoscli/tables.py @@ -301,7 +301,6 @@ def package_search_table(search_results): ('NAME', lambda p: p['name']), ('VERSION', lambda p: p['currentVersion']), ('FRAMEWORK', lambda p: p['framework']), - ('SOURCE', lambda p: p['source']), ('DESCRIPTION', lambda p: p['description']) ]) @@ -309,14 +308,12 @@ def package_search_table(search_results): for result in search_results: for package in result['packages']: package_ = copy.deepcopy(package) - package_['source'] = result['source'] packages.append(package_) tb = table(fields, packages, sortby="NAME") tb.align['NAME'] = 'l' tb.align['VERSION'] = 'l' tb.align['FRAMEWORK'] = 'l' - tb.align['SOURCE'] = 'l' tb.align['DESCRIPTION'] = 'l' return tb diff --git a/cli/tests/data/config/missing_params_dcos.toml b/cli/tests/data/config/missing_params_dcos.toml index ffed882..5e68b86 100644 --- a/cli/tests/data/config/missing_params_dcos.toml +++ b/cli/tests/data/config/missing_params_dcos.toml @@ -2,5 +2,4 @@ reporting = false email = "test@mail.com" [package] -sources = [ "git://github.com/mesosphere/universe.git", "https://github.com/mesosphere/universe/archive/master.zip",] -cache = "true" +cosmos_url = "http://localhost:7070" diff --git a/cli/tests/data/dcos.toml b/cli/tests/data/dcos.toml index 27a5a85..b1e9422 100644 --- a/cli/tests/data/dcos.toml +++ b/cli/tests/data/dcos.toml @@ -2,8 +2,5 @@ reporting = false email = "test@mail.com" timeout = 5 -ssl_verify = "false" dcos_url = "http://dcos.snakeoil.mesosphere.com" -[package] -sources = [ "https://github.com/mesosphere/universe/archive/cli-test-3.zip",] -cache = "tmp/cache" +ssl_verify = "false" diff --git a/cli/tests/data/help/config.txt b/cli/tests/data/help/config.txt index b93d109..63b5ee7 100644 --- a/cli/tests/data/help/config.txt +++ b/cli/tests/data/help/config.txt @@ -2,19 +2,15 @@ Get and set DCOS CLI configuration properties Usage: dcos config --info - dcos config append - dcos config prepend dcos config set dcos config show [] - dcos config unset [--index=] + dcos config unset dcos config validate Options: -h, --help Show this screen --info Show a short description of this subcommand --version Show version - --index= Index into the list. The first element in the list has an - index of zero Positional Arguments: The name of the property diff --git a/cli/tests/data/help/package.txt b/cli/tests/data/help/package.txt index a15a333..50a1598 100644 --- a/cli/tests/data/help/package.txt +++ b/cli/tests/data/help/package.txt @@ -7,19 +7,20 @@ Usage: [--render] [--package-versions] [--options=] - [--package-version=] + [--package-version=] - dcos package install [--cli | [--app --app-id=]] - [--package-version=] + dcos package install [--cli | [--app --app-id=]] + [--package-version=] [--options=] [--yes] - dcos package list [--json --endpoints --app-id= ] + dcos package list [--json --app-id= ] dcos package search [--json ] - dcos package sources + dcos package repo add [--index=] + dcos package repo remove (--repo-name= | --repo-url=) + dcos package repo list dcos package uninstall [--cli | [--app --app-id= --all]] - dcos package update [--validate] Options: --all @@ -41,13 +42,16 @@ Options: -h, --help Show this screen + --index= + Index into the list. The first element in the list has an index of zero + --info Show a short description of this subcommand --options= Path to a JSON file containing package installation options - --package-version= + --package-version= Package version to install --package-versions @@ -58,8 +62,11 @@ Options: values from config.json and --options. If not provided, print the raw templates. - --validate - Validate package content when updating sources + --repo-name= + Name for repository + + --repo-url= + URL of repository of DCOS packages. E.g. https://universe.mesosphere.com/repo --version Show version @@ -73,3 +80,9 @@ Positional Arguments: Pattern to use for searching for package + + + Name for repository + + + URL of repository of DCOS packages. E.g. https://universe.mesosphere.com/repo diff --git a/cli/tests/data/marathon/groups/complicated.json b/cli/tests/data/marathon/groups/complicated.json index 298f9a4..ba8c404 100644 --- a/cli/tests/data/marathon/groups/complicated.json +++ b/cli/tests/data/marathon/groups/complicated.json @@ -13,7 +13,7 @@ "cmd": "sleep 10", "id": "sleep10", "instances": 0, - "dependencies": ["/product/database", "../backend"] + "dependencies": ["/product/database"] } ], "id": "app" diff --git a/cli/tests/data/marathon/missing_marathon_params.toml b/cli/tests/data/marathon/missing_marathon_params.toml index c2ff18b..2e4a447 100644 --- a/cli/tests/data/marathon/missing_marathon_params.toml +++ b/cli/tests/data/marathon/missing_marathon_params.toml @@ -3,5 +3,4 @@ reporting = false email = "test@mail.com" [marathon] [package] -sources = [ "git://github.com/mesosphere/universe.git", "https://github.com/mesosphere/universe/archive/master.zip",] -cache = "true" +cosmos_url = "http://localhost:7070" diff --git a/cli/tests/data/package/json/cassandra_single_node.json b/cli/tests/data/package/json/cassandra_single_node.json new file mode 100644 index 0000000..4a2f219 --- /dev/null +++ b/cli/tests/data/package/json/cassandra_single_node.json @@ -0,0 +1,12 @@ +{ + "cassandra": { + "resources": { + "cpus": 0.1, + "mem": 512, + "disk": 272 + }, + "health-check-interval-seconds": 15, + "node-count": 1, + "seed-count": 1 + } +} diff --git a/cli/tests/data/package/json/test_describe_app_cli.json b/cli/tests/data/package/json/test_describe_app_cli.json index c1067d6..840b399 100644 --- a/cli/tests/data/package/json/test_describe_app_cli.json +++ b/cli/tests/data/package/json/test_describe_app_cli.json @@ -14,8 +14,8 @@ 0 ], "uris": [ - "{{resource.assets.uris.cassandra-mesos-tar-gz}}", - "{{resource.assets.uris.jre-7u76-linux-x64}}" + "{{resource.assets.uris.cassandra-mesos-0-2-0-1-tar-gz}}", + "{{resource.assets.uris.jre-7u76-linux-x64-tar-gz}}" ], "healthChecks": [ { diff --git a/cli/tests/data/package/json/test_describe_app_marathon.json b/cli/tests/data/package/json/test_describe_app_marathon.json index c62cbcb..33a9ee1 100644 --- a/cli/tests/data/package/json/test_describe_app_marathon.json +++ b/cli/tests/data/package/json/test_describe_app_marathon.json @@ -26,7 +26,7 @@ "container": { "type": "DOCKER", "docker": { - "image": "mesosphere/marathon:v0.11.1", + "image": "{{resource.assets.container.docker.5e187be16235}}", "network": "HOST" } }, diff --git a/cli/tests/data/package/json/test_describe_app_options.json b/cli/tests/data/package/json/test_describe_app_options.json index e2d7477..6857edb 100644 --- a/cli/tests/data/package/json/test_describe_app_options.json +++ b/cli/tests/data/package/json/test_describe_app_options.json @@ -8,7 +8,7 @@ ], "container": { "docker": { - "image": "mesosphere/marathon:v0.11.1", + "image": "docker.io/mesosphere/marathon:v0.11.1", "network": "HOST" }, "type": "DOCKER" @@ -33,11 +33,11 @@ "labels": { "DCOS_PACKAGE_FRAMEWORK_NAME": "marathon-user", "DCOS_PACKAGE_IS_FRAMEWORK": "true", - "DCOS_PACKAGE_METADATA": "eyJkZXNjcmlwdGlvbiI6ICJBIGNsdXN0ZXItd2lkZSBpbml0IGFuZCBjb250cm9sIHN5c3RlbSBmb3Igc2VydmljZXMgaW4gY2dyb3VwcyBvciBEb2NrZXIgY29udGFpbmVycy4iLCAiZnJhbWV3b3JrIjogdHJ1ZSwgImltYWdlcyI6IHsiaWNvbi1sYXJnZSI6ICJodHRwczovL2Rvd25sb2Fkcy5tZXNvc3BoZXJlLmlvL21hcmF0aG9uL2Fzc2V0cy9pY29uLXNlcnZpY2UtbWFyYXRob24tbGFyZ2UucG5nIiwgImljb24tbWVkaXVtIjogImh0dHBzOi8vZG93bmxvYWRzLm1lc29zcGhlcmUuaW8vbWFyYXRob24vYXNzZXRzL2ljb24tc2VydmljZS1tYXJhdGhvbi1tZWRpdW0ucG5nIiwgImljb24tc21hbGwiOiAiaHR0cHM6Ly9kb3dubG9hZHMubWVzb3NwaGVyZS5pby9tYXJhdGhvbi9hc3NldHMvaWNvbi1zZXJ2aWNlLW1hcmF0aG9uLXNtYWxsLnBuZyJ9LCAibGljZW5zZXMiOiBbeyJuYW1lIjogIkFwYWNoZSBMaWNlbnNlIFZlcnNpb24gMi4wIiwgInVybCI6ICJodHRwczovL2dpdGh1Yi5jb20vbWVzb3NwaGVyZS9tYXJhdGhvbi9ibG9iL21hc3Rlci9MSUNFTlNFIn1dLCAibWFpbnRhaW5lciI6ICJzdXBwb3J0QG1lc29zcGhlcmUuaW8iLCAibmFtZSI6ICJtYXJhdGhvbiIsICJwb3N0SW5zdGFsbE5vdGVzIjogIk1hcmF0aG9uIERDT1MgU2VydmljZSBoYXMgYmVlbiBzdWNjZXNzZnVsbHkgaW5zdGFsbGVkIVxuXG5cdERvY3VtZW50YXRpb246IGh0dHBzOi8vbWVzb3NwaGVyZS5naXRodWIuaW8vbWFyYXRob25cblx0SXNzdWVzOiBodHRwczovZ2l0aHViLmNvbS9tZXNvc3BoZXJlL21hcmF0aG9uL2lzc3Vlc1xuIiwgInBvc3RVbmluc3RhbGxOb3RlcyI6ICJUaGUgTWFyYXRob24gRENPUyBTZXJ2aWNlIGhhcyBiZWVuIHVuaW5zdGFsbGVkIGFuZCB3aWxsIG5vIGxvbmdlciBydW4uXG5QbGVhc2UgZm9sbG93IHRoZSBpbnN0cnVjdGlvbnMgYXQgaHR0cDovL2RvY3MubWVzb3NwaGVyZS5jb20vc2VydmljZXMvbWFyYXRob24vI3VuaW5zdGFsbCB0byBjbGVhbiB1cCBhbnkgcGVyc2lzdGVkIHN0YXRlIiwgInByZUluc3RhbGxOb3RlcyI6ICJXZSByZWNvbW1lbmQgYSBtaW5pbXVtIG9mIG9uZSBub2RlIHdpdGggYXQgbGVhc3QgMiBDUFUncyBhbmQgMUdCIG9mIFJBTSBhdmFpbGFibGUgZm9yIHRoZSBNYXJhdGhvbiBTZXJ2aWNlLiIsICJzY20iOiAiaHR0cHM6Ly9naXRodWIuY29tL21lc29zcGhlcmUvbWFyYXRob24uZ2l0IiwgInRhZ3MiOiBbImluaXQiLCAibG9uZy1ydW5uaW5nIl0sICJ2ZXJzaW9uIjogIjAuMTEuMSJ9", + "DCOS_PACKAGE_METADATA": "eyJsaWNlbnNlcyI6W3sibmFtZSI6IkFwYWNoZSBMaWNlbnNlIFZlcnNpb24gMi4wIiwidXJsIjoiaHR0cHM6Ly9naXRodWIuY29tL21lc29zcGhlcmUvbWFyYXRob24vYmxvYi9tYXN0ZXIvTElDRU5TRSJ9XSwibmFtZSI6Im1hcmF0aG9uIiwicG9zdEluc3RhbGxOb3RlcyI6Ik1hcmF0aG9uIERDT1MgU2VydmljZSBoYXMgYmVlbiBzdWNjZXNzZnVsbHkgaW5zdGFsbGVkIVxuXG5cdERvY3VtZW50YXRpb246IGh0dHBzOi8vbWVzb3NwaGVyZS5naXRodWIuaW8vbWFyYXRob25cblx0SXNzdWVzOiBodHRwczovZ2l0aHViLmNvbS9tZXNvc3BoZXJlL21hcmF0aG9uL2lzc3Vlc1xuIiwic2NtIjoiaHR0cHM6Ly9naXRodWIuY29tL21lc29zcGhlcmUvbWFyYXRob24uZ2l0IiwiZGVzY3JpcHRpb24iOiJBIGNsdXN0ZXItd2lkZSBpbml0IGFuZCBjb250cm9sIHN5c3RlbSBmb3Igc2VydmljZXMgaW4gY2dyb3VwcyBvciBEb2NrZXIgY29udGFpbmVycy4iLCJwYWNrYWdpbmdWZXJzaW9uIjoiMi4wIiwidGFncyI6WyJpbml0IiwibG9uZy1ydW5uaW5nIl0sInBvc3RVbmluc3RhbGxOb3RlcyI6IlRoZSBNYXJhdGhvbiBEQ09TIFNlcnZpY2UgaGFzIGJlZW4gdW5pbnN0YWxsZWQgYW5kIHdpbGwgbm8gbG9uZ2VyIHJ1bi5cblBsZWFzZSBmb2xsb3cgdGhlIGluc3RydWN0aW9ucyBhdCBodHRwOi8vZG9jcy5tZXNvc3BoZXJlLmNvbS9zZXJ2aWNlcy9tYXJhdGhvbi8jdW5pbnN0YWxsIHRvIGNsZWFuIHVwIGFueSBwZXJzaXN0ZWQgc3RhdGUiLCJtYWludGFpbmVyIjoic3VwcG9ydEBtZXNvc3BoZXJlLmlvIiwiZnJhbWV3b3JrIjp0cnVlLCJ2ZXJzaW9uIjoiMC4xMS4xIiwicHJlSW5zdGFsbE5vdGVzIjoiV2UgcmVjb21tZW5kIGEgbWluaW11bSBvZiBvbmUgbm9kZSB3aXRoIGF0IGxlYXN0IDIgQ1BVJ3MgYW5kIDFHQiBvZiBSQU0gYXZhaWxhYmxlIGZvciB0aGUgTWFyYXRob24gU2VydmljZS4iLCJpbWFnZXMiOnsiaWNvbi1zbWFsbCI6Imh0dHBzOi8vZG93bmxvYWRzLm1lc29zcGhlcmUuY29tL21hcmF0aG9uL2Fzc2V0cy9pY29uLXNlcnZpY2UtbWFyYXRob24tc21hbGwucG5nIiwiaWNvbi1tZWRpdW0iOiJodHRwczovL2Rvd25sb2Fkcy5tZXNvc3BoZXJlLmNvbS9tYXJhdGhvbi9hc3NldHMvaWNvbi1zZXJ2aWNlLW1hcmF0aG9uLW1lZGl1bS5wbmciLCJpY29uLWxhcmdlIjoiaHR0cHM6Ly9kb3dubG9hZHMubWVzb3NwaGVyZS5jb20vbWFyYXRob24vYXNzZXRzL2ljb24tc2VydmljZS1tYXJhdGhvbi1sYXJnZS5wbmciLCJzY3JlZW5zaG90cyI6bnVsbH19", "DCOS_PACKAGE_NAME": "marathon", - "DCOS_PACKAGE_REGISTRY_VERSION": "2.0.0-rc1", - "DCOS_PACKAGE_RELEASE": "6", - "DCOS_PACKAGE_SOURCE": "https://github.com/mesosphere/universe/archive/cli-test-3.zip", + "DCOS_PACKAGE_REGISTRY_VERSION": "2.0", + "DCOS_PACKAGE_RELEASE": "0", + "DCOS_PACKAGE_SOURCE": "https://github.com/mesosphere/universe/archive/cli-test-4.zip", "DCOS_PACKAGE_VERSION": "0.11.1" }, "mem": 1024.0, diff --git a/cli/tests/data/package/json/test_describe_marathon.json b/cli/tests/data/package/json/test_describe_marathon.json index c86c1dc..d661f34 100644 --- a/cli/tests/data/package/json/test_describe_marathon.json +++ b/cli/tests/data/package/json/test_describe_marathon.json @@ -1,11 +1,6 @@ { "description": "A cluster-wide init and control system for services in cgroups or Docker containers.", "framework": true, - "images": { - "icon-large": "https://downloads.mesosphere.io/marathon/assets/icon-service-marathon-large.png", - "icon-medium": "https://downloads.mesosphere.io/marathon/assets/icon-service-marathon-medium.png", - "icon-small": "https://downloads.mesosphere.io/marathon/assets/icon-service-marathon-small.png" - }, "licenses": [ { "name": "Apache License Version 2.0", @@ -14,6 +9,7 @@ ], "maintainer": "support@mesosphere.io", "name": "marathon", + "packagingVersion": "2.0", "postInstallNotes": "Marathon DCOS Service has been successfully installed!\n\n\tDocumentation: https://mesosphere.github.io/marathon\n\tIssues: https:/github.com/mesosphere/marathon/issues\n", "postUninstallNotes": "The Marathon DCOS Service has been uninstalled and will no longer run.\nPlease follow the instructions at http://docs.mesosphere.com/services/marathon/#uninstall to clean up any persisted state", "preInstallNotes": "We recommend a minimum of one node with at least 2 CPU's and 1GB of RAM available for the Marathon Service.", @@ -22,5 +18,5 @@ "init", "long-running" ], - "version": "0.11.1" + "versions": ["0.11.1"] } diff --git a/cli/tests/data/package/json/test_describe_marathon_0.8.1.json b/cli/tests/data/package/json/test_describe_marathon_0.11.1.json similarity index 71% rename from cli/tests/data/package/json/test_describe_marathon_0.8.1.json rename to cli/tests/data/package/json/test_describe_marathon_0.11.1.json index c95ebe5..f24c224 100644 --- a/cli/tests/data/package/json/test_describe_marathon_0.8.1.json +++ b/cli/tests/data/package/json/test_describe_marathon_0.11.1.json @@ -1,11 +1,6 @@ { "description": "A cluster-wide init and control system for services in cgroups or Docker containers.", "framework": true, - "images": { - "icon-large": "https://downloads.mesosphere.io/marathon/assets/icon-service-marathon-large.png", - "icon-medium": "https://downloads.mesosphere.io/marathon/assets/icon-service-marathon-medium.png", - "icon-small": "https://downloads.mesosphere.io/marathon/assets/icon-service-marathon-small.png" - }, "licenses": [ { "name": "Apache License Version 2.0", @@ -14,13 +9,14 @@ ], "maintainer": "support@mesosphere.io", "name": "marathon", + "packagingVersion": "2.0", "postInstallNotes": "Marathon DCOS Service has been successfully installed!\n\n\tDocumentation: https://mesosphere.github.io/marathon\n\tIssues: https:/github.com/mesosphere/marathon/issues\n", "postUninstallNotes": "The Marathon DCOS Service has been uninstalled and will no longer run.\nPlease follow the instructions at http://docs.mesosphere.com/services/marathon/#uninstall to clean up any persisted state", "preInstallNotes": "We recommend a minimum of one node with at least 2 CPU's and 1GB of RAM available for the Marathon Service.", "scm": "https://github.com/mesosphere/marathon.git", "tags": [ - "mesosphere", - "framework" + "init", + "long-running" ], - "version": "0.8.1" + "version": "0.11.1" } diff --git a/cli/tests/data/package/json/test_describe_marathon_app_render.json b/cli/tests/data/package/json/test_describe_marathon_app_render.json index e8ead20..1041350 100644 --- a/cli/tests/data/package/json/test_describe_marathon_app_render.json +++ b/cli/tests/data/package/json/test_describe_marathon_app_render.json @@ -8,7 +8,7 @@ ], "container": { "docker": { - "image": "mesosphere/marathon:v0.11.1", + "image": "docker.io/mesosphere/marathon:v0.11.1", "network": "HOST" }, "type": "DOCKER" @@ -33,11 +33,11 @@ "labels": { "DCOS_PACKAGE_FRAMEWORK_NAME": "marathon-user", "DCOS_PACKAGE_IS_FRAMEWORK": "true", - "DCOS_PACKAGE_METADATA": "eyJkZXNjcmlwdGlvbiI6ICJBIGNsdXN0ZXItd2lkZSBpbml0IGFuZCBjb250cm9sIHN5c3RlbSBmb3Igc2VydmljZXMgaW4gY2dyb3VwcyBvciBEb2NrZXIgY29udGFpbmVycy4iLCAiZnJhbWV3b3JrIjogdHJ1ZSwgImltYWdlcyI6IHsiaWNvbi1sYXJnZSI6ICJodHRwczovL2Rvd25sb2Fkcy5tZXNvc3BoZXJlLmlvL21hcmF0aG9uL2Fzc2V0cy9pY29uLXNlcnZpY2UtbWFyYXRob24tbGFyZ2UucG5nIiwgImljb24tbWVkaXVtIjogImh0dHBzOi8vZG93bmxvYWRzLm1lc29zcGhlcmUuaW8vbWFyYXRob24vYXNzZXRzL2ljb24tc2VydmljZS1tYXJhdGhvbi1tZWRpdW0ucG5nIiwgImljb24tc21hbGwiOiAiaHR0cHM6Ly9kb3dubG9hZHMubWVzb3NwaGVyZS5pby9tYXJhdGhvbi9hc3NldHMvaWNvbi1zZXJ2aWNlLW1hcmF0aG9uLXNtYWxsLnBuZyJ9LCAibGljZW5zZXMiOiBbeyJuYW1lIjogIkFwYWNoZSBMaWNlbnNlIFZlcnNpb24gMi4wIiwgInVybCI6ICJodHRwczovL2dpdGh1Yi5jb20vbWVzb3NwaGVyZS9tYXJhdGhvbi9ibG9iL21hc3Rlci9MSUNFTlNFIn1dLCAibWFpbnRhaW5lciI6ICJzdXBwb3J0QG1lc29zcGhlcmUuaW8iLCAibmFtZSI6ICJtYXJhdGhvbiIsICJwb3N0SW5zdGFsbE5vdGVzIjogIk1hcmF0aG9uIERDT1MgU2VydmljZSBoYXMgYmVlbiBzdWNjZXNzZnVsbHkgaW5zdGFsbGVkIVxuXG5cdERvY3VtZW50YXRpb246IGh0dHBzOi8vbWVzb3NwaGVyZS5naXRodWIuaW8vbWFyYXRob25cblx0SXNzdWVzOiBodHRwczovZ2l0aHViLmNvbS9tZXNvc3BoZXJlL21hcmF0aG9uL2lzc3Vlc1xuIiwgInBvc3RVbmluc3RhbGxOb3RlcyI6ICJUaGUgTWFyYXRob24gRENPUyBTZXJ2aWNlIGhhcyBiZWVuIHVuaW5zdGFsbGVkIGFuZCB3aWxsIG5vIGxvbmdlciBydW4uXG5QbGVhc2UgZm9sbG93IHRoZSBpbnN0cnVjdGlvbnMgYXQgaHR0cDovL2RvY3MubWVzb3NwaGVyZS5jb20vc2VydmljZXMvbWFyYXRob24vI3VuaW5zdGFsbCB0byBjbGVhbiB1cCBhbnkgcGVyc2lzdGVkIHN0YXRlIiwgInByZUluc3RhbGxOb3RlcyI6ICJXZSByZWNvbW1lbmQgYSBtaW5pbXVtIG9mIG9uZSBub2RlIHdpdGggYXQgbGVhc3QgMiBDUFUncyBhbmQgMUdCIG9mIFJBTSBhdmFpbGFibGUgZm9yIHRoZSBNYXJhdGhvbiBTZXJ2aWNlLiIsICJzY20iOiAiaHR0cHM6Ly9naXRodWIuY29tL21lc29zcGhlcmUvbWFyYXRob24uZ2l0IiwgInRhZ3MiOiBbImluaXQiLCAibG9uZy1ydW5uaW5nIl0sICJ2ZXJzaW9uIjogIjAuMTEuMSJ9", + "DCOS_PACKAGE_METADATA": "eyJsaWNlbnNlcyI6W3sibmFtZSI6IkFwYWNoZSBMaWNlbnNlIFZlcnNpb24gMi4wIiwidXJsIjoiaHR0cHM6Ly9naXRodWIuY29tL21lc29zcGhlcmUvbWFyYXRob24vYmxvYi9tYXN0ZXIvTElDRU5TRSJ9XSwibmFtZSI6Im1hcmF0aG9uIiwicG9zdEluc3RhbGxOb3RlcyI6Ik1hcmF0aG9uIERDT1MgU2VydmljZSBoYXMgYmVlbiBzdWNjZXNzZnVsbHkgaW5zdGFsbGVkIVxuXG5cdERvY3VtZW50YXRpb246IGh0dHBzOi8vbWVzb3NwaGVyZS5naXRodWIuaW8vbWFyYXRob25cblx0SXNzdWVzOiBodHRwczovZ2l0aHViLmNvbS9tZXNvc3BoZXJlL21hcmF0aG9uL2lzc3Vlc1xuIiwic2NtIjoiaHR0cHM6Ly9naXRodWIuY29tL21lc29zcGhlcmUvbWFyYXRob24uZ2l0IiwiZGVzY3JpcHRpb24iOiJBIGNsdXN0ZXItd2lkZSBpbml0IGFuZCBjb250cm9sIHN5c3RlbSBmb3Igc2VydmljZXMgaW4gY2dyb3VwcyBvciBEb2NrZXIgY29udGFpbmVycy4iLCJwYWNrYWdpbmdWZXJzaW9uIjoiMi4wIiwidGFncyI6WyJpbml0IiwibG9uZy1ydW5uaW5nIl0sInBvc3RVbmluc3RhbGxOb3RlcyI6IlRoZSBNYXJhdGhvbiBEQ09TIFNlcnZpY2UgaGFzIGJlZW4gdW5pbnN0YWxsZWQgYW5kIHdpbGwgbm8gbG9uZ2VyIHJ1bi5cblBsZWFzZSBmb2xsb3cgdGhlIGluc3RydWN0aW9ucyBhdCBodHRwOi8vZG9jcy5tZXNvc3BoZXJlLmNvbS9zZXJ2aWNlcy9tYXJhdGhvbi8jdW5pbnN0YWxsIHRvIGNsZWFuIHVwIGFueSBwZXJzaXN0ZWQgc3RhdGUiLCJtYWludGFpbmVyIjoic3VwcG9ydEBtZXNvc3BoZXJlLmlvIiwiZnJhbWV3b3JrIjp0cnVlLCJ2ZXJzaW9uIjoiMC4xMS4xIiwicHJlSW5zdGFsbE5vdGVzIjoiV2UgcmVjb21tZW5kIGEgbWluaW11bSBvZiBvbmUgbm9kZSB3aXRoIGF0IGxlYXN0IDIgQ1BVJ3MgYW5kIDFHQiBvZiBSQU0gYXZhaWxhYmxlIGZvciB0aGUgTWFyYXRob24gU2VydmljZS4iLCJpbWFnZXMiOnsiaWNvbi1zbWFsbCI6Imh0dHBzOi8vZG93bmxvYWRzLm1lc29zcGhlcmUuY29tL21hcmF0aG9uL2Fzc2V0cy9pY29uLXNlcnZpY2UtbWFyYXRob24tc21hbGwucG5nIiwiaWNvbi1tZWRpdW0iOiJodHRwczovL2Rvd25sb2Fkcy5tZXNvc3BoZXJlLmNvbS9tYXJhdGhvbi9hc3NldHMvaWNvbi1zZXJ2aWNlLW1hcmF0aG9uLW1lZGl1bS5wbmciLCJpY29uLWxhcmdlIjoiaHR0cHM6Ly9kb3dubG9hZHMubWVzb3NwaGVyZS5jb20vbWFyYXRob24vYXNzZXRzL2ljb24tc2VydmljZS1tYXJhdGhvbi1sYXJnZS5wbmciLCJzY3JlZW5zaG90cyI6bnVsbH19", "DCOS_PACKAGE_NAME": "marathon", - "DCOS_PACKAGE_REGISTRY_VERSION": "2.0.0-rc1", - "DCOS_PACKAGE_RELEASE": "6", - "DCOS_PACKAGE_SOURCE": "https://github.com/mesosphere/universe/archive/cli-test-3.zip", + "DCOS_PACKAGE_REGISTRY_VERSION": "2.0", + "DCOS_PACKAGE_RELEASE": "0", + "DCOS_PACKAGE_SOURCE": "https://github.com/mesosphere/universe/archive/cli-test-4.zip", "DCOS_PACKAGE_VERSION": "0.11.1" }, "mem": 1024.0, diff --git a/cli/tests/data/package/json/test_describe_marathon_package_version.json b/cli/tests/data/package/json/test_describe_marathon_package_version.json index 9989eb3..f24c224 100644 --- a/cli/tests/data/package/json/test_describe_marathon_package_version.json +++ b/cli/tests/data/package/json/test_describe_marathon_package_version.json @@ -1,23 +1,22 @@ { - "name": "marathon", - "version": "0.8.1", - "scm": "https://github.com/mesosphere/marathon.git", - "maintainer": "support@mesosphere.io", "description": "A cluster-wide init and control system for services in cgroups or Docker containers.", "framework": true, - "images": { - "icon-small": "https://downloads.mesosphere.io/marathon/assets/icon-service-marathon-small.png", - "icon-medium": "https://downloads.mesosphere.io/marathon/assets/icon-service-marathon-medium.png", - "icon-large": "https://downloads.mesosphere.io/marathon/assets/icon-service-marathon-large.png" - }, "licenses": [ { "name": "Apache License Version 2.0", "url": "https://github.com/mesosphere/marathon/blob/master/LICENSE" } ], - "tags": ["mesosphere", "framework"], - "preInstallNotes": "We recommend a minimum of one node with at least 2 CPU's and 1GB of RAM available for the Marathon Service.", + "maintainer": "support@mesosphere.io", + "name": "marathon", + "packagingVersion": "2.0", "postInstallNotes": "Marathon DCOS Service has been successfully installed!\n\n\tDocumentation: https://mesosphere.github.io/marathon\n\tIssues: https:/github.com/mesosphere/marathon/issues\n", - "postUninstallNotes": "The Marathon DCOS Service has been uninstalled and will no longer run.\nPlease follow the instructions at http://docs.mesosphere.com/services/marathon/#uninstall to clean up any persisted state" + "postUninstallNotes": "The Marathon DCOS Service has been uninstalled and will no longer run.\nPlease follow the instructions at http://docs.mesosphere.com/services/marathon/#uninstall to clean up any persisted state", + "preInstallNotes": "We recommend a minimum of one node with at least 2 CPU's and 1GB of RAM available for the Marathon Service.", + "scm": "https://github.com/mesosphere/marathon.git", + "tags": [ + "init", + "long-running" + ], + "version": "0.11.1" } diff --git a/cli/tests/data/package/json/test_describe_marathon_package_versions.json b/cli/tests/data/package/json/test_describe_marathon_package_versions.json index 6f0856b..f4983d9 100644 --- a/cli/tests/data/package/json/test_describe_marathon_package_versions.json +++ b/cli/tests/data/package/json/test_describe_marathon_package_versions.json @@ -1,7 +1,3 @@ -0.11.1 -0.11.0 -0.10.1 -0.9.2 -0.9.0 -0.9.0-RC3 -0.8.1 +[ + "0.11.1" +] diff --git a/cli/tests/data/package/json/test_list_chronos.json b/cli/tests/data/package/json/test_list_chronos.json new file mode 100644 index 0000000..4b2b328 --- /dev/null +++ b/cli/tests/data/package/json/test_list_chronos.json @@ -0,0 +1,26 @@ +{ + "apps": [ + "/chronos" + ], + "description": "A fault tolerant job scheduler for Mesos which handles dependencies and ISO8601 based schedules.", + "framework": true, + "licenses": [ + { + "name": "Apache License Version 2.0", + "url": "https://github.com/mesos/chronos/blob/master/LICENSE" + } + ], + "maintainer": "support@mesosphere.io", + "name": "chronos", + "packagingVersion": "2.0", + "postInstallNotes": "Chronos DCOS Service has been successfully installed!\n\n\tDocumentation: http://mesos.github.io/chronos\n\tIssues: https://github.com/mesos/chronos/issues", + "postUninstallNotes": "The Chronos DCOS Service has been uninstalled and will no longer run.\nPlease follow the instructions at http://docs.mesosphere.com/services/chronos/#uninstall to clean up any persisted state", + "preInstallNotes": "We recommend a minimum of one node with at least 1 CPU and 2GB of RAM available for the Chronos Service.", + "scm": "https://github.com/mesos/chronos.git", + "tags": [ + "cron", + "analytics", + "batch" + ], + "version": "2.4.0" +} diff --git a/cli/tests/data/package/json/test_list_chronos_two_users.json b/cli/tests/data/package/json/test_list_chronos_two_users.json new file mode 100644 index 0000000..11b6435 --- /dev/null +++ b/cli/tests/data/package/json/test_list_chronos_two_users.json @@ -0,0 +1,28 @@ +{ + "apps": [ + "/chronos-user-1", + "/chronos-user-2" + ], + "description": "A fault tolerant job scheduler for Mesos which handles dependencies and ISO8601 based schedules.", + "framework": true, + "licenses": [ + { + "name": "Apache License Version 2.0", + "url": "https://github.com/mesos/chronos/blob/master/LICENSE" + } + ], + "maintainer": "support@mesosphere.io", + "name": "chronos", + "packagingVersion": "2.0", + "postInstallNotes": "Chronos DCOS Service has been successfully installed!\n\n\tDocumentation: http://mesos.github.io/chronos\n\tIssues: https://github.com/mesos/chronos/issues", + "postUninstallNotes": "The Chronos DCOS Service has been uninstalled and will no longer run.\nPlease follow the instructions at http://docs.mesosphere.com/services/chronos/#uninstall to clean up any persisted state", + "preInstallNotes": "We recommend a minimum of one node with at least 1 CPU and 2GB of RAM available for the Chronos Service.", + "scm": "https://github.com/mesos/chronos.git", + "tags": [ + "cron", + "analytics", + "batch" + ], + "version": "2.4.0" +} + diff --git a/cli/tests/data/package/json/test_list_chronos_user_1.json b/cli/tests/data/package/json/test_list_chronos_user_1.json new file mode 100644 index 0000000..bfb2bc8 --- /dev/null +++ b/cli/tests/data/package/json/test_list_chronos_user_1.json @@ -0,0 +1,26 @@ +{ + "apps": [ + "/chronos-user-1" + ], + "description": "A fault tolerant job scheduler for Mesos which handles dependencies and ISO8601 based schedules.", + "framework": true, + "licenses": [ + { + "name": "Apache License Version 2.0", + "url": "https://github.com/mesos/chronos/blob/master/LICENSE" + } + ], + "maintainer": "support@mesosphere.io", + "name": "chronos", + "packagingVersion": "2.0", + "postInstallNotes": "Chronos DCOS Service has been successfully installed!\n\n\tDocumentation: http://mesos.github.io/chronos\n\tIssues: https://github.com/mesos/chronos/issues", + "postUninstallNotes": "The Chronos DCOS Service has been uninstalled and will no longer run.\nPlease follow the instructions at http://docs.mesosphere.com/services/chronos/#uninstall to clean up any persisted state", + "preInstallNotes": "We recommend a minimum of one node with at least 1 CPU and 2GB of RAM available for the Chronos Service.", + "scm": "https://github.com/mesos/chronos.git", + "tags": [ + "cron", + "analytics", + "batch" + ], + "version": "2.4.0" +} diff --git a/cli/tests/data/package/json/test_list_chronos_user_2.json b/cli/tests/data/package/json/test_list_chronos_user_2.json new file mode 100644 index 0000000..697533a --- /dev/null +++ b/cli/tests/data/package/json/test_list_chronos_user_2.json @@ -0,0 +1,27 @@ +{ + "apps": [ + "/chronos-user-2" + ], + "description": "A fault tolerant job scheduler for Mesos which handles dependencies and ISO8601 based schedules.", + "framework": true, + "licenses": [ + { + "name": "Apache License Version 2.0", + "url": "https://github.com/mesos/chronos/blob/master/LICENSE" + } + ], + "maintainer": "support@mesosphere.io", + "name": "chronos", + "packagingVersion": "2.0", + "postInstallNotes": "Chronos DCOS Service has been successfully installed!\n\n\tDocumentation: http://mesos.github.io/chronos\n\tIssues: https://github.com/mesos/chronos/issues", + "postUninstallNotes": "The Chronos DCOS Service has been uninstalled and will no longer run.\nPlease follow the instructions at http://docs.mesosphere.com/services/chronos/#uninstall to clean up any persisted state", + "preInstallNotes": "We recommend a minimum of one node with at least 1 CPU and 2GB of RAM available for the Chronos Service.", + "scm": "https://github.com/mesos/chronos.git", + "tags": [ + "cron", + "analytics", + "batch" + ], + "version": "2.4.0" +} + diff --git a/cli/tests/data/ssl/ssl.toml b/cli/tests/data/ssl/ssl.toml index f87c58a..b076a22 100644 --- a/cli/tests/data/ssl/ssl.toml +++ b/cli/tests/data/ssl/ssl.toml @@ -1,6 +1,5 @@ [package] -cache = "tmp/cache" -sources = [ "https://github.com/mesosphere/universe/archive/cli-test-3.zip",] +cosmos_url = "http://localhost:7070" [core] timeout = 5 dcos_url = "https://dcos.snakeoil.mesosphere.com" diff --git a/cli/tests/fixtures/package.py b/cli/tests/fixtures/package.py index 32ac6ce..e31a8c6 100644 --- a/cli/tests/fixtures/package.py +++ b/cli/tests/fixtures/package.py @@ -1,6 +1,3 @@ -from dcos.package import HttpSource, IndexEntries - - def package_fixture(): """ DCOS package fixture. @@ -38,108 +35,106 @@ def search_result_fixture(): :rtype: dict """ - return IndexEntries( - HttpSource( - "https://github.com/mesosphere/universe/archive/master.zip"), - [ - { - "currentVersion": "0.1.0-SNAPSHOT-447-master-3ad1bbf8f7", - "description": "Apache Cassandra running on Apache Mesos", - "framework": True, - "name": "cassandra", - "tags": [ - "mesosphere", - "framework" - ], - "versions": [ - "0.1.0-SNAPSHOT-447-master-3ad1bbf8f7" - ] - }, - { - "currentVersion": "2.3.4", - "description": ("A fault tolerant job scheduler for Mesos " + - "which handles dependencies and ISO8601 " + - "based schedules."), - "framework": True, - "name": "chronos", - "tags": [ - "mesosphere", - "framework" - ], - "versions": [ - "2.3.4" - ] - }, - { - "currentVersion": "0.1.1", - "description": ("Hadoop Distributed File System (HDFS), " + - "Highly Available"), - "framework": True, - "name": "hdfs", - "tags": [ - "mesosphere", - "framework", - "filesystem" - ], - "versions": [ - "0.1.1" - ] - }, - { - "currentVersion": "0.1.0", - "description": "Example DCOS application package", - "framework": False, - "name": "helloworld", - "tags": [ - "mesosphere", - "example", - "subcommand" - ], - "versions": [ - "0.1.0" - ] - }, - { - "currentVersion": "0.9.0-beta", - "description": "Apache Kafka running on top of Apache Mesos", - "framework": True, - "name": "kafka", - "tags": [ - "mesosphere", - "framework", - "bigdata" - ], - "versions": [ - "0.9.0-beta" - ] - }, - { - "currentVersion": "0.8.1", - "description": ("A cluster-wide init and control system for " + - "services in cgroups or Docker containers."), - "framework": True, - "name": "marathon", - "tags": [ - "mesosphere", - "framework" - ], - "versions": [ - "0.8.1" - ] - }, - { - "currentVersion": "1.4.0-SNAPSHOT", - "description": ("Spark is a fast and general cluster " + - "computing system for Big Data"), - "framework": True, - "name": "spark", - "tags": [ - "mesosphere", - "framework", - "bigdata" - ], - "versions": [ - "1.4.0-SNAPSHOT" - ] - } - ]).as_dict() + return {"packages": [ + { + "currentVersion": "0.1.0-SNAPSHOT-447-master-3ad1bbf8f7", + "description": "Apache Cassandra running on Apache Mesos", + "framework": True, + "name": "cassandra", + "tags": [ + "mesosphere", + "framework" + ], + "versions": [ + "0.1.0-SNAPSHOT-447-master-3ad1bbf8f7" + ] + }, + { + "currentVersion": "2.3.4", + "description": ("A fault tolerant job scheduler for Mesos " + + "which handles dependencies and ISO8601 " + + "based schedules."), + "framework": True, + "name": "chronos", + "tags": [ + "mesosphere", + "framework" + ], + "versions": [ + "2.3.4" + ] + }, + { + "currentVersion": "0.1.1", + "description": ("Hadoop Distributed File System (HDFS), " + + "Highly Available"), + "framework": True, + "name": "hdfs", + "tags": [ + "mesosphere", + "framework", + "filesystem" + ], + "versions": [ + "0.1.1" + ] + }, + { + "currentVersion": "0.1.0", + "description": "Example DCOS application package", + "framework": False, + "name": "helloworld", + "tags": [ + "mesosphere", + "example", + "subcommand" + ], + "versions": [ + "0.1.0" + ] + }, + { + "currentVersion": "0.9.0-beta", + "description": "Apache Kafka running on top of Apache Mesos", + "framework": True, + "name": "kafka", + "tags": [ + "mesosphere", + "framework", + "bigdata" + ], + "versions": [ + "0.9.0-beta" + ] + }, + { + "currentVersion": "0.8.1", + "description": ("A cluster-wide init and control system for " + + "services in cgroups or Docker containers."), + "framework": True, + "name": "marathon", + "tags": [ + "mesosphere", + "framework" + ], + "versions": [ + "0.8.1" + ] + }, + { + "currentVersion": "1.4.0-SNAPSHOT", + "description": ("Spark is a fast and general cluster " + + "computing system for Big Data"), + "framework": True, + "name": "spark", + "tags": [ + "mesosphere", + "framework", + "bigdata" + ], + "versions": [ + "1.4.0-SNAPSHOT" + ] + } + ] + } diff --git a/cli/tests/integrations/common.py b/cli/tests/integrations/common.py index 4723748..5890e39 100644 --- a/cli/tests/integrations/common.py +++ b/cli/tests/integrations/common.py @@ -249,7 +249,7 @@ def remove_app(app_id): :rtype: None """ - assert_command(['dcos', 'marathon', 'app', 'remove', app_id]) + assert_command(['dcos', 'marathon', 'app', 'remove', '--force', app_id]) def package_install(package, deploy=False, args=[]): @@ -593,21 +593,17 @@ def config_set(key, value, env=None): assert stderr == b'' -def config_unset(key, index=None, env=None): +def config_unset(key, env=None): """ dcos config unset --index= :param key: :type key: str - :param index: - :type index: str :param env: env vars :type env: dict :rtype: None """ cmd = ['dcos', 'config', 'unset', key] - if index is not None: - cmd.append('--index={}'.format(index)) returncode, stdout, stderr = exec_command(cmd, env=env) diff --git a/cli/tests/integrations/test_config.py b/cli/tests/integrations/test_config.py index bf9c803..c83aa8c 100644 --- a/cli/tests/integrations/test_config.py +++ b/cli/tests/integrations/test_config.py @@ -51,15 +51,12 @@ def test_version(): stdout=stdout) -def test_list_property(env): +def _test_list_property(env): stdout = b"""core.dcos_url=http://dcos.snakeoil.mesosphere.com core.email=test@mail.com core.reporting=False core.ssl_verify=false core.timeout=5 -package.cache=tmp/cache -package.sources=['https://github.com/mesosphere/universe/archive/\ -cli-test-3.zip'] """ assert_command(['dcos', 'config', 'show'], stdout=stdout, @@ -91,13 +88,16 @@ def test_invalid_dcos_url(env): def test_get_top_property(env): stderr = ( - b"Property 'package' doesn't fully specify a value - " + b"Property 'core' doesn't fully specify a value - " b"possible properties are:\n" - b"package.cache\n" - b"package.sources\n" + b"core.dcos_url\n" + b"core.email\n" + b"core.reporting\n" + b"core.ssl_verify\n" + b"core.timeout\n" ) - assert_command(['dcos', 'config', 'show', 'package'], + assert_command(['dcos', 'config', 'show', 'core'], stderr=stderr, returncode=1) @@ -143,7 +143,7 @@ def test_set_same_output(env): def test_set_new_output(env): - config_unset('core.dcos_url', None, env) + config_unset('core.dcos_url', env) assert_command( ['dcos', 'config', 'set', 'core.dcos_url', 'http://dcos.snakeoil.mesosphere.com:5081'], @@ -153,80 +153,8 @@ def test_set_new_output(env): config_set('core.dcos_url', 'http://dcos.snakeoil.mesosphere.com', env) -def test_append_empty_list(env): - config_set('package.sources', '[]', env) - _append_value( - 'package.sources', - 'https://github.com/mesosphere/universe/archive/cli-test-3.zip', - env) - _get_value( - 'package.sources', - ['https://github.com/mesosphere/universe/archive/cli-test-3.zip'], - env) - - -def test_prepend_empty_list(env): - config_set('package.sources', '[]', env) - _prepend_value( - 'package.sources', - 'https://github.com/mesosphere/universe/archive/cli-test-3.zip', - env) - _get_value( - 'package.sources', - ['https://github.com/mesosphere/universe/archive/cli-test-3.zip'], - env) - - -def test_append_list(env): - _append_value( - 'package.sources', - 'https://universe.mesosphere.com/repo', - env) - _get_value( - 'package.sources', - ['https://github.com/mesosphere/universe/archive/cli-test-3.zip', - 'https://universe.mesosphere.com/repo'], - env) - config_unset('package.sources', '1', env) - - -def test_prepend_list(env): - _prepend_value( - 'package.sources', - 'https://universe.mesosphere.com/repo', - env) - _get_value( - 'package.sources', - ['https://universe.mesosphere.com/repo', - 'https://github.com/mesosphere/universe/archive/cli-test-3.zip'], - env) - config_unset('package.sources', '0', env) - - -def test_append_non_list(env): - stderr = (b"Append/Prepend not supported on 'core.dcos_url' " - b"properties - use 'dcos config set core.dcos_url new_uri'\n") - - assert_command( - ['dcos', 'config', 'append', 'core.dcos_url', 'new_uri'], - returncode=1, - stderr=stderr, - env=env) - - -def test_prepend_non_list(env): - stderr = (b"Append/Prepend not supported on 'core.dcos_url' " - b"properties - use 'dcos config set core.dcos_url new_uri'\n") - - assert_command( - ['dcos', 'config', 'prepend', 'core.dcos_url', 'new_uri'], - returncode=1, - stderr=stderr, - env=env) - - def test_unset_property(env): - config_unset('core.reporting', None, env) + config_unset('core.reporting', env) _get_missing_value('core.reporting', env) config_set('core.reporting', 'false', env) @@ -246,85 +174,19 @@ def test_unset_output(env): config_set('core.reporting', 'false', env) -def test_unset_index_output(env): - stdout = ( - b"[package.sources]: removed element " - b"'https://github.com/mesosphere/universe/archive/cli-test-3.zip' " - b"at index '0'\n" - ) - - assert_command(['dcos', 'config', 'unset', 'package.sources', '--index=0'], - stdout=stdout, - env=env) - - _prepend_value( - 'package.sources', - 'https://github.com/mesosphere/universe/archive/cli-test-3.zip', - env) - - -def test_set_whole_list(env): - config_set( - 'package.sources', - '["https://github.com/mesosphere/universe/archive/cli-test-3.zip"]', - env) - - def test_unset_top_property(env): stderr = ( - b"Property 'package' doesn't fully specify a value - " + b"Property 'core' doesn't fully specify a value - " b"possible properties are:\n" - b"package.cache\n" - b"package.sources\n" + b"core.dcos_url\n" + b"core.email\n" + b"core.reporting\n" + b"core.ssl_verify\n" + b"core.timeout\n" ) assert_command( - ['dcos', 'config', 'unset', 'package'], - returncode=1, - stderr=stderr, - env=env) - - -def test_unset_list_index(env): - config_unset('package.sources', '0', env) - _get_value( - 'package.sources', - [], - env) - _prepend_value( - 'package.sources', - 'https://github.com/mesosphere/universe/archive/cli-test-3.zip', - env) - - -def test_unset_outbound_index(env): - stderr = ( - b'Index (3) is out of bounds - possible values are ' - b'between 0 and 0\n' - ) - - assert_command( - ['dcos', 'config', 'unset', '--index=3', 'package.sources'], - returncode=1, - stderr=stderr, - env=env) - - -def test_unset_bad_index(env): - stderr = b'Error parsing string as int\n' - - assert_command( - ['dcos', 'config', 'unset', '--index=number', 'package.sources'], - returncode=1, - stderr=stderr, - env=env) - - -def test_unset_index_from_string(env): - stderr = b'Unsetting based on an index is only supported for lists\n' - - assert_command( - ['dcos', 'config', 'unset', '--index=0', 'core.dcos_url'], + ['dcos', 'config', 'unset', 'core'], returncode=1, stderr=stderr, env=env) @@ -336,20 +198,6 @@ def test_validate(env): env=env, stdout=stdout) -def test_validation_error(env): - source = ["https://github.com/mesosphere/universe/archive/cli-test-3.zip"] - config_unset('package.sources', None, env) - - stdout = b"Error: missing required property 'sources'.\n" - assert_command(['dcos', 'config', 'validate'], - returncode=1, - stdout=stdout, - env=env) - - config_set('package.sources', json.dumps(source), env) - _get_value('package.sources', source, env) - - def test_set_property_key(env): assert_command( ['dcos', 'config', 'set', 'path.to.value', 'cool new value'], @@ -361,7 +209,7 @@ def test_set_property_key(env): def test_set_missing_property(missing_env): config_set('core.dcos_url', 'http://localhost:8080', missing_env) _get_value('core.dcos_url', 'http://localhost:8080', missing_env) - config_unset('core.dcos_url', None, missing_env) + config_unset('core.dcos_url', missing_env) def test_set_core_property(env): @@ -374,85 +222,32 @@ def test_url_validation(env): key = 'core.dcos_url' default_value = 'http://dcos.snakeoil.mesosphere.com' + key2 = 'package.cosmos_url' + config_set(key, 'http://localhost', env) config_set(key, 'https://localhost', env) config_set(key, 'http://dcos-1234', env) - config_set(key, 'http://dcos-1234.mydomain.com', env) + config_set(key2, 'http://dcos-1234.mydomain.com', env) config_set(key, 'http://localhost:5050', env) config_set(key, 'https://localhost:5050', env) config_set(key, 'http://mesos-1234:5050', env) - config_set(key, 'http://mesos-1234.mydomain.com:5050', env) + config_set(key2, 'http://mesos-1234.mydomain.com:5050', env) config_set(key, 'http://localhost:8080', env) config_set(key, 'https://localhost:8080', env) config_set(key, 'http://marathon-1234:8080', env) - config_set(key, 'http://marathon-1234.mydomain.com:5050', env) + config_set(key2, 'http://marathon-1234.mydomain.com:5050', env) config_set(key, 'http://user@localhost:8080', env) config_set(key, 'http://u-ser@localhost:8080', env) config_set(key, 'http://user123_@localhost:8080', env) config_set(key, 'http://user:p-ssw_rd@localhost:8080', env) config_set(key, 'http://user123:password321@localhost:8080', env) - config_set(key, 'http://us%r1$3:pa#sw*rd321@localhost:8080', env) + config_set(key2, 'http://us%r1$3:pa#sw*rd321@localhost:8080', env) config_set(key, default_value, env) - - -def test_append_url_validation(env): - default_value = ('["https://github.com/mesosphere/universe/archive/' - 'cli-test-3.zip"]') - - config_set('package.sources', '[]', env) - _append_value( - 'package.sources', - 'https://github.com/mesosphere/universe/archive/cli-test-3.zip', - env) - _append_value( - 'package.sources', - 'git@github.com:mesosphere/test.git', - env) - _append_value( - 'package.sources', - 'https://github.com/mesosphere/test.git', - env) - _append_value( - 'package.sources', - 'file://some-domain.com/path/to/file.extension', - env) - _append_value( - 'package.sources', - 'file:///path/to/file.extension', - env) - config_set('package.sources', default_value, env) - - -def test_prepend_url_validation(env): - default_value = ('["https://github.com/mesosphere/universe/archive/' - 'cli-test-3.zip"]') - - config_set('package.sources', '[]', env) - _prepend_value( - 'package.sources', - 'https://github.com/mesosphere/universe/archive/cli-test-3.zip', - env) - _prepend_value( - 'package.sources', - 'git@github.com:mesosphere/test.git', - env) - _prepend_value( - 'package.sources', - 'https://github.com/mesosphere/test.git', - env) - _prepend_value( - 'package.sources', - 'file://some-domain.com/path/to/file.extension', - env) - _prepend_value( - 'package.sources', - 'file:///path/to/file.extension', - env) - config_set('package.sources', default_value, env) + config_unset(key2, env) def test_fail_url_validation(env): @@ -467,14 +262,6 @@ def test_bad_port_fail_url_validation(env): 'http://localhost:bad_port/', env) -def test_append_fail_validation(env): - _fail_validation('append', 'package.sources', 'bad_url', env) - - -def test_prepend_fail_validation(env): - _fail_validation('prepend', 'package.sources', 'bad_url', env) - - def test_timeout(missing_env): config_set('marathon.url', 'http://1.2.3.4', missing_env) config_set('core.timeout', '1', missing_env) @@ -486,8 +273,8 @@ def test_timeout(missing_env): assert stdout == b'' assert "(connect timeout=1)".encode('utf-8') in stderr - config_unset('core.timeout', None, missing_env) - config_unset('marathon.url', None, missing_env) + config_unset('core.timeout', missing_env) + config_unset('marathon.url', missing_env) def test_parse_error(): @@ -513,28 +300,6 @@ def _fail_url_validation(command, key, value, env): 'Unable to parse {!r} as a url'.format(value)).encode('utf-8')) -def _fail_validation(command, key, value, env): - returncode_, stdout_, stderr_ = exec_command( - ['dcos', 'config', command, key, value], env=env) - - assert returncode_ == 1 - assert stdout_ == b'' - assert stderr_.startswith(str( - 'Error: {!r} does not match'.format(value)).encode('utf-8')) - - -def _append_value(key, value, env): - assert_command( - ['dcos', 'config', 'append', key, value], - env=env) - - -def _prepend_value(key, value, env): - assert_command( - ['dcos', 'config', 'prepend', key, value], - env=env) - - def _get_value(key, value, env): returncode, stdout, stderr = exec_command( ['dcos', 'config', 'show', key], diff --git a/cli/tests/integrations/test_marathon.py b/cli/tests/integrations/test_marathon.py index 8ef5571..ad70add 100644 --- a/cli/tests/integrations/test_marathon.py +++ b/cli/tests/integrations/test_marathon.py @@ -428,6 +428,8 @@ def test_killing_with_host_app(): assert len(expected_to_be_killed.intersection(new_tasks)) == 0 +@pytest.mark.skipif( + True, reason='https://github.com/mesosphere/marathon/issues/3251') def test_kill_stopped_app(): with _zero_instance_app(): returncode, stdout, stderr = exec_command( diff --git a/cli/tests/integrations/test_package.py b/cli/tests/integrations/test_package.py index b8e969e..5bd8fe8 100644 --- a/cli/tests/integrations/test_package.py +++ b/cli/tests/integrations/test_package.py @@ -1,15 +1,12 @@ import base64 import contextlib import json -import os import pkg_resources import six -from dcos import package, subcommand -from dcos.errors import DCOSException +from dcos import subcommand import pytest -from mock import patch from .common import (assert_command, assert_lines, delete_zk_node, delete_zk_nodes, exec_command, file_bytes, file_json, @@ -17,69 +14,26 @@ from .common import (assert_command, assert_lines, delete_zk_node, service_shutdown, wait_for_service, watch_all_deployments) +def setup_module(module): + assert_command( + ['dcos', 'package', 'repo', 'remove', '--repo-name=Universe']) + repo = "https://github.com/mesosphere/universe/archive/cli-test-4.zip" + assert_command(['dcos', 'package', 'repo', 'add', 'test4', repo]) + + +def teardown_module(module): + assert_command( + ['dcos', 'package', 'repo', 'remove', '--repo-name=test4']) + repo = "https://universe.mesosphere.com/repo" + assert_command(['dcos', 'package', 'repo', 'add', 'Universe', repo]) + + @pytest.fixture(scope="module") def zk_znode(request): request.addfinalizer(delete_zk_nodes) return request -def _chronos_description(app_ids): - """ - :param app_ids: a list of application id - :type app_ids: [str] - :returns: a binary string representing the chronos description - :rtype: str - """ - - result = [ - {"apps": app_ids, - "description": "A fault tolerant job scheduler for Mesos which " - "handles dependencies and ISO8601 based schedules.", - "framework": True, - "images": { - "icon-large": "https://downloads.mesosphere.io/chronos/assets/" - "icon-service-chronos-large.png", - "icon-medium": "https://downloads.mesosphere.io/chronos/assets/" - "icon-service-chronos-medium.png", - "icon-small": "https://downloads.mesosphere.io/chronos/assets/" - "icon-service-chronos-small.png" - }, - "licenses": [ - { - "name": "Apache License Version 2.0", - "url": "https://github.com/mesos/chronos/blob/master/LICENSE" - } - ], - "maintainer": "support@mesosphere.io", - "name": "chronos", - "packageSource": "https://github.com/mesosphere/universe/archive/\ -cli-test-3.zip", - "postInstallNotes": "Chronos DCOS Service has been successfully " - "installed!\n\n\tDocumentation: http://mesos." - "github.io/chronos\n\tIssues: https://github.com/" - "mesos/chronos/issues", - "postUninstallNotes": "The Chronos DCOS Service has been uninstalled " - "and will no longer run.\nPlease follow the " - "instructions at http://docs.mesosphere." - "com/services/chronos/#uninstall to clean up " - "any persisted state", - "preInstallNotes": "We recommend a minimum of one node with at least " - "1 CPU and 2GB of RAM available for the Chronos " - "Service.", - "releaseVersion": "1", - "scm": "https://github.com/mesos/chronos.git", - "tags": [ - "cron", - "analytics", - "batch" - ], - "version": "2.4.0" - }] - - return (json.dumps(result, sort_keys=True, indent=2).replace(' \n', '\n') + - '\n').encode('utf-8') - - def test_package(): stdout = pkg_resources.resource_string( 'tests', @@ -98,32 +52,71 @@ def test_version(): stdout=b'dcos-package version SNAPSHOT\n') -def test_sources_list(): - stdout = b"fd40db7f075490e0c92ec6fcd62ec1caa361b313 " + \ - b"https://github.com/mesosphere/universe/archive/cli-test-3.zip\n" - assert_command(['dcos', 'package', 'sources'], - stdout=stdout) +def test_repo_list(): + repo_list = b"""\ +test4: https://github.com/mesosphere/universe/archive/cli-test-4.zip +""" + assert_command(['dcos', 'package', 'repo', 'list'], stdout=repo_list) -def test_update_without_validation(): - returncode, stdout, stderr = exec_command(['dcos', 'package', 'update']) - - assert returncode == 0 - assert b'source' in stdout - assert b'Validating package definitions...' not in stdout - assert b'OK' not in stdout - assert stderr == b'' +def test_repo_add(): + repo = \ + "https://github.com/mesosphere/universe/archive/cli-test-3.zip" + repo_list = b"""\ +test4: https://github.com/mesosphere/universe/archive/cli-test-4.zip +test: https://github.com/mesosphere/universe/archive/cli-test-3.zip +""" + args = ["test", repo] + _repo_add(args, repo_list) -def test_update_with_validation(): +def test_repo_add_index(): + repo = \ + "https://github.com/mesosphere/universe/archive/cli-test-2.zip" + repo_list = b"""\ +test4: https://github.com/mesosphere/universe/archive/cli-test-4.zip +test2: https://github.com/mesosphere/universe/archive/cli-test-2.zip +test: https://github.com/mesosphere/universe/archive/cli-test-3.zip +""" + args = ["test2", repo, '--index=1'] + _repo_add(args, repo_list) + + +def test_repo_remove_by_repo_name(): + repo_list = b"""\ +test4: https://github.com/mesosphere/universe/archive/cli-test-4.zip +test2: https://github.com/mesosphere/universe/archive/cli-test-2.zip +""" + _repo_remove(['--repo-name=test'], repo_list) + + +def test_repo_remove_by_package_repo(): + repo = \ + "https://github.com/mesosphere/universe/archive/cli-test-2.zip" + repo_list = b"""\ +test4: https://github.com/mesosphere/universe/archive/cli-test-4.zip +""" + _repo_remove(['--repo-url={}'.format(repo)], repo_list) + + +def test_repo_empty(): + assert_command( + ['dcos', 'package', 'repo', 'remove', '--repo-name=test4']) + returncode, stdout, stderr = exec_command( - ['dcos', 'package', 'update', '--validate']) + ['dcos', 'package', 'repo', 'list']) + stderr_msg = (b"There are currently no repos configured. " + b"Please use `dcos package repo add` to add a repo\n") + assert returncode == 1 + assert stdout == b'' + assert stderr == stderr_msg - assert returncode == 0 - assert b'source' in stdout - assert b'Validating package definitions...' in stdout - assert b'OK' in stdout - assert stderr == b'' + repo = \ + "https://github.com/mesosphere/universe/archive/cli-test-4.zip" + repo_list = b"""\ +test4: https://github.com/mesosphere/universe/archive/cli-test-4.zip +""" + _repo_add(["test4", repo], repo_list) def test_describe_nonexistent(): @@ -133,7 +126,7 @@ def test_describe_nonexistent(): def test_describe_nonexistent_version(): - stderr = b'Version a.b.c of package [marathon] is not available\n' + stderr = b'Version [a.b.c] of package [marathon] not found\n' assert_command(['dcos', 'package', 'describe', 'marathon', '--package-version=a.b.c'], stderr=stderr, @@ -143,8 +136,14 @@ def test_describe_nonexistent_version(): def test_describe(): stdout = file_json( 'tests/data/package/json/test_describe_marathon.json') - assert_command(['dcos', 'package', 'describe', 'marathon'], - stdout=stdout) + + returncode_, stdout_, stderr_ = exec_command( + ['dcos', 'package', 'describe', 'marathon']) + + assert returncode_ == 0 + output = json.loads(stdout_.decode('utf-8')) + assert _remove_nulls(output) == json.loads(stdout.decode('utf-8')) + assert stderr_ == b'' def test_describe_cli(): @@ -169,23 +168,42 @@ def test_describe_config(): def test_describe_render(): + # DCOS_PACKAGE_METADATA label will need to be changed after issue 431 stdout = file_json( 'tests/data/package/json/test_describe_marathon_app_render.json') - assert_command( - ['dcos', 'package', 'describe', 'marathon', '--app', '--render'], - stdout=stdout) + stdout = json.loads(stdout.decode('utf-8')) + expected_labels = stdout.pop("labels", None) + + returncode, stdout_, stderr = exec_command( + ['dcos', 'package', 'describe', 'marathon', '--app', '--render']) + + stdout_ = json.loads(stdout_.decode('utf-8')) + actual_labels = stdout_.pop("labels", None) + + for label, value in expected_labels.items(): + assert value == actual_labels.get(label) + + assert stdout == stdout_ + assert stderr == b'' + assert returncode == 0 def test_describe_package_version(): stdout = file_json( 'tests/data/package/json/test_describe_marathon_package_version.json') - assert_command( - ['dcos', 'package', 'describe', 'marathon', '--package-version=0.8.1'], - stdout=stdout) + + returncode_, stdout_, stderr_ = exec_command( + ['dcos', 'package', 'describe', 'marathon', + '--package-version=0.11.1']) + + assert returncode_ == 0 + output = json.loads(stdout_.decode('utf-8')) + assert _remove_nulls(output) == json.loads(stdout.decode('utf-8')) + assert stderr_ == b'' def test_describe_package_version_missing(): - stderr = b'Version bogus of package [marathon] is not available\n' + stderr = b'Version [bogus] of package [marathon] not found\n' assert_command( ['dcos', 'package', 'describe', 'marathon', '--package-version=bogus'], returncode=1, @@ -213,9 +231,22 @@ def test_describe_package_versions_others(): def test_describe_options(): stdout = file_json( 'tests/data/package/json/test_describe_app_options.json') - assert_command(['dcos', 'package', 'describe', '--app', '--options', - 'tests/data/package/marathon.json', 'marathon'], - stdout=stdout) + stdout = json.loads(stdout.decode('utf-8')) + expected_labels = stdout.pop("labels", None) + + returncode, stdout_, stderr = exec_command( + ['dcos', 'package', 'describe', '--app', '--options', + 'tests/data/package/marathon.json', 'marathon']) + + stdout_ = json.loads(stdout_.decode('utf-8')) + actual_labels = stdout_.pop("labels", None) + + for label, value in expected_labels.items(): + assert value == actual_labels.get(label) + + assert stdout == stdout_ + assert stderr == b'' + assert returncode == 0 def test_describe_app_cli(): @@ -228,27 +259,28 @@ def test_describe_app_cli(): def test_describe_specific_version(): stdout = file_bytes( - 'tests/data/package/json/test_describe_marathon_0.8.1.json') - assert_command(['dcos', 'package', 'describe', '--package-version=0.8.1', - 'marathon'], - stdout=stdout) + 'tests/data/package/json/test_describe_marathon_0.11.1.json') + + returncode_, stdout_, stderr_ = exec_command( + ['dcos', 'package', 'describe', '--package-version=0.11.1', + 'marathon']) + + assert returncode_ == 0 + output = json.loads(stdout_.decode('utf-8')) + assert _remove_nulls(output) == json.loads(stdout.decode('utf-8')) + assert stderr_ == b'' def test_bad_install(): args = ['--options=tests/data/package/chronos-bad.json', '--yes'] - stderr = b"""Error: False is not of type 'string' -Path: chronos.zk-hosts -Value: false - + stdout = b"" + stderr = """\ Please create a JSON file with the appropriate options, and pass the \ /path/to/file as an --options argument. """ - - _install_chronos(args=args, - returncode=1, - stdout=b'', - stderr=stderr, - postInstallNotes=b'') + _install_bad_chronos(args=args, + stdout=stdout, + stderr=stderr) def test_install(zk_znode): @@ -262,6 +294,33 @@ def test_install(zk_znode): if service['name'] == 'chronos']) == 0 +def test_bad_install_marathon_msg(): + stdout = (b'A sample pre-installation message\n' + b'Installing Marathon app for package [helloworld] version ' + b'[0.1.0] with app id [/foo]\n' + b'Installing CLI subcommand for package [helloworld] ' + b'version [0.1.0]\n' + b'New command available: dcos helloworld\n' + b'A sample post-installation message\n') + + _install_helloworld(['--yes', '--app-id=/foo'], + stdout=stdout) + + stdout2 = (b'A sample pre-installation message\n' + b'Installing Marathon app for package [helloworld] version ' + b'[0.1.0] with app id [/foo/bar]\n') + + stderr = (b'Object is not valid\n' + b'Groups and Applications may not have the same ' + b'identifier: /foo\n') + + _install_helloworld(['--yes', '--app-id=/foo/bar'], + stdout=stdout2, + stderr=stderr, + returncode=1) + _uninstall_helloworld() + + def test_install_missing_options_file(): """Test that a missing options file results in the expected stderr message.""" @@ -276,32 +335,33 @@ def test_install_specific_version(): stdout = (b'We recommend a minimum of one node with at least 2 ' b'CPU\'s and 1GB of RAM available for the Marathon Service.\n' b'Installing Marathon app for package [marathon] ' - b'version [0.8.1]\n' + b'version [0.11.1]\n' b'Marathon DCOS Service has been successfully installed!\n\n' b'\tDocumentation: https://mesosphere.github.io/marathon\n' b'\tIssues: https:/github.com/mesosphere/marathon/issues\n\n') uninstall_stderr = ( - b'Uninstalled package [marathon] version [0.8.1]\n' - b'The Marathon DCOS Service has been uninstalled and will no longer ' - b'run.\nPlease follow the instructions at http://docs.mesosphere.com/' - b'services/marathon/#uninstall to clean up any persisted state\n' + b'Uninstalled package [marathon] version [0.11.1]\n' + b'The Marathon DCOS Service has been uninstalled and will no ' + b'longer run.\nPlease follow the instructions at http://docs.' + b'mesosphere.com/services/marathon/#uninstall to clean up any ' + b'persisted state\n' ) with _package('marathon', stdout=stdout, uninstall_stderr=uninstall_stderr, - args=['--yes', '--package-version=0.8.1']): + args=['--yes', '--package-version=0.11.1']): returncode, stdout, stderr = exec_command( ['dcos', 'package', 'list', 'marathon', '--json']) assert returncode == 0 assert stderr == b'' - assert json.loads(stdout.decode('utf-8'))[0]['version'] == "0.8.1" + assert json.loads(stdout.decode('utf-8'))[0]['version'] == "0.11.1" def test_install_bad_package_version(): - stderr = b'Version a.b.c of package [cassandra] is not available\n' + stderr = b'Version [a.b.c] of package [cassandra] not found\n' assert_command( ['dcos', 'package', 'install', 'cassandra', '--package-version=a.b.c'], @@ -313,25 +373,23 @@ def test_package_metadata(): _install_helloworld() # test marathon labels - expected_metadata = b"""eyJkZXNjcmlwdGlvbiI6ICJFeGFtcGxlIERDT1MgYXBwbGljYX\ -Rpb24gcGFja2FnZSIsICJtYWludGFpbmVyIjogInN1cHBvcnRAbWVzb3NwaGVyZS5pbyIsICJuYW1l\ -IjogImhlbGxvd29ybGQiLCAicG9zdEluc3RhbGxOb3RlcyI6ICJBIHNhbXBsZSBwb3N0LWluc3RhbG\ -xhdGlvbiBtZXNzYWdlIiwgInByZUluc3RhbGxOb3RlcyI6ICJBIHNhbXBsZSBwcmUtaW5zdGFsbGF0\ -aW9uIG1lc3NhZ2UiLCAidGFncyI6IFsibWVzb3NwaGVyZSIsICJleGFtcGxlIiwgInN1YmNvbW1hbm\ -QiXSwgInZlcnNpb24iOiAiMC4xLjAiLCAid2Vic2l0ZSI6ICJodHRwczovL2dpdGh1Yi5jb20vbWVz\ -b3NwaGVyZS9kY29zLWhlbGxvd29ybGQifQ==""" + expected_metadata = b"""eyJ3ZWJzaXRlIjoiaHR0cHM6Ly9naXRodWIuY29tL21lc29zcG\ +hlcmUvZGNvcy1oZWxsb3dvcmxkIiwibmFtZSI6ImhlbGxvd29ybGQiLCJwb3N0SW5zdGFsbE5vdGVz\ +IjoiQSBzYW1wbGUgcG9zdC1pbnN0YWxsYXRpb24gbWVzc2FnZSIsImRlc2NyaXB0aW9uIjoiRXhhbX\ +BsZSBEQ09TIGFwcGxpY2F0aW9uIHBhY2thZ2UiLCJwYWNrYWdpbmdWZXJzaW9uIjoiMi4wIiwidGFn\ +cyI6WyJtZXNvc3BoZXJlIiwiZXhhbXBsZSIsInN1YmNvbW1hbmQiXSwibWFpbnRhaW5lciI6InN1cH\ +BvcnRAbWVzb3NwaGVyZS5pbyIsInZlcnNpb24iOiIwLjEuMCIsInByZUluc3RhbGxOb3RlcyI6IkEg\ +c2FtcGxlIHByZS1pbnN0YWxsYXRpb24gbWVzc2FnZSJ9""" - expected_command = b"""eyJwaXAiOiBbImRjb3M8MS4wIiwgImdpdCtodHRwczovL2dpdGh\ -1Yi5jb20vbWVzb3NwaGVyZS9kY29zLWhlbGxvd29ybGQuZ2l0I2Rjb3MtaGVsbG93b3JsZD0wLjEuM\ -CJdfQ==""" + expected_command = b"""eyJwaXAiOlsiZGNvczwxLjAiLCJnaXQraHR0cHM6Ly9naXRodWI\ +uY29tL21lc29zcGhlcmUvZGNvcy1oZWxsb3dvcmxkLmdpdCNkY29zLWhlbGxvd29ybGQ9MC4xLjAiX\ +X0=""" expected_source = b"""https://github.com/mesosphere/universe/archive/\ -cli-test-3.zip""" +cli-test-4.zip""" expected_labels = { - 'DCOS_PACKAGE_METADATA': expected_metadata, - 'DCOS_PACKAGE_COMMAND': expected_command, - 'DCOS_PACKAGE_REGISTRY_VERSION': b'2.0.0-rc1', + 'DCOS_PACKAGE_REGISTRY_VERSION': b'2.0', 'DCOS_PACKAGE_NAME': b'helloworld', 'DCOS_PACKAGE_VERSION': b'0.1.0', 'DCOS_PACKAGE_SOURCE': expected_source, @@ -339,15 +397,22 @@ cli-test-3.zip""" } app_labels = _get_app_labels('helloworld') - for label, value in expected_labels.items(): assert value == six.b(app_labels.get(label)) + # these labels are different for cosmos b/c of null problem + # we have cosmos tests for test, and will fix in issue 431 + assert expected_metadata == six.b( + app_labels.get('DCOS_PACKAGE_METADATA')) + assert expected_command == six.b( + app_labels.get('DCOS_PACKAGE_COMMAND')) + # test local package.json package = { "description": "Example DCOS application package", "maintainer": "support@mesosphere.io", "name": "helloworld", + "packagingVersion": "2.0", "postInstallNotes": "A sample post-installation message", "preInstallNotes": "A sample pre-installation message", "tags": ["mesosphere", "example", "subcommand"], @@ -355,22 +420,10 @@ cli-test-3.zip""" "website": "https://github.com/mesosphere/dcos-helloworld", } - package_dir = subcommand.package_dir('helloworld') + helloworld_subcommand = subcommand.InstalledSubcommand("helloworld") # test local package.json - package_path = os.path.join(package_dir, 'package.json') - with open(package_path) as f: - assert json.load(f) == package - - # test local source - source_path = os.path.join(package_dir, 'source') - with open(source_path) as f: - assert six.b(f.read()) == expected_source - - # test local version - version_path = os.path.join(package_dir, 'version') - with open(version_path) as f: - assert six.b(f.read()) == b'0' + assert _remove_nulls(helloworld_subcommand.package_json()) == package # uninstall helloworld _uninstall_helloworld() @@ -394,6 +447,7 @@ def test_images_in_metadata(): b'Please follow the instructions at http://docs.mesosphere.com/' b'services/cassandra/#uninstall to clean up any persisted ' b'state\n') + package_uninstall('cassandra', stderr=stderr) assert_command(['dcos', 'marathon', 'group', 'remove', '/cassandra']) delete_zk_node('cassandra-mesos') @@ -412,9 +466,7 @@ def test_install_with_id(zk_znode): def test_install_missing_package(): - stderr = b"""Package [missing-package] not found -You may need to run 'dcos package update' to update your repositories -""" + stderr = b'Package [missing-package] not found\n' assert_command(['dcos', 'package', 'install', 'missing-package'], returncode=1, stderr=stderr) @@ -426,14 +478,13 @@ def test_uninstall_with_id(zk_znode): def test_uninstall_all(zk_znode): _uninstall_chronos(args=['--all']) - get_services(expected_count=1, args=['--inactive']) def test_uninstall_missing(): - stderr = 'Package [chronos] is not installed.\n' + stderr = 'Package [chronos] is not installed\n' _uninstall_chronos(returncode=1, stderr=stderr) - stderr = 'Package [chronos] with id [chronos-1] is not installed.\n' + stderr = 'Package [chronos] with id [/chronos-1] is not installed\n' _uninstall_chronos( args=['--app-id=chronos-1'], returncode=1, @@ -448,9 +499,9 @@ def test_uninstall_subcommand(): def test_uninstall_cli(): _install_helloworld() - _uninstall_helloworld(args=['--cli']) + _uninstall_cli_helloworld(args=['--cli']) - stdout = b"""[ + stdout = b""" { "apps": [ "/helloworld" @@ -458,11 +509,9 @@ def test_uninstall_cli(): "description": "Example DCOS application package", "maintainer": "support@mesosphere.io", "name": "helloworld", - "packageSource": "https://github.com/mesosphere/universe/archive/\ -cli-test-3.zip", + "packagingVersion": "2.0", "postInstallNotes": "A sample post-installation message", "preInstallNotes": "A sample pre-installation message", - "releaseVersion": "0", "tags": [ "mesosphere", "example", @@ -471,9 +520,13 @@ cli-test-3.zip", "version": "0.1.0", "website": "https://github.com/mesosphere/dcos-helloworld" } -] """ - _list(stdout=stdout) + returncode_, stdout_, stderr_ = exec_command( + ['dcos', 'package', 'list', '--json']) + assert stderr_ == b'' + assert returncode_ == 0 + output = json.loads(stdout_.decode('utf-8'))[0] + assert _remove_nulls(output) == json.loads(stdout.decode('utf-8')) _uninstall_helloworld() @@ -504,10 +557,13 @@ def test_uninstall_multiple_apps(): b"[/helloworld-1, /helloworld-2].\n" b"Please use --app-id to specify the ID of the app " b"to uninstall, or use --all to uninstall all apps.\n") - _uninstall_helloworld(stderr=stderr, - returncode=1) + returncode = 1 - assert_command(['dcos', 'package', 'uninstall', 'helloworld', '--all']) + _uninstall_helloworld(stderr=stderr, + returncode=returncode, + uninstalled=b'') + + _uninstall_helloworld(args=['--all'], stdout=b'', stderr=b'', returncode=0) watch_all_deployments() @@ -518,13 +574,14 @@ def test_list(zk_znode): _list(args=['--app-id=/xyzzy', '--json']) _install_chronos() - expected_output = _chronos_description(['/chronos']) - _list(stdout=expected_output) - _list(args=['--json', 'chronos'], - stdout=expected_output) - _list(args=['--json', '--app-id=/chronos'], - stdout=expected_output) + expected_output = file_json( + 'tests/data/package/json/test_list_chronos.json') + + _list_remove_nulls(stdout=expected_output) + _list_remove_nulls(args=['--json', 'chronos'], stdout=expected_output) + _list_remove_nulls(args=['--json', '--app-id=/chronos'], + stdout=expected_output) _list(args=['--json', 'ceci-nest-pas-une-package']) _list(args=['--json', '--app-id=/ceci-nest-pas-une-package']) @@ -562,10 +619,7 @@ def test_install_no(): def test_list_cli(): - _install_helloworld() - - stdout = b"""\ -[ + stdout = b""" { "apps": [ "/helloworld" @@ -576,11 +630,9 @@ def test_list_cli(): "description": "Example DCOS application package", "maintainer": "support@mesosphere.io", "name": "helloworld", - "packageSource": "https://github.com/mesosphere/universe/archive/\ -cli-test-3.zip", + "packagingVersion": "2.0", "postInstallNotes": "A sample post-installation message", "preInstallNotes": "A sample pre-installation message", - "releaseVersion": "0", "tags": [ "mesosphere", "example", @@ -589,9 +641,9 @@ cli-test-3.zip", "version": "0.1.0", "website": "https://github.com/mesosphere/dcos-helloworld" } -] """ - _list(stdout=stdout) + _install_helloworld() + _list_remove_nulls(stdout=stdout) _uninstall_helloworld() stdout = (b"A sample pre-installation message\n" @@ -602,7 +654,6 @@ cli-test-3.zip", _install_helloworld(args=['--cli', '--yes'], stdout=stdout) stdout = b"""\ -[ { "command": { "name": "helloworld" @@ -610,11 +661,9 @@ cli-test-3.zip", "description": "Example DCOS application package", "maintainer": "support@mesosphere.io", "name": "helloworld", - "packageSource": "https://github.com/mesosphere/universe/archive/\ -cli-test-3.zip", + "packagingVersion": "2.0", "postInstallNotes": "A sample post-installation message", "preInstallNotes": "A sample pre-installation message", - "releaseVersion": "0", "tags": [ "mesosphere", "example", @@ -623,10 +672,9 @@ cli-test-3.zip", "version": "0.1.0", "website": "https://github.com/mesosphere/dcos-helloworld" } -] """ - _list(stdout=stdout) - _uninstall_helloworld() + _list_remove_nulls(stdout=stdout) + _uninstall_cli_helloworld() def test_uninstall_multiple_frameworknames(zk_znode): @@ -637,35 +685,35 @@ def test_uninstall_multiple_frameworknames(zk_znode): watch_all_deployments() - expected_output = _chronos_description( - ['/chronos-user-1', '/chronos-user-2']) + expected_output = file_json( + 'tests/data/package/json/test_list_chronos_two_users.json') + + # issue 431 + _list_remove_nulls(stdout=expected_output) + _list_remove_nulls(args=['--json', 'chronos'], stdout=expected_output) + _list_remove_nulls(args=['--json', '--app-id=/chronos-user-1'], + stdout=file_json( + 'tests/data/package/json/test_list_chronos_user_1.json')) + + _list_remove_nulls(args=['--json', '--app-id=/chronos-user-2'], + stdout=file_json( + 'tests/data/package/json/test_list_chronos_user_2.json')) - _list(stdout=expected_output) - _list(args=['--json', 'chronos'], stdout=expected_output) - _list(args=['--json', '--app-id=/chronos-user-1'], - stdout=_chronos_description(['/chronos-user-1'])) - _list(args=['--json', '--app-id=/chronos-user-2'], - stdout=_chronos_description(['/chronos-user-2'])) _uninstall_chronos( args=['--app-id=chronos-user-1'], returncode=1, stderr='Uninstalled package [chronos] version [2.4.0]\n' - 'The Chronos DCOS Service has been uninstalled and will no ' - 'longer run.\nPlease follow the instructions at http://docs.' - 'mesosphere.com/services/chronos/#uninstall to clean up any ' - 'persisted state\n' - 'Unable to shutdown the framework for [chronos-user] because ' - 'there are multiple frameworks with the same name: ') + 'Unable to shutdown [chronos] service framework with name ' + '[chronos-user] because there are multiple framework ids ' + 'matching this name: ') + _uninstall_chronos( args=['--app-id=chronos-user-2'], returncode=1, stderr='Uninstalled package [chronos] version [2.4.0]\n' - 'The Chronos DCOS Service has been uninstalled and will no ' - 'longer run.\nPlease follow the instructions at http://docs.' - 'mesosphere.com/services/chronos/#uninstall to clean up any ' - 'persisted state\n' - 'Unable to shutdown the framework for [chronos-user] because ' - 'there are multiple frameworks with the same name: ') + 'Unable to shutdown [chronos] service framework with name ' + '[chronos-user] because there are multiple framework ids ' + 'matching this name: ') for framework in get_services(args=['--inactive']): if framework['name'] == 'chronos-user': @@ -685,8 +733,6 @@ def test_search(): assert returncode == 0 assert b'"packages": []' in stdout - assert b'"source": "https://github.com/mesosphere/universe/archive/\ -cli-test-3.zip"' in stdout assert stderr == b'' returncode, stdout, stderr = exec_command( @@ -730,7 +776,9 @@ def test_search_ends_with_wildcard(): registries = json.loads(stdout.decode('utf-8')) for registry in registries: - assert len(registry['packages']) == 2 + # cosmos matches wildcards in name/description/tags + # so will find more results (3 instead of 2) + assert len(registry['packages']) >= 2 def test_search_start_with_wildcard(): @@ -759,22 +807,6 @@ def test_search_middle_with_wildcard(): assert len(registry['packages']) == 1 -@patch('dcos.package.Package.package_json') -@patch('dcos.package.Package.config_json') -def test_bad_config_schema_msg(config_mock, package_mock): - pkg = package.Package("", "/") - config_mock.return_value = {} - package_mock.return_value = {'maintainer': 'support@test'} - - with pytest.raises(DCOSException) as e: - pkg.options("1", {}) - - msg = ("An object in the package's config.json is missing the " - "required 'properties' feature:\n {}" - "\nPlease contact the project maintainer: support@test") - assert e.exconly().split(':', 1)[1].strip() == msg - - def _get_app_labels(app_id): returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'show', app_id]) @@ -795,16 +827,30 @@ def _install_helloworld( b'version [0.1.0]\n' b'New command available: dcos helloworld\n' b'A sample post-installation message\n', + stderr=b'', returncode=0, stdin=None): assert_command( ['dcos', 'package', 'install', 'helloworld'] + args, stdout=stdout, returncode=returncode, - stdin=stdin) + stdin=stdin, + stderr=stderr) def _uninstall_helloworld( + args=[], + stdout=b'', + stderr=b'', + returncode=0, + uninstalled=b'Uninstalled package [helloworld] version [0.1.0]\n'): + assert_command(['dcos', 'package', 'uninstall', 'helloworld'] + args, + stdout=stdout, + stderr=uninstalled+stderr, + returncode=returncode) + + +def _uninstall_cli_helloworld( args=[], stdout=b'', stderr=b'', @@ -824,6 +870,19 @@ def _uninstall_chronos(args=[], returncode=0, stdout=b'', stderr=''): assert result_stderr.decode('utf-8').startswith(stderr) +def _install_bad_chronos(args=['--yes'], + stdout=b'', + stderr=''): + cmd = ['dcos', 'package', 'install', 'chronos'] + args + returncode_, stdout_, stderr_ = exec_command(cmd) + assert returncode_ == 1 + assert stderr in stderr_.decode('utf-8') + preInstallNotes = (b'We recommend a minimum of one node with at least 1 ' + b'CPU and 2GB of RAM available for the Chronos ' + b'Service.\n') + assert stdout_ == preInstallNotes + + def _install_chronos( args=['--yes'], returncode=0, @@ -855,6 +914,16 @@ def _list(args=['--json'], stdout=stdout) +def _list_remove_nulls(args=['--json'], stdout=b'[]\n'): + returncode_, stdout_, stderr_ = exec_command( + ['dcos', 'package', 'list'] + args) + + assert returncode_ == 0 + output = json.loads(stdout_.decode('utf-8'))[0] + assert _remove_nulls(output) == json.loads(stdout.decode('utf-8')) + assert stderr_ == b'' + + def _helloworld(): stdout = b'''A sample pre-installation message Installing Marathon app for package [helloworld] version [0.1.0] @@ -862,8 +931,11 @@ Installing CLI subcommand for package [helloworld] version [0.1.0] New command available: dcos helloworld A sample post-installation message ''' + + stderr = b'Uninstalled package [helloworld] version [0.1.0]\n' return _package('helloworld', - stdout=stdout) + stdout=stdout, + uninstall_stderr=stderr) @contextlib.contextmanager @@ -894,3 +966,26 @@ def _package(name, ['dcos', 'package', 'uninstall', name], stderr=uninstall_stderr) watch_all_deployments() + + +def _repo_add(args=[], repo_list=[]): + assert_command(['dcos', 'package', 'repo', 'add'] + args) + assert_command(['dcos', 'package', 'repo', 'list'], stdout=repo_list) + + +def _repo_remove(args=[], repo_list=[]): + assert_command(['dcos', 'package', 'repo', 'remove'] + args) + assert_command(['dcos', 'package', 'repo', 'list'], stdout=repo_list) + + +# issue 431 +def _remove_nulls(output): + """Remove nulls from dict. Temporary until we fix this in cosmos + + :param output: dict with possible null values + :type output: dict + :returns: dict without null + :rtype: dict + """ + + return {k: v for k, v in output.items() if v} diff --git a/cli/tests/integrations/test_ssl.py b/cli/tests/integrations/test_ssl.py index 09f0dee..116b1ba 100644 --- a/cli/tests/integrations/test_ssl.py +++ b/cli/tests/integrations/test_ssl.py @@ -40,7 +40,7 @@ def test_dont_verify_ssl_with_config(env): assert returncode == 0 assert stderr == b'' - config_unset('core.ssl_verify', None, env) + config_unset('core.ssl_verify', env) def test_verify_ssl_without_cert_env_var(env): @@ -62,7 +62,7 @@ def test_verify_ssl_without_cert_config(env): assert returncode == 1 assert "certificate verify failed" in stderr.decode('utf-8') - config_unset('core.ssl_verify', None, env) + config_unset('core.ssl_verify', env) def test_verify_ssl_with_bad_cert_env_var(env): @@ -84,7 +84,7 @@ def test_verify_ssl_with_bad_cert_config(env): assert returncode == 1 assert "PEM lib" in stderr.decode('utf-8') # wrong private key - config_unset('core.ssl_verify', None, env) + config_unset('core.ssl_verify', env) def test_verify_ssl_with_good_cert_env_var(env): @@ -106,4 +106,4 @@ def test_verify_ssl_with_good_cert_config(env): assert returncode == 0 assert stderr == b'' - config_unset('core.ssl_verify', None, env) + config_unset('core.ssl_verify', env) diff --git a/cli/tests/integrations/test_task.py b/cli/tests/integrations/test_task.py index 4c660c2..b00b061 100644 --- a/cli/tests/integrations/test_task.py +++ b/cli/tests/integrations/test_task.py @@ -263,8 +263,9 @@ def test_log_file_unavailable(): def test_ls(): + stdout = b'stderr stderr.logrotate.conf stdout stdout.logrotate.conf\n' assert_command(['dcos', 'task', 'ls', 'test-app1'], - stdout=b'stderr stdout\n') + stdout=stdout) def test_ls_multiple_tasks(): @@ -278,7 +279,7 @@ def test_ls_multiple_tasks(): def test_ls_long(): - assert_lines(['dcos', 'task', 'ls', '--long', 'test-app1'], 2) + assert_lines(['dcos', 'task', 'ls', '--long', 'test-app1'], 4) def test_ls_path(): diff --git a/cli/tests/unit/data/package_search.txt b/cli/tests/unit/data/package_search.txt index 3a02bc3..3b26c17 100644 --- a/cli/tests/unit/data/package_search.txt +++ b/cli/tests/unit/data/package_search.txt @@ -1,8 +1,8 @@ -NAME VERSION FRAMEWORK SOURCE DESCRIPTION -cassandra 0.1.0-SNAPSHOT-447-master-3ad1bbf8f7 True https://github.com/mesosphere/universe/archive/master.zip Apache Cassandra running on Apache Mesos -chronos 2.3.4 True https://github.com/mesosphere/universe/archive/master.zip A fault tolerant job scheduler for Mesos which handles dependencies and ISO8601 based schedules. -hdfs 0.1.1 True https://github.com/mesosphere/universe/archive/master.zip Hadoop Distributed File System (HDFS), Highly Available -helloworld 0.1.0 False https://github.com/mesosphere/universe/archive/master.zip Example DCOS application package -kafka 0.9.0-beta True https://github.com/mesosphere/universe/archive/master.zip Apache Kafka running on top of Apache Mesos -marathon 0.8.1 True https://github.com/mesosphere/universe/archive/master.zip A cluster-wide init and control system for services in cgroups or Docker containers. -spark 1.4.0-SNAPSHOT True https://github.com/mesosphere/universe/archive/master.zip Spark is a fast and general cluster computing system for Big Data \ No newline at end of file +NAME VERSION FRAMEWORK DESCRIPTION +cassandra 0.1.0-SNAPSHOT-447-master-3ad1bbf8f7 True Apache Cassandra running on Apache Mesos +chronos 2.3.4 True A fault tolerant job scheduler for Mesos which handles dependencies and ISO8601 based schedules. +hdfs 0.1.1 True Hadoop Distributed File System (HDFS), Highly Available +helloworld 0.1.0 False Example DCOS application package +kafka 0.9.0-beta True Apache Kafka running on top of Apache Mesos +marathon 0.8.1 True A cluster-wide init and control system for services in cgroups or Docker containers. +spark 1.4.0-SNAPSHOT True Spark is a fast and general cluster computing system for Big Data \ No newline at end of file diff --git a/dcos/config.py b/dcos/config.py index 48389eb..a5708df 100644 --- a/dcos/config.py +++ b/dcos/config.py @@ -3,7 +3,6 @@ import copy import json import pkg_resources -import six import toml from dcos import emitting, jsonitem, subcommand, util from dcos.errors import DCOSException @@ -58,57 +57,6 @@ def set_val(name, value): return toml_config -def unset(name, index): - """ - :param name: name of paramater - :type name: str - :param index: index in list to unset - :type param: int - :rtype: None - """ - - toml_config = util.get_config(True) - toml_config_pre = copy.deepcopy(toml_config) - section = name.split(".", 1)[0] - if section not in toml_config_pre._dictionary: - toml_config_pre._dictionary[section] = {} - value = toml_config.pop(name, None) - if value is None: - raise DCOSException("Property {!r} doesn't exist".format(name)) - elif isinstance(value, collections.Mapping): - raise DCOSException(generate_choice_msg(name, value)) - elif ((isinstance(value, collections.Sequence) and - not isinstance(value, six.string_types)) and - index is not None): - index = util.parse_int(index) - - if not value: - raise DCOSException( - 'Index ({}) is out of bounds - [{}] is empty'.format( - index, - name)) - if index < 0 or index >= len(value): - raise DCOSException( - 'Index ({}) is out of bounds - possible values are ' - 'between {} and {}'.format(index, 0, len(value) - 1)) - - popped_value = value.pop(index) - emitter.publish( - "[{}]: removed element '{}' at index '{}'".format( - name, popped_value, index)) - - toml_config[name] = value - save(toml_config) - return - elif index is not None: - raise DCOSException( - 'Unsetting based on an index is only supported for lists') - else: - emitter.publish("Removed [{}]".format(name)) - save(toml_config) - return - - def load_from_path(path, mutable=False): """Loads a TOML file from the path @@ -158,6 +106,48 @@ def _get_path(toml_config, path): return toml_config +def unset(name): + """ + :param name: name of config value to unset + :type name: str + :returns: process status + :rtype: None + """ + + toml_config = util.get_config(True) + toml_config_pre = copy.deepcopy(toml_config) + section = name.split(".", 1)[0] + if section not in toml_config_pre._dictionary: + toml_config_pre._dictionary[section] = {} + value = toml_config.pop(name, None) + if value is None: + raise DCOSException("Property {!r} doesn't exist".format(name)) + elif isinstance(value, collections.Mapping): + raise DCOSException(_generate_choice_msg(name, value)) + else: + emitter.publish("Removed [{}]".format(name)) + save(toml_config) + return + + +def _generate_choice_msg(name, value): + """ + :param name: name of the property + :type name: str + :param value: dictionary for the value + :type value: dcos.config.Toml + :returns: an error message for top level properties + :rtype: str + """ + + message = ("Property {!r} doesn't fully specify a value - " + "possible properties are:").format(name) + for key, _ in sorted(value.property_items()): + message += '\n{}.{}'.format(name, key) + + return message + + def _iterator(parent, dictionary): """ :param parent: Path to the value parameter diff --git a/dcos/cosmospackage.py b/dcos/cosmospackage.py index 9292511..7b997f9 100644 --- a/dcos/cosmospackage.py +++ b/dcos/cosmospackage.py @@ -1,5 +1,10 @@ +import functools +import json + +import pystache from dcos import emitting, http, util -from dcos.errors import DCOSAuthenticationException +from dcos.errors import (DCOSAuthenticationException, DCOSException, + DCOSHTTPException, DefaultError) from six.moves import urllib @@ -8,7 +13,7 @@ logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() -class Cosmos: +class Cosmos(): """Implementation of Package Manager using Cosmos""" def __init__(self, cosmos_url): @@ -23,7 +28,7 @@ class Cosmos: try: url = urllib.parse.urljoin(self.cosmos_url, 'capabilities') response = http.get(url, - headers=_get_cosmos_header("capabilities")) + headers=_get_capabilities_header()) # return `Authentication failed` error messages, but all other errors # are treated as endpoint not available except DCOSAuthenticationException: @@ -34,6 +39,401 @@ class Cosmos: return response.status_code == 200 + def install_app(self, pkg, options, app_id): + """Installs a package's application + + :param pkg: the package to install + :type pkg: CosmosPackageVersion + :param options: user supplied package parameters + :type options: dict + :param app_id: app ID for installation of this package + :type app_id: str + :rtype: None + """ + + params = {"packageName": pkg.name(), "packageVersion": pkg.version()} + if options is not None: + params["options"] = options + if app_id is not None: + params["appId"] = app_id + + self.cosmos_post("install", params) + + def uninstall_app(self, package_name, remove_all, app_id): + """Uninstalls an app. + + :param package_name: The package to uninstall + :type package_name: str + :param remove_all: Whether to remove all instances of the named app + :type remove_all: boolean + :param app_id: App ID of the app instance to uninstall + :type app_id: str + :returns: whether uninstall was successful or not + :rtype: bool + """ + + params = {"packageName": package_name} + if remove_all is True: + params["all"] = True + if app_id is not None: + params["appId"] = app_id + + response = self.cosmos_post("uninstall", params) + results = response.json().get("results") + + uninstalled_versions = [] + for res in results: + version = res.get("packageVersion") + if version not in uninstalled_versions: + emitter.publish( + DefaultError( + 'Uninstalled package [{}] version [{}]'.format( + res.get("packageName"), + res.get("packageVersion")))) + uninstalled_versions += [res.get("packageVersion")] + + if res.get("postUninstallNotes") is not None: + emitter.publish( + DefaultError(res.get("postUninstallNotes"))) + + return True + + def search_sources(self, query): + """package search + + :param query: query to search + :type query: str + :returns: list of package indicies of matching packages + :rtype: [packages] + """ + response = self.cosmos_post("search", {"query": query}) + return [response.json()] + + def get_package_version(self, package_name, package_version): + """Returns PackageVersion of specified package + + :param package_name: package name + :type package_name: str + :param package_version: version of package + :type package_version: str | None + :rtype: PackageVersion + + """ + + return CosmosPackageVersion(package_name, package_version, + self.cosmos_url) + + def installed_apps(self, package_name, app_id): + """List installed packages + + { + 'appId': , + .... + } + + :param package_name: the optional package to list + :type package_name: str + :param app_id: the optional application id to list + :type app_id: str + :rtype: [dict] + """ + + params = {} + if package_name is not None: + params["packageName"] = package_name + if app_id is not None: + params["appId"] = app_id + + list_response = self.cosmos_post("list", params).json() + + packages = [] + for pkg in list_response['packages']: + result = pkg['packageInformation']['packageDefinition'] + + result['appId'] = pkg['appId'] + packages.append(result) + + return packages + + def get_repos(self): + """List locations of repos + + :returns: the list of repos, in resolution order + :rtype: [str] + """ + + response = self.cosmos_post("repository/list", params={}) + repos = ["{}: {}".format(repo.get("name"), repo.get("uri")) + for repo in response.json().get("repositories")] + return "\n".join(repos) + + def add_repo(self, name, package_repo, index): + """Add package repo and update repo with new repo + + :param name: name to call repo + :type name: str + :param package_repo: location of repo to add + :type package_repo: str + :param index: index to add this repo + :type index: int + :rtype: None + """ + + params = {"name": name, "uri": package_repo} + if index is not None: + params["index"] = index + response = self.cosmos_post("repository/add", params=params) + return response.json() + + def remove_repo(self, name, package_repo): + """Remove package repo and update repo + + :param package_repo: location of repo to remove + :type package_repo: str + :rtype: None + """ + + params = {"name": name, "uri": package_repo} + response = self.cosmos_post("repository/delete", params=params) + return response.json() + + def cosmos_error(fn): + """Decorator for errors returned from cosmos + + :param fn: function to check for errors from cosmos + :type fn: function + :rtype: Response + :returns: Response + """ + + @functools.wraps(fn) + def check_for_cosmos_error(*args, **kwargs): + """Returns response from cosmos or raises exception + + :param response: response from cosmos + :type response: Response + :returns: Response or raises Exception + :rtype: valid response + """ + + response = fn(*args, **kwargs) + content_type = response.headers.get('Content-Type') + if content_type is None: + raise DCOSHTTPException(response) + elif _get_header("error") in content_type: + error_msg = _format_error_message(response.json()) + raise DCOSException(error_msg) + return response + + return check_for_cosmos_error + + @cosmos_error + def cosmos_post(self, request, params): + """Request to cosmos server + + :param request: type of request + :type requet: str + :param params: body of request + :type params: dict + :returns: Response + :rtype: Response + """ + + url = urllib.parse.urljoin(self.cosmos_url, + 'package/{}'.format(request)) + try: + response = http.post(url, json=params, + headers=_get_cosmos_header(request)) + if not _check_cosmos_header(request, response): + raise DCOSException( + "Server returned incorrect response type: {}".format( + response.headers)) + except DCOSHTTPException as e: + # let the response be handled by `cosmos_error` so we can expose + # errors reported by cosmos + response = e.response + return response + + +class CosmosPackageVersion(): + """Interface to a specific package version from cosmos""" + + def __init__(self, name, package_version, url): + self._name = name + self._cosmos_url = url + + params = {"packageName": name} + if package_version is not None: + params["packageVersion"] = package_version + response = Cosmos(url).cosmos_post("describe", params) + + package_info = response.json() + self._package_json = package_info.get("package") + self._package_version = package_version or \ + self._package_json.get("version") + self._config_json = package_info.get("config") + self._command_json = package_info.get("command") + self._resource_json = package_info.get("resource") + self._marathon_template = package_info.get("marathonMustache") + + def registry(self): + """Cosmos only supports one registry right now, so default to cosmos + + :returns: registry + :rtype: str + """ + + return "cosmos" + + def version(self): + """Returns the package version. + + :returns: The version of this package + :rtype: str + """ + + return self._package_version + + def name(self): + """Returns the package name. + + :returns: The name of this package + :rtype: str + """ + + return self._name + + def revision(self): + """We aren't exposing revisions for cosmos right now, so make + custom string. + + :returns: revision + :rtype: str + """ + return "cosmos" + self._package_version + + def cosmos_url(self): + """ + Returns location of cosmos server + + :returns: revision + :rtype: str + """ + + return self._cosmos_url + + def package_json(self): + """Returns the JSON content of the package.json file. + + :returns: Package data + :rtype: dict + """ + + return self._package_json + + def config_json(self): + """Returns the JSON content of the config.json file. + + :returns: Package config schema + :rtype: dict + """ + + return self._config_json + + def _resource_json(self): + """Returns the JSON content of the resource.json file. + + :returns: Package resources + :rtype: dict + """ + + return self._resource_json + + def command_template(self): + """ Returns raw data from command.json + + :returns: raw data from command.json + :rtype: str + """ + return self._command_json + + def marathon_template(self): + """Returns raw data from marathon.json + + :returns: raw data from marathon.json + :rtype: str + """ + + return self._marathon_template + + def marathon_json(self, options): + """Returns the JSON content of the marathon.json template, after + rendering it with options. + + :param options: the template options to use in rendering + :type options: dict + :rtype: dict + """ + + params = {"packageName": self._name} + params["packageVersion"] = self._package_version + params["options"] = options + response = Cosmos(self._cosmos_url).cosmos_post("render", params) + return response.json().get("marathonJson") + + def has_mustache_definition(self): + """Dummy method since all packages in cosmos must have mustache + definition. + """ + + return True + + def options(self, user_options): + """Makes sure user supplied options are valid, and returns valid options + + :param options: the template options to use in rendering + :type options: dict + :rtype: dict + """ + + self.marathon_json(user_options) + + return user_options + + def has_command_definition(self): + """Returns true if the package defines a command; false otherwise. + + :rtype: bool + """ + + return self._command_json is not None + + def command_json(self, options): + """Returns the JSON content of the command.json template, after + rendering it with options. + + :param options: the template options to use in rendering + :type options: dict + :returns: Package data + :rtype: dict + """ + + rendered = pystache.render(json.dumps(self._command_json), options) + return util.load_jsons(rendered) + + def package_versions(self): + """Returns a list of available versions for this package + + :returns: package version + :rtype: [] + """ + + params = {"packageName": self.name(), "includePackageVersions": True} + response = Cosmos(self._cosmos_url).cosmos_post( + "list-versions", params) + + return list(response.json().get("results").keys()) + def _get_header(request_type): """Returns header str for talking with cosmos @@ -57,5 +457,92 @@ def _get_cosmos_header(request_name): :rtype: {} """ + request_name = request_name.replace("/", ".") return {"Accept": _get_header("{}-response".format(request_name)), "Content-Type": _get_header("{}-request".format(request_name))} + + +def _get_capabilities_header(): + """Returns header fields needed for a valid request to cosmos capabilities + endpoint + + :returns: header information + :rtype: str + """ + header = "application/vnd.dcos.capabilities+json;charset=utf-8;version=v1" + return {"Accept": header, "Content-Type": header} + + +def _check_cosmos_header(request_name, response): + """Validate that cosmos returned correct header for request + + :param request_type: name of specified request (ie uninstall-request) + :type request_type: str + :param response: response object + :type response: Response + :returns: whether or not we got expected response + :rtype: bool + """ + + request_name = request_name.replace("/", ".") + rsp = "{}-response".format(request_name) + return _get_header(rsp) in response.headers.get('Content-Type') + + +def _format_error_message(error): + """Returns formatted error message based on error type + + :param error: cosmos error + :type error: dict + :returns: formatted error + :rtype: str + """ + if error.get("type") == "AmbiguousAppId": + helper = (".\nPlease use --app-id to specify the ID of the app " + "to uninstall, or use --all to uninstall all apps.") + error_message = error.get("message") + helper + elif error.get("type") == "MultipleFrameworkIds": + helper = ". Manually shut them down using 'dcos service shutdown'" + error_message = error.get("message") + helper + elif error.get("type") == "JsonSchemaMismatch": + error_message = _format_json_schema_mismatch_message(error) + elif error.get("type") == "MarathonBadResponse": + error_message = _format_marathon_bad_response_message(error) + else: + error_message = error.get("message") + + return error_message + + +def _format_json_schema_mismatch_message(error): + """Returns the formatted error message for JsonSchemaMismatch + + :param error: cosmos JsonSchemMismatch error + :type error: dict + :returns: formatted error + :rtype: str + """ + + error_messages = ["Error: {}".format(error.get("message"))] + for err in error.get("data").get("errors"): + found = "Found: {}\n".format(err.get("found")) + expected = "Expected: {}\n".format(",".join(err.get("expected"))) + pointer = err.get("instance").get("pointer") + formatted_path = pointer.lstrip("/").replace("/", ".") + path = "Path: {}".format(formatted_path) + error_messages += [found + expected + path] + + error_messages += [ + "\nPlease create a JSON file with the appropriate options, and" + " pass the /path/to/file as an --options argument." + ] + + return "\n".join(error_messages) + + +def _format_marathon_bad_response_message(error): + data = error.get("data") + error_messages = [error.get("message")] + if data is not None: + error_messages += [err.get("error") for err in data.get("errors")] + return "\n".join(error_messages) diff --git a/dcos/marathon.py b/dcos/marathon.py index 7da1255..90ab54d 100644 --- a/dcos/marathon.py +++ b/dcos/marathon.py @@ -273,6 +273,18 @@ class Client(object): response = _http_req(http.get, url, timeout=self._timeout) return response.json()['apps'] + def get_apps_for_framework(self, framework_name): + """ Return all apps running the given framework. + + :param framework_name: framework name + :type framework_name: str + :rtype: [dict] + """ + + return [app for app in self.get_apps() + if app.get('labels', {}).get( + 'DCOS_PACKAGE_FRAMEWORK_NAME') == framework_name] + def add_app(self, app_resource): """Add a new application. diff --git a/dcos/package.py b/dcos/package.py index af8f762..22e40b5 100644 --- a/dcos/package.py +++ b/dcos/package.py @@ -1,151 +1,18 @@ -import abc -import base64 import collections -import copy -import hashlib -import json -import os -import re -import shutil -import stat -import subprocess -import zipfile -from distutils.version import LooseVersion -import git -import portalocker -import pystache -import six -from dcos import (constants, emitting, errors, http, marathon, mesos, - subcommand, util) -from dcos.errors import DCOSException, DefaultError - -from six.moves import urllib +from dcos import emitting, subcommand, util +from dcos.errors import DCOSException logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() -PACKAGE_METADATA_KEY = 'DCOS_PACKAGE_METADATA' -PACKAGE_NAME_KEY = 'DCOS_PACKAGE_NAME' -PACKAGE_VERSION_KEY = 'DCOS_PACKAGE_VERSION' -PACKAGE_SOURCE_KEY = 'DCOS_PACKAGE_SOURCE' -PACKAGE_FRAMEWORK_KEY = 'DCOS_PACKAGE_IS_FRAMEWORK' -PACKAGE_RELEASE_KEY = 'DCOS_PACKAGE_RELEASE' -PACKAGE_COMMAND_KEY = 'DCOS_PACKAGE_COMMAND' -PACKAGE_REGISTRY_VERSION_KEY = 'DCOS_PACKAGE_REGISTRY_VERSION' -PACKAGE_FRAMEWORK_NAME_KEY = 'DCOS_PACKAGE_FRAMEWORK_NAME' - - -def install_app(pkg, revision, init_client, options, app_id): - """Installs a package's application - - :param pkg: the package to install - :type pkg: Package - :param revision: the package revision to install - :type revision: str - :param init_client: the program to use to run the package - :type init_client: object - :param options: package parameters - :type options: dict - :param app_id: app ID for installation of this package - :type app_id: str - :rtype: None - """ - - # Insert option parameters into the init template - init_desc = pkg.marathon_json(revision, options) - - if app_id is not None: - logger.debug('Setting app ID to "%s" (was "%s")', - app_id, - init_desc['id']) - init_desc['id'] = app_id - - # Send the descriptor to init - init_client.add_app(init_desc) - - -def _make_package_labels(pkg, revision, options): - """Returns Marathon app labels for a package. - - :param pkg: The package to install - :type pkg: Package - :param revision: The package revision to install - :type revision: str - :param options: package parameters - :type options: dict - :returns: Marathon app labels - :rtype: dict - """ - - metadata = pkg.package_json(revision) - # add images to package.json metadata for backwards compatability in the UI - if pkg._has_resource_definition(revision): - images = {"images": pkg._resource_json(revision)["images"]} - metadata.update(images) - - encoded_metadata = _base64_encode(metadata) - - is_framework = metadata.get('framework') - if not is_framework: - is_framework = False - - package_registry_version = pkg.registry.get_version() - - package_labels = { - PACKAGE_METADATA_KEY: encoded_metadata, - PACKAGE_NAME_KEY: metadata['name'], - PACKAGE_VERSION_KEY: metadata['version'], - PACKAGE_SOURCE_KEY: pkg.registry.source.url, - PACKAGE_FRAMEWORK_KEY: json.dumps(is_framework), - PACKAGE_REGISTRY_VERSION_KEY: package_registry_version, - PACKAGE_RELEASE_KEY: revision - } - - if pkg.has_command_definition(revision): - command = pkg.command_json(revision, options) - package_labels[PACKAGE_COMMAND_KEY] = _base64_encode(command) - - # Run a heuristic that determines the hint for the framework name - framework_name = _find_framework_name(pkg.name(), options) - if framework_name: - package_labels[PACKAGE_FRAMEWORK_NAME_KEY] = framework_name - - return package_labels - - -def _find_framework_name(package_name, options): - """ - :param package_name: the name of the package - :type package_name: str - :param options: the options object - :type options: dict - :returns: the name of framework if found; None otherwise - :rtype: str - """ - - return options.get(package_name, {}).get('framework-name', None) - - -def _base64_encode(dictionary): - """Returns base64(json(dictionary)). - - :param dictionary: dict to encode - :type dictionary: dict - :returns: base64 encoding - :rtype: str - """ - - json_str = json.dumps(dictionary, sort_keys=True) - str_bytes = six.b(json_str) - return base64.b64encode(str_bytes).decode('utf-8') - - -def uninstall(package_name, remove_all, app_id, cli, app): +def uninstall(pkg, package_name, remove_all, app_id, cli, app): """Uninstalls a package. + :param pkg: package manager to uninstall with + :type pkg: PackageManager :param package_name: The package to uninstall :type package_name: str :param remove_all: Whether to remove all instances of the named app @@ -161,19 +28,18 @@ def uninstall(package_name, remove_all, app_id, cli, app): cli = app = True uninstalled = False - if cli: + installed = installed_packages(pkg, app_id, package_name) + installed_cli = next((True for installed_pkg in installed + if installed_pkg.get("command")), False) + installed_app = next((True for installed_pkg in installed + if installed_pkg.get("apps")), False) + + if cli and installed_cli: if subcommand.uninstall(package_name): uninstalled = True - if app: - num_apps = uninstall_app( - package_name, - remove_all, - app_id, - marathon.create_client(), - mesos.DCOSClient()) - - if num_apps > 0: + if app and installed_app: + if pkg.uninstall_app(package_name, remove_all, app_id): uninstalled = True if uninstalled: @@ -181,8 +47,9 @@ def uninstall(package_name, remove_all, app_id, cli, app): else: msg = 'Package [{}]'.format(package_name) if app_id is not None: + app_id = util.normalize_app_id(app_id) msg += " with id [{}]".format(app_id) - msg += " is not installed." + msg += " is not installed" raise DCOSException(msg) @@ -198,101 +65,6 @@ def uninstall_subcommand(distribution_name): return subcommand.uninstall(distribution_name) -def uninstall_app(app_name, remove_all, app_id, init_client, dcos_client): - """Uninstalls an app. - - :param app_name: The app to uninstall - :type app_name: str - :param remove_all: Whether to remove all instances of the named app - :type remove_all: boolean - :param app_id: App ID of the app instance to uninstall - :type app_id: str - :param init_client: The program to use to run the app - :type init_client: object - :param dcos_client: the DCOS client - :type dcos_client: dcos.mesos.DCOSClient - :returns: number of apps uninstalled - :rtype: int - """ - - apps = init_client.get_apps() - - def is_match(app): - encoding = 'utf-8' # We normalize encoding for byte-wise comparison - name_label = app.get('labels', {}).get(PACKAGE_NAME_KEY, u'') - name_label_enc = name_label.encode(encoding) - app_name_enc = app_name.encode(encoding) - name_matches = name_label_enc == app_name_enc - - if app_id is not None: - pkg_app_id = app.get('id', '') - normalized_app_id = init_client.normalize_app_id(app_id) - return name_matches and pkg_app_id == normalized_app_id - else: - return name_matches - - matching_apps = [a for a in apps if is_match(a)] - - if not remove_all and len(matching_apps) > 1: - app_ids = [a.get('id') for a in matching_apps] - raise DCOSException( - ("Multiple apps named [{}] are installed: [{}].\n" + - "Please use --app-id to specify the ID of the app to uninstall," + - " or use --all to uninstall all apps.").format( - app_name, - ', '.join(app_ids))) - - for app in matching_apps: - package_json = _decode_and_add_context( - app['id'], - app.get('labels', {})) - - # First, remove the app from Marathon - init_client.remove_app(app['id'], force=True) - - # Second, shutdown the framework with Mesos - framework_name = app.get('labels', {}).get(PACKAGE_FRAMEWORK_NAME_KEY) - if framework_name is not None: - logger.info( - 'Trying to shutdown framework {}'.format(framework_name)) - frameworks = mesos.Master(dcos_client.get_master_state()) \ - .frameworks(inactive=True) - - # Look up all the framework names - framework_ids = [ - framework['id'] - for framework in frameworks - if framework['name'] == framework_name - ] - - logger.info( - 'Found the following frameworks: {}'.format(framework_ids)) - - # Emit post uninstall notes - emitter.publish( - DefaultError( - 'Uninstalled package [{}] version [{}]'.format( - package_json['name'], - package_json['version']))) - - if 'postUninstallNotes' in package_json: - emitter.publish( - DefaultError(package_json['postUninstallNotes'])) - - if len(framework_ids) == 1: - dcos_client.shutdown_framework(framework_ids[0]) - elif len(framework_ids) > 1: - raise DCOSException( - "Unable to shutdown the framework for [{}] because there " - "are multiple frameworks with the same name: [{}]. " - "Manually shut them down using 'dcos service " - "shutdown'.".format( - framework_name, - ', '.join(framework_ids))) - - return len(matching_apps) - - class InstalledPackage(object): """Represents an intalled DCOS package. One of `app` and `subcommand` must be supplied. @@ -332,14 +104,11 @@ class InstalledPackage(object): ret['command'] = {'name': self.subcommand.name} if self.apps: - ret['apps'] = [app['appId'] for app in self.apps] + ret['apps'] = sorted([app['appId'] for app in self.apps]) if self.subcommand: package_json = self.subcommand.package_json() ret.update(package_json) - - ret['packageSource'] = self.subcommand.package_source() - ret['releaseVersion'] = self.subcommand.package_revision() else: ret.update(self.apps[0]) ret.pop('appId') @@ -347,7 +116,21 @@ class InstalledPackage(object): return ret -def installed_packages(init_client, endpoints): +def _matches_package_name(name, command_name): + """ + :param name: the name of the package + :type name: str + :param command_name: the name of the command + :type command_name: str + :returns: True if the name is not defined or the package matches that name; + False otherwise + :rtype: bool + """ + + return name is None or command_name == name + + +def installed_packages(package_manager, app_id, package_name): """Returns all installed packages in the format: [{ @@ -360,32 +143,32 @@ def installed_packages(init_client, endpoints): :param init_client: The program to use to list packages :type init_client: object - :param endpoints: Whether to include a list of - endpoints as port-host pairs - :type endpoints: boolean - :returns: A list of installed packages - :rtype: [InstalledPackage] + :param app_id: App ID of app to show + :type app_id: str + :param package_name: The package to show + :type package_name: str + :returns: A list of installed packages matching criteria + :rtype: [dict] """ - apps = installed_apps(init_client, endpoints) - subcommands = installed_subcommands() - dicts = collections.defaultdict(lambda: {'apps': [], 'command': None}) + apps = package_manager.installed_apps(package_name, app_id) for app in apps: - key = (app['name'], app['releaseVersion'], app['packageSource']) + key = app['name'] dicts[key]['apps'].append(app) + subcommands = installed_subcommands() for subcmd in subcommands: - package_revision = subcmd.package_revision() - package_source = subcmd.package_source() - key = (subcmd.name, package_revision, package_source) - dicts[key]['command'] = subcmd + if _matches_package_name(package_name, subcmd.name): + dicts[subcmd.name]['command'] = subcmd - return [ + installed = [ InstalledPackage(pkg['apps'], pkg['command']) for pkg in dicts.values() ] + return [pkg.dict() for pkg in installed] + def installed_subcommands(): """Returns all installed subcommands. @@ -396,1243 +179,3 @@ def installed_subcommands(): return [subcommand.InstalledSubcommand(name) for name in subcommand.distributions()] - - -def installed_apps(init_client, endpoints=False): - """ - Returns all installed apps. An app is of the format: - - { - 'appId': , - 'packageSource': , - 'registryVersion': , - 'releaseVersion': - 'endpoints' (optional): [{ - 'host': , - 'ports': , - }] - .... - } - - :param init_client: The program to use to list packages - :type init_client: object - :param endpoints: Whether to include a list of - endpoints as port-host pairs - :type endpoints: boolean - :returns: all installed apps - :rtype: [dict] - """ - - apps = init_client.get_apps() - - encoded_apps = [(a['id'], a['labels']) - for a in apps - if a.get('labels', {}).get(PACKAGE_METADATA_KEY)] - - # Filter elements that failed to parse correctly as JSON - valid_apps = [] - for app_id, labels in encoded_apps: - try: - decoded = _decode_and_add_context(app_id, labels) - except Exception: - logger.exception( - 'Unable to decode package metadata during install: %s', - app_id) - - valid_apps.append(decoded) - - if endpoints: - for app in valid_apps: - tasks = init_client.get_tasks(app["appId"]) - app['endpoints'] = [{"host": t["host"], "ports": t["ports"]} - for t in tasks] - - return valid_apps - - -def _decode_and_add_context(app_id, labels): - """ Create an enhanced package JSON from Marathon labels - - { - 'appId': , - 'packageSource': , - 'registryVersion': , - 'releaseVersion': , - .... - } - - :param app_id: Marathon application id - :type app_id: str - :param labels: Marathon label dictionary - :type labels: dict - :rtype: dict - """ - - encoded = labels.get(PACKAGE_METADATA_KEY, {}) - decoded = base64.b64decode(six.b(encoded)).decode() - - decoded_json = util.load_jsons(decoded) - decoded_json['appId'] = app_id - decoded_json['packageSource'] = labels.get(PACKAGE_SOURCE_KEY) - decoded_json['releaseVersion'] = labels.get(PACKAGE_RELEASE_KEY) - - return decoded_json - - -def search(query, cfg): - """Returns a list of index entry collections, one for each registry in - the supplied config. - - :param query: The search term - :type query: str - :param cfg: Configuration dictionary - :type cfg: dcos.config.Toml - :rtype: [IndexEntries] - """ - - threshold = 0.5 # Minimum rank required to appear in results - results = [] - - def clean_package_entry(entry): - result = entry.copy() - result.update({ - 'versions': list(entry['versions'].keys()) - }) - return result - - for registry in registries(cfg): - source_results = [] - index = registry.get_index() - - for pkg in index['packages']: - rank = _search_rank(pkg, query) - if rank >= threshold: - source_results.append(clean_package_entry(pkg)) - - entries = IndexEntries(registry.source, source_results) - results.append(entries) - - return results - - -def _search_rank(pkg, query): - """ - :param pkg: Index entry to rank for affinity with the search term - :type pkg: object - :param query: Search term - :type query: str - :rtype: float - """ - result = 0.0 - - wildcard_symbol = '*' - regex_pattern = '.*' - - q = query.lower() - if wildcard_symbol in q: - q = q.replace(wildcard_symbol, regex_pattern) - if q.endswith(wildcard_symbol): - q = '^{}'.format(q) - else: - q = '{}$'.format(q) - - if re.match(q, pkg['name'].lower()): - result += 2.0 - return result - - if q in pkg['name'].lower(): - result += 2.0 - for tag in pkg['tags']: - if q in tag.lower(): - result += 1.0 - - if q in pkg['description'].lower(): - result += 0.5 - - return result - - -def _extract_default_values(config_schema): - """ - :param config_schema: A json-schema describing configuration options. - :type config_schema: dict - :returns: a dictionary with the default specified by the schema - :rtype: dict | None - """ - - defaults = {} - if 'properties' not in config_schema: - return None - - for key, value in config_schema['properties'].items(): - if isinstance(value, dict) and 'default' in value: - defaults[key] = value['default'] - elif isinstance(value, dict) and value.get('type', '') == 'object': - # Generate the default value from the embedded schema - defaults[key] = _extract_default_values(value) - - return defaults - - -def _merge_options(first, second, overrides=True): - """Merges the :code:`second` dictionary into the :code:`first` dictionary. - If both dictionaries have the same key and both values are dictionaries - then it recursively merges those two dictionaries. - - :param first: first dictionary - :type first: dict - :param second: second dictionary - :type second: dict - :param overrides: allow second to override first if both have same key - :type overrides: bool - :returns: merged dictionary - :rtype: dict - """ - - result = copy.deepcopy(first) - for key, second_value in second.items(): - if key in first: - first_value = first[key] - - if (isinstance(first_value, collections.Mapping) and - isinstance(second_value, collections.Mapping)): - result[key] = _merge_options(first_value, second_value) - elif not overrides and first_value != second_value: - raise DCOSException( - "Trying to override package.json's key {} to {}".format( - key, second_value)) - else: - result[key] = second_value - else: - result[key] = second_value - - return result - - -def resolve_package(package_name, config=None): - """Returns the first package with the supplied name found by looking at - the configured sources in the order they are defined. - - :param package_name: The name of the package to resolve - :type package_name: str - :param config: dcos config - :type config: dcos.config.Toml | None - :returns: The named package, if found - :rtype: Package - """ - - if not config: - config = util.get_config() - - for registry in registries(config): - package = registry.get_package(package_name) - if package: - return package - - return None - - -def registries(config): - """Returns configured cached package registries. - - :param config: Configuration dictionary - :type config: dcos.config.Toml - :returns: The list of registries, in resolution order - :rtype: [Registry] - """ - - sources = list_sources(config) - return [Registry(source, source.local_cache(config)) for source in sources] - - -def list_sources(config): - """List configured package sources. - - :param config: Configuration dictionary - :type config: dcos.config.Toml - :returns: The list of sources, in resolution order - :rtype: [Source] - """ - - source_uris = util.get_config_vals(['package.sources'], config)[0] - - sources = [url_to_source(s) for s in source_uris] - - errs = [source for source in sources if isinstance(source, Error)] - if errs: - raise DCOSException('\n'.join(err.error() for err in errs)) - - return sources - - -def url_to_source(url): - """Creates a package source from the supplied URL. - - :param url: Location of the package source - :type url: str - :returns: A Source backed by the supplied URL - :rtype: Source | Error - """ - - parse_result = urllib.parse.urlparse(url) - scheme = parse_result.scheme - - if scheme == 'file': - return FileSource(url) - elif scheme == 'http' or scheme == 'https': - return HttpSource(url) - elif scheme == 'git': - return GitSource(url) - else: - return Error("Source URL uses unsupported protocol [{}]".format(url)) - - -def _acquire_file_lock(lock_file_path): - """Acquires an exclusive lock on the supplied file. - - :param lock_file_path: Path to the lock file - :type lock_file_path: str - :returns: Lock file - :rtype: File - """ - - try: - lock_file = open(lock_file_path, 'w') - except IOError as e: - logger.exception('Failed to open lock file: %s', lock_file_path) - - raise util.io_exception(lock_file_path, e.errno) - - acquire_mode = portalocker.LOCK_EX | portalocker.LOCK_NB - - try: - portalocker.lock(lock_file, acquire_mode) - return lock_file - except portalocker.LockException: - logger.exception( - 'Failure while tring to aquire file lock: %s', - lock_file_path) - - lock_file.close() - raise DCOSException('Unable to acquire the package cache lock') - - -def update_sources(config, validate=False): - """Overwrites the local package cache with the latest source data. - - :param config: Configuration dictionary - :type config: dcos.config.Toml - :rtype: None - """ - - errors = [] - - # ensure the cache directory is properly configured - cache_dir = os.path.expanduser( - util.get_config_vals(['package.cache'], config)[0]) - - # ensure the cache directory exists - if not os.path.exists(cache_dir): - os.makedirs(cache_dir) - - if not os.path.isdir(cache_dir): - raise DCOSException( - 'Cache directory does not exist! [{}]'.format(cache_dir)) - - # obtain an exclusive file lock on $CACHE/.lock - lock_path = os.path.join(cache_dir, '.lock') - - with _acquire_file_lock(lock_path): - - # list sources - sources = list_sources(config) - - for source in sources: - - emitter.publish('Updating source [{}]'.format(source)) - - # create a temporary staging directory - with util.tempdir() as tmp_dir: - - stage_dir = os.path.join(tmp_dir, source.hash()) - - # copy to the staging directory - try: - source.copy_to_cache(stage_dir) - except DCOSException as e: - logger.exception( - 'Failed to copy universe source %s to cache %s', - source.url, - stage_dir) - - errors.append(str(e)) - continue - - # check version - # TODO(jsancio): move this to the validation when it is forced - Registry(source, stage_dir).check_version( - LooseVersion('1.0'), - LooseVersion('3.0')) - - # validate content - if validate: - validation_errors = Registry(source, stage_dir).validate() - if len(validation_errors) > 0: - errors += validation_errors - continue # keep updating the other sources - - # remove the $CACHE/source.hash() directory - target_dir = os.path.join(cache_dir, source.hash()) - try: - if os.path.exists(target_dir): - shutil.rmtree(target_dir, - onerror=_rmtree_on_error, - ignore_errors=False) - except OSError: - logger.exception( - 'Error removing target directory before move: %s', - target_dir) - - err = "Could not remove directory [{}]".format(target_dir) - errors.append(err) - continue # keep updating the other sources - - # move the staging directory to $CACHE/source.hash() - shutil.move(stage_dir, target_dir) - - if errors: - raise DCOSException(util.list_to_err(errors)) - - -class Source: - """A source of DCOS packages.""" - - @property - @abc.abstractmethod - def url(self): - """ - :returns: Location of the package source - :rtype: str - """ - - raise NotImplementedError - - def hash(self): - """Returns a cryptographically secure hash derived from this source. - - :returns: a hexadecimal string - :rtype: str - """ - - return hashlib.sha1(self.url.encode('utf-8')).hexdigest() - - def local_cache(self, config): - """Returns the file system path to this source's local cache. - - :param config: Configuration dictionary - :type config: dcos.config.Toml - :returns: Path to this source's local cache on disk - :rtype: str or None - """ - - cache_dir = os.path.expanduser( - util.get_config_vals(['package.cache'], config)[0]) - return os.path.join(cache_dir, self.hash()) - - def copy_to_cache(self, target_dir): - """Copies the source content to the supplied local directory. - - :param target_dir: Path to the destination directory. - :type target_dir: str - :rtype: None - """ - - raise NotImplementedError - - def __repr__(self): - - return self.url - - -class FileSource(Source): - """A registry of DCOS packages. - - :param url: Location of the package source - :type url: str - """ - - def __init__(self, url): - self._url = url - - @property - def url(self): - """ - :returns: Location of the package source - :rtype: str - """ - - return self._url - - def copy_to_cache(self, target_dir): - """Copies the source content to the supplied local directory. - - :param target_dir: Path to the destination directory. - :type target_dir: str - :rtype: None - """ - - # copy the source to the target_directory - parse_result = urllib.parse.urlparse(self._url) - source_dir = parse_result.path - try: - shutil.copytree(source_dir, target_dir) - return None - except OSError: - logger.exception( - 'Error copying source director [%s] to target directory [%s].', - source_dir, - target_dir) - - raise DCOSException( - 'Unable to fetch packages from [{}]'.format(self.url)) - - -class HttpSource(Source): - """A registry of DCOS packages. - - :param url: Location of the package source - :type url: str - """ - - def __init__(self, url): - self._url = url - - @property - def url(self): - """ - :returns: Location of the package source - :rtype: str - """ - - return self._url - - def copy_to_cache(self, target_dir): - """Copies the source content to the supplied local directory. - - :param target_dir: Path to the destination directory. - :type target_dir: str - :returns: The error, if one occurred - :rtype: None - """ - - try: - with util.tempdir() as tmp_dir: - - tmp_file = os.path.join(tmp_dir, 'packages.zip') - # Download the zip file. - req = http.get(self.url) - if req.status_code == 200: - with open(tmp_file, 'wb') as f: - for chunk in req.iter_content(1024): - f.write(chunk) - else: - raise Exception( - 'HTTP GET for {} did not return 200: {}'.format( - self.url, - req.status_code)) - - # Unzip the downloaded file. - packages_zip = zipfile.ZipFile(tmp_file, 'r') - packages_zip.extractall(tmp_dir) - - # Move the enclosing directory to the target directory - enclosing_dirs = [item - for item in os.listdir(tmp_dir) - if os.path.isdir( - os.path.join(tmp_dir, item))] - - # There should only be one directory present after extracting. - assert(len(enclosing_dirs) is 1) - - enclosing_dir = os.path.join(tmp_dir, enclosing_dirs[0]) - - shutil.copytree(enclosing_dir, target_dir) - - # Set appropriate file permissions on the scripts. - x_mode = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | - stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP) - - scripts_dir = os.path.join(target_dir, 'scripts') - scripts = os.listdir(scripts_dir) - - for script in scripts: - script_path = os.path.join(scripts_dir, script) - if os.path.isfile(script_path): - os.chmod(script_path, x_mode) - - return None - - except Exception: - logger.exception('Unable to fetch packages from URL: %s', self.url) - - raise DCOSException( - 'Unable to fetch packages from [{}]'.format(self.url)) - - -class GitSource(Source): - """A registry of DCOS packages. - - :param url: Location of the package source - :type url: str - """ - - def __init__(self, url): - self._url = url - - @property - def url(self): - """ - :returns: Location of the package source - :rtype: str - """ - - return self._url - - def copy_to_cache(self, target_dir): - """Copies the source content to the supplied local directory. - - :param target_dir: Path to the destination directory. - :type target_dir: str - :returns: The error, if one occurred - :rtype: None - """ - - try: - # TODO(SS): add better url parsing - - # Ensure git is installed properly. - git_program = util.which('git') - if git_program is None: - raise DCOSException("""Could not locate the git program. Make sure \ -it is installed and on the system search path. -PATH = {}""".format(os.environ[constants.PATH_ENV])) - - # Clone git repo into the supplied target directory. - git.Repo.clone_from(self._url, - to_path=target_dir, - progress=None, - branch='master') - - # Remove .git directory to save space. - shutil.rmtree(os.path.join(target_dir, ".git"), - onerror=_rmtree_on_error) - return None - - except git.exc.GitCommandError: - logger.exception('Unable to fetch packages from git: %s', self.url) - - raise DCOSException( - 'Unable to fetch packages from [{}]'.format(self.url)) - - -def _rmtree_on_error(func, path, exc_info): - """Error handler for ``shutil.rmtree``. - If the error is due to an access error (read only file) - it attempts to add write permission and then retries. - If the error is for another reason it re-raises the error. - - Usage : ``shutil.rmtree(path, onerror=onerror)``. - - :param func: Function which raised the exception. - :type func: function - :param path: The path name passed to ``shutil.rmtree`` function. - :type path: str - :param exc_info: Information about the last raised exception. - :type exc_info: tuple - :rtype: None - """ - import stat - if not os.access(path, os.W_OK): - os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - func(path) - else: - raise - - -class Error(errors.Error): - """Class for describing errors during packaging operations. - - :param message: Error message - :type message: str - """ - - def __init__(self, message): - self._message = message - - def error(self): - """Return error message - - :returns: The error message - :rtype: str - """ - - return self._message - - -class Registry(): - """Represents a package registry on disk. - - :param base_path: Path to the registry - :type base_path: str - :param source: The associated package source - :type source: Source - """ - - def __init__(self, source, base_path): - self._base_path = base_path - self._source = source - - def validate(self): - """Validates a package registry. - - :returns: Validation errors - :rtype: [str] - """ - - # TODO(CD): implement these checks in pure Python? - scripts_dir = os.path.join(self._base_path, 'scripts') - if util.is_windows_platform(): - validate_script = os.path.join(scripts_dir, - '1-validate-packages.ps1') - cmd = ['powershell', '-ExecutionPolicy', - 'ByPass', '-File', validate_script] - result = subprocess.call(cmd) - else: - validate_script = os.path.join(scripts_dir, - '1-validate-packages.sh') - result = subprocess.call(validate_script) - if result is not 0: - return ["Source tree is not valid [{}]".format(self._base_path)] - else: - return [] - - @property - def source(self): - """Returns the associated upstream package source for this registry. - - :rtype: Source - """ - - return self._source - - def check_version(self, min_version, max_version): - """Checks that the version is [min_version, max_version) - - :param min_version: the min version inclusive - :type min_version: LooseVersion - :param max_version: the max version exclusive - :type max_version: LooseVersion - :returns: None - """ - - version = LooseVersion(self.get_version()) - if not (version >= min_version and - version < max_version): - raise DCOSException(( - 'Unable to update source [{}] because version {} is ' - 'not supported. Supported versions are between {} and ' - '{}. Please update your DCOS CLI.').format( - self._source.url, - version, - min_version, - max_version)) - - def get_version(self): - """Returns the version of this registry. - - :rtype: str - """ - - # The package version is found in $BASE/repo/meta/version.json - index_path = os.path.join( - self._base_path, - 'repo', - 'meta', - 'version.json') - - if not os.path.isfile(index_path): - raise DCOSException('Path [{}] is not a file'.format(index_path)) - - try: - with util.open_file(index_path) as fd: - version_json = json.load(fd) - return version_json.get('version') - except ValueError: - logger.exception('Unable to parse JSON: %s', index_path) - - raise DCOSException('Unable to parse [{}]'.format(index_path)) - - def get_index(self): - """Returns the index of packages in this registry. - - :rtype: dict - """ - - # The package index is found in $BASE/repo/meta/index.json - index_path = os.path.join( - self._base_path, - 'repo', - 'meta', - 'index.json') - - if not os.path.isfile(index_path): - raise DCOSException('Path [{}] is not a file'.format(index_path)) - - try: - with util.open_file(index_path) as fd: - return json.load(fd) - except ValueError: - logger.exception('Unable to parse JSON: %s', index_path) - - raise DCOSException('Unable to parse [{}]'.format(index_path)) - - def get_package(self, package_name): - """Returns the named package, if it exists. - - :param package_name: The name of the package to fetch - :type package_name: str - :returns: The requested package - :rtype: Package - """ - - if len(package_name) is 0: - raise DCOSException('Package name must not be empty.') - - # Packages are found in $BASE/repo/package// - first_character = package_name[0].title() - - package_path = os.path.join( - self._base_path, - 'repo', - 'packages', - first_character, - package_name) - - if not os.path.isdir(package_path): - return None - - try: - return Package(self, package_path) - except: - logger.exception('Unable to read package: %s', package_path) - - raise DCOSException( - 'Could not read package [{}]'.format(package_name)) - - -class Package(): - """Interface to a package on disk. - - :param registry: The containing registry for this package. - :type registry: Registry - :param path: Path to the package description on disk - :type path: str - """ - - def __init__(self, registry, path): - assert os.path.isdir(path) - self._registry = registry - self.path = path - - def name(self): - """Returns the package name. - - :returns: The name of this package - :rtype: str - """ - - return os.path.basename(self.path) - - def options(self, revision, user_options): - """Merges package options with user supplied options, validates, and - returns the result. - - :param revision: the package revision to install - :type revision: str - :param user_options: package parameters - :type user_options: dict - :returns: a dictionary with the user supplied options - :rtype: dict - """ - - if user_options is None: - user_options = {} - - config_schema = self.config_json(revision) - default_options = _extract_default_values(config_schema) - if default_options is None: - pkg = self.package_json(revision) - msg = ("An object in the package's config.json is missing the " - "required 'properties' feature:\n {}".format(config_schema)) - if 'maintainer' in pkg: - msg += "\nPlease contact the project maintainer: {}".format( - pkg['maintainer']) - raise DCOSException(msg) - - logger.info('Generated default options: %r', default_options) - - # Merge option overrides, second argument takes precedence - options = _merge_options(default_options, user_options) - - logger.info('Merged options: %r', options) - - # Validate options with the config schema - errs = util.validate_json(options, config_schema) - if len(errs) != 0: - raise DCOSException( - "{}\n\n{}".format( - util.list_to_err(errs), - 'Please create a JSON file with the appropriate options, ' - 'and pass the /path/to/file as an --options argument.')) - - return options - - @property - def registry(self): - """Returns the containing registry for this package. - - :rtype: Registry - """ - - return self._registry - - def has_definition(self, revision, filename): - """Returns true if the package defines filename; false otherwise. - - :param revision: package revision - :type revision: str - :param filename: file in package definition - :type filename: str - :returns: whether filename is defined - :rtype: bool - """ - - return os.path.isfile( - os.path.join( - self.path, - os.path.join(revision, filename))) - - def has_command_definition(self, revision): - """Returns true if the package defines a command; false otherwise. - - :param revision: package revision - :type revision: str - :rtype: bool - """ - - return self.has_definition(revision, 'command.json') - - def _has_resource_definition(self, revision): - """Returns true if the package defines a resource; false otherwise. - - :param revision: package revision - :type revision: str - :rtype: bool - """ - - return self.has_definition(revision, 'resource.json') - - def has_marathon_definition(self, revision): - """Returns true if the package defines a Marathon json. false otherwise. - - :param revision: package revision - :type revision: str - :rtype: bool - """ - - return self.has_definition(revision, 'marathon.json') - - def has_marathon_mustache_definition(self, revision): - """Returns true if the package defines a Marathon.json.mustache false - otherwise. - - :param revision: package revision - :type revision: str - :rtype: bool - """ - - return self.has_definition(revision, 'marathon.json.mustache') - - def _get_marathon_json_file(self, revision): - """Returns the file name of Marathon json - - :param revision: package revision - :type revision: str - :returns: Marathon file name - :rtype: str - """ - if self.has_marathon_definition(revision): - return 'marathon.json' - elif self.has_marathon_mustache_definition(revision): - return 'marathon.json.mustache' - else: - raise DCOSException("Missing Marathon json definition of package") - - def config_json(self, revision): - """Returns the JSON content of the config.json file. - - :param revision: package revision - :type revision: str - :returns: Package config schema - :rtype: dict - """ - - return self._json(revision, 'config.json') - - def package_json(self, revision): - """Returns the JSON content of the package.json file. - - :param revision: the package revision - :type revision: str - :returns: Package data - :rtype: dict - """ - - return self._json(revision, 'package.json') - - def _resource_json(self, revision): - """Returns the JSON content of the resource.json file. - - :param revision: the package revision - :type revision: str - :returns: Package data - :rtype: dict - """ - - return self._json(revision, 'resource.json') - - def marathon_json(self, revision, options): - """Returns the JSON content of the marathon.json template, after - rendering it with options. - - :param revision: the package revision - :type revision: str - :param options: the template options to use in rendering - :type options: dict - :rtype: dict - """ - - marathon_file = self._get_marathon_json_file(revision) - if self.has_marathon_mustache_definition(revision) and \ - self._has_resource_definition(revision): - resources = {"resource": self._resource_json(revision)} - options = _merge_options(options, resources, False) - init_desc = self._render_template( - marathon_file, - revision, - options) - - # Add package metadata - package_labels = _make_package_labels(self, revision, options) - - # Preserve existing labels - labels = init_desc.get('labels', {}) - - labels.update(package_labels) - init_desc['labels'] = labels - - return init_desc - - def command_json(self, revision, options): - """Returns the JSON content of the comand.json template, after - rendering it with options. - - :param revision: the package revision - :type revision: str - :param options: the template options to use in rendering - :type options: dict - :returns: Package data - :rtype: dict - """ - - template = self._data(revision, 'command.json') - rendered = pystache.render(template, options) - return json.loads(rendered) - - def marathon_template(self, revision): - """ Returns raw data from marathon.json - - :param revision: the package revision - :type revision: str - :returns: raw data from marathon.json - :rtype: str - """ - return self._data(revision, self._get_marathon_json_file(revision)) - - def command_template(self, revision): - """ Returns raw data from command.json - - :param revision: the package revision - :type revision: str - :returns: raw data from command.json - :rtype: str - """ - return self._data(revision, 'command.json') - - def _render_template(self, name, revision, options): - """Render a template. - - :param name: the file name of the template - :type name: str - :param revision: the package revision - :type revision: str - :param options: the template options to use in rendering - :type options: dict - :rtype: dict - """ - - template = self._data(revision, name) - return util.render_mustache_json(template, options) - - def _json(self, revision, name): - """Returns the json content of the file named `name` in the directory - named `revision` - - :param revision: the package revision - :type revision: str - :param name: file name - :type name: str - :rtype: dict - """ - - data = self._data(revision, name) - return util.load_jsons(data) - - def _data(self, revision, name): - """Returns the content of the file named `name` in the directory named - `revision` - - :param revision: the package revision - :type revision: str - :param name: file name - :type name: str - :returns: File content of the supplied path - :rtype: str - """ - - path = os.path.join(revision, name) - full_path = os.path.join(self.path, path) - return util.read_file(full_path) - - def package_revisions(self): - """Returns all of the available package revisions, most recent first. - - :returns: Available revisions of this package - :rtype: [str] - """ - - vs = sorted((f for f in os.listdir(self.path) - if not f.startswith('.')), key=int, reverse=True) - return vs - - def package_revisions_map(self): - """Returns an ordered mapping from the package revision to the package - version, sorted by package revision. - - :returns: Map from package revision to package version - :rtype: OrderedDict - - """ - - package_version_map = collections.OrderedDict() - for rev in self.package_revisions(): - pkg_json = self.package_json(rev) - package_version_map[rev] = pkg_json['version'] - return package_version_map - - def latest_package_revision(self, package_version=None): - """Returns the most recent package revision, for a - given package version if specified. - - :param package_version: a given package version - :type package_version: str - :returns: package revision - :rtype: str | None - """ - - if package_version: - pkg_rev_map = self.package_revisions_map() - # depends on package_revisions() returning an OrderedDict - if package_version in pkg_rev_map.values(): - return next(pkg_rev for pkg_rev in reversed(pkg_rev_map) - if pkg_rev_map[pkg_rev] == package_version) - else: - return None - else: - pkg_revisions = self.package_revisions() - revision = pkg_revisions[0] - - return revision - - def __repr__(self): - - rev = self.latest_package_revision() - pkg_json = self.package_json(rev) - - return json.dumps(pkg_json) - - -class IndexEntries(): - """A collection of package index entries from a single source. - Each entry is a dict as described by the JSON schema for the package index: - https://github.com/mesosphere/universe/blob/master/repo/meta/schema/index-schema.json - - :param source: The source of these index entries - :type source: Source - :param packages: The index entries - :type packages: [dict] - """ - - def __init__(self, source, packages): - self._source = source - self._packages = packages - - @property - def source(self): - """Returns the source of these index entries. - - :rtype: Source - """ - - return self._source - - @property - def packages(self): - """Returns the package index entries. - - :rtype: list of dict - """ - - return self._packages - - def as_dict(self): - """ - :rtype: dict - """ - - return {'source': self.source.url, 'packages': self.packages} - - -def get_apps_for_framework(framework_name, client): - """ Return all apps running the given framework. - - :param framework_name: framework name - :type framework_name: str - :param client: marathon client - :type client: marathon.Client - :rtype: [dict] - """ - - return [app for app in client.get_apps() - if app.get('labels', {}).get( - PACKAGE_FRAMEWORK_NAME_KEY) == framework_name] diff --git a/dcos/subcommand.py b/dcos/subcommand.py index 05ba9d2..ef7ca36 100644 --- a/dcos/subcommand.py +++ b/dcos/subcommand.py @@ -45,7 +45,7 @@ def get_package_commands(package_name): :returns: list of all the dcos program paths in package :rtype: [str] """ - bin_dir = os.path.join(package_dir(package_name), + bin_dir = os.path.join(_package_dir(package_name), constants.DCOS_SUBCOMMAND_VIRTUALENV_SUBDIR, BIN_DIRECTORY) @@ -178,75 +178,37 @@ def noun(executable_path): return noun -def _write_package_json(pkg, revision): +def _write_package_json(pkg): """ Write package.json locally. :param pkg: the package being installed - :type pkg: Package - :param revision: the package revision to install - :type revision: str + :type pkg: PackageVersion :rtype: None """ - pkg_dir = package_dir(pkg.name()) + pkg_dir = _package_dir(pkg.name()) package_path = os.path.join(pkg_dir, 'package.json') - package_json = pkg.package_json(revision) + package_json = pkg.package_json() with util.open_file(package_path, 'w') as package_file: json.dump(package_json, package_file) -def _write_package_revision(pkg, revision): - """ Write package revision locally. - - :param pkg: the package being installed - :type pkg: Package - :param revision: the package revision to install - :type revision: str - :rtype: None - """ - - pkg_dir = package_dir(pkg.name()) - - revision_path = os.path.join(pkg_dir, 'version') - - with util.open_file(revision_path, 'w') as revision_file: - revision_file.write(revision) - - -def _write_package_source(pkg): - """ Write package source locally. - - :param pkg: the package being installed - :type pkg: Package - :rtype: None - """ - - pkg_dir = package_dir(pkg.name()) - - source_path = os.path.join(pkg_dir, 'source') - - with util.open_file(source_path, 'w') as source_file: - source_file.write(pkg.registry.source.url) - - -def _install_env(pkg, revision, options): +def _install_env(pkg, options): """ Install subcommand virtual env. :param pkg: the package to install - :type pkg: Package - :param revision: the package revision to install - :type revision: str + :type pkg: PackageVersion :param options: package parameters :type options: dict :rtype: None """ - pkg_dir = package_dir(pkg.name()) + pkg_dir = _package_dir(pkg.name()) - install_operation = pkg.command_json(revision, options) + install_operation = pkg.command_json(options) env_dir = os.path.join(pkg_dir, constants.DCOS_SUBCOMMAND_VIRTUALENV_SUBDIR) @@ -261,26 +223,22 @@ def _install_env(pkg, revision, options): install_operation.keys())) -def install(pkg, revision, options): +def install(pkg, options): """Installs the dcos cli subcommand :param pkg: the package to install :type pkg: Package - :param revision: the package revision to install - :type revision: str :param options: package parameters :type options: dict :rtype: None """ - pkg_dir = package_dir(pkg.name()) + pkg_dir = _package_dir(pkg.name()) util.ensure_dir_exists(pkg_dir) - _write_package_json(pkg, revision) - _write_package_revision(pkg, revision) - _write_package_source(pkg) + _write_package_json(pkg) - _install_env(pkg, revision, options) + _install_env(pkg, options) def _subcommand_dir(): @@ -290,8 +248,7 @@ def _subcommand_dir(): constants.DCOS_SUBCOMMAND_SUBDIR)) -# TODO(mgummelt): should be made private after "dcos subcommand" is removed -def package_dir(name): +def _package_dir(name): """ Returns ~/.dcos/subcommands/ :param name: package name @@ -311,7 +268,7 @@ def uninstall(package_name): :rtype: bool """ - pkg_dir = package_dir(package_name) + pkg_dir = _package_dir(package_name) if os.path.isdir(pkg_dir): shutil.rmtree(pkg_dir) @@ -449,16 +406,16 @@ class InstalledSubcommand(object): :rtype: str """ - return package_dir(self.name) + return _package_dir(self.name) def package_revision(self): """ - :returns: this subcommand's revision. + :returns: this subcommand's version. :rtype: str """ - revision_path = os.path.join(self._dir(), 'version') - return util.read_file(revision_path) + version_path = os.path.join(self._dir(), 'version') + return util.read_file(version_path) def package_source(self): """ diff --git a/dcos/util.py b/dcos/util.py index a5c9331..fab9ca8 100644 --- a/dcos/util.py +++ b/dcos/util.py @@ -19,6 +19,8 @@ import six from dcos import constants from dcos.errors import DCOSException +from six.moves import urllib + def get_logger(name): """Get a logger @@ -700,4 +702,16 @@ def validate_png(filename): 'Unable to validate [{}] as a PNG file'.format(filename)) +def normalize_app_id(app_id): + """Normalizes the application id. + + :param app_id: raw application ID + :type app_id: str + :returns: normalized application ID + :rtype: str + """ + + return urllib.parse.quote('/' + app_id.strip('/')) + + logger = get_logger(__name__) diff --git a/tests/test_package.py b/tests/test_package.py deleted file mode 100644 index 095d8d5..0000000 --- a/tests/test_package.py +++ /dev/null @@ -1,117 +0,0 @@ -import collections - -from dcos import package -from dcos.errors import DCOSException - -import pytest - -MergeData = collections.namedtuple( - 'MergeData', - ['first', 'second', 'expected']) - - -@pytest.fixture(params=[ - MergeData( - first={}, - second={'a': 1}, - expected={'a': 1}), - MergeData( - first={'a': 'a'}, - second={'a': 1}, - expected={'a': 1}), - MergeData( - first={'b': 'b'}, - second={'a': 1}, - expected={'b': 'b', 'a': 1}), - MergeData( - first={'b': 'b'}, - second={}, - expected={'b': 'b'}), - MergeData( - first={'b': {'a': 'a'}}, - second={'b': {'c': 'c'}}, - expected={'b': {'c': 'c', 'a': 'a'}}), - MergeData( - first={'b': 'c'}, - second={'b': 'd'}, - expected={'b': 'd'}), - ]) -def merge_data(request): - return request.param - - -def test_options_merge_wont_override(): - with pytest.raises(DCOSException): - package._merge_options({'b': 'c'}, {'b': 'd'}, False) - - -def test_option_merge(merge_data): - assert merge_data.expected == package._merge_options( - merge_data.first, - merge_data.second) - - -DefaultConfigValues = collections.namedtuple( - 'DefaultConfigValue', - ['schema', 'expected']) - - -@pytest.fixture(params=[ - DefaultConfigValues( - schema={ - "type": "object", - "properties": { - "foo": { - "type": "object", - "properties": { - "bar": { - "type": "string", - "description": "A bar name." - }, - "baz": { - "type": "integer", - "description": "How many times to do baz.", - "minimum": 0, - "maximum": 16, - "required": False, - "default": 4 - } - } - }, - "fiz": { - "type": "boolean", - "default": True, - }, - "buz": { - "type": "string" - } - } - }, - expected={'foo': {'baz': 4}, 'fiz': True}), - DefaultConfigValues( - schema={ - "type": "object", - "properties": { - "fiz": { - "type": "boolean", - "default": True, - }, - "additionalProperties": False - } - }, - expected={'fiz': True}), - DefaultConfigValues( - schema={ - "type": "object", - }, - expected=None)]) -def config_value(request): - return request.param - - -def test_extract_default_values(config_value): - try: - result = package._extract_default_values(config_value.schema) - except DCOSException as e: - result = str(e) - assert result == config_value.expected diff --git a/win_bin/install/install-dcos-cli.ps1 b/win_bin/install/install-dcos-cli.ps1 index 9ce0eca..133bb3c 100644 --- a/win_bin/install/install-dcos-cli.ps1 +++ b/win_bin/install/install-dcos-cli.ps1 @@ -103,10 +103,6 @@ $env:DCOS_CONFIG = $DCOS_CONFIG dcos config set core.reporting true dcos config set core.dcos_url $dcos_url dcos config set core.timeout 5 -dcos config set package.cache $env:temp\dcos\package-cache -dcos config set package.sources '[\"https://universe.mesosphere.com/repo\"]' - -dcos package update $ACTIVATE_PATH="$installation_path\Scripts\activate.ps1" diff --git a/win_bin/install/legacy/install-legacy-dcos-cli.ps1 b/win_bin/install/legacy/install-legacy-dcos-cli.ps1 new file mode 100644 index 0000000..ed7ce39 --- /dev/null +++ b/win_bin/install/legacy/install-legacy-dcos-cli.ps1 @@ -0,0 +1,147 @@ +param([Parameter(Mandatory=$true,ValueFromPipeline=$true)] + [string] + $installation_path, + [Parameter(Mandatory=$true,ValueFromPipeline=$true)] + [string] + $dcos_url, + [string] + $add_path + ) + +if (-Not(Get-Command python -errorAction SilentlyContinue)) +{ + echo "The program 'python' could not be found. Make sure that 'python' is installed and that its directory is included in the PATH system variable." + exit 1 +} + +$PYTHON_VERSION = (python --version) 2>&1 + +if ($PYTHON_VERSION -match "[0-9]+.[0-9]+") { + $PYTHON_VERSION = $matches[0] + + if (-Not (($PYTHON_VERSION -eq "2.7") -Or ($PYTHON_VERSION -eq "3.4"))) { + echo "Python must be version 2.7 or 3.4. Aborting." + exit 1 + } +} + +if (-Not(Get-Command pip -errorAction SilentlyContinue)) +{ + echo "The program 'pip' could not be found. Make sure that 'pip' is installed and that its directory (eg 'C:\Python27\Scripts') is included in the PATH system variable." + exit 1 +} + +$PIP_VERSION = (pip -V) + +"$PIP_VERSION" -match "[0-9]+\.[0-9]+" + +if ([double]$matches[0] -le 1.4) { + echo "Pip version must be greater than 1.4. Aborting." + exit 1 +} + +if (-Not(Get-Command virtualenv -errorAction SilentlyContinue)) +{ + echo "The program 'virtualenv' could not be found. Make sure that it has been installed with the 'pip' Python package program." + exit 1 +} + +$VIRTUAL_ENV_VERSION = (virtualenv --version) + +$VIRTUAL_ENV_VERSION -match "[0-9]+" + +if ($matches[0] -lt 12) { + echo "Virtualenv version must be 12 or greater. Aborting." + exit 1 +} + +if (-Not(Get-Command git -errorAction SilentlyContinue)) +{ + echo "The program 'git' could not be found. Make sure that 'git' is installed and that its directory is included in the PATH system variable." + exit 1 +} + +echo "Installing DCOS CLI from PyPI..." +echo "" + +if (-Not([System.IO.Path]::IsPathRooted("$installation_path"))) { + $installation_path = Join-Path (pwd) $installation_path +} + +if (-Not( Test-Path $installation_path)) { + mkdir $installation_path +} + +& virtualenv $installation_path +& $installation_path\Scripts\activate + +[int]$PYTHON_ARCHITECTURE=(python -c 'import struct;print( 8 * struct.calcsize(\"P\"))') + +if ($PYTHON_ARCHITECTURE -eq 64) { + & $installation_path\Scripts\easy_install "http://downloads.sourceforge.net/project/pywin32/pywin32/Build%20219/pywin32-219.win-amd64-py$PYTHON_VERSION.exe" 2>&1 | out-null +} else { + & $installation_path\Scripts\easy_install "http://downloads.sourceforge.net/project/pywin32/pywin32/Build%20219/pywin32-219.win32-py$PYTHON_VERSION.exe" 2>&1 | out-null +} + +if ($env:DCOS_CLI_VERSION) { + & $installation_path\Scripts\pip install --quiet "dcoscli==$env:DCOS_CLI_VERSION" +} else { + & $installation_path\Scripts\pip install --quiet "dcoscli<0.4.0" +} + +$env:Path="$env:Path;$installation_path\Scripts\" + +$DCOS_CONFIG="$env:USERPROFILE\.dcos\dcos.toml" + +if (-Not(Test-Path $DCOS_CONFIG)) { + mkdir "$env:USERPROFILE\.dcos" + New-Item $DCOS_CONFIG -type file +} +[Environment]::SetEnvironmentVariable("DCOS_CONFIG", "$DCOS_CONFIG", "User") +$env:DCOS_CONFIG = $DCOS_CONFIG + +dcos config set core.reporting true +dcos config set core.dcos_url $dcos_url +dcos config set core.timeout 5 +dcos config set package.cache $env:temp\dcos\package-cache +dcos config set package.sources '[\"https://github.com/mesosphere/universe/archive/version-1.x.zip\"]' + +dcos package update + +$ACTIVATE_PATH="$installation_path\Scripts\activate.ps1" + +function AddToPath ($AddedLocation) +{ + $Reg = "Registry::HKCU\Environment" + $OldPath = (Get-ItemProperty -Path "$Reg" -Name PATH).Path + $NewPath = $OldPath + ';' + $AddedLocation + Set-ItemProperty -Path "$Reg" -Name PATH –Value $NewPath + $script:ACTIVATE_PATH="activate.ps1" +} + +function PromptAddToPath ($AddedLocation) +{ + $message = "Modify your Environment to add DCOS to your PATH?" + $yes = New-Object System.Management.Automation.Host.ChoiceDescription "&Yes", ` + "Yes, add DCOS to PATH." + $no = New-Object System.Management.Automation.Host.ChoiceDescription "&No", ` + "No, do not add DCOS to PATH." + $options = [System.Management.Automation.Host.ChoiceDescription[]]($yes, $no) + $result = $host.ui.PromptForChoice("", $message, $options, 0) + if ($result -eq 0) + { + AddToPath "$AddedLocation" + } +} + +switch -regex ($add_path) +{ + "[Yy].*" {AddToPath "$installation_path\Scripts"; break} + "[Nn].*" {break} + default {PromptAddToPath "$installation_path\Scripts"} +} + +echo "Finished installing and configuring DCOS CLI." +echo "" +echo "Run this command to set up your environment and to get started:" +echo "& $ACTIVATE_PATH; dcos help"