Run tests with CCM

Dockerfile.test

@@ -0,0 +1,16 @@
+# This image runs the dcos-cli test suite.
+
+FROM ubuntu:15.04
+MAINTAINER support@mesosphere.com
+
+RUN apt-get update && apt-get install -y \
+  httpie \
+  jq \
+  make \
+  virtualenv \
+  openssh-client \
+  git \
+  sudo
+
+ADD . /dcos-cli
+WORKDIR /dcos-cli

bin/start_tests.sh

@@ -1,7 +1,12 @@
 #!/bin/bash -x
 
-# move the dcos package
-cd dcos-cli
+# This script expects the following env vars:
+#   DCOS_URL
+#   DCOS_CONFIG (this path will be overwritten)
+#   CLI_TEST_SSH_KEY_PATH (path to cluster ssh key)
+#   CLI_TEST_MASTER_PROXY (true or false, depending on if DCOS_URL points to an AWS cluster)
+#
+# CWD is assumed to be the dcos-cli repo root
 
 make clean env
 source env/bin/activate
@@ -12,7 +17,6 @@ deactivate
 cd cli
 
 cp tests/data/dcos.toml $DCOS_CONFIG
-echo "$VBOX_IP dcos.snakeoil.mesosphere.com" | sudo tee -a /etc/hosts > /dev/null
 
 make clean env
 source env/bin/activate

ccm/delete_ccm_cluster.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# This script expects the following env var:
+#   CCM_AUTH_TOKEN
+#   CLUSTER_ID
+
+set -e
+set -o pipefail
+set -x
+
+http --ignore-stdin DELETE https://ccm.mesosphere.com/api/cluster/${CLUSTER_ID}/ Authorization:"Token ${CCM_AUTH_TOKEN}"

ccm/start_ccm_cluster.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# This script expects the following env var:
+#   CLUSTER_NAME
+#   CCM_AUTH_TOKEN
+
+set -e
+set -x
+
+# create cluster
+CLUSTER_ID=$(http --ignore-stdin \https://ccm.mesosphere.com/api/cluster/ \
+     Authorization:"Token ${CCM_AUTH_TOKEN}" \
+     name=$CLUSTER_NAME \
+     cloud_provider=0 \
+     region=us-west-2 \
+     time=60 \
+     channel=stable \
+     cluster_desc="DCOS CLI testing cluster" \
+     template=single-master.cloudformation.json \
+     adminlocation=0.0.0.0/0 \
+     public_agents=0 \
+     private_agents=1 | \
+     jq ".id");
+
+echo $CLUSTER_ID

ccm/tests_ccm_cluster.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# This script expects the following env var:
+#   CLI_TEST_SSH_KEY_PATH
+
+set -e
+set -o pipefail
+set -x
+
+# run tests
+DCOS_PRODUCTION=false \
+CLI_TEST_MASTER_PROXY=true \
+CLI_TEST_SSH_KEY_PATH=/dcos-cli/mesosphere-aws.key \
+./bin/start_tests.sh

ccm/wait_for_ccm_cluster.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# This script expects the following env var:
+#   CCM_AUTH_TOKEN
+#   CLUSTER_ID
+
+set -e
+set -o pipefail
+set -x
+
+# wait for cluster to come up
+while true; do
+    STATUS=$(http --ignore-stdin \
+                  https://ccm.mesosphere.com/api/cluster/${CLUSTER_ID}/ \
+                  Authorization:"Token ${CCM_AUTH_TOKEN}" | \
+                    jq ".status");
+    if [ $STATUS -eq 0 ]; then
+        break;
+    fi;
+    sleep 10;
+done;
+
+# get dcos_url
+CLUSTER_INFO=$(http GET https://ccm.mesosphere.com/api/cluster/${CLUSTER_ID}/ Authorization:"Token ${CCM_AUTH_TOKEN}" | jq ".cluster_info")
+eval CLUSTER_INFO=$CLUSTER_INFO  # unescape json
+
+DCOS_URL=$(echo "$CLUSTER_INFO" | jq ".MastersIpAddresses[0]")
+DCOS_URL=${DCOS_URL:1:-1} # remove JSON string quotes
+echo $DCOS_URL

cli/dcoscli/data/config-schema/core.json

@@ -43,10 +43,10 @@
             "type": "string"
         },
         "ssl_verify": {
-                "type": "string",
-                "default": "false",
-                "title": "SSL Verification",
-                "description": "Whether or not to verify SSL certs for HTTPS or path to cert(s)"
+            "type": "string",
+            "default": "false",
+            "title": "SSL Verification",
+            "description": "Whether or not to verify SSL certs for HTTPS or path to cert(s)"
         }
     },
     "type": "object"

cli/tests/data/file/follow.json

@@ -1,7 +1,7 @@
 {
-  "id": "follow",
-  "cmd": "sleep 5 && echo \"follow_this\" && sleep 1000",
-  "cpus": 0.1,
-  "mem": 16,
-  "instances": 1
+    "id": "follow",
+    "cmd": "while true; do echo 'follow_this'; sleep 1; done;",
+    "cpus": 0.1,
+    "mem": 16,
+    "instances": 1
 }

cli/tests/data/file/two_tasks_follow.json

@@ -1,7 +1,7 @@
 {
-  "id": "two-tasks-follow",
-  "cmd": "sleep 5 && echo \"follow_this\" && sleep 1000",
-  "cpus": 0.1,
-  "mem": 16,
-  "instances": 2
+    "id": "two-tasks-follow",
+    "cmd": "while true; do echo 'follow_this'; sleep 1; done;",
+    "cpus": 0.1,
+    "mem": 16,
+    "instances": 2
 }

cli/tests/data/marathon/apps/sleep_many_instances.json

@@ -3,7 +3,7 @@
   "cmd": "sleep 1000",
   "cpus": 0.1,
   "mem": 16,
-  "instances": 10,
+  "instances": 100,
   "labels": {
     "PACKAGE_ID": "test-app",
     "PACKAGE_VERSION": "1.2.3"

cli/tests/fixtures/node.py

@@ -28,7 +28,6 @@ def slave_fixture():
         "pid": "slave(1)@172.17.8.101:5051",
         "registered_time": 1435625024.42234,
         "reregistered_time": 1435625024.42234,
-        "reserved_resources": {},
         "resources": {
             "cpus": 4.0,
             "disk": 10823.0,
@@ -36,12 +35,8 @@ def slave_fixture():
             "ports": ("[1025-2180, 2182-3887, 3889-5049, 5052-8079, " +
                       "8082-8180, 8182-65535]")
         },
-        "unreserved_resources": {
-            "cpus": 4.0,
-            "disk": 10823.0,
-            "mem": 2933.0,
-            "ports": "[1025-2180, 2182-3887, 3889-5049, 5052-32000]"
-        },
+        "reserved_resources": {},
+        "unreserved_resources": {},
         "used_resources": {
             "cpus": 0.0,
             "disk": 0.0,

cli/tests/integrations/common.py

@@ -495,6 +495,7 @@ def package(package_name, deploy=False, args=[]):
         yield
     finally:
         package_uninstall(package_name)
+        watch_all_deployments()
 
 
 @contextlib.contextmanager
@@ -540,21 +541,29 @@ def popen_tty(cmd):
 
 
 def ssh_output(cmd):
-    """ Runs an SSH command and returns the stdout/stderr.
+    """ Runs an SSH command and returns the stdout/stderr/returncode.
 
     :param cmd: command to run
     :type cmd: str
-    :rtype: (str, str)
+    :rtype: (str, str, int)
     """
 
+    print('SSH COMMAND: {}'.format(cmd))
+
     # ssh must run with stdin attached to a tty
     proc, master = popen_tty(cmd)
 
     # wait for the ssh connection
     time.sleep(8)
 
+    proc.poll()
+    returncode = proc.returncode
+
     # kill the whole process group
-    os.killpg(os.getpgid(proc.pid), 15)
+    try:
+        os.killpg(os.getpgid(proc.pid), 15)
+    except OSError:
+        pass
 
     os.close(master)
     stdout, stderr = proc.communicate()
@@ -562,7 +571,7 @@ def ssh_output(cmd):
     print('SSH STDOUT: {}'.format(stdout.decode('utf-8')))
     print('SSH STDERR: {}'.format(stderr.decode('utf-8')))
 
-    return stdout, stderr
+    return stdout, stderr, returncode
 
 
 def config_set(key, value, env=None):

cli/tests/integrations/test_marathon.py

@@ -556,9 +556,11 @@ def test_watching_missing_deployment():
 
 def test_watching_deployment():
     with _zero_instance_app():
-        _start_app('zero-instance-app', 10)
+        _start_app('zero-instance-app', _ZERO_INSTANCE_APP_INSTANCES)
         result = list_deployments(1, 'zero-instance-app')
         watch_deployment(result[0]['id'], 60)
+        assert_command(
+            ['dcos', 'marathon', 'deployment', 'stop', result[0]['id']])
         list_deployments(0, 'zero-instance-app')
 
 

cli/tests/integrations/test_node.py

@@ -97,26 +97,27 @@ def test_node_ssh_master():
 
 def test_node_ssh_slave():
     slave_id = mesos.DCOSClient().get_state_summary()['slaves'][0]['id']
-    _node_ssh(['--slave={}'.format(slave_id)])
+    _node_ssh(['--slave={}'.format(slave_id), '--master-proxy'])
 
 
 def test_node_ssh_option():
-    stdout, stderr = _node_ssh_output(
+    stdout, stderr, _ = _node_ssh_output(
         ['--master', '--option', 'Protocol=0'])
     assert stdout == b''
     assert b'ignoring bad proto spec' in stderr
 
 
 def test_node_ssh_config_file():
-    stdout, stderr = _node_ssh_output(
+    stdout, stderr, _ = _node_ssh_output(
         ['--master', '--config-file', 'tests/data/node/ssh_config'])
     assert stdout == b''
     assert b'ignoring bad proto spec' in stderr
 
 
 def test_node_ssh_user():
-    stdout, stderr = _node_ssh_output(
-        ['--master', '--user=bogus', '--option', 'PasswordAuthentication=no'])
+    stdout, stderr, _ = _node_ssh_output(
+        ['--master-proxy', '--master', '--user=bogus', '--option',
+         'PasswordAuthentication=no'])
     assert stdout == b''
     assert b'Permission denied' in stderr
 
@@ -142,6 +143,7 @@ def test_node_ssh_master_proxy():
 
 def _node_ssh_output(args):
     cli_test_ssh_key_path = os.environ['CLI_TEST_SSH_KEY_PATH']
+
     cmd = ('ssh-agent /bin/bash -c "ssh-add {} 2> /dev/null && ' +
            'dcos node ssh --option StrictHostKeyChecking=no {}"').format(
                cli_test_ssh_key_path,
@@ -151,7 +153,12 @@ def _node_ssh_output(args):
 
 
 def _node_ssh(args):
-    stdout, stderr = _node_ssh_output(args)
+    if os.environ.get('CLI_TEST_MASTER_PROXY') and \
+            '--master-proxy' not in args:
+        args.append('--master-proxy')
+
+    stdout, stderr, returncode = _node_ssh_output(args)
+    assert returncode is None
 
     assert stdout
     assert b"Running `" in stderr
@@ -165,6 +172,15 @@ def _node_ssh(args):
 def _get_schema(slave):
     schema = create_schema(slave)
     schema['required'].remove('reregistered_time')
+
+    schema['required'].remove('reserved_resources')
+    schema['properties']['reserved_resources']['required'] = []
+    schema['properties']['reserved_resources']['additionalProperties'] = True
+
+    schema['required'].remove('unreserved_resources')
+    schema['properties']['unreserved_resources']['required'] = []
+    schema['properties']['unreserved_resources']['additionalProperties'] = True
+
     schema['properties']['used_resources']['required'].remove('ports')
     schema['properties']['offered_resources']['required'].remove('ports')
     schema['properties']['attributes']['additionalProperties'] = True

cli/tests/integrations/test_package.py

@@ -868,3 +868,4 @@ def _package(name,
         assert_command(
             ['dcos', 'package', 'uninstall', name],
             stderr=uninstall_stderr)
+        watch_all_deployments()

cli/tests/integrations/test_service.py

@@ -151,7 +151,7 @@ def test_log_marathon_file():
 
 
 def test_log_marathon_config():
-    stdout, stderr = ssh_output(
+    stdout, stderr, _ = ssh_output(
         'dcos service log marathon ' +
         '--ssh-config-file=tests/data/node/ssh_config')
 
@@ -159,6 +159,10 @@ def test_log_marathon_config():
     assert b'ignoring bad proto spec' in stderr
 
 
+@pytest.mark.skipif(True,
+                    reason=(
+                        "Now that we test against an AWS cluster, this test "
+                        "is blocked on DCOS-3104requires python3.3"))
 def test_log_marathon():
     stdout, stderr = ssh_output(
         'dcos service log marathon ' +
@@ -184,15 +188,21 @@ def test_log_follow():
     wait_for_service('chronos')
     proc = subprocess.Popen(['dcos', 'service', 'log', 'chronos', '--follow'],
                             preexec_fn=os.setsid,
-                            stdout=subprocess.PIPE)
-
-    time.sleep(5)
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.PIPE)
+    time.sleep(10)
 
     proc.poll()
     assert proc.returncode is None
 
     os.killpg(os.getpgid(proc.pid), 15)
-    assert len(proc.stdout.read().decode('utf-8').split('\n')) > 3
+
+    stdout = proc.stdout.read()
+    stderr = proc.stderr.read()
+
+    print('STDOUT: {}'.format(stdout))
+    print('STDERR: {}'.format(stderr))
+    assert len(stdout.decode('utf-8').split('\n')) > 3
 
 
 def test_log_lines():

cli/tests/integrations/test_ssl.py

@@ -88,7 +88,7 @@ def test_verify_ssl_with_bad_cert_config(env):
 
 
 def test_verify_ssl_with_good_cert_env_var(env):
-    env[constants.DCOS_SSL_VERIFY_ENV] = '/adminrouter/snakeoil.crt'
+    env[constants.DCOS_SSL_VERIFY_ENV] = '/dcos-cli/adminrouter/snakeoil.crt'
 
     returncode, stdout, stderr = exec_command(
         ['dcos', 'marathon', 'app', 'list'], env)
@@ -99,7 +99,7 @@ def test_verify_ssl_with_good_cert_env_var(env):
 
 
 def test_verify_ssl_with_good_cert_config(env):
-    config_set('core.ssl_verify', '/adminrouter/snakeoil.crt', env)
+    config_set('core.ssl_verify', '/dcos-cli/adminrouter/snakeoil.crt', env)
 
     returncode, stdout, stderr = exec_command(
         ['dcos', 'marathon', 'app', 'list'], env)

cli/tests/integrations/test_task.py

@@ -159,12 +159,12 @@ def test_log_follow():
         _mark_non_blocking(proc.stdout)
 
         # wait for data to be output
-        time.sleep(1)
+        time.sleep(10)
 
         # assert lines before and after sleep
-        assert len(proc.stdout.read().decode('utf-8').split('\n')) == 5
-        time.sleep(8)
-        assert len(proc.stdout.read().decode('utf-8').split('\n')) == 2
+        assert len(proc.stdout.read().decode('utf-8').split('\n')) >= 5
+        time.sleep(5)
+        assert len(proc.stdout.read().decode('utf-8').split('\n')) >= 3
 
         proc.kill()
 
@@ -195,17 +195,17 @@ def test_log_two_tasks_follow():
         _mark_non_blocking(proc.stdout)
 
         # wait for data to be output
-        time.sleep(1)
+        time.sleep(10)
 
         # get output before and after the task's sleep
         first_lines = proc.stdout.read().decode('utf-8').split('\n')
-        time.sleep(8)
+        time.sleep(5)
         second_lines = proc.stdout.read().decode('utf-8').split('\n')
 
         # assert both tasks have printed the expected amount of output
-        assert len(first_lines) >= 11
+        assert len(first_lines) >= 5
         # assert there is some difference after sleeping
-        assert len(second_lines) > 0
+        assert len(second_lines) >= 3
 
         proc.kill()
 

cli/tox.ini

@@ -9,7 +9,7 @@ deps =
   mock
   pytz
   -e..
-passenv = DCOS_* CI_FLAGS EXHIBITOR_URL VBOX_IP CLI_TEST_SSH_KEY_PATH
+passenv = DCOS_* CI_FLAGS EXHIBITOR_URL VBOX_IP CLI_TEST_SSH_KEY_PATH CLI_TEST_MASTER_PROXY
 
 [testenv:syntax]
 deps =