From a0448fd3093987dc9e73664e9cd1c169396d0c8c Mon Sep 17 00:00:00 2001 From: Charles Ruhland Date: Mon, 12 Dec 2016 16:48:45 -0800 Subject: [PATCH] pods: enable integration tests (#851) --- cli/dcoscli/log.py | 4 +- cli/dcoscli/task/main.py | 4 +- cli/tests/data/marathon/pods/double.json | 5 ++ .../data/marathon/pods/doubleplusgood.json | 10 ++++ cli/tests/data/marathon/pods/good_status.json | 4 +- cli/tests/integrations/common.py | 14 ++--- cli/tests/integrations/test_marathon.py | 1 - cli/tests/integrations/test_marathon_debug.py | 51 +++++++++---------- cli/tests/integrations/test_marathon_pod.py | 13 +---- cli/tests/integrations/test_service.py | 27 +++++----- cli/tests/integrations/test_task.py | 10 ++-- 11 files changed, 73 insertions(+), 70 deletions(-) diff --git a/cli/dcoscli/log.py b/cli/dcoscli/log.py index 711bf1c..bc9d11b 100644 --- a/cli/dcoscli/log.py +++ b/cli/dcoscli/log.py @@ -5,7 +5,7 @@ import time import six from dcos import emitting, util -from dcos.errors import DCOSException +from dcos.errors import DCOSException, DefaultError logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() @@ -116,6 +116,8 @@ def _output(curr_header, output_header, header, lines): if lines: if output_header and header != curr_header: emitter.publish('===> {} <==='.format(header)) + if lines == ['']: + emitter.publish(DefaultError('No logs for this task')) for line in lines: emitter.publish(line) return header diff --git a/cli/dcoscli/task/main.py b/cli/dcoscli/task/main.py index 3c484f9..70afd2f 100644 --- a/cli/dcoscli/task/main.py +++ b/cli/dcoscli/task/main.py @@ -151,10 +151,10 @@ def _log(follow, completed, lines, task, file_): mesos_files = _mesos_files(tasks, file_, client) if not mesos_files: if fltr is None: - msg = "No tasks found. Exiting" + msg = "No tasks found. Exiting." else: msg = "No matching tasks. Exiting." - raise DCOSException('No matching tasks. Exiting.') + raise DCOSException(msg) log.log_files(mesos_files, follow, lines) diff --git a/cli/tests/data/marathon/pods/double.json b/cli/tests/data/marathon/pods/double.json index a384607..eef945c 100644 --- a/cli/tests/data/marathon/pods/double.json +++ b/cli/tests/data/marathon/pods/double.json @@ -15,6 +15,11 @@ }, { "name": "thing-2", + "exec": { + "command": { + "shell": "sleep 1000" + } + }, "resources": { "cpus": 0.1, "mem": 16.0 diff --git a/cli/tests/data/marathon/pods/doubleplusgood.json b/cli/tests/data/marathon/pods/doubleplusgood.json index aca7d72..6716347 100644 --- a/cli/tests/data/marathon/pods/doubleplusgood.json +++ b/cli/tests/data/marathon/pods/doubleplusgood.json @@ -3,6 +3,11 @@ "containers": [ { "name": "thing-1", + "exec": { + "command": { + "shell": "sleep 1000" + } + }, "resources": { "cpus": 0.1, "mem": 16.0 @@ -10,6 +15,11 @@ }, { "name": "thing-2", + "exec": { + "command": { + "shell": "sleep 1000" + } + }, "resources": { "cpus": 0.1, "mem": 16.0 diff --git a/cli/tests/data/marathon/pods/good_status.json b/cli/tests/data/marathon/pods/good_status.json index acbcf31..e213843 100644 --- a/cli/tests/data/marathon/pods/good_status.json +++ b/cli/tests/data/marathon/pods/good_status.json @@ -8,7 +8,7 @@ "status": "TASK_RUNNING" } ], - "status": "stable" + "status": "STABLE" } ], "spec": { @@ -37,5 +37,5 @@ "instances": 1 } }, - "status": "stable" + "status": "STABLE" } diff --git a/cli/tests/integrations/common.py b/cli/tests/integrations/common.py index 36e6ceb..5002073 100644 --- a/cli/tests/integrations/common.py +++ b/cli/tests/integrations/common.py @@ -252,20 +252,20 @@ def package_install(package, deploy=False, args=[]): watch_all_deployments() -def package_uninstall(package, args=[], stderr=b''): +def package_uninstall(package_name, args=[], stderr=b''): """ Calls `dcos package uninstall` - :param package: name of the package to uninstall - :type package: str + :param package_name: name of the package to uninstall + :type package_name: str :param args: extra CLI args :type args: [str] :param stderr: expected string in stderr for package uninstall - :type stderr: str + :type stderr: bytes :rtype: None """ assert_command( - ['dcos', 'package', 'uninstall', package] + args, + ['dcos', 'package', 'uninstall', package_name] + args, stderr=stderr) @@ -676,7 +676,9 @@ def package(package_name, deploy=False, args=[]): try: yield finally: - package_uninstall(package_name) + command = ['dcos', 'package', 'uninstall', package_name] + returncode, _, _ = exec_command(command) + assert returncode == 0 watch_all_deployments() diff --git a/cli/tests/integrations/test_marathon.py b/cli/tests/integrations/test_marathon.py index ae0644a..1ca3888 100644 --- a/cli/tests/integrations/test_marathon.py +++ b/cli/tests/integrations/test_marathon.py @@ -659,7 +659,6 @@ def test_stop_task(): _stop_task(task_id) -@pytest.mark.skip(reason="https://mesosphere.atlassian.net/browse/DCOS-10325") def test_stop_task_wipe(): with _zero_instance_app(): _start_app('zero-instance-app', 1) diff --git a/cli/tests/integrations/test_marathon_debug.py b/cli/tests/integrations/test_marathon_debug.py index 55b317d..dcf0715 100644 --- a/cli/tests/integrations/test_marathon_debug.py +++ b/cli/tests/integrations/test_marathon_debug.py @@ -1,18 +1,14 @@ import contextlib -import os +import json import re -import pytest - from .common import (app, exec_command, pod) from .test_marathon import (_list_tasks) -_PODS_ENABLED = 'DCOS_PODS_ENABLED' in os.environ list_regex = '/stuck-(?:sleep|pod)\W+[^Z]+Z\W+9\W+(?:True|False)' \ '\W+\d\W+\d\W+[^Z]+Z\W+[^Z]+Z' -@pytest.mark.skipif(not _PODS_ENABLED, reason="Requires pods") def test_debug_list(): with _stuck_app(): returncode, stdout, stderr = exec_command( @@ -31,7 +27,6 @@ def test_debug_list(): assert re.search(list_regex, decoded) is not None -@pytest.mark.skipif(not _PODS_ENABLED, reason="Requires pods") def test_debug_list_json(): with _stuck_app(): returncode, stdout, stderr = exec_command( @@ -46,7 +41,6 @@ def test_debug_list_json(): assert '"reason": "UnfulfilledConstraint"' in decoded -@pytest.mark.skipif(not _PODS_ENABLED, reason="Requires pods") def test_debug_list_pod(): with _stuck_pod(): returncode, stdout, stderr = exec_command( @@ -65,7 +59,6 @@ def test_debug_list_pod(): assert re.search(list_regex, decoded) is not None -@pytest.mark.skipif(not _PODS_ENABLED, reason="Requires pods") def test_debug_list_pod_json(): with _stuck_pod(): returncode, stdout, stderr = exec_command( @@ -80,7 +73,6 @@ def test_debug_list_pod_json(): assert '"reason": "UnfulfilledConstraint"' in decoded -@pytest.mark.skipif(not _PODS_ENABLED, reason="Requires pods") def test_debug_summary(): with _stuck_app(): returncode, stdout, stderr = exec_command( @@ -95,7 +87,6 @@ def test_debug_summary(): assert '0.00%' in decoded -@pytest.mark.skipif(not _PODS_ENABLED, reason="Requires pods") def test_debug_summary_json(): with _stuck_app(): returncode, stdout, stderr = exec_command( @@ -110,7 +101,6 @@ def test_debug_summary_json(): in decoded -@pytest.mark.skipif(not _PODS_ENABLED, reason="Requires pods") def test_debug_summary_pod(): with _stuck_pod(): returncode, stdout, stderr = exec_command( @@ -126,7 +116,6 @@ def test_debug_summary_pod(): assert '0.00%' in decoded -@pytest.mark.skipif(not _PODS_ENABLED, reason="Requires pods") def test_debug_summary_pod_json(): with _stuck_pod(): returncode, stdout, stderr = exec_command( @@ -141,7 +130,6 @@ def test_debug_summary_pod_json(): in decoded -@pytest.mark.skipif(not _PODS_ENABLED, reason="Requires pods") def test_debug_details(): with _stuck_app(): returncode, stdout, stderr = exec_command( @@ -154,17 +142,22 @@ def test_debug_details(): """The public agent has all resources to launch this task, but not the matching role, therefore the output should be: ok - ok ok ok ok - To avoid formatting issues, whitspaces are ignored. + To avoid formatting issues, whitespaces are ignored. """ assert 'ok-okokokok' in decoded.replace(' ', '') - """We do have 3 lines. The headline and two lines for the agents. - If we split the decoded output by line break, there should be four - entries in the array. The additional entry is empty. + + returncode, stdout, stderr = exec_command(['dcos', 'node', '--json']) + + assert returncode == 0 + assert stderr == b'' + agent_count = len(json.loads(stdout.decode('utf-8'))) + + """The extra two lines come from the heading and the empty line at the + end of the table. """ - assert len(decoded.split('\n')) == 4 + assert len(decoded.split('\n')) == agent_count + 2 -@pytest.mark.skipif(not _PODS_ENABLED, reason="Requires pods") def test_debug_details_json(): with _stuck_app(): returncode, stdout, stderr = exec_command( @@ -179,7 +172,6 @@ def test_debug_details_json(): in decoded -@pytest.mark.skipif(not _PODS_ENABLED, reason="Requires pods") def test_debug_details_pod(): with _stuck_pod(): returncode, stdout, stderr = exec_command( @@ -192,17 +184,22 @@ def test_debug_details_pod(): """The public agent has all resources to launch this task, but not the matching role, therefore the output should be: ok - ok ok ok ok - To avoid formatting issues, whitspaces are ignored. + To avoid formatting issues, whitespaces are ignored. """ assert 'ok-okokokok' in decoded.replace(' ', '') - """We do have 3 lines. The headline and two lines for the agents. - If we split the decoded output by line break, there should be four - entries in the array. The additional entry is empty. + + returncode, stdout, stderr = exec_command(['dcos', 'node', '--json']) + + assert returncode == 0 + assert stderr == b'' + agent_count = len(json.loads(stdout.decode('utf-8'))) + + """The extra two lines come from the heading and the empty line at the + end of the table. """ - assert len(decoded.split('\n')) == 4 + assert len(decoded.split('\n')) == agent_count + 2 -@pytest.mark.skipif(not _PODS_ENABLED, reason="Requires pods") def test_debug_details_pod_json(): with _stuck_pod(): returncode, stdout, stderr = exec_command( @@ -224,7 +221,7 @@ def _stuck_app(max_count=300): count = 0 while count < max_count: tasks = _list_tasks(app_id='stuck-sleep') - if (len(tasks) == 1): + if len(tasks) == 1: break yield diff --git a/cli/tests/integrations/test_marathon_pod.py b/cli/tests/integrations/test_marathon_pod.py index 718e2d3..f6730d0 100644 --- a/cli/tests/integrations/test_marathon_pod.py +++ b/cli/tests/integrations/test_marathon_pod.py @@ -1,10 +1,7 @@ import json -import os import re import time -import pytest - from .common import (add_pod, assert_command, exec_command, file_json_ast, pod, pods, remove_pod, watch_all_deployments) @@ -16,8 +13,6 @@ from ..fixtures.marathon import (DOUBLE_POD_FILE_PATH, DOUBLE_POD_ID, TRIPLE_POD_ID, UNGOOD_POD_FILE_PATH, UPDATED_GOOD_POD_FILE_PATH) -_PODS_ENABLED = 'DCOS_PODS_ENABLED' in os.environ - _POD_BASE_CMD = ['dcos', 'marathon', 'pod'] _POD_ADD_CMD = _POD_BASE_CMD + ['add'] _POD_KILL_CMD = _POD_BASE_CMD + ['kill'] @@ -27,21 +22,18 @@ _POD_SHOW_CMD = _POD_BASE_CMD + ['show'] _POD_UPDATE_CMD = _POD_BASE_CMD + ['update'] -@pytest.mark.skipif(not _PODS_ENABLED, reason="Requires pods") def test_pod_add_from_file(): add_pod(GOOD_POD_FILE_PATH) remove_pod(GOOD_POD_ID, force=False) watch_all_deployments() -@pytest.mark.skipif(not _PODS_ENABLED, reason="Requires pods") def test_pod_add_from_stdin(): _pod_add_from_stdin(GOOD_POD_FILE_PATH) remove_pod(GOOD_POD_ID) watch_all_deployments() -@pytest.mark.skipif(not _PODS_ENABLED, reason="Requires pods") def test_pod_list(): expected_json = pod_list_fixture() @@ -53,7 +45,6 @@ def test_pod_list(): _assert_pod_list_table() -@pytest.mark.skipif(not _PODS_ENABLED, reason="Requires pods") def test_pod_show(): expected_json = file_json_ast(GOOD_POD_STATUS_FILE_PATH) @@ -70,7 +61,6 @@ def test_pod_update_does_not_support_properties(): assert stderr == b'' -@pytest.mark.skipif(not _PODS_ENABLED, reason="Requires pods") def test_pod_update_from_stdin(): with pod(GOOD_POD_FILE_PATH, GOOD_POD_ID): # The deployment will never complete @@ -86,7 +76,6 @@ def test_pod_update_from_stdin(): watch_all_deployments() -@pytest.mark.skipif(not _PODS_ENABLED, reason="Requires pods") def test_pod_kill(): with pod(POD_KILL_FILE_PATH, POD_KILL_ID): kill_1, keep, kill_2 = _get_pod_instance_ids(POD_KILL_ID, 3) @@ -207,7 +196,7 @@ def _assert_pod_list_table(): stdout_lines = stdout.decode('utf-8').split('\n') - pattern = r'ID\+TASKS +INSTANCES +VERSION +STATUS +STATUS SINCE *' + pattern = r'ID\+TASKS +INSTANCES +VERSION +STATUS +STATUS SINCE +WAITING *' assert re.fullmatch(pattern, stdout_lines[0]) assert stdout_lines[1].startswith('/double-pod') diff --git a/cli/tests/integrations/test_service.py b/cli/tests/integrations/test_service.py index c074734..eb0e76d 100644 --- a/cli/tests/integrations/test_service.py +++ b/cli/tests/integrations/test_service.py @@ -11,7 +11,7 @@ from dcos.util import create_schema from .common import (assert_command, assert_lines, delete_zk_node, delete_zk_nodes, exec_command, get_services, - package_install, remove_app, service_shutdown, + package, package_install, remove_app, service_shutdown, setup_universe_server, ssh_output, teardown_universe_server, wait_for_service) from ..fixtures.service import framework_fixture @@ -106,22 +106,23 @@ def test_service_completed(): def test_log(): - package_install('cassandra', args=['--package-version=0.2.0-1']) + with package('cassandra', deploy=True, args=['--package-version=0.2.0-1']): - returncode, stdout, stderr = exec_command( - ['dcos', 'service', 'log', 'cassandra.dcos']) + returncode, stdout, stderr = exec_command( + ['dcos', 'service', 'log', 'cassandra.dcos']) - assert returncode == 0 - assert len(stdout.decode('utf-8').split('\n')) > 1 - assert stderr == b'' + assert returncode == 0 + assert len(stdout.decode('utf-8').split('\n')) > 1 + assert stderr == b'' - returncode, stdout, stderr = exec_command( - ['dcos', 'service', 'log', 'cassandra.dcos', 'stderr']) + returncode, stdout, stderr = exec_command( + ['dcos', 'service', 'log', 'cassandra.dcos', 'stderr']) - assert returncode == 0 - assert len(stdout.decode('utf-8').split('\n')) > 1 - assert stderr == b'' - exec_command(['dcos', 'package', 'uninstall', 'cassandra']) + assert returncode == 0 + assert len(stdout.decode('utf-8').split('\n')) > 1 + assert stderr == b'' + + # Package was uninstalled but its group needs to be removed separately exec_command(['dcos', 'marathon', 'group', 'remove', 'cassandra']) diff --git a/cli/tests/integrations/test_task.py b/cli/tests/integrations/test_task.py index fc8e8a4..05120bb 100644 --- a/cli/tests/integrations/test_task.py +++ b/cli/tests/integrations/test_task.py @@ -122,8 +122,6 @@ def test_log_single_file(): assert len(stdout.decode('utf-8').split('\n')) > 0 -@pytest.mark.skipif('DCOS_PODS_ENABLED' not in os.environ, - reason="Requires pods") def test_log_pod_task(): good_pod_file = 'tests/data/marathon/pods/good.json' with pod(good_pod_file, 'good-pod'): @@ -132,11 +130,11 @@ def test_log_pod_task(): ['dcos', 'task', 'log', 'good-container', 'stderr']) # pod task log are not executor logs, so normal executor stderr - # logs shouldn't be seen and this pod shoudn't have any logging + # logs shouldn't be seen and this pod shouldn't have any logging # to stderr - assert returncode == 1 - assert stderr == b'No files exist. Exiting.\n' - assert stdout == b'' + assert returncode == 0 + assert stderr == b'No logs for this task\n' + assert stdout == b'\n' def test_log_missing_file():