From ffb2684f16f336058dd5751e6179d8ad755a9aaf Mon Sep 17 00:00:00 2001 From: Ken Sipe Date: Mon, 3 Apr 2017 12:27:02 -0500 Subject: [PATCH] job: update `list` last run to match the UI (#959) --- cli/dcoscli/job/main.py | 72 +++++++------- cli/dcoscli/tables.py | 62 ++++++++---- cli/tests/fixtures/metronome.py | 136 +++++++++++++++++++++++++++ cli/tests/unit/data/job_history.txt | 3 + cli/tests/unit/data/job_list.txt | 3 + cli/tests/unit/data/job_runs.txt | 3 + cli/tests/unit/data/job_schedule.txt | 2 + cli/tests/unit/test_tables.py | 26 +++++ dcos/errors.py | 13 +++ dcos/http.py | 5 +- dcos/metronome.py | 38 +++++--- dcos/packagemanager.py | 5 +- 12 files changed, 297 insertions(+), 71 deletions(-) create mode 100644 cli/tests/fixtures/metronome.py create mode 100644 cli/tests/unit/data/job_history.txt create mode 100644 cli/tests/unit/data/job_list.txt create mode 100644 cli/tests/unit/data/job_runs.txt create mode 100644 cli/tests/unit/data/job_schedule.txt diff --git a/cli/dcoscli/job/main.py b/cli/dcoscli/job/main.py index 3b58e96..9231593 100644 --- a/cli/dcoscli/job/main.py +++ b/cli/dcoscli/job/main.py @@ -21,7 +21,17 @@ logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() DEFAULT_TIMEOUT = 180 -METRONOME_EMBEDDED = '?embed=activeRuns&embed=schedules&embed=history' + +# single job, not a lot of data +EMBEDS_FOR_JOB_HISTORY = [ + metronome.EMBED_ACTIVE_RUNS, + metronome.EMBED_SCHEDULES, + metronome.EMBED_HISTORY] + +# unknown number of jobs, using history summary +EMBEDS_FOR_JOBS_HISTORY = [ + metronome.EMBED_ACTIVE_RUNS, + metronome.EMBED_HISTORY_SUMMARY] def main(argv): @@ -239,16 +249,12 @@ def _kill(job_id, run_id, all=False): def _list(json_flag=False): - """ + """ Provides a list of jobs along with their active runs and history summary :returns: process return code :rtype: int """ - - try: - client = metronome.create_client() - json_list = client.get_jobs() - except DCOSException as e: - raise DCOSException(e) + client = metronome.create_client() + json_list = client.get_jobs(EMBEDS_FOR_JOBS_HISTORY) if json_flag: emitter.publish(json_list) @@ -266,44 +272,32 @@ def _history(job_id, json_flag=False, show_failures=False): :returns: process return code :rtype: int """ - response = None - url = urllib.parse.urljoin(_get_api_url('v1/jobs/'), - job_id + METRONOME_EMBEDDED) - try: - response = _do_request(url, 'GET') - except DCOSHTTPException as e: - raise DCOSException("Job ID does NOT exist.") - except DCOSException as e: - raise DCOSException(e) + + client = metronome.create_client() + json_history = client.get_job(job_id, EMBEDS_FOR_JOB_HISTORY) + + if 'history' not in json_history: + return 0 + + if json_flag: + emitter.publish(json_history) else: + emitter.publish(_get_history_message(json_history, job_id)) + table = tables.job_history_table( + json_history['history']['successfulFinishedRuns']) + output = six.text_type(table) + if output: + emitter.publish(output) - if response.status_code is not 200: - raise DCOSException("Job ID does NOT exist.") - - json_history = _read_http_response_body(response) - - if 'history' not in json_history: - return 0 - - if json_flag: - emitter.publish(json_history) - else: - emitter.publish(_get_history_message(json_history, job_id)) + if show_failures: + emitter.publish(_get_history_message( + json_history, job_id, False)) table = tables.job_history_table( - json_history['history']['successfulFinishedRuns']) + json_history['history']['failedFinishedRuns']) output = six.text_type(table) if output: emitter.publish(output) - if show_failures: - emitter.publish(_get_history_message( - json_history, job_id, False)) - table = tables.job_history_table( - json_history['history']['failedFinishedRuns']) - output = six.text_type(table) - if output: - emitter.publish(output) - return 0 diff --git a/cli/dcoscli/tables.py b/cli/dcoscli/tables.py index 1d7cca4..93250c5 100644 --- a/cli/dcoscli/tables.py +++ b/cli/dcoscli/tables.py @@ -275,21 +275,13 @@ def job_table(job_list): fields = OrderedDict([ ('id', lambda s: s['id']), - ('Description', lambda s: - s['description'] if 'description' in s else ''), ('Status', lambda s: _job_status(s)), - ('Last Succesful Run', lambda s: s['history']['lastSuccessAt'] - if 'history' in s else 'N/A'), + ('Last Run', lambda s: _last_run_status(s)), ]) - limits = { - "Description": 35 - } - - tb = truncate_table(fields, job_list, limits, sortby="ID") - tb.align['ID'] = 'l' - tb.align["DESCRIPTION"] = 'l' + tb = truncate_table(fields, job_list, None, sortby="ID") tb.align["STATUS"] = 'l' + tb.align["LAST RUN"] = 'l' return tb @@ -308,7 +300,8 @@ def job_history_table(schedule_list): ('finished', lambda s: s['finishedAt']), ]) tb = table(fields, schedule_list, sortby="STARTED") - tb.align['ID'] = 'l' + tb.align["STARTED"] = 'l' + tb.align["FINISHED"] = 'l' return tb @@ -326,12 +319,14 @@ def schedule_table(schedule_list): ('id', lambda s: s['id']), ('cron', lambda s: s['cron']), ('enabled', lambda s: s['enabled']), - ('next run', lambda s: s['nextRunAt']), ('concurrency policy', lambda s: s['concurrencyPolicy']), + ('next run', lambda s: s['nextRunAt']), ]) tb = table(fields, schedule_list) - tb.align['ID'] = 'l' tb.align['CRON'] = 'l' + tb.align['ENABLED'] = 'l' + tb.align['NEXT RUN'] = 'l' + tb.align['CONCURRENCY POLICY'] = 'l' return tb @@ -345,17 +340,52 @@ def job_runs_table(runs_list): :rtype: PrettyTable """ fields = OrderedDict([ - ('job id', lambda s: s['jobId']), ('id', lambda s: s['id']), + ('job id', lambda s: s['jobId']), ('started at', lambda s: s['createdAt']), ]) tb = table(fields, runs_list) - tb.align['ID'] = 'l' tb.align['JOB ID'] = 'l' + tb.align['STARTED AT'] = 'l' return tb +def _str_to_datetime(datetime_str): + """ Takes a JSON date of `2017-03-30T15:50:16.187+0000` format and + Returns a datetime. + + :param datetime_str: JSON date + :type datetime_str: str + :rtype: datetime + """ + if not datetime_str: + return None + datetime_str = datetime_str.split('+')[0] + return datetime.datetime.strptime(datetime_str, "%Y-%m-%dT%H:%M:%S.%f") + + +def _last_run_status(job): + """ Provided a job with embedded history it Returns a status based on the + following rules: + 0 Runs = 'N/A' + last success is > last failure = 'Success' otherwise 'Failed' + + :param job: JSON job with embedded history + :type job: dict + :rtype: str + """ + last_success = _str_to_datetime(job['historySummary']['lastSuccessAt']) + last_failure = _str_to_datetime(job['historySummary']['lastFailureAt']) + if not last_success and not last_failure: + return 'N/A' + elif ((last_success and not last_failure) or + (last_success and last_success > last_failure)): + return 'Success' + else: + return 'Failed' + + def _job_status(job): """Utility function that returns the status of a job diff --git a/cli/tests/fixtures/metronome.py b/cli/tests/fixtures/metronome.py new file mode 100644 index 0000000..2c4c534 --- /dev/null +++ b/cli/tests/fixtures/metronome.py @@ -0,0 +1,136 @@ + +def job_list_fixture(): + """Job list fixture + + :rtype: dict + """ + + return [ + { + "description": "sleeping is what is do", + "historySummary": { + "failureCount": 0, + "lastFailureAt": None, + "lastSuccessAt": None, + "successCount": 0 + }, + "id": "snorlax", + "labels": {}, + "run": { + "artifacts": [], + "cmd": "sleep 10", + "cpus": 0.01, + "disk": 0, + "env": {}, + "maxLaunchDelay": 3600, + "mem": 32, + "placement": { + "constraints": [] + }, + "restart": { + "policy": "NEVER" + }, + "volumes": [] + } + }, + { + "description": "electrifying rodent", + "historySummary": { + "failureCount": 0, + "lastFailureAt": None, + "lastSuccessAt": "2017-03-31T14:22:01.541+0000", + "successCount": 1 + }, + "id": "pikachu", + "labels": {}, + "run": { + "artifacts": [], + "cmd": "sleep 10", + "cpus": 0.01, + "disk": 0, + "env": {}, + "maxLaunchDelay": 3600, + "mem": 32, + "placement": { + "constraints": [] + }, + "restart": { + "policy": "NEVER" + }, + "volumes": [] + } + }] + + +def job_run_fixture(): + """Job run fixture + + :rtype: dict + """ + + return [ + { + "completedAt": None, + "createdAt": "2017-03-31T21:05:30.613+0000", + "id": "20170331210530QHpRU", + "jobId": "pikachu", + "status": "ACTIVE", + "tasks": [ + { + "id": "pikachu_20170331210530QHpRU.c5e4b1e7-1655-11e7-8bd5-6ef119b8e20f", # NOQA + "startedAt": "2017-03-31T21:05:31.499+0000", + "status": "TASK_RUNNING" + } + ] + }, + { + "completedAt": None, + "createdAt": "2017-03-31T21:05:32.422+0000", + "id": "20170331210532uxgVF", + "jobId": "pikachu", + "status": "ACTIVE", + "tasks": [ + { + "id": "pikachu_20170331210532uxgVF.c8e324d8-1655-11e7-8bd5-6ef119b8e20f", # NOQA + "startedAt": "2017-03-31T21:05:36.417+0000", + "status": "TASK_RUNNING" + } + ] + }] + + +def job_history_fixture(): + """Job history fixture + + :rtype: dict + """ + + return [ + { + "createdAt": "2017-03-31T21:05:32.422+0000", + "finishedAt": "2017-03-31T21:05:46.805+0000", + "id": "20170331210532uxgVF" + }, + { + "createdAt": "2017-03-31T21:05:30.613+0000", + "finishedAt": "2017-03-31T21:05:41.740+0000", + "id": "20170331210530QHpRU" + }] + + +def job_schedule_fixture(): + """Job schedule fixture + + :rtype: dict + """ + + return [ + { + "concurrencyPolicy": "ALLOW", + "cron": "20 0 * * *", + "enabled": True, + "id": "nightly", + "nextRunAt": "2017-04-01T00:20:00.000+0000", + "startingDeadlineSeconds": 900, + "timezone": "UTC" + }] diff --git a/cli/tests/unit/data/job_history.txt b/cli/tests/unit/data/job_history.txt new file mode 100644 index 0000000..15b8faa --- /dev/null +++ b/cli/tests/unit/data/job_history.txt @@ -0,0 +1,3 @@ + ID STARTED FINISHED +20170331210530QHpRU 2017-03-31T21:05:30.613+0000 2017-03-31T21:05:41.740+0000 +20170331210532uxgVF 2017-03-31T21:05:32.422+0000 2017-03-31T21:05:46.805+0000 \ No newline at end of file diff --git a/cli/tests/unit/data/job_list.txt b/cli/tests/unit/data/job_list.txt new file mode 100644 index 0000000..e19f67f --- /dev/null +++ b/cli/tests/unit/data/job_list.txt @@ -0,0 +1,3 @@ + ID STATUS LAST RUN +pikachu Unscheduled Success +snorlax Unscheduled N/A \ No newline at end of file diff --git a/cli/tests/unit/data/job_runs.txt b/cli/tests/unit/data/job_runs.txt new file mode 100644 index 0000000..e879b07 --- /dev/null +++ b/cli/tests/unit/data/job_runs.txt @@ -0,0 +1,3 @@ + ID JOB ID STARTED AT +20170331210530QHpRU pikachu 2017-03-31T21:05:30.613+0000 +20170331210532uxgVF pikachu 2017-03-31T21:05:32.422+0000 \ No newline at end of file diff --git a/cli/tests/unit/data/job_schedule.txt b/cli/tests/unit/data/job_schedule.txt new file mode 100644 index 0000000..6311ae4 --- /dev/null +++ b/cli/tests/unit/data/job_schedule.txt @@ -0,0 +1,2 @@ + ID CRON ENABLED CONCURRENCY POLICY NEXT RUN +nightly 20 0 * * * True ALLOW 2017-04-01T00:20:00.000+0000 \ No newline at end of file diff --git a/cli/tests/unit/test_tables.py b/cli/tests/unit/test_tables.py index 1ee5c77..8da4534 100644 --- a/cli/tests/unit/test_tables.py +++ b/cli/tests/unit/test_tables.py @@ -17,6 +17,8 @@ from ..fixtures.marathon import (app_fixture, app_task_fixture, from ..fixtures.metrics import (agent_metrics_node_details_fixture, agent_metrics_node_summary_fixture, agent_metrics_task_details_fixture) +from ..fixtures.metronome import (job_history_fixture, job_list_fixture, + job_run_fixture, job_schedule_fixture) from ..fixtures.node import slave_fixture from ..fixtures.package import package_fixture, search_result_fixture from ..fixtures.service import framework_fixture @@ -83,6 +85,30 @@ def test_auth_providers_table(): 'tests/unit/data/auth_provider.txt') +def test_job_list_table(): + _test_table(tables.job_table, + job_list_fixture(), + 'tests/unit/data/job_list.txt') + + +def test_job_runs_table(): + _test_table(tables.job_runs_table, + job_run_fixture(), + 'tests/unit/data/job_runs.txt') + + +def test_job_history_table(): + _test_table(tables.job_history_table, + job_history_fixture(), + 'tests/unit/data/job_history.txt') + + +def test_job_schedule_table(): + _test_table(tables.schedule_table, + job_schedule_fixture(), + 'tests/unit/data/job_schedule.txt') + + def test_pod_table(): _test_table(tables.pod_table, pod_list_fixture(), diff --git a/dcos/errors.py b/dcos/errors.py index 6068891..ac6e36f 100644 --- a/dcos/errors.py +++ b/dcos/errors.py @@ -70,6 +70,19 @@ class DCOSAuthorizationException(DCOSHTTPException): return "You are not authorized to perform this operation" +class DCOSConnectionError(DCOSException): + """An Error object for when a connection attempt fails. + + :param url: URL for the Request + :type url: str + """ + def __init__(self, url): + self.url = url + + def __str__(self): + return 'URL [{0}] is unreachable.'.format(self.url) + + class DCOSBadRequest(DCOSHTTPException): """A wrapper around Response objects for HTTP Bad Request (400). diff --git a/dcos/http.py b/dcos/http.py index fb6a4bf..0d1190c 100644 --- a/dcos/http.py +++ b/dcos/http.py @@ -1,4 +1,5 @@ import requests + from requests.auth import AuthBase from six.moves.urllib.parse import urlparse @@ -6,7 +7,7 @@ from six.moves.urllib.parse import urlparse from dcos import config, util from dcos.errors import (DCOSAuthenticationException, DCOSAuthorizationException, DCOSBadRequest, - DCOSException, DCOSHTTPException, + DCOSConnectionError, DCOSException, DCOSHTTPException, DCOSUnprocessableException) @@ -107,7 +108,7 @@ def _request(method, raise DCOSException(msg) except requests.exceptions.ConnectionError as e: logger.exception("HTTP Connection Error") - raise DCOSException('URL [{0}] is unreachable: {1}'.format(url, e)) + raise DCOSConnectionError(url) except requests.exceptions.Timeout as e: logger.exception("HTTP Timeout") raise DCOSException('Request to URL [{0}] timed out.'.format(url)) diff --git a/dcos/metronome.py b/dcos/metronome.py index 045adba..2444731 100644 --- a/dcos/metronome.py +++ b/dcos/metronome.py @@ -7,6 +7,11 @@ from dcos.errors import DCOSException logger = util.get_logger(__name__) +EMBED_ACTIVE_RUNS = 'activeRuns' +EMBED_SCHEDULES = 'schedules' +EMBED_HISTORY = 'history' +EMBED_HISTORY_SUMMARY = 'historySummary' + def create_client(toml_config=None): """Creates a Metronome client with the supplied configuration. @@ -28,6 +33,10 @@ def create_client(toml_config=None): return Client(rpc_client) +def _get_embed_query_string(embed_list): + return '?{}'.format('&'.join('embed=%s' % (item) for item in embed_list)) + + def _get_metronome_url(toml_config=None): """ :param toml_config: configuration dictionary @@ -71,32 +80,35 @@ class Client(object): return response.json() - def get_job(self, job_id): - """Returns a representation of the requested application version. If - version is None the return the latest version. + def get_job(self, job_id, embed_with=None): + """Returns a representation of the requested job. - :param app_id: the ID of the application - :type app_id: str - :param version: application version as a ISO8601 datetime - :type version: str + :param job_id: the ID of the application + :type job_id: str + :param embed_with: list of strings to ?embed=str&embed=str2... + :type embed_with: [str] :returns: the requested Metronome job :rtype: dict """ # refactor util name it isn't marathon specific job_id = util.normalize_marathon_id_path(job_id) - path = 'v1/jobs{}'.format(job_id) - - response = self._rpc.http_req(http.get, path) + embeds = _get_embed_query_string(embed_with) if embed_with else None + url = ('v1/jobs{}{}'.format(job_id, embeds) + if embeds else 'v1/jobs{}'.format(job_id)) + response = self._rpc.http_req(http.get, url) return response.json() - def get_jobs(self): + def get_jobs(self, embed_with=None): """Get a list of known jobs. + :param embed_with: list of strings to ?embed=str&embed=str2... + :type embed_with: [str] :returns: list of known jobs :rtype: [dict] """ - - response = self._rpc.http_req(http.get, 'v1/jobs') + embeds = _get_embed_query_string(embed_with) if embed_with else None + url = 'v1/jobs{}'.format(embeds) if embeds else 'v1/jobs' + response = self._rpc.http_req(http.get, url) return response.json() def add_job(self, job_resource): diff --git a/dcos/packagemanager.py b/dcos/packagemanager.py index 0188c02..08e554a 100644 --- a/dcos/packagemanager.py +++ b/dcos/packagemanager.py @@ -8,7 +8,8 @@ from six.moves import urllib from dcos import cosmos, emitting, util from dcos.errors import (DCOSAuthenticationException, DCOSAuthorizationException, DCOSBadRequest, - DCOSException, DCOSHTTPException, DefaultError) + DCOSConnectionError, DCOSException, DCOSHTTPException, + DefaultError) logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() @@ -71,6 +72,8 @@ class PackageManager: raise except DCOSAuthorizationException: raise + except DCOSConnectionError: + raise except Exception as e: logger.exception(e) return False