Browse Source

Add deployment status generation to Shipyard

This patch set adds functionality to Shipyard's action_helper module
to generate a set of "deployment-status" data. This data can be used
to denote the status and result of a Shipyard action and will be
utilized in a future feature to help track actions/deployments.

This new functionality is not yet being used, and actual Shipyard
functionality should not be altered with this patch set.

Change-Id: I838abfd2144f08708b3fc5a53cc76f124aedff04
changes/37/659637/15
Carter, Matthew (mc981n) 2 years ago
committed by Michael Beaver
parent
commit
c19580378b
  1. 143
      src/bin/shipyard_airflow/shipyard_airflow/control/helpers/action_helper.py
  2. 9
      src/bin/shipyard_airflow/shipyard_airflow/dags/dag_names.py
  3. 357
      src/bin/shipyard_airflow/tests/unit/control/test_action_helper.py

143
src/bin/shipyard_airflow/shipyard_airflow/control/helpers/action_helper.py

@ -17,7 +17,11 @@ import falcon
import logging
from shipyard_airflow.common.notes.notes import MIN_VERBOSITY
from shipyard_airflow.control.helpers.design_reference_helper import (
DesignRefHelper
)
from shipyard_airflow.control.helpers.notes import NOTES as notes_helper
from shipyard_airflow.dags.dag_names import CRITICAL_DAG_STEPS
from shipyard_airflow.db.db import AIRFLOW_DB, SHIPYARD_DB
from shipyard_airflow.errors import ApiError
@ -90,6 +94,73 @@ def format_step(action_id, step, index, notes):
}
def get_deployment_status(action, force_completed=False):
"""Given a set of action data, make/return a set of deployment status data
:param dict action: A dictionary of data about an action.
The keys of the dict should include:
- committed_rev_id
- context_marker
- dag_status
- id
- timestamp
- user
Any missing key will result in a piece of the
deployment status being "unknown"
:param bool force_completed: optional. If True the status will be forced
to "completed". This will be useful when the
last step of the DAG wants to get the status
of the deployment, and knows that the
DAG/deployment/action really is completed even
though it does not appear to be.
:returns: A dict of deployment status data
:rtype: dict
"""
# Create a URL to get the documents in Deckhand, using the Deckhand
# revision ID associated with the action
document_url = DesignRefHelper().get_design_reference_href(
action.get('committed_rev_id', 'unknown'))
# Create the "status" and "result" of the deployment
dag_status = action.get('dag_status', '').upper()
status = 'running'
result = 'unknown'
status_result_map = {
'FAILED': 'failed',
'UPSTREAM_FAILED': 'failed',
'SKIPPED': 'failed',
'REMOVED': 'failed',
'SHUTDOWN': 'failed',
'SUCCESS': 'successful'
}
if dag_status in status_result_map:
# We don't need to check anything else, we know the status of the
# deployment is completed and we can map directly to a result
status = 'completed'
result = status_result_map[dag_status]
else:
# The DAG could still be running, or be in some other state, so we
# need to dig into the DAG to determine what the result really is
# Use an ActionsHelper to check all the DAG steps
helper = ActionsHelper(action.get('id'))
result = helper.get_result_from_dag_steps()
# Check to see if we need to override the status to "completed"
if force_completed:
status = "completed"
# Return the dictionary of data
return {
'status': status,
'results': result,
'context-marker': action.get('context_marker', 'unknown'),
'action': action.get('id', 'unknown'),
'document_url': document_url,
'user': action.get('user', 'unknown'),
'date': action.get('timestamp', 'unknown')
}
class ActionsHelper(object):
"""
A helper class for Shipyard actions
@ -164,6 +235,78 @@ class ActionsHelper(object):
return steps
def _get_latest_steps(self):
"""Get all the steps for the action, and return the latest try on each
:returns: All dictionary of task_id (a string name) to step details
:rtype: dict
"""
latest_steps = {}
for step in self._get_all_steps():
task_id = step['task_id']
if task_id in latest_steps:
# We already have this step, see if this one is more recent
# than the one we already have
if step['try_number'] > latest_steps[task_id]['try_number']:
latest_steps[task_id] = step
else:
# Step we have not seen yet
latest_steps[task_id] = step
return latest_steps
def get_result_from_dag_steps(self):
"""Look up the DAG steps, and create a result string based on the state
of the DAG steps
We will only check "critical" steps, as these are what's most
important, and won't even run if other steps fail in the first place
If any critical steps have a state that falls under our success states
and no other critical steps have a state that falls under the failed or
running states, result will be "successful"
If any critical steps have a state that fall under our failed states,
result will be "failed"
If any critical steps have a state that fall under our running states,
result will be "pending"
If no critical steps have states that fall under any of the state sets,
result will be "unknown"
:returns: The result of the DAG based on the DAG's steps
:rtype: str
"""
result = 'unknown'
running_states = [
None,
'None',
'scheduled',
'queued',
'running',
'up_for_retry',
'up_for_reschedule'
]
failed_states = ['shutdown', 'failed', 'upstream_failed']
success_states = ['skipped', 'success']
latest_steps = self._get_latest_steps()
for step_id in CRITICAL_DAG_STEPS:
if step_id in latest_steps:
state = latest_steps[step_id]['state']
if state in success_states and result == 'unknown':
result = 'successful'
elif state in failed_states:
result = 'failed'
break
elif state in running_states:
result = 'pending'
elif state not in success_states:
# The state we are looking at doesn't fall under any of our
# known states
LOG.warning('Found DAG step with unexpected state: {}'.
format(state))
result = 'unknown'
break
return result
def get_step(self, step_id, try_number=None):
"""
:param step_id: Step ID - task_id in db

9
src/bin/shipyard_airflow/shipyard_airflow/dags/dag_names.py

@ -31,3 +31,12 @@ GET_RENDERED_DOC = 'get_rendered_doc'
SKIP_UPGRADE_AIRFLOW = 'skip_upgrade_airflow'
UPGRADE_AIRFLOW = 'upgrade_airflow'
DESTROY_SERVER = 'destroy_nodes'
# Define a list of critical steps, used to determine successfulness of a
# still-running DAG
CRITICAL_DAG_STEPS = [
ARMADA_BUILD_DAG_NAME,
SKIP_UPGRADE_AIRFLOW,
UPGRADE_AIRFLOW,
ARMADA_TEST_RELEASES
]

357
src/bin/shipyard_airflow/tests/unit/control/test_action_helper.py

@ -13,9 +13,142 @@
# limitations under the License.
""" Tests for the action_helper.py module """
from shipyard_airflow.control.helpers import action_helper
from unittest.mock import patch
import yaml
from shipyard_airflow.control.helpers import action_helper
from shipyard_airflow.control.helpers.design_reference_helper import (
DesignRefHelper
)
def get_repeated_steps():
"""Returns a list of fake step dictionaries with repeated steps (tries)
For use in testing getting the latest of a step
Currently, for tests that use this, the only thing that matters is the
task ID and the try number. If this function gets used by more/future tests
more data may need to be added
task_A tries: 1, 2, 3
task_B tries: 1
task_C tries: 1, 2
task_D tries: 1
task_E tries: 1
:returns: A list of fake (and incomplete) step dictionaries, some of which
are repeated across multiple tries
:rtype: list
"""
return [
{
'task_id': 'task_A',
'try_number': 1
},
{
'task_id': 'task_A',
'try_number': 2
},
{
'task_id': 'task_A',
'try_number': 3
},
{
'task_id': 'task_B',
'try_number': 1
},
{
'task_id': 'task_C',
'try_number': 2
},
{
'task_id': 'task_C',
'try_number': 1
},
{
'task_id': 'task_D',
'try_number': 1
},
{
'task_id': 'task_E',
'try_number': 1
}
]
def get_fake_latest_step_dict_failed():
"""Make a fake dictionary of "latest" steps that represent a failed dag
The only key required by the tests calling this function is "state", so
the steps contained in the returned dict are incomplete
:returns: A dictionary of "latest" steps that represent a failed dag
:rtype: dict
"""
return {
'armada_build': {'state': 'failed'},
'arbitrary_step': {'state': 'success'},
'another_arbitrary_step': {'state': 'running'},
'upgrade_airflow': {'state': 'success'},
'concurrency_check': {'state': 'success'}
}
def get_fake_latest_step_dict_running():
"""Make a fake dictionary of "latest" steps that represent a running dag
The only key required by the tests calling this function is "state", so
the steps contained in the returned dict are incomplete
:returns: A dictionary of "latest" steps that represent a running dag
:rtype: dict
"""
return {
'armada_build': {'state': 'queued'},
'arbitrary_step': {'state': 'success'},
'another_arbitrary_step': {'state': 'running'},
'upgrade_airflow': {'state': 'running'},
'concurrency_check': {'state': 'success'}
}
def get_fake_latest_step_dict_successful():
"""Make a fake dictionary of "latest" steps that represent a successful dag
The only key required by the tests calling this function is "state", so
the steps contained in the returned dict are incomplete
:returns: A dictionary of "latest" steps that represent a successful dag
:rtype: dict
"""
return {
'armada_build': {'state': 'success'},
'arbitrary_step': {'state': 'success'},
'another_arbitrary_step': {'state': 'success'},
'upgrade_airflow': {'state': 'skipped'},
'concurrency_check': {'state': 'success'}
}
def get_fake_latest_step_dict_unknown():
"""Make a fake dictionary of "latest" steps that represent a dag of unknown
result
The only key required by the tests calling this function is "state", so
the steps contained in the returned dict are incomplete
:returns: A dictionary of "latest" steps that represent a dag of unknown
result
:rtype: dict
"""
return {
'armada_build': {'state': 'success'},
'arbitrary_step': {'state': 'what'},
'another_arbitrary_step': {'state': 'are'},
'upgrade_airflow': {'state': 'these'},
'concurrency_check': {'state': 'states?'}
}
def test_determine_lifecycle():
dag_statuses = [
@ -168,3 +301,225 @@ def test_get_step():
try_number = 2
step = actions_helper.get_step(step_id, try_number)
assert(step['hostname'].startswith('airflow-worker-1'))
@patch('shipyard_airflow.control.helpers.deckhand_client.DeckhandClient.'
'get_path')
@patch('shipyard_airflow.control.helpers.design_reference_helper.'
'DesignRefHelper.get_design_reference_href', return_value='href')
def test_get_deployment_status_no_action_helper_completed_failed(get_href,
get_path):
action = {
'committed_rev_id': 'rev_id',
'context_marker': 'markofcontext',
'dag_status': 'FAILED',
'id': 'action_id',
'timestamp': 'my_timestamp',
'user': 'cool-person'
}
expected_data = {
'status': 'completed',
'results': 'failed',
'context-marker': action['context_marker'],
'action': action['id'],
'document_url': 'href',
'user': action['user'],
'date': action['timestamp']
}
deployment_status = action_helper.get_deployment_status(action)
assert deployment_status['status'] == expected_data['status']
assert deployment_status['results'] == expected_data['results']
assert (deployment_status['context-marker'] ==
expected_data['context-marker'])
assert deployment_status['action'] == expected_data['action']
assert deployment_status['document_url'] == expected_data['document_url']
assert deployment_status['user'] == expected_data['user']
assert deployment_status['date'] == expected_data['date']
get_href.assert_called_once_with(action['committed_rev_id'])
assert get_path.called # This means we created a DesignRefHelper object
@patch('shipyard_airflow.control.helpers.deckhand_client.DeckhandClient.'
'get_path')
@patch('shipyard_airflow.control.helpers.design_reference_helper.'
'DesignRefHelper.get_design_reference_href', return_value='href')
def test_get_deployment_status_no_action_helper_completed_success(get_href,
get_path):
action = {
'committed_rev_id': 'rev_id',
'context_marker': 'markofcontext',
'dag_status': 'SUCCESS',
'id': 'action_id',
'timestamp': 'my_timestamp',
'user': 'cool-person'
}
expected_data = {
'status': 'completed',
'results': 'successful',
'context-marker': action['context_marker'],
'action': action['id'],
'document_url': 'href',
'user': action['user'],
'date': action['timestamp']
}
deployment_status = action_helper.get_deployment_status(action)
assert deployment_status['status'] == expected_data['status']
assert deployment_status['results'] == expected_data['results']
assert (deployment_status['context-marker'] ==
expected_data['context-marker'])
assert deployment_status['action'] == expected_data['action']
assert deployment_status['document_url'] == expected_data['document_url']
assert deployment_status['user'] == expected_data['user']
assert deployment_status['date'] == expected_data['date']
get_href.assert_called_once_with(action['committed_rev_id'])
assert get_path.called # This means we created a DesignRefHelper object
@patch.object(action_helper.ActionsHelper,
'get_result_from_dag_steps',
return_value='result')
@patch('shipyard_airflow.control.helpers.deckhand_client.DeckhandClient.'
'get_path')
@patch('shipyard_airflow.control.helpers.design_reference_helper.'
'DesignRefHelper.get_design_reference_href', return_value='href')
def test_get_deployment_status_use_action_helper(get_href,
get_path,
get_result):
action = {
'committed_rev_id': 'rev_id',
'context_marker': 'markofcontext',
'dag_status': 'ASDFJKL:',
'id': 'action_id',
'timestamp': 'my_timestamp',
'user': 'cool-person'
}
expected_data = {
'status': 'running',
'results': 'result',
'context-marker': action['context_marker'],
'action': action['id'],
'document_url': 'href',
'user': action['user'],
'date': action['timestamp']
}
deployment_status = action_helper.get_deployment_status(action)
assert deployment_status['status'] == expected_data['status']
assert deployment_status['results'] == expected_data['results']
assert (deployment_status['context-marker'] ==
expected_data['context-marker'])
assert deployment_status['action'] == expected_data['action']
assert deployment_status['document_url'] == expected_data['document_url']
assert deployment_status['user'] == expected_data['user']
assert deployment_status['date'] == expected_data['date']
get_href.assert_called_once_with(action['committed_rev_id'])
assert get_result.called
assert get_path.called # This means we created a DesignRefHelper object
@patch.object(action_helper.ActionsHelper,
'get_result_from_dag_steps',
return_value='result')
@patch('shipyard_airflow.control.helpers.deckhand_client.DeckhandClient.'
'get_path')
@patch('shipyard_airflow.control.helpers.design_reference_helper.'
'DesignRefHelper.get_design_reference_href', return_value='href')
def test_get_deployment_status_use_action_helper_force_completed(get_href,
get_path,
get_result):
action = {
'committed_rev_id': 'rev_id',
'context_marker': 'markofcontext',
'dag_status': 'ASDFJKL:',
'id': 'action_id',
'timestamp': 'my_timestamp',
'user': 'cool-person'
}
expected_data = {
'status': 'completed',
'results': 'result',
'context-marker': action['context_marker'],
'action': action['id'],
'document_url': 'href',
'user': action['user'],
'date': action['timestamp']
}
deployment_status = action_helper.get_deployment_status(action, True)
assert deployment_status['status'] == expected_data['status']
assert deployment_status['results'] == expected_data['results']
assert (deployment_status['context-marker'] ==
expected_data['context-marker'])
assert deployment_status['action'] == expected_data['action']
assert deployment_status['document_url'] == expected_data['document_url']
assert deployment_status['user'] == expected_data['user']
assert deployment_status['date'] == expected_data['date']
get_href.assert_called_once_with(action['committed_rev_id'])
assert get_result.called
assert get_path.called # This means we created a DesignRefHelper object
@patch.object(action_helper.ActionsHelper, '_get_action_info')
@patch.object(action_helper.ActionsHelper, '_get_all_steps',
return_value=get_repeated_steps())
def test__get_latest_steps(get_all_steps, get_action_info):
helper = action_helper.ActionsHelper(action_id='irrelevant')
latest_steps_dict = helper._get_latest_steps()
assert latest_steps_dict['task_A']['try_number'] == 3
assert latest_steps_dict['task_B']['try_number'] == 1
assert latest_steps_dict['task_C']['try_number'] == 2
assert latest_steps_dict['task_D']['try_number'] == 1
assert latest_steps_dict['task_E']['try_number'] == 1
assert get_all_steps.called
assert get_action_info.called
@patch.object(action_helper.ActionsHelper, '_get_action_info')
@patch.object(action_helper.ActionsHelper, '_get_latest_steps',
return_value=get_fake_latest_step_dict_successful())
def test_get_result_from_dag_steps_success(get_latest_steps, get_action_info):
helper = action_helper.ActionsHelper(action_id='irrelevant')
result = helper.get_result_from_dag_steps()
assert result == 'successful'
assert get_latest_steps.called
assert get_action_info.called
@patch.object(action_helper.ActionsHelper, '_get_action_info')
@patch.object(action_helper.ActionsHelper, '_get_latest_steps',
return_value=get_fake_latest_step_dict_failed())
def test_get_result_from_dag_steps_failed(get_latest_steps, get_action_info):
helper = action_helper.ActionsHelper(action_id='irrelevant')
result = helper.get_result_from_dag_steps()
assert result == 'failed'
assert get_latest_steps.called
assert get_action_info.called
@patch.object(action_helper.ActionsHelper, '_get_action_info')
@patch.object(action_helper.ActionsHelper, '_get_latest_steps',
return_value=get_fake_latest_step_dict_running())
def test_get_result_from_dag_steps_running(get_latest_steps, get_action_info):
helper = action_helper.ActionsHelper(action_id='irrelevant')
result = helper.get_result_from_dag_steps()
assert result == 'pending'
assert get_latest_steps.called
assert get_action_info.called
@patch('logging.Logger.warning')
@patch.object(action_helper.ActionsHelper, '_get_action_info')
@patch.object(action_helper.ActionsHelper, '_get_latest_steps',
return_value=get_fake_latest_step_dict_unknown())
def test_get_result_from_dag_steps_unknown(get_latest_steps,
get_action_info,
log_warning_patch):
helper = action_helper.ActionsHelper(action_id='irrelevant')
result = helper.get_result_from_dag_steps()
assert result == 'unknown'
assert get_latest_steps.called
assert get_action_info.called
# Each critical step that had an unknown state should log a warning:
assert log_warning_patch.call_count == 1
Loading…
Cancel
Save