airship
/
shipyard
Code Issues Proposed changes

Defect fix validation calls to UCP components 

Fixes the content type of the call to validate docs from yaml to json
Fixes bug in formulating deckhand reference
Adds some logging to the validations flow
Refactors the response handling for validations to be more resilient

Change-Id: Ie0b044539f2bc98eaa0ab84e444b6b4ae361ae8c
This commit is contained in:
Bryan Strassner 2017-11-28 16:45:39 -06:00
parent 5f8513b4f1
commit 0b5684d549
3 changed files with 71 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
shipyard_airflow/control/configdocs/configdocs_helper.py
View File

@ -346,8 +346,8 @@ class ConfigdocsHelper(object):
# Constructs the design reference as json for use by other components # Constructs the design reference as json for use by other components
design_reference = { design_reference = {
"rel": "design", "rel": "design",
"href": "deckhand+{}/rendered-documents".format( "href": "deckhand+{}".format(DeckhandClient.get_path(
DeckhandPaths.RENDERED_REVISION_DOCS.value.format(revision_id) DeckhandPaths.RENDERED_REVISION_DOCS).format(revision_id)
), ),
"type": "application/x-yaml" "type": "application/x-yaml"
} }
@ -374,20 +374,18 @@ class ConfigdocsHelper(object):
# create a holder for things we need back from the threads # create a holder for things we need back from the threads
response = {'response': None} response = {'response': None}
exception = {'exception': None} exception = {'exception': None}
design_ref = ConfigdocsHelper._get_design_reference(revision_id)
validation_threads.append( validation_threads.append(
{ {
'thread': threading.Thread( 'thread': threading.Thread(
target=ConfigdocsHelper._get_validations_for_component, target=ConfigdocsHelper._get_validations_for_component,
args=(
endpoint['url'],
ConfigdocsHelper._get_design_reference(
revision_id
),
response,
exception,
ctx.external_marker
),
kwargs={ kwargs={
'url': endpoint['url'],
'design_reference': design_ref,
'response': response,
'exception': exception,
'context_marker': ctx.external_marker,
'thread_name': endpoint['name'],
'log_extra': { 'log_extra': {
'req_id': ctx.request_id, 'req_id': ctx.request_id,
'external_ctx': ctx.external_marker, 'external_ctx': ctx.external_marker,
@ -409,26 +407,44 @@ class ConfigdocsHelper(object):
response, response,
exception, exception,
context_marker, context_marker,
thread_name,
**kwargs): **kwargs):
# Invoke the POST for validation # Invoke the POST for validation
try: try:
headers = { headers = {
'X-Context-Marker': context_marker, 'X-Context-Marker': context_marker,
'X-Auth-Token': get_token(), 'X-Auth-Token': get_token(),
'content-type': 'application/x-yaml' 'content-type': 'application/json'
} }
http_resp = requests.post(url, http_resp = requests.post(url,
headers=headers, headers=headers,
data=design_reference, data=design_reference,
timeout=(5, 30)) timeout=(5, 30))
http_resp.raise_for_status() # 400 response is "valid" failure to validate. > 400 is a problem.
raw_response = http_resp.decode('utf-8') if http_resp.status_code > 400:
response_dict = json.loads(raw_response) http_resp.raise_for_status()
response_dict = http_resp.json()
response['response'] = response_dict response['response'] = response_dict
except Exception as ex: except Exception as ex:
# catch anything exceptional as a failure to validate # catch anything exceptional as a failure to run validations
LOG.error(ex) unable_str = '{} unable to validate configdocs'.format(thread_name)
LOG.error("%s. Exception follows.", unable_str)
LOG.error(str(ex))
response['response'] = {
'details': {
'messageList': [
{
'message': unable_str,
'error': True
},
{
'message': str(ex),
'error': True
}
]
}
}
exception['exception'] = ex exception['exception'] = ex
def get_validations_for_revision(self, revision_id): def get_validations_for_revision(self, revision_id):
@ -456,37 +472,25 @@ class ConfigdocsHelper(object):
# check on the response, extract the validations # check on the response, extract the validations
error_count = 0 error_count = 0
for validation_thread in validation_threads: for validation_thread in validation_threads:
val_response = validation_thread.get('response')['response'] th_name = validation_thread.get('name')
if (not val_response or val_response = validation_thread.get('response',
validation_thread.get('exception')['exception']): {}).get('response')
# exception was raised, or no body was returned. LOG.debug("Validation from: %s response: %s",
raise AppError( th_name, str(val_response))
title='Unable to properly validate configdocs', if validation_thread.get('exception', {}).get('exception'):
description=( LOG.error('Invocation of validation by %s has failed', th_name)
'Invocation of validation by {} has failed'.format(
validation_thread.get('name')
)
),
status=falcon.HTTP_500,
retry=False,
)
if not val_response:
raise AppError(
title='An invalid response was returned by validation',
description='No valid response status from {}'.format(
validation_thread.get('name')),
status=falcon.HTTP_500,
retry=False,
)
# invalid status needs collection of messages # invalid status needs collection of messages
# valid status means that it passed. No messages to collect # valid status means that it passed. No messages to collect
msg_list = val_response.get('details').get('messageList') if val_response.get('details') is None:
msg_list = [{'message': str(val_response), 'error': True}]
else:
msg_list = val_response.get('details').get('messageList', [])
for msg in msg_list: for msg in msg_list:
if msg.get('error'): if msg.get('error'):
error_count = error_count + 1 error_count = error_count + 1
resp_msgs.append( resp_msgs.append(
{ {
'name': validation_thread.get('name'), 'name': th_name,
'message': msg.get('message'), 'message': msg.get('message'),
'error': True 'error': True
} }
@ -494,7 +498,7 @@ class ConfigdocsHelper(object):
else: else:
resp_msgs.append( resp_msgs.append(
{ {
'name': validation_thread.get('name'), 'name': th_name,
'message': msg.get('message'), 'message': msg.get('message'),
'error': False 'error': False
} }

2
shipyard_airflow/control/configdocs/deckhand_client.py
View File

@ -153,7 +153,7 @@ class DeckhandClient(object):
).format(revision_id, tag) ).format(revision_id, tag)
response = self._post_request(url) response = self._post_request(url)
response.raise_for_status() self._handle_bad_response(response)
return yaml.safe_load(response.text) return yaml.safe_load(response.text)
def rollback(self, target_revision_id): def rollback(self, target_revision_id):

43
tests/unit/control/test_configdocs_helper.py
View File

@ -26,6 +26,7 @@ from shipyard_airflow.control.configdocs.configdocs_helper import (
) )
from shipyard_airflow.control.configdocs.deckhand_client import ( from shipyard_airflow.control.configdocs.deckhand_client import (
DeckhandClient, DeckhandClient,
DeckhandPaths,
DeckhandResponseError, DeckhandResponseError,
NoRevisionsExistError NoRevisionsExistError
) )
@ -539,25 +540,29 @@ def test_get_validations_for_revision():
""" """
Tets the functionality of the get_validations_for_revision method Tets the functionality of the get_validations_for_revision method
""" """
helper = ConfigdocsHelper(CTX) with patch('shipyard_airflow.control.configdocs.deckhand_client.'
hold_ve = helper.__class__._get_validation_endpoints 'DeckhandClient.get_path') as mock_get_path:
hold_vfc = helper.__class__._get_validations_for_component mock_get_path.return_value = 'path{}'
helper.__class__._get_validation_endpoints = ( helper = ConfigdocsHelper(CTX)
_fake_get_validation_endpoints hold_ve = helper.__class__._get_validation_endpoints
) hold_vfc = helper.__class__._get_validations_for_component
helper.__class__._get_validations_for_component = ( helper.__class__._get_validation_endpoints = (
_fake_get_validations_for_component _fake_get_validation_endpoints
) )
helper._get_deckhand_validations = lambda revision_id: [] helper.__class__._get_validations_for_component = (
try: _fake_get_validations_for_component
val_status = helper.get_validations_for_revision(3) )
err_count = val_status['details']['errorCount'] helper._get_deckhand_validations = lambda revision_id: []
err_list_count = len(val_status['details']['messageList']) try:
assert err_count == err_list_count val_status = helper.get_validations_for_revision(3)
assert val_status['details']['errorCount'] == 4 err_count = val_status['details']['errorCount']
finally: err_list_count = len(val_status['details']['messageList'])
helper.__class__._get_validation_endpoints = hold_ve assert err_count == err_list_count
helper.__class__._get_validations_for_component = hold_vfc assert val_status['details']['errorCount'] == 4
finally:
helper.__class__._get_validation_endpoints = hold_ve
helper.__class__._get_validations_for_component = hold_vfc
mock_get_path.assert_called_with(DeckhandPaths.RENDERED_REVISION_DOCS)
FK_VAL_BASE_RESP = FakeResponse(status_code=200, text=""" FK_VAL_BASE_RESP = FakeResponse(status_code=200, text="""