airship/drydock
Code Issues Proposed changes

Enhance validation message flow

Browse Source 
- Update validation message format to latest UCP standard
- Support message levels of ERROR, WARN, INFO
- Update ingestion and schema validation to use new validation
  message format

Change-Id: Ic463badaea8025c5dece88b3999075ead3419bec
This commit is contained in:
Scott Hussey 2018-03-02 14:34:20 -06:00
parent cff99e4d1c
commit 47a27981ec
7 changed files with 199 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
drydock_provisioner/error.py
View File

@ -53,6 +53,14 @@ class InvalidDesignReference(DesignError):
"""
pass
class UnsupportedDocumentType(DesignError):
"""
**Message:** *Site definition document in an unknown format*.
**Troubleshoot:**
"""
pass
class StateError(Exception):
pass

65
drydock_provisioner/ingester/plugins/deckhand.py
View File

@ -74,8 +74,8 @@ class DeckhandIngester(IngesterPlugin):
raise errors.IngesterError("Error parsing YAML: %s" % (err))
# tracking processing status to provide a complete summary of issues
ps = objects.TaskStatus()
ps.set_status(hd_fields.ActionResult.Success)
ps = objects.Validation()
ps.set_status(hd_fields.ValidationResult.Success)
for d in parsed_data:
try:
(schema_ns, doc_kind, doc_version) = d.get('schema',
@ -87,45 +87,56 @@ class DeckhandIngester(IngesterPlugin):
continue
if schema_ns == 'drydock':
try:
doc_ref = objects.DocumentReference(
doc_type=hd_fields.DocumentType.Deckhand,
doc_schema=d.get('schema'),
doc_name=d.get('metadata', {}).get('name', 'Unknown'))
doc_errors = self.validate_drydock_document(d)
if len(doc_errors) > 0:
doc_ctx = d.get('metadata', {}).get('name', 'Unknown')
for e in doc_errors:
ps.add_status_msg(
msg="%s:%s validation error: %s" %
(doc_kind, doc_version, e),
error=True,
ctx_type='document',
ctx=doc_ctx)
ps.add_detail_msg(
objects.ValidationMessage(
msg="%s:%s schema validation error: %s" %
(doc_kind, doc_version, e),
name="DD001",
docs=[doc_ref],
error=True,
level=hd_fields.MessageLevels.ERROR,
diagnostic=
"Invalid input file - see Drydock Troubleshooting Guide for DD001"
))
ps.set_status(hd_fields.ActionResult.Failure)
continue
model = self.process_drydock_document(d)
ps.add_status_msg(
msg="Successfully processed Drydock document type %s."
% doc_kind,
error=False,
ctx_type='document',
ctx=model.get_id())
model.doc_ref = doc_ref
models.append(model)
except errors.IngesterError as ie:
msg = "Error processing document: %s" % str(ie)
self.logger.warning(msg)
if d.get('metadata', {}).get('name', None) is not None:
ctx = d.get('metadata').get('name')
else:
ctx = 'Unknown'
ps.add_status_msg(
msg=msg, error=True, ctx_type='document', ctx=ctx)
ps.add_detail_msg(
objects.ValidationMessage(
msg=msg,
name="DD000",
error=True,
level=hd_fields.MessageLevels.ERROR,
docs=[doc_ref],
diagnostic="Exception during document processing "
"- see Drydock Troubleshooting Guide "
"for DD000"))
ps.set_status(hd_fields.ActionResult.Failure)
except Exception as ex:
msg = "Unexpected error processing document: %s" % str(ex)
self.logger.error(msg, exc_info=True)
if d.get('metadata', {}).get('name', None) is not None:
ctx = d.get('metadata').get('name')
else:
ctx = 'Unknown'
ps.add_status_msg(
msg=msg, error=True, ctx_type='document', ctx=ctx)
ps.add_detail_msg(
objects.ValidationMessage(
msg=msg,
name="DD000",
error=True,
level=hd_fields.MessageLevels.ERROR,
docs=[doc_ref],
diagnostic="Unexpected exception during document "
"processing - see Drydock Troubleshooting "
"Guide for DD000"))
ps.set_status(hd_fields.ActionResult.Failure)
return (ps, models)

1
drydock_provisioner/objects/__init__.py
View File

@ -32,6 +32,7 @@ def register_all():
importlib.import_module('drydock_provisioner.objects.bootaction')
importlib.import_module('drydock_provisioner.objects.task')
importlib.import_module('drydock_provisioner.objects.builddata')
importlib.import_module('drydock_provisioner.objects.validation')
# Utility class for calculating inheritance

5
drydock_provisioner/objects/base.py
View File

@ -34,6 +34,11 @@ class DrydockObject(base.VersionedObject):
OBJ_PROJECT_NAMESPACE = 'drydock_provisioner.objects'
# Maintain a reference to the source document for the model
fields = {
'doc_ref': obj_fields.ObjectField('DocumentReference', nullable=True)
}
# Return None for undefined attributes
def obj_load_attr(self, attrname):
if attrname in self.fields.keys():

10
drydock_provisioner/objects/fields.py
View File

@ -193,3 +193,13 @@ class NetworkLinkTrunkingModeField(fields.BaseEnumField):
class ValidationResult(BaseDrydockEnum):
Success = 'success'
Failure = 'failure'
class MessageLevels(BaseDrydockEnum):
INFO = 'Info'
WARN = 'Warning'
ERROR = 'Error'
class DocumentType(BaseDrydockEnum):
Deckhand = 'deckhand'

120
drydock_provisioner/objects/validation.py Normal file
View File

@ -0,0 +1,120 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Models for representing asynchronous tasks."""
from datetime import datetime
import oslo_versionedobjects.fields as ovo_fields
from drydock_provisioner import objects
import drydock_provisioner.objects.base as base
import drydock_provisioner.error as errors
import drydock_provisioner.objects.fields as hd_fields
from .task import TaskStatus, TaskStatusMessage
class Validation(TaskStatus):
"""Specialized status for design validation status."""
def __init__(self):
super().__init__()
def add_detail_msg(self, msg=None):
"""Add a detailed validation message.
:param msg: instance of ValidationMessage
"""
self.message_list.append(msg)
if msg.error or msg.level == "Error":
self.error_count = self.error_count + 1
def to_dict(self):
return {
'kind': 'Status',
'apiVersion': 'v1.0',
'metadata': {},
'message': self.message,
'reason': self.reason,
'status': self.status,
'details': {
'errorCount': self.error_count,
'messageList': [x.to_dict() for x in self.message_list],
}
}
class ValidationMessage(TaskStatusMessage):
"""Message describing details of a validation."""
def __init__(self, msg, name, error=False, level=None, docs=None, diagnostic=None):
self.name = name
self.message = msg
self.error = error
self.level = level
self.diagnostic = diagnostic
self.ts = datetime.utcnow()
self.docs = docs
def to_dict(self):
"""Convert to a dictionary in prep for JSON/YAML serialization."""
_dict = {
'kind': 'ValidationMessage',
'name': self.name,
'message': self.message,
'error': self.error,
'level': self.level,
'diagnostic': self.diagnostic,
'ts': str(self.ts),
'documents': [x.to_dict() for x in self.docs]
}
return _dict
@base.DrydockObjectRegistry.register
class DocumentReference(base.DrydockObject):
"""Keep a reference to the original document that data was loaded from."""
VERSION = '1.0'
fields = {
'doc_type': ovo_fields.StringField(),
'doc_schema': ovo_fields.StringField(nullable=True),
'doc_name': ovo_fields.StringField(nullable=True),
}
def __init__(self, **kwargs):
super().__init__(**kwargs)
if (self.doc_type == hd_fields.DocumentType.Deckhand):
if not all([self.doc_schema, self.doc_name]):
raise ValueError("doc_schema and doc_name required for Deckhand sources.")
else:
raise errors.UnsupportedDocumentType(
"Document type %s not supported." % self.doc_type)
def to_dict(self):
"""Serialize to a dictionary for further serialization."""
d = dict()
if self.doc_type == hd_fields.DocumentType.Deckhand:
d['schema'] = self.doc_schema
d['name'] = self.doc_name
return d
# Emulate OVO object registration
setattr(objects, Validation.obj_name(), Validation)
setattr(objects, ValidationMessage.obj_name(), ValidationMessage)

25
tests/unit/test_ingester.py
View File

@ -19,8 +19,6 @@ import drydock_provisioner.objects as objects
class TestClass(object):
def test_ingest_deckhand(self, input_files, setup, deckhand_ingester):
objects.register_all()
input_file = input_files.join("deckhand_fullsite.yaml")
design_state = DrydockState()
@ -29,14 +27,26 @@ class TestClass(object):
design_status, design_data = deckhand_ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
print("%s" % str(design_status.to_dict()))
assert design_status.status == objects.fields.ActionResult.Success
assert design_status.status == objects.fields.ValidationResult.Success
assert len(design_data.host_profiles) == 2
assert len(design_data.baremetal_nodes) == 2
def test_ingest_yaml(self, input_files, setup, yaml_ingester):
objects.register_all()
def test_ingest_deckhand_docref_exists(self, input_files, setup, deckhand_ingester):
"""Test that each processed document has a doc_ref."""
input_file = input_files.join('deckhand_fullsite.yaml')
design_state = DrydockState()
design_ref = "file://%s" % str(input_file)
design_status, design_data = deckhand_ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
assert design_status.status == objects.fields.ValidationResult.Success
for p in design_data.host_profiles:
assert p.doc_ref is not None
assert p.doc_ref.doc_schema == 'drydock/HostProfile/v1'
assert p.doc_ref.doc_name is not None
def test_ingest_yaml(self, input_files, setup, yaml_ingester):
input_file = input_files.join("fullsite.yaml")
design_state = DrydockState()
@ -45,7 +55,6 @@ class TestClass(object):
design_status, design_data = yaml_ingester.ingest_data(
design_state=design_state, design_ref=design_ref)
print("%s" % str(design_status.to_dict()))
assert design_status.status == objects.fields.ActionResult.Success
assert design_status.status == objects.fields.ValidationResult.Success
assert len(design_data.host_profiles) == 2
assert len(design_data.baremetal_nodes) == 2