diff --git a/deckhand/engine/document.py b/deckhand/engine/document.py deleted file mode 100644 index b1e1a292..00000000 --- a/deckhand/engine/document.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class Document(object): - """Object wrapper for documents. - - After "raw" documents undergo schema validation, they can be wrapped with - this class to allow nested dictionary entries to be quickly retrieved. - """ - - def __init__(self, data): - """Constructor for ``Document``. - - :param data: Dictionary of all document data (includes metadata, data, - schema, etc.). - """ - self._inner = data - - def to_dict(self): - return self._inner - - def is_abstract(self): - """Return whether the document is abstract. - - Not all documents contain this property; in that case they are - concrete. - """ - try: - return self._inner['metadata']['layeringDefinition']['abstract'] - except Exception: - return False - - def get_schema(self): - try: - return self._inner['schema'] - except Exception: - return '' - - def get_name(self): - try: - return self._inner['metadata']['name'] - except Exception: - return '' - - def get_layer(self): - try: - return self._inner['metadata']['layeringDefinition']['layer'] - except Exception: - return '' - - def get_parent_selector(self): - """Return the `parentSelector` for the document. - - The topmost document defined by the `layerOrder` in the LayeringPolicy - does not have a `parentSelector` as it has no parent. - - :returns: `parentSelector` for the document if present, else None. - """ - try: - return self._inner['metadata']['layeringDefinition'][ - 'parentSelector'] - except Exception: - return {} - - def get_labels(self): - return self._inner['metadata']['labels'] - - def get_substitutions(self): - try: - return self._inner['metadata'].get('substitutions', []) - except Exception: - return [] - - def get_actions(self): - try: - return self._inner['metadata']['layeringDefinition']['actions'] - except Exception: - return [] - - def get_children(self, nested=False): - """Get document children, if any. - - :param nested: Recursively retrieve all children for each child - document. - :type nested: boolean - :returns: List of children of type `Document`. - """ - if not nested: - return self._inner.get('children', []) - else: - return self._get_nested_children(self, []) - - def _get_nested_children(self, doc, nested_children): - for child in doc.get('children', []): - nested_children.append(child) - if 'children' in child._inner: - self._get_nested_children(child, nested_children) - return nested_children - - def get(self, k, default=None): - return self.__getitem__(k, default=default) - - def __getitem__(self, k, default=None): - return self._inner.get(k, default) - - def __setitem__(self, k, val): - self._inner[k] = val - - def __delitem__(self, k): - if self.__contains__(k): - del self._inner[k] - - def __contains__(self, k): - return self.get(k, default=None) is not None - - def __missing__(self, k): - return not self.__contains__(k) - - def __repr__(self): - return '(%s, %s)' % (self.get_schema(), self.get_name()) - - def __str__(self): - return str(self._inner) diff --git a/deckhand/engine/document_validation.py b/deckhand/engine/document_validation.py index ffb1642f..cae804ce 100644 --- a/deckhand/engine/document_validation.py +++ b/deckhand/engine/document_validation.py @@ -19,6 +19,7 @@ import jsonschema from oslo_log import log as logging import six +from deckhand.engine import document_wrapper from deckhand.engine.schema import base_schema from deckhand.engine.schema import v1_0 from deckhand import errors @@ -140,9 +141,9 @@ class SchemaValidator(BaseValidator): return matching_schemas def matches(self, document): - if is_abstract(document) is True: + if document.is_abstract: LOG.info('Skipping schema validation for abstract document [%s]: ' - '%s.', document['schema'], document['metadata']['name']) + '%s.', document.schema, document.name) return False return True @@ -167,7 +168,7 @@ class SchemaValidator(BaseValidator): schemas_to_use = self._get_schemas(document) if not schemas_to_use and use_fallback_schema: LOG.debug('Document schema %s not recognized. Using "fallback" ' - 'schema.', document['schema']) + 'schema.', document.schema) schemas_to_use = [SchemaValidator._fallback_schema] for schema_to_use in schemas_to_use: @@ -191,8 +192,8 @@ class SchemaValidator(BaseValidator): for error in errors: LOG.error( 'Failed schema validation for document [%s] %s. ' - 'Details: %s.', document['schema'], - document['metadata']['name'], error.message) + 'Details: %s.', document.schema, document.name, + error.message) parent_path = root_path + '.'.join( [six.text_type(x) for x in error.path]) yield error.message, parent_path @@ -211,9 +212,9 @@ class DataSchemaValidator(SchemaValidator): for data_schema in data_schemas: # Ensure that each `DataSchema` document has required properties # before they themselves can be used to validate other documents. - if 'name' not in data_schema.get('metadata', {}): + if 'name' not in data_schema.metadata: continue - if self._schema_re.match(data_schema['metadata']['name']) is None: + if self._schema_re.match(data_schema.name) is None: continue if 'data' not in data_schema: continue @@ -221,16 +222,16 @@ class DataSchemaValidator(SchemaValidator): 'metadata.name') class Schema(object): - schema = data_schema['data'] + schema = data_schema.data schema_map[schema_version].setdefault(schema_prefix, Schema()) return schema_map def matches(self, document): - if is_abstract(document) is True: + if document.is_abstract: LOG.info('Skipping schema validation for abstract document [%s]: ' - '%s.', document['schema'], document['metadata']['name']) + '%s.', document.schema, document.name) return False schema_prefix, schema_version = get_schema_parts(document) return schema_prefix in self._schema_map.get(schema_version, {}) @@ -257,25 +258,26 @@ class DocumentValidation(object): revisions to be used the "data" section of each document in ``documents``. Additional ``DataSchema`` documents in ``documents`` are combined with these. - :type existing_data_schemas: List[dict] + :type existing_data_schemas: dict or List[dict] """ self.documents = [] existing_data_schemas = existing_data_schemas or [] - data_schemas = existing_data_schemas[:] - db_data_schemas = {d['metadata']['name']: d for d in data_schemas} + data_schemas = [document_wrapper.DocumentDict(d) + for d in existing_data_schemas] + _data_schema_map = {d.name: d for d in data_schemas} - if not isinstance(documents, (list, tuple)): + if not isinstance(documents, list): documents = [documents] - for document in documents: - if document.get('schema', '').startswith(types.DATA_SCHEMA_SCHEMA): + if not isinstance(document, document_wrapper.DocumentDict): + document = document_wrapper.DocumentDict(document) + if document.schema.startswith(types.DATA_SCHEMA_SCHEMA): data_schemas.append(document) # If a newer version of the same DataSchema was passed in, # only use the new one and discard the old one. - document_name = document.get('metadata', {}).get('name') - if document_name in db_data_schemas: - data_schemas.remove(db_data_schemas.pop(document_name)) + if document.name in _data_schema_map: + data_schemas.remove(_data_schema_map.pop(document.name)) self.documents.append(document) # NOTE(fmontei): The order of the validators is important. The @@ -346,8 +348,8 @@ class DocumentValidation(object): if error_messages: for error_msg, error_path in error_messages: result['errors'].append({ - 'schema': document['schema'], - 'name': document['metadata']['name'], + 'schema': document.schema, + 'name': document.name, 'message': error_msg, 'path': error_path }) @@ -418,13 +420,6 @@ class DocumentValidation(object): return validations -def is_abstract(document): - try: - return document['metadata']['layeringDefinition']['abstract'] - except Exception: - return False - - def get_schema_parts(document, schema_key='schema'): schema_parts = utils.jsonpath_parse(document, schema_key).split('/') schema_prefix = '/'.join(schema_parts[:2]) diff --git a/deckhand/engine/document_wrapper.py b/deckhand/engine/document_wrapper.py new file mode 100644 index 00000000..75eb2e72 --- /dev/null +++ b/deckhand/engine/document_wrapper.py @@ -0,0 +1,84 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_serialization import jsonutils as json + +from deckhand import utils + + +class DocumentDict(dict): + """Wrapper for a document. + + Implements convenient properties for nested, commonly accessed document + keys. Property setters are only implemented for mutable data. + + Useful for accessing nested dictionary keys without having to worry about + exceptions getting thrown. + + """ + + @property + def is_abstract(self): + return utils.jsonpath_parse( + self, 'metadata.layeringDefinition.abstract') is True + + @property + def schema(self): + return self.get('schema', '') + + @property + def metadata(self): + return self.get('metadata', {}) + + @property + def data(self): + return self.get('data', {}) + + @data.setter + def data(self, value): + self['data'] = value + + @property + def name(self): + return utils.jsonpath_parse(self, 'metadata.name') + + @property + def layer(self): + return utils.jsonpath_parse( + self, 'metadata.layeringDefinition.layer') + + @property + def parent_selector(self): + return utils.jsonpath_parse( + self, 'metadata.layeringDefinition.parentSelector') or {} + + @property + def labels(self): + return utils.jsonpath_parse(self, 'metadata.labels') or {} + + @property + def substitutions(self): + return utils.jsonpath_parse(self, 'metadata.substitutions') or [] + + @substitutions.setter + def substitutions(self, value): + return utils.jsonpath_replace(self, value, 'metadata.substitutions') + + @property + def actions(self): + return utils.jsonpath_parse( + self, 'metadata.layeringDefinition.actions') or [] + + def __hash__(self): + return hash(json.dumps(self, sort_keys=True)) diff --git a/deckhand/engine/layering.py b/deckhand/engine/layering.py index c77096c8..0c39eed9 100644 --- a/deckhand/engine/layering.py +++ b/deckhand/engine/layering.py @@ -16,9 +16,8 @@ import collections import copy from oslo_log import log as logging -import six -from deckhand.engine import document +from deckhand.engine import document_wrapper from deckhand.engine import secrets_manager from deckhand.engine import utils from deckhand import errors @@ -62,21 +61,25 @@ class DocumentLayering(object): document contains a "children" property in addition to original data. List of documents returned is ordered from highest to lowest layer. - :rtype: list of deckhand.engine.document.Document objects. + :rtype: List[:class:`DocumentDict`] :raises IndeterminateDocumentParent: If more than one parent document was found for a document. """ layered_docs = list( - filter(lambda x: 'layeringDefinition' in x['metadata'], + filter(lambda x: 'layeringDefinition' in x.metadata, self._documents)) # ``all_children`` is a counter utility for verifying that each # document has exactly one parent. all_children = collections.Counter() + # Mapping of (doc.name, doc.metadata.name) => children, where children + # are the documents whose `parentSelector` references the doc. + self._children = {} + def _get_children(doc): children = [] - doc_layer = doc.get_layer() + doc_layer = doc.layer try: next_layer_idx = self._layer_order.index(doc_layer) + 1 children_doc_layer = self._layer_order[next_layer_idx] @@ -89,16 +92,16 @@ class DocumentLayering(object): # Documents with different schemas are never layered together, # so consider only documents with same schema as candidates. is_potential_child = ( - other_doc.get_layer() == children_doc_layer and - other_doc.get_schema() == doc.get_schema() + other_doc.layer == children_doc_layer and + other_doc.schema == doc.schema ) if (is_potential_child): # A document can have many labels but should only have one # explicit label for the parentSelector. - parent_sel = other_doc.get_parent_selector() + parent_sel = other_doc.parent_selector parent_sel_key = list(parent_sel.keys())[0] parent_sel_val = list(parent_sel.values())[0] - doc_labels = doc.get_labels() + doc_labels = doc.labels if (parent_sel_key in doc_labels and parent_sel_val == doc_labels[parent_sel_key]): @@ -108,18 +111,19 @@ class DocumentLayering(object): for layer in self._layer_order: docs_by_layer = list(filter( - (lambda x: x.get_layer() == layer), layered_docs)) + (lambda x: x.layer == layer), layered_docs)) for doc in docs_by_layer: children = _get_children(doc) if children: all_children.update(children) - doc.to_dict().setdefault('children', children) + self._children.setdefault((doc.name, doc.schema), + children) all_children_elements = list(all_children.elements()) secondary_docs = list( - filter(lambda d: d.get_layer() != self._layer_order[0], + filter(lambda d: d.layer != self._layer_order[0], layered_docs)) for doc in secondary_docs: # Unless the document is the topmost document in the @@ -128,32 +132,30 @@ class DocumentLayering(object): if doc not in all_children_elements: LOG.info('Could not find parent for document with name=%s, ' 'schema=%s, layer=%s, parentSelector=%s.', - doc.get_name(), doc.get_schema(), doc.get_layer(), - doc.get_parent_selector()) + doc.name, doc.schema, doc.layer, doc.parent_selector) self._parentless_documents.append(doc) # If the document is a child document of more than 1 parent, then # the document has too many parents, which is a validation error. - elif all_children[doc] != 1: + elif all_children[doc] > 1: LOG.info('%d parent documents were found for child document ' 'with name=%s, schema=%s, layer=%s, parentSelector=%s' - '. Each document must only have 1 parent.', - all_children[doc], doc.get_name(), doc.get_schema(), - doc.get_layer(), doc.get_parent_selector()) + '. Each document must have exactly 1 parent.', + all_children[doc], doc.name, doc.schema, doc.layer, + doc.parent_selector) raise errors.IndeterminateDocumentParent(document=doc) return layered_docs def _extract_layering_policy(self, documents): - documents = copy.deepcopy(documents) for doc in documents: if doc['schema'].startswith(types.LAYERING_POLICY_SCHEMA): layering_policy = doc - documents.remove(doc) return ( - document.Document(layering_policy), - [document.Document(d) for d in documents] + document_wrapper.DocumentDict(layering_policy), + [document_wrapper.DocumentDict(d) for d in documents + if d is not layering_policy] ) - return None, [document.Document(d) for d in documents] + return None, [document_wrapper.DocumentDict(d) for d in documents] def __init__(self, documents, substitution_sources=None): """Contructor for ``DocumentLayering``. @@ -256,15 +258,29 @@ class DocumentLayering(object): return overall_data + def _get_children(self, document): + """Recursively retrieve all children. + + Used in the layering module when calculating children for each + document. + + :returns: List of nested children. + :rtype: Generator[:class:`DocumentDict`] + """ + for child in self._children.get((document.name, document.schema), []): + yield child + grandchildren = self._get_children(child) + for grandchild in grandchildren: + yield grandchild + def _apply_substitutions(self, document): try: secrets_substitution = secrets_manager.SecretsSubstitution( document, self._substitution_sources) return secrets_substitution.substitute_all() - except errors.DocumentNotFound as e: + except errors.SubstitutionDependencyNotFound: LOG.error('Failed to render the documents because a secret ' 'document could not be found.') - LOG.exception(six.text_type(e)) def render(self): """Perform layering on the list of documents passed to ``__init__``. @@ -281,72 +297,63 @@ class DocumentLayering(object): """ # ``rendered_data_by_layer`` tracks the set of changes across all # actions across each layer for a specific document. - rendered_data_by_layer = {} + rendered_data_by_layer = document_wrapper.DocumentDict() # NOTE(fmontei): ``global_docs`` represents the topmost documents in # the system. It should probably be impossible for more than 1 # top-level doc to exist, but handle multiple for now. global_docs = [doc for doc in self._layered_documents - if doc.get_layer() == self._layer_order[0]] + if doc.layer == self._layer_order[0]] for doc in global_docs: - layer_idx = self._layer_order.index(doc.get_layer()) - if doc.get_substitutions(): - substituted_data = self._apply_substitutions(doc.to_dict()) - rendered_data_by_layer[layer_idx] = substituted_data[0] + layer_idx = self._layer_order.index(doc.layer) + if doc.substitutions: + substituted_data = self._apply_substitutions(doc) + if substituted_data: + rendered_data_by_layer[layer_idx] = substituted_data[0] else: - rendered_data_by_layer[layer_idx] = doc.to_dict() + rendered_data_by_layer[layer_idx] = doc # Keep iterating as long as a child exists. - for child in doc.get_children(nested=True): + for child in self._get_children(doc): # Retrieve the most up-to-date rendered_data (by # referencing the child's parent's data). - child_layer_idx = self._layer_order.index(child.get_layer()) + child_layer_idx = self._layer_order.index(child.layer) rendered_data = rendered_data_by_layer[child_layer_idx - 1] # Apply each action to the current document. - for action in child.get_actions(): + for action in child.actions: LOG.debug('Applying action %s to child document with ' 'name=%s, schema=%s, layer=%s.', action, - child.get_name(), child.get_schema(), - child.get_layer()) + child.name, child.schema, child.layer) rendered_data = self._apply_action( - action, child.to_dict(), rendered_data) + action, child, rendered_data) # Update the actual document data if concrete. - if not child.is_abstract(): - if child.get_substitutions(): - rendered_data['metadata'][ - 'substitutions'] = child.get_substitutions() + if not child.is_abstract: + if child.substitutions: + rendered_data.substitutions = child.substitutions substituted_data = self._apply_substitutions( rendered_data) if substituted_data: rendered_data = substituted_data[0] child_index = self._layered_documents.index(child) - self._layered_documents[child_index]['data'] = ( - rendered_data['data']) + self._layered_documents[child_index].data = ( + rendered_data.data) # Update ``rendered_data_by_layer`` for this layer so that # children in deeper layers can reference the most up-to-date # changes. rendered_data_by_layer[child_layer_idx] = rendered_data - if 'children' in doc: - del doc['children'] - # Handle edge case for parentless documents that require substitution. # If a document has no parent, then the for loop above doesn't iterate # over the parentless document, so substitution must be done here for # parentless documents. for doc in self._parentless_documents: - if not doc.is_abstract(): - substituted_data = self._apply_substitutions(doc.to_dict()) + if not doc.is_abstract and doc.substitutions: + substituted_data = self._apply_substitutions(doc) if substituted_data: - # TODO(fmontei): Use property after implementing it in - # document wrapper class. - doc._inner = substituted_data[0] + doc = substituted_data[0] - return ( - [d.to_dict() for d in self._layered_documents] + - [self._layering_policy.to_dict()] - ) + return self._layered_documents + [self._layering_policy] diff --git a/deckhand/engine/secrets_manager.py b/deckhand/engine/secrets_manager.py index 470f036d..52c481fa 100644 --- a/deckhand/engine/secrets_manager.py +++ b/deckhand/engine/secrets_manager.py @@ -16,7 +16,7 @@ from oslo_log import log as logging import six from deckhand.barbican import driver -from deckhand.engine import document as document_wrapper +from deckhand.engine import document_wrapper from deckhand import errors from deckhand import utils @@ -111,18 +111,19 @@ class SecretsSubstitution(object): sources for substitution. Should only include concrete documents. :type substitution_sources: List[dict] """ - if not isinstance(documents, (list, tuple)): + + if not isinstance(documents, list): documents = [documents] - self.substitution_documents = [] - self.substitution_sources = substitution_sources or [] + self._documents = [] + self._substitution_sources = substitution_sources or [] for document in documents: - if not isinstance(document, document_wrapper.Document): - document_obj = document_wrapper.Document(document) - # If the document has substitutions include it. - if document_obj.get_substitutions(): - self.substitution_documents.append(document_obj) + if not isinstance(document, document_wrapper.DocumentDict): + document = document_wrapper.DocumentDict(document) + # If the document has substitutions include it. + if document.substitutions: + self._documents.append(document) def substitute_all(self): """Substitute all documents that have a `metadata.substitutions` field. @@ -133,16 +134,19 @@ class SecretsSubstitution(object): from a document in the site layer. :returns: List of fully substituted documents. + :rtype: List[:class:`DocumentDict`] + :raises SubstitutionDependencyNotFound: If a substitution source wasn't + found or something else went wrong during substitution. """ - LOG.debug('Substituting secrets for documents: %s', - self.substitution_documents) + LOG.debug('Performing substitution on following documents: %s', + ', '.join(['[%s] %s' % (d.schema, d.name) + for d in self._documents])) substituted_docs = [] - for doc in self.substitution_documents: - LOG.debug( - 'Checking for substitutions in schema=%s, metadata.name=%s', - doc.get_name(), doc.get_schema()) - for sub in doc.get_substitutions(): + for document in self._documents: + LOG.debug('Checking for substitutions for document [%s] %s.', + document.schema, document.name) + for sub in document.substitutions: src_schema = sub['src']['schema'] src_name = sub['src']['name'] src_path = sub['src']['path'] @@ -151,7 +155,7 @@ class SecretsSubstitution(object): x['metadata']['name'] == src_name) try: src_doc = next( - iter(filter(is_match, self.substitution_sources))) + iter(filter(is_match, self._substitution_sources))) except StopIteration: src_doc = {} @@ -172,16 +176,16 @@ class SecretsSubstitution(object): src_name, src_path, dest_path, dest_pattern) try: substituted_data = utils.jsonpath_replace( - doc['data'], src_secret, dest_path, dest_pattern) + document['data'], src_secret, dest_path, dest_pattern) if isinstance(substituted_data, dict): - doc['data'].update(substituted_data) + document['data'].update(substituted_data) else: - doc['data'] = substituted_data + document['data'] = substituted_data except Exception as e: LOG.error('Unexpected exception occurred while attempting ' 'secret substitution. %s', six.text_type(e)) raise errors.SubstitutionDependencyNotFound( details=six.text_type(e)) - substituted_docs.append(doc.to_dict()) - return substituted_docs + substituted_docs.append(document) + return substituted_docs diff --git a/deckhand/tests/unit/engine/test_document_layering_negative.py b/deckhand/tests/unit/engine/test_document_layering_negative.py index ba14bf99..b3e46e31 100644 --- a/deckhand/tests/unit/engine/test_document_layering_negative.py +++ b/deckhand/tests/unit/engine/test_document_layering_negative.py @@ -125,13 +125,10 @@ class TestDocumentLayeringNegative( # same unique parent identifier referenced by `parentSelector`. doc_factory = factories.DocumentFactory(3, [1, 1, 1]) documents = doc_factory.gen_test({}, site_abstract=False) + documents.append(documents[2]) # Copy region layer. - # 1 is global layer, 2 is region layer. - for idx in (1, 2): - documents.append(documents[idx]) - self.assertRaises(errors.IndeterminateDocumentParent, - layering.DocumentLayering, documents) - documents.pop(-1) # Remove the just-appended duplicate. + self.assertRaises(errors.IndeterminateDocumentParent, + layering.DocumentLayering, documents) @mock.patch.object(layering, 'LOG', autospec=True) def test_layering_document_references_itself(self, mock_log):