Simplify document wrapper class
This PS simplifies the DocumentWrapper class by changing the way it is designed. The purpose of the class was to make it easier to retrieve nested dictionary attributes from a document. The class previously inherited from `object` meaning that the object could not directly be treated as a dictionary, complicating usage of the class. With this change, the class now inherits from a `dict` meaning that it can be manipulated the same way a dictionary can, while still able to return nested dictionary attributes without having to worry about exceptions getting thrown. Each property implemented by `DocumentWrapper` uses jsonpath_parse implements in `deckhand.utils` to retrieve nested attributes or else self.get() to retrieve first-level dictionary attributes. Change-Id: I1d73a79aa4c3117be31aab978c20258c1052ad6d
This commit is contained in:
Felipe Monteiro
parent
feb3dd57e2
commit
3dc3f4c47b
|
|
@ -1,135 +0,0 @@
|
|||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
class Document(object):
|
||||
"""Object wrapper for documents.
|
||||
|
||||
After "raw" documents undergo schema validation, they can be wrapped with
|
||||
this class to allow nested dictionary entries to be quickly retrieved.
|
||||
"""
|
||||
|
||||
def __init__(self, data):
|
||||
"""Constructor for ``Document``.
|
||||
|
||||
:param data: Dictionary of all document data (includes metadata, data,
|
||||
schema, etc.).
|
||||
"""
|
||||
self._inner = data
|
||||
|
||||
def to_dict(self):
|
||||
return self._inner
|
||||
|
||||
def is_abstract(self):
|
||||
"""Return whether the document is abstract.
|
||||
|
||||
Not all documents contain this property; in that case they are
|
||||
concrete.
|
||||
"""
|
||||
try:
|
||||
return self._inner['metadata']['layeringDefinition']['abstract']
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def get_schema(self):
|
||||
try:
|
||||
return self._inner['schema']
|
||||
except Exception:
|
||||
return ''
|
||||
|
||||
def get_name(self):
|
||||
try:
|
||||
return self._inner['metadata']['name']
|
||||
except Exception:
|
||||
return ''
|
||||
|
||||
def get_layer(self):
|
||||
try:
|
||||
return self._inner['metadata']['layeringDefinition']['layer']
|
||||
except Exception:
|
||||
return ''
|
||||
|
||||
def get_parent_selector(self):
|
||||
"""Return the `parentSelector` for the document.
|
||||
|
||||
The topmost document defined by the `layerOrder` in the LayeringPolicy
|
||||
does not have a `parentSelector` as it has no parent.
|
||||
|
||||
:returns: `parentSelector` for the document if present, else None.
|
||||
"""
|
||||
try:
|
||||
return self._inner['metadata']['layeringDefinition'][
|
||||
'parentSelector']
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def get_labels(self):
|
||||
return self._inner['metadata']['labels']
|
||||
|
||||
def get_substitutions(self):
|
||||
try:
|
||||
return self._inner['metadata'].get('substitutions', [])
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def get_actions(self):
|
||||
try:
|
||||
return self._inner['metadata']['layeringDefinition']['actions']
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def get_children(self, nested=False):
|
||||
"""Get document children, if any.
|
||||
|
||||
:param nested: Recursively retrieve all children for each child
|
||||
document.
|
||||
:type nested: boolean
|
||||
:returns: List of children of type `Document`.
|
||||
"""
|
||||
if not nested:
|
||||
return self._inner.get('children', [])
|
||||
else:
|
||||
return self._get_nested_children(self, [])
|
||||
|
||||
def _get_nested_children(self, doc, nested_children):
|
||||
for child in doc.get('children', []):
|
||||
nested_children.append(child)
|
||||
if 'children' in child._inner:
|
||||
self._get_nested_children(child, nested_children)
|
||||
return nested_children
|
||||
|
||||
def get(self, k, default=None):
|
||||
return self.__getitem__(k, default=default)
|
||||
|
||||
def __getitem__(self, k, default=None):
|
||||
return self._inner.get(k, default)
|
||||
|
||||
def __setitem__(self, k, val):
|
||||
self._inner[k] = val
|
||||
|
||||
def __delitem__(self, k):
|
||||
if self.__contains__(k):
|
||||
del self._inner[k]
|
||||
|
||||
def __contains__(self, k):
|
||||
return self.get(k, default=None) is not None
|
||||
|
||||
def __missing__(self, k):
|
||||
return not self.__contains__(k)
|
||||
|
||||
def __repr__(self):
|
||||
return '(%s, %s)' % (self.get_schema(), self.get_name())
|
||||
|
||||
def __str__(self):
|
||||
return str(self._inner)
|
||||
|
|
@ -19,6 +19,7 @@ import jsonschema
|
|||
from oslo_log import log as logging
|
||||
import six
|
||||
|
||||
from deckhand.engine import document_wrapper
|
||||
from deckhand.engine.schema import base_schema
|
||||
from deckhand.engine.schema import v1_0
|
||||
from deckhand import errors
|
||||
|
|
@ -140,9 +141,9 @@ class SchemaValidator(BaseValidator):
|
|||
return matching_schemas
|
||||
|
||||
def matches(self, document):
|
||||
if is_abstract(document) is True:
|
||||
if document.is_abstract:
|
||||
LOG.info('Skipping schema validation for abstract document [%s]: '
|
||||
'%s.', document['schema'], document['metadata']['name'])
|
||||
'%s.', document.schema, document.name)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
|
@ -167,7 +168,7 @@ class SchemaValidator(BaseValidator):
|
|||
schemas_to_use = self._get_schemas(document)
|
||||
if not schemas_to_use and use_fallback_schema:
|
||||
LOG.debug('Document schema %s not recognized. Using "fallback" '
|
||||
'schema.', document['schema'])
|
||||
'schema.', document.schema)
|
||||
schemas_to_use = [SchemaValidator._fallback_schema]
|
||||
|
||||
for schema_to_use in schemas_to_use:
|
||||
|
|
@ -191,8 +192,8 @@ class SchemaValidator(BaseValidator):
|
|||
for error in errors:
|
||||
LOG.error(
|
||||
'Failed schema validation for document [%s] %s. '
|
||||
'Details: %s.', document['schema'],
|
||||
document['metadata']['name'], error.message)
|
||||
'Details: %s.', document.schema, document.name,
|
||||
error.message)
|
||||
parent_path = root_path + '.'.join(
|
||||
[six.text_type(x) for x in error.path])
|
||||
yield error.message, parent_path
|
||||
|
|
@ -211,9 +212,9 @@ class DataSchemaValidator(SchemaValidator):
|
|||
for data_schema in data_schemas:
|
||||
# Ensure that each `DataSchema` document has required properties
|
||||
# before they themselves can be used to validate other documents.
|
||||
if 'name' not in data_schema.get('metadata', {}):
|
||||
if 'name' not in data_schema.metadata:
|
||||
continue
|
||||
if self._schema_re.match(data_schema['metadata']['name']) is None:
|
||||
if self._schema_re.match(data_schema.name) is None:
|
||||
continue
|
||||
if 'data' not in data_schema:
|
||||
continue
|
||||
|
|
@ -221,16 +222,16 @@ class DataSchemaValidator(SchemaValidator):
|
|||
'metadata.name')
|
||||
|
||||
class Schema(object):
|
||||
schema = data_schema['data']
|
||||
schema = data_schema.data
|
||||
|
||||
schema_map[schema_version].setdefault(schema_prefix, Schema())
|
||||
|
||||
return schema_map
|
||||
|
||||
def matches(self, document):
|
||||
if is_abstract(document) is True:
|
||||
if document.is_abstract:
|
||||
LOG.info('Skipping schema validation for abstract document [%s]: '
|
||||
'%s.', document['schema'], document['metadata']['name'])
|
||||
'%s.', document.schema, document.name)
|
||||
return False
|
||||
schema_prefix, schema_version = get_schema_parts(document)
|
||||
return schema_prefix in self._schema_map.get(schema_version, {})
|
||||
|
|
@ -257,25 +258,26 @@ class DocumentValidation(object):
|
|||
revisions to be used the "data" section of each document in
|
||||
``documents``. Additional ``DataSchema`` documents in ``documents``
|
||||
are combined with these.
|
||||
:type existing_data_schemas: List[dict]
|
||||
:type existing_data_schemas: dict or List[dict]
|
||||
"""
|
||||
|
||||
self.documents = []
|
||||
existing_data_schemas = existing_data_schemas or []
|
||||
data_schemas = existing_data_schemas[:]
|
||||
db_data_schemas = {d['metadata']['name']: d for d in data_schemas}
|
||||
data_schemas = [document_wrapper.DocumentDict(d)
|
||||
for d in existing_data_schemas]
|
||||
_data_schema_map = {d.name: d for d in data_schemas}
|
||||
|
||||
if not isinstance(documents, (list, tuple)):
|
||||
if not isinstance(documents, list):
|
||||
documents = [documents]
|
||||
|
||||
for document in documents:
|
||||
if document.get('schema', '').startswith(types.DATA_SCHEMA_SCHEMA):
|
||||
if not isinstance(document, document_wrapper.DocumentDict):
|
||||
document = document_wrapper.DocumentDict(document)
|
||||
if document.schema.startswith(types.DATA_SCHEMA_SCHEMA):
|
||||
data_schemas.append(document)
|
||||
# If a newer version of the same DataSchema was passed in,
|
||||
# only use the new one and discard the old one.
|
||||
document_name = document.get('metadata', {}).get('name')
|
||||
if document_name in db_data_schemas:
|
||||
data_schemas.remove(db_data_schemas.pop(document_name))
|
||||
if document.name in _data_schema_map:
|
||||
data_schemas.remove(_data_schema_map.pop(document.name))
|
||||
self.documents.append(document)
|
||||
|
||||
# NOTE(fmontei): The order of the validators is important. The
|
||||
|
|
@ -346,8 +348,8 @@ class DocumentValidation(object):
|
|||
if error_messages:
|
||||
for error_msg, error_path in error_messages:
|
||||
result['errors'].append({
|
||||
'schema': document['schema'],
|
||||
'name': document['metadata']['name'],
|
||||
'schema': document.schema,
|
||||
'name': document.name,
|
||||
'message': error_msg,
|
||||
'path': error_path
|
||||
})
|
||||
|
|
@ -418,13 +420,6 @@ class DocumentValidation(object):
|
|||
return validations
|
||||
|
||||
|
||||
def is_abstract(document):
|
||||
try:
|
||||
return document['metadata']['layeringDefinition']['abstract']
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def get_schema_parts(document, schema_key='schema'):
|
||||
schema_parts = utils.jsonpath_parse(document, schema_key).split('/')
|
||||
schema_prefix = '/'.join(schema_parts[:2])
|
||||
|
|
|
|||
|
|
@ -0,0 +1,84 @@
|
|||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from oslo_serialization import jsonutils as json
|
||||
|
||||
from deckhand import utils
|
||||
|
||||
|
||||
class DocumentDict(dict):
|
||||
"""Wrapper for a document.
|
||||
|
||||
Implements convenient properties for nested, commonly accessed document
|
||||
keys. Property setters are only implemented for mutable data.
|
||||
|
||||
Useful for accessing nested dictionary keys without having to worry about
|
||||
exceptions getting thrown.
|
||||
|
||||
"""
|
||||
|
||||
@property
|
||||
def is_abstract(self):
|
||||
return utils.jsonpath_parse(
|
||||
self, 'metadata.layeringDefinition.abstract') is True
|
||||
|
||||
@property
|
||||
def schema(self):
|
||||
return self.get('schema', '')
|
||||
|
||||
@property
|
||||
def metadata(self):
|
||||
return self.get('metadata', {})
|
||||
|
||||
@property
|
||||
def data(self):
|
||||
return self.get('data', {})
|
||||
|
||||
@data.setter
|
||||
def data(self, value):
|
||||
self['data'] = value
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return utils.jsonpath_parse(self, 'metadata.name')
|
||||
|
||||
@property
|
||||
def layer(self):
|
||||
return utils.jsonpath_parse(
|
||||
self, 'metadata.layeringDefinition.layer')
|
||||
|
||||
@property
|
||||
def parent_selector(self):
|
||||
return utils.jsonpath_parse(
|
||||
self, 'metadata.layeringDefinition.parentSelector') or {}
|
||||
|
||||
@property
|
||||
def labels(self):
|
||||
return utils.jsonpath_parse(self, 'metadata.labels') or {}
|
||||
|
||||
@property
|
||||
def substitutions(self):
|
||||
return utils.jsonpath_parse(self, 'metadata.substitutions') or []
|
||||
|
||||
@substitutions.setter
|
||||
def substitutions(self, value):
|
||||
return utils.jsonpath_replace(self, value, 'metadata.substitutions')
|
||||
|
||||
@property
|
||||
def actions(self):
|
||||
return utils.jsonpath_parse(
|
||||
self, 'metadata.layeringDefinition.actions') or []
|
||||
|
||||
def __hash__(self):
|
||||
return hash(json.dumps(self, sort_keys=True))
|
||||
|
|
@ -16,9 +16,8 @@ import collections
|
|||
import copy
|
||||
|
||||
from oslo_log import log as logging
|
||||
import six
|
||||
|
||||
from deckhand.engine import document
|
||||
from deckhand.engine import document_wrapper
|
||||
from deckhand.engine import secrets_manager
|
||||
from deckhand.engine import utils
|
||||
from deckhand import errors
|
||||
|
|
@ -62,21 +61,25 @@ class DocumentLayering(object):
|
|||
document contains a "children" property in addition to original
|
||||
data. List of documents returned is ordered from highest to lowest
|
||||
layer.
|
||||
:rtype: list of deckhand.engine.document.Document objects.
|
||||
:rtype: List[:class:`DocumentDict`]
|
||||
:raises IndeterminateDocumentParent: If more than one parent document
|
||||
was found for a document.
|
||||
"""
|
||||
layered_docs = list(
|
||||
filter(lambda x: 'layeringDefinition' in x['metadata'],
|
||||
filter(lambda x: 'layeringDefinition' in x.metadata,
|
||||
self._documents))
|
||||
|
||||
# ``all_children`` is a counter utility for verifying that each
|
||||
# document has exactly one parent.
|
||||
all_children = collections.Counter()
|
||||
|
||||
# Mapping of (doc.name, doc.metadata.name) => children, where children
|
||||
# are the documents whose `parentSelector` references the doc.
|
||||
self._children = {}
|
||||
|
||||
def _get_children(doc):
|
||||
children = []
|
||||
doc_layer = doc.get_layer()
|
||||
doc_layer = doc.layer
|
||||
try:
|
||||
next_layer_idx = self._layer_order.index(doc_layer) + 1
|
||||
children_doc_layer = self._layer_order[next_layer_idx]
|
||||
|
|
@ -89,16 +92,16 @@ class DocumentLayering(object):
|
|||
# Documents with different schemas are never layered together,
|
||||
# so consider only documents with same schema as candidates.
|
||||
is_potential_child = (
|
||||
other_doc.get_layer() == children_doc_layer and
|
||||
other_doc.get_schema() == doc.get_schema()
|
||||
other_doc.layer == children_doc_layer and
|
||||
other_doc.schema == doc.schema
|
||||
)
|
||||
if (is_potential_child):
|
||||
# A document can have many labels but should only have one
|
||||
# explicit label for the parentSelector.
|
||||
parent_sel = other_doc.get_parent_selector()
|
||||
parent_sel = other_doc.parent_selector
|
||||
parent_sel_key = list(parent_sel.keys())[0]
|
||||
parent_sel_val = list(parent_sel.values())[0]
|
||||
doc_labels = doc.get_labels()
|
||||
doc_labels = doc.labels
|
||||
|
||||
if (parent_sel_key in doc_labels and
|
||||
parent_sel_val == doc_labels[parent_sel_key]):
|
||||
|
|
@ -108,18 +111,19 @@ class DocumentLayering(object):
|
|||
|
||||
for layer in self._layer_order:
|
||||
docs_by_layer = list(filter(
|
||||
(lambda x: x.get_layer() == layer), layered_docs))
|
||||
(lambda x: x.layer == layer), layered_docs))
|
||||
|
||||
for doc in docs_by_layer:
|
||||
children = _get_children(doc)
|
||||
|
||||
if children:
|
||||
all_children.update(children)
|
||||
doc.to_dict().setdefault('children', children)
|
||||
self._children.setdefault((doc.name, doc.schema),
|
||||
children)
|
||||
|
||||
all_children_elements = list(all_children.elements())
|
||||
secondary_docs = list(
|
||||
filter(lambda d: d.get_layer() != self._layer_order[0],
|
||||
filter(lambda d: d.layer != self._layer_order[0],
|
||||
layered_docs))
|
||||
for doc in secondary_docs:
|
||||
# Unless the document is the topmost document in the
|
||||
|
|
@ -128,32 +132,30 @@ class DocumentLayering(object):
|
|||
if doc not in all_children_elements:
|
||||
LOG.info('Could not find parent for document with name=%s, '
|
||||
'schema=%s, layer=%s, parentSelector=%s.',
|
||||
doc.get_name(), doc.get_schema(), doc.get_layer(),
|
||||
doc.get_parent_selector())
|
||||
doc.name, doc.schema, doc.layer, doc.parent_selector)
|
||||
self._parentless_documents.append(doc)
|
||||
# If the document is a child document of more than 1 parent, then
|
||||
# the document has too many parents, which is a validation error.
|
||||
elif all_children[doc] != 1:
|
||||
elif all_children[doc] > 1:
|
||||
LOG.info('%d parent documents were found for child document '
|
||||
'with name=%s, schema=%s, layer=%s, parentSelector=%s'
|
||||
'. Each document must only have 1 parent.',
|
||||
all_children[doc], doc.get_name(), doc.get_schema(),
|
||||
doc.get_layer(), doc.get_parent_selector())
|
||||
'. Each document must have exactly 1 parent.',
|
||||
all_children[doc], doc.name, doc.schema, doc.layer,
|
||||
doc.parent_selector)
|
||||
raise errors.IndeterminateDocumentParent(document=doc)
|
||||
|
||||
return layered_docs
|
||||
|
||||
def _extract_layering_policy(self, documents):
|
||||
documents = copy.deepcopy(documents)
|
||||
for doc in documents:
|
||||
if doc['schema'].startswith(types.LAYERING_POLICY_SCHEMA):
|
||||
layering_policy = doc
|
||||
documents.remove(doc)
|
||||
return (
|
||||
document.Document(layering_policy),
|
||||
[document.Document(d) for d in documents]
|
||||
document_wrapper.DocumentDict(layering_policy),
|
||||
[document_wrapper.DocumentDict(d) for d in documents
|
||||
if d is not layering_policy]
|
||||
)
|
||||
return None, [document.Document(d) for d in documents]
|
||||
return None, [document_wrapper.DocumentDict(d) for d in documents]
|
||||
|
||||
def __init__(self, documents, substitution_sources=None):
|
||||
"""Contructor for ``DocumentLayering``.
|
||||
|
|
@ -256,15 +258,29 @@ class DocumentLayering(object):
|
|||
|
||||
return overall_data
|
||||
|
||||
def _get_children(self, document):
|
||||
"""Recursively retrieve all children.
|
||||
|
||||
Used in the layering module when calculating children for each
|
||||
document.
|
||||
|
||||
:returns: List of nested children.
|
||||
:rtype: Generator[:class:`DocumentDict`]
|
||||
"""
|
||||
for child in self._children.get((document.name, document.schema), []):
|
||||
yield child
|
||||
grandchildren = self._get_children(child)
|
||||
for grandchild in grandchildren:
|
||||
yield grandchild
|
||||
|
||||
def _apply_substitutions(self, document):
|
||||
try:
|
||||
secrets_substitution = secrets_manager.SecretsSubstitution(
|
||||
document, self._substitution_sources)
|
||||
return secrets_substitution.substitute_all()
|
||||
except errors.DocumentNotFound as e:
|
||||
except errors.SubstitutionDependencyNotFound:
|
||||
LOG.error('Failed to render the documents because a secret '
|
||||
'document could not be found.')
|
||||
LOG.exception(six.text_type(e))
|
||||
|
||||
def render(self):
|
||||
"""Perform layering on the list of documents passed to ``__init__``.
|
||||
|
|
@ -281,72 +297,63 @@ class DocumentLayering(object):
|
|||
"""
|
||||
# ``rendered_data_by_layer`` tracks the set of changes across all
|
||||
# actions across each layer for a specific document.
|
||||
rendered_data_by_layer = {}
|
||||
rendered_data_by_layer = document_wrapper.DocumentDict()
|
||||
|
||||
# NOTE(fmontei): ``global_docs`` represents the topmost documents in
|
||||
# the system. It should probably be impossible for more than 1
|
||||
# top-level doc to exist, but handle multiple for now.
|
||||
global_docs = [doc for doc in self._layered_documents
|
||||
if doc.get_layer() == self._layer_order[0]]
|
||||
if doc.layer == self._layer_order[0]]
|
||||
|
||||
for doc in global_docs:
|
||||
layer_idx = self._layer_order.index(doc.get_layer())
|
||||
if doc.get_substitutions():
|
||||
substituted_data = self._apply_substitutions(doc.to_dict())
|
||||
rendered_data_by_layer[layer_idx] = substituted_data[0]
|
||||
layer_idx = self._layer_order.index(doc.layer)
|
||||
if doc.substitutions:
|
||||
substituted_data = self._apply_substitutions(doc)
|
||||
if substituted_data:
|
||||
rendered_data_by_layer[layer_idx] = substituted_data[0]
|
||||
else:
|
||||
rendered_data_by_layer[layer_idx] = doc.to_dict()
|
||||
rendered_data_by_layer[layer_idx] = doc
|
||||
|
||||
# Keep iterating as long as a child exists.
|
||||
for child in doc.get_children(nested=True):
|
||||
for child in self._get_children(doc):
|
||||
# Retrieve the most up-to-date rendered_data (by
|
||||
# referencing the child's parent's data).
|
||||
child_layer_idx = self._layer_order.index(child.get_layer())
|
||||
child_layer_idx = self._layer_order.index(child.layer)
|
||||
rendered_data = rendered_data_by_layer[child_layer_idx - 1]
|
||||
|
||||
# Apply each action to the current document.
|
||||
for action in child.get_actions():
|
||||
for action in child.actions:
|
||||
LOG.debug('Applying action %s to child document with '
|
||||
'name=%s, schema=%s, layer=%s.', action,
|
||||
child.get_name(), child.get_schema(),
|
||||
child.get_layer())
|
||||
child.name, child.schema, child.layer)
|
||||
rendered_data = self._apply_action(
|
||||
action, child.to_dict(), rendered_data)
|
||||
action, child, rendered_data)
|
||||
|
||||
# Update the actual document data if concrete.
|
||||
if not child.is_abstract():
|
||||
if child.get_substitutions():
|
||||
rendered_data['metadata'][
|
||||
'substitutions'] = child.get_substitutions()
|
||||
if not child.is_abstract:
|
||||
if child.substitutions:
|
||||
rendered_data.substitutions = child.substitutions
|
||||
substituted_data = self._apply_substitutions(
|
||||
rendered_data)
|
||||
if substituted_data:
|
||||
rendered_data = substituted_data[0]
|
||||
child_index = self._layered_documents.index(child)
|
||||
self._layered_documents[child_index]['data'] = (
|
||||
rendered_data['data'])
|
||||
self._layered_documents[child_index].data = (
|
||||
rendered_data.data)
|
||||
|
||||
# Update ``rendered_data_by_layer`` for this layer so that
|
||||
# children in deeper layers can reference the most up-to-date
|
||||
# changes.
|
||||
rendered_data_by_layer[child_layer_idx] = rendered_data
|
||||
|
||||
if 'children' in doc:
|
||||
del doc['children']
|
||||
|
||||
# Handle edge case for parentless documents that require substitution.
|
||||
# If a document has no parent, then the for loop above doesn't iterate
|
||||
# over the parentless document, so substitution must be done here for
|
||||
# parentless documents.
|
||||
for doc in self._parentless_documents:
|
||||
if not doc.is_abstract():
|
||||
substituted_data = self._apply_substitutions(doc.to_dict())
|
||||
if not doc.is_abstract and doc.substitutions:
|
||||
substituted_data = self._apply_substitutions(doc)
|
||||
if substituted_data:
|
||||
# TODO(fmontei): Use property after implementing it in
|
||||
# document wrapper class.
|
||||
doc._inner = substituted_data[0]
|
||||
doc = substituted_data[0]
|
||||
|
||||
return (
|
||||
[d.to_dict() for d in self._layered_documents] +
|
||||
[self._layering_policy.to_dict()]
|
||||
)
|
||||
return self._layered_documents + [self._layering_policy]
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ from oslo_log import log as logging
|
|||
import six
|
||||
|
||||
from deckhand.barbican import driver
|
||||
from deckhand.engine import document as document_wrapper
|
||||
from deckhand.engine import document_wrapper
|
||||
from deckhand import errors
|
||||
from deckhand import utils
|
||||
|
||||
|
|
@ -111,18 +111,19 @@ class SecretsSubstitution(object):
|
|||
sources for substitution. Should only include concrete documents.
|
||||
:type substitution_sources: List[dict]
|
||||
"""
|
||||
if not isinstance(documents, (list, tuple)):
|
||||
|
||||
if not isinstance(documents, list):
|
||||
documents = [documents]
|
||||
|
||||
self.substitution_documents = []
|
||||
self.substitution_sources = substitution_sources or []
|
||||
self._documents = []
|
||||
self._substitution_sources = substitution_sources or []
|
||||
|
||||
for document in documents:
|
||||
if not isinstance(document, document_wrapper.Document):
|
||||
document_obj = document_wrapper.Document(document)
|
||||
# If the document has substitutions include it.
|
||||
if document_obj.get_substitutions():
|
||||
self.substitution_documents.append(document_obj)
|
||||
if not isinstance(document, document_wrapper.DocumentDict):
|
||||
document = document_wrapper.DocumentDict(document)
|
||||
# If the document has substitutions include it.
|
||||
if document.substitutions:
|
||||
self._documents.append(document)
|
||||
|
||||
def substitute_all(self):
|
||||
"""Substitute all documents that have a `metadata.substitutions` field.
|
||||
|
|
@ -133,16 +134,19 @@ class SecretsSubstitution(object):
|
|||
from a document in the site layer.
|
||||
|
||||
:returns: List of fully substituted documents.
|
||||
:rtype: List[:class:`DocumentDict`]
|
||||
:raises SubstitutionDependencyNotFound: If a substitution source wasn't
|
||||
found or something else went wrong during substitution.
|
||||
"""
|
||||
LOG.debug('Substituting secrets for documents: %s',
|
||||
self.substitution_documents)
|
||||
LOG.debug('Performing substitution on following documents: %s',
|
||||
', '.join(['[%s] %s' % (d.schema, d.name)
|
||||
for d in self._documents]))
|
||||
substituted_docs = []
|
||||
|
||||
for doc in self.substitution_documents:
|
||||
LOG.debug(
|
||||
'Checking for substitutions in schema=%s, metadata.name=%s',
|
||||
doc.get_name(), doc.get_schema())
|
||||
for sub in doc.get_substitutions():
|
||||
for document in self._documents:
|
||||
LOG.debug('Checking for substitutions for document [%s] %s.',
|
||||
document.schema, document.name)
|
||||
for sub in document.substitutions:
|
||||
src_schema = sub['src']['schema']
|
||||
src_name = sub['src']['name']
|
||||
src_path = sub['src']['path']
|
||||
|
|
@ -151,7 +155,7 @@ class SecretsSubstitution(object):
|
|||
x['metadata']['name'] == src_name)
|
||||
try:
|
||||
src_doc = next(
|
||||
iter(filter(is_match, self.substitution_sources)))
|
||||
iter(filter(is_match, self._substitution_sources)))
|
||||
except StopIteration:
|
||||
src_doc = {}
|
||||
|
||||
|
|
@ -172,16 +176,16 @@ class SecretsSubstitution(object):
|
|||
src_name, src_path, dest_path, dest_pattern)
|
||||
try:
|
||||
substituted_data = utils.jsonpath_replace(
|
||||
doc['data'], src_secret, dest_path, dest_pattern)
|
||||
document['data'], src_secret, dest_path, dest_pattern)
|
||||
if isinstance(substituted_data, dict):
|
||||
doc['data'].update(substituted_data)
|
||||
document['data'].update(substituted_data)
|
||||
else:
|
||||
doc['data'] = substituted_data
|
||||
document['data'] = substituted_data
|
||||
except Exception as e:
|
||||
LOG.error('Unexpected exception occurred while attempting '
|
||||
'secret substitution. %s', six.text_type(e))
|
||||
raise errors.SubstitutionDependencyNotFound(
|
||||
details=six.text_type(e))
|
||||
|
||||
substituted_docs.append(doc.to_dict())
|
||||
return substituted_docs
|
||||
substituted_docs.append(document)
|
||||
return substituted_docs
|
||||
|
|
|
|||
|
|
@ -125,13 +125,10 @@ class TestDocumentLayeringNegative(
|
|||
# same unique parent identifier referenced by `parentSelector`.
|
||||
doc_factory = factories.DocumentFactory(3, [1, 1, 1])
|
||||
documents = doc_factory.gen_test({}, site_abstract=False)
|
||||
documents.append(documents[2]) # Copy region layer.
|
||||
|
||||
# 1 is global layer, 2 is region layer.
|
||||
for idx in (1, 2):
|
||||
documents.append(documents[idx])
|
||||
self.assertRaises(errors.IndeterminateDocumentParent,
|
||||
layering.DocumentLayering, documents)
|
||||
documents.pop(-1) # Remove the just-appended duplicate.
|
||||
self.assertRaises(errors.IndeterminateDocumentParent,
|
||||
layering.DocumentLayering, documents)
|
||||
|
||||
@mock.patch.object(layering, 'LOG', autospec=True)
|
||||
def test_layering_document_references_itself(self, mock_log):
|
||||
|
|
|
|||
Loading…
Reference in New Issue