Merge pull request #10 from att-comdev/revisions-api

Revisions database and API implementation

deckhand/common/timeutils.py

@@ -1,88 +0,0 @@
-# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Time related utilities and helper functions.
-"""
-
-import datetime
-
-import iso8601
-from monotonic import monotonic as now  # noqa
-from oslo_utils import encodeutils
-
-# ISO 8601 extended time format with microseconds
-_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
-_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
-PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND
-
-
-def isotime(at=None, subsecond=False):
-    """Stringify time in ISO 8601 format."""
-    if not at:
-        at = utcnow()
-    st = at.strftime(_ISO8601_TIME_FORMAT
-                     if not subsecond
-                     else _ISO8601_TIME_FORMAT_SUBSECOND)
-    tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
-    st += ('Z' if tz == 'UTC' else tz)
-    return st
-
-
-def parse_isotime(timestr):
-    """Parse time from ISO 8601 format."""
-    try:
-        return iso8601.parse_date(timestr)
-    except iso8601.ParseError as e:
-        raise ValueError(encodeutils.exception_to_unicode(e))
-    except TypeError as e:
-        raise ValueError(encodeutils.exception_to_unicode(e))
-
-
-def utcnow(with_timezone=False):
-    """Overridable version of utils.utcnow that can return a TZ-aware datetime.
-    """
-    if utcnow.override_time:
-        try:
-            return utcnow.override_time.pop(0)
-        except AttributeError:
-            return utcnow.override_time
-    if with_timezone:
-        return datetime.datetime.now(tz=iso8601.iso8601.UTC)
-    return datetime.datetime.utcnow()
-
-
-def normalize_time(timestamp):
-    """Normalize time in arbitrary timezone to UTC naive object."""
-    offset = timestamp.utcoffset()
-    if offset is None:
-        return timestamp
-    return timestamp.replace(tzinfo=None) - offset
-
-
-def iso8601_from_timestamp(timestamp, microsecond=False):
-    """Returns an iso8601 formatted date from timestamp."""
-    return isotime(datetime.datetime.utcfromtimestamp(timestamp), microsecond)
-
-utcnow.override_time = None
-
-
-def delta_seconds(before, after):
-    """Return the difference between two timing objects.
-
-    Compute the difference in seconds between two date, time, or
-    datetime objects (as a float, to microsecond resolution).
-    """
-    delta = after - before
-    return datetime.timedelta.total_seconds(delta)

deckhand/control/README.rst

@@ -7,13 +7,150 @@ Deckhand-managed data.
 v1.0 Endpoints
 --------------
 
-/api/v1.0/documents
+POST `/documents`
-~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~
 
-POST - Create a new YAML document and return a revision number. If the YAML
+Accepts a multi-document YAML body and creates a new revision which adds
-document already exists, then the document will be replaced and a new
+those documents. Updates are detected based on exact match to an existing
-revision number will be returned.
+document of `schema` + `metadata.name`. Documents are "deleted" by including
+documents with the tombstone metadata schema, such as:
 
+```yaml
+---
+schema: any-namespace/AnyKind/v1
+metadata:
+  schema: metadata/Tombstone/v1
+  name: name-to-delete
+...
+```
+
+This endpoint is the only way to add, update, and delete documents. This
+triggers Deckhand's internal schema validations for all documents.
+
+If no changes are detected, a new revision should not be created. This allows
+services to periodically re-register their schemas without creating
+unnecessary revisions.
+
+Sample response:
+
+```yaml
+---
+created_at: '2017-07-31T14:46:46.119853'
+data:
+  path:
+    to:
+      merge:
+        into:
+          ignored: {data: here}
+          parent: {foo: bar}
+  substitution: {target: null}
+deleted: false
+deleted_at: null
+id: f99630d9-a89c-4aad-9aaa-7c44462047c1
+metadata:
+  labels: {genesis: enabled, master: enabled}
+  layeringDefinition:
+    abstract: false
+    actions:
+    - {method: merge, path: .path.to.merge.into.parent}
+    - {method: delete, path: .path.to.delete}
+    layer: region
+    parentSelector: {required_key_a: required_label_a, required_key_b: required_label_b}
+  name: unique-name-given-schema
+  schema: metadata/Document/v1
+  storagePolicy: cleartext
+  substitutions:
+  - dest: {path: .substitution.target}
+    src: {name: name-of-source-document, path: .source.path, schema: another-service/SourceType/v1}
+name: unique-name-given-schema
+revision_id: 0206088a-c9e9-48e1-8725-c9bdac15d6b7
+schema: some-service/ResourceType/v1
+updated_at: '2017-07-31T14:46:46.119858'
+```
+
+GET `/revisions`
+~~~~~~~~~~~~~~~~
+
+Lists existing revisions and reports basic details including a summary of
+validation status for each `deckhand/ValidationPolicy` that is part of that
+revision.
+
+Sample response:
+
+```yaml
+---
+count: 7
+next: https://deckhand/api/v1.0/revisions?limit=2&offset=2
+prev: null
+results:
+  - id: 0
+    url: https://deckhand/api/v1.0/revisions/0
+    createdAt: 2017-07-14T21:23Z
+    validationPolicies:
+      site-deploy-validation:
+        status: failed
+  - id: 1
+    url: https://deckhand/api/v1.0/revisions/1
+    createdAt: 2017-07-16T01:15Z
+    validationPolicies:
+      site-deploy-validation:
+        status: succeeded
+...
+```
+
+GET `/revisions/{revision_id}/documents`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Returns a multi-document YAML response containing all the documents matching
+the filters specified via query string parameters. Returned documents will be
+as originally posted with no substitutions or layering applied.
+
+Supported query string parameters:
+
+* `schema` - string, optional - The top-level `schema` field to select. This
+  may be partially specified by section, e.g., `schema=promenade` would select all
+  `kind` and `version` schemas owned by promenade, or `schema=promenade/Node`
+  which would select all versions of `promenade/Node` documents. One may not
+  partially specify the namespace or kind, so `schema=promenade/No` would not
+  select `promenade/Node/v1` documents, and `schema=prom` would not select
+  `promenade` documents.
+* `metadata.name` - string, optional
+* `metadata.layeringDefinition.abstract` - string, optional - Valid values are
+  the "true" and "false".
+* `metadata.layeringDefinition.layer` - string, optional - Only return documents from
+  the specified layer.
+* `metadata.label` - string, optional, repeatable - Uses the format
+  `metadata.label=key=value`. Repeating this parameter indicates all
+  requested labels must apply (AND not OR).
+
+Sample response:
+
+```yaml
+created_at: '2017-07-31T14:36:00.352701'
+data: {foo: bar}
+deleted: false
+deleted_at: null
+id: ffba233a-326b-4eed-9b21-079ebd2a53f0
+metadata:
+  labels: {genesis: enabled, master: enabled}
+  layeringDefinition:
+    abstract: false
+    actions:
+    - {method: merge, path: .path.to.merge.into.parent}
+    - {method: delete, path: .path.to.delete}
+    layer: region
+    parentSelector: {required_key_a: required_label_a, required_key_b: required_label_b}
+  name: foo-name-given-schema
+  schema: metadata/Document/v1
+  storagePolicy: cleartext
+  substitutions:
+  - dest: {path: .substitution.target}
+    src: {name: name-of-source-document, path: .source.path, schema: another-service/SourceType/v1}
+name: foo-name-given-schema
+revision_id: d3428d6a-d8c4-4a5b-8006-aba974cc36a2
+schema: some-service/ResourceType/v1
+updated_at: '2017-07-31T14:36:00.352705'
+```
 
 Testing
 -------
@@ -22,7 +159,9 @@ Document creation can be tested locally using (from root deckhand directory):
 
 .. code-block:: console
 
-    curl -i -X POST localhost:9000/api/v1.0/documents \
+    $ curl -i -X POST localhost:9000/api/v1.0/documents \
-        -H "Content-Type: application/x-yaml" \
+         -H "Content-Type: application/x-yaml" \
-        --data-binary "@deckhand/tests/unit/resources/sample.yaml"
+         --data-binary "@deckhand/tests/unit/resources/sample.yaml"
 
+    # revision_id copy/pasted from previous response.
+    $ curl -i -X GET localhost:9000/api/v1.0/revisions/0e99c8b9-bab4-4fc7-8405-7dbd22c33a30/documents

deckhand/control/api.py

@@ -21,6 +21,8 @@ from oslo_log import log as logging
 from deckhand.conf import config
 from deckhand.control import base as api_base
 from deckhand.control import documents
+from deckhand.control import revision_documents
+from deckhand.control import revisions
 from deckhand.control import secrets
 from deckhand.db.sqlalchemy import api as db_api
 
@@ -68,6 +70,10 @@ def start_api(state_manager=None):
 
     v1_0_routes = [
         ('documents', documents.DocumentsResource()),
+        ('revisions', revisions.RevisionsResource()),
+        ('revisions/{revision_id}', revisions.RevisionsResource()),
+        ('revisions/{revision_id}/documents',
+         revision_documents.RevisionDocumentsResource()),
         ('secrets', secrets.SecretsResource())
     ]
 

deckhand/control/base.py

@@ -12,14 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import uuid
+import yaml
 
 import falcon
 from falcon import request
+from oslo_log import log as logging
+from oslo_serialization import jsonutils as json
+import six
 
 from deckhand import errors
 
+LOG = logging.getLogger(__name__)
+
 
 class BaseResource(object):
     """Base resource class for implementing API resources."""
@@ -72,15 +77,29 @@ class BaseResource(object):
 
     def return_error(self, resp, status_code, message="", retry=False):
         resp.body = json.dumps(
-            {'type': 'error', 'message': message, 'retry': retry})
+            {'type': 'error', 'message': six.text_type(message),
+             'retry': retry})
         resp.status = status_code
 
+    def to_yaml_body(self, dict_body):
+        """Converts JSON body into YAML response body.
+
+        :dict_body: response body to be converted to YAML.
+        :returns: YAML encoding of `dict_body`.
+        """
+        if isinstance(dict_body, dict):
+            return yaml.safe_dump(dict_body)
+        elif isinstance(dict_body, list):
+            return yaml.safe_dump_all(dict_body)
+        raise TypeError('Unrecognized dict_body type when converting response '
+                        'body to YAML format.')
+
 
 class DeckhandRequestContext(object):
 
     def __init__(self):
         self.user = None
-        self.roles = ['anyone']
+        self.roles = []
         self.request_id = str(uuid.uuid4())
 
     def set_user(self, user):

deckhand/control/common.py

@@ -0,0 +1,31 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import string
+
+
+def to_camel_case(s):
+    return (s[0].lower() + string.capwords(s, sep='_').replace('_', '')[1:]
+            if s else s)
+
+
+class ViewBuilder(object):
+    """Model API responses as dictionaries."""
+
+    _collection_name = None
+
+    def _gen_url(self, revision):
+        # TODO: Use a config-based url for the base url below.
+        base_url = 'https://deckhand/api/v1.0/%s/%s'
+        return base_url % (self._collection_name, revision.get('id'))

deckhand/control/documents.py

@@ -32,16 +32,6 @@ LOG = logging.getLogger(__name__)
 class DocumentsResource(api_base.BaseResource):
     """API resource for realizing CRUD endpoints for Documents."""
 
-    def __init__(self, **kwargs):
-        super(DocumentsResource, self).__init__(**kwargs)
-        self.authorized_roles = ['user']
-
-    def on_get(self, req, resp):
-        pass
-
-    def on_head(self, req, resp):
-        pass
-
     def on_post(self, req, resp):
         """Create a document. Accepts YAML data only."""
         if req.content_type != 'application/x-yaml':
@@ -57,10 +47,11 @@ class DocumentsResource(api_base.BaseResource):
             LOG.error(error_msg)
             return self.return_error(resp, falcon.HTTP_400, message=error_msg)
 
-        # Validate the document before doing anything with it.
+        # All concrete documents in the payload must successfully pass their
+        # JSON schema validations. Otherwise raise an error.
         try:
             for doc in documents:
-                document_validation.DocumentValidation(doc)
+                document_validation.DocumentValidation(doc).pre_validate()
         except deckhand_errors.InvalidFormat as e:
             return self.return_error(resp, falcon.HTTP_400, message=e)
 
@@ -72,7 +63,5 @@ class DocumentsResource(api_base.BaseResource):
             return self.return_error(resp, falcon.HTTP_500, message=e)
 
         resp.status = falcon.HTTP_201
-        resp.body = json.dumps(created_documents)
+        resp.append_header('Content-Type', 'application/x-yaml')
-
+        resp.body = self.to_yaml_body(created_documents)
-    def _check_document_exists(self):
-        pass

deckhand/control/revision_documents.py

@@ -0,0 +1,42 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import falcon
+from oslo_db import exception as db_exc
+
+from deckhand.control import base as api_base
+from deckhand.db.sqlalchemy import api as db_api
+from deckhand import errors
+
+
+class RevisionDocumentsResource(api_base.BaseResource):
+    """API resource for realizing CRUD endpoints for Document Revisions."""
+
+    def on_get(self, req, resp, revision_id):
+        """Returns all documents for a `revision_id`.
+        
+        Returns a multi-document YAML response containing all the documents
+        matching the filters specified via query string parameters. Returned
+        documents will be as originally posted with no substitutions or
+        layering applied.
+        """
+        params = req.params
+        try:
+            documents = db_api.revision_get_documents(revision_id, **params)
+        except errors.RevisionNotFound as e:
+            return self.return_error(resp, falcon.HTTP_404, message=e)
+
+        resp.status = falcon.HTTP_200
+        resp.append_header('Content-Type', 'application/x-yaml')
+        resp.body = self.to_yaml_body(documents)

deckhand/control/revisions.py

@@ -0,0 +1,60 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import falcon
+
+from deckhand.control import base as api_base
+from deckhand.control.views import revision as revision_view
+from deckhand.db.sqlalchemy import api as db_api
+from deckhand import errors
+
+
+class RevisionsResource(api_base.BaseResource):
+    """API resource for realizing CRUD operations for revisions."""
+
+    def on_get(self, req, resp, revision_id=None):
+        """Returns list of existing revisions.
+        
+        Lists existing revisions and reports basic details including a summary
+        of validation status for each `deckhand/ValidationPolicy` that is part
+        of each revision.
+        """
+        if revision_id:
+            self._show_revision(req, resp, revision_id=revision_id)
+        else:
+            self._list_revisions(req, resp)
+
+    def _show_revision(self, req, resp, revision_id):
+        """Returns detailed description of a particular revision.
+
+        The status of each ValidationPolicy belonging to the revision is also
+        included.
+        """
+        try:
+            revision = db_api.revision_get(revision_id)
+        except errors.RevisionNotFound as e:
+            return self.return_error(resp, falcon.HTTP_404, message=e)
+
+        revision_resp = revision_view.ViewBuilder().show(revision)
+        resp.status = falcon.HTTP_200
+        resp.append_header('Content-Type', 'application/x-yaml')
+        resp.body = self.to_yaml_body(revision_resp)
+
+    def _list_revisions(self, req, resp):
+        revisions = db_api.revision_get_all()
+        revisions_resp = revision_view.ViewBuilder().list(revisions)
+
+        resp.status = falcon.HTTP_200
+        resp.append_header('Content-Type', 'application/x-yaml')
+        resp.body = self.to_yaml_body(revisions_resp)

deckhand/control/views/revision.py

@@ -0,0 +1,47 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from deckhand.control import common
+
+
+class ViewBuilder(common.ViewBuilder):
+	"""Model revision API responses as a python dictionary."""
+
+	_collection_name = 'revisions'
+
+	def list(self, revisions):
+	    resp_body = {
+	        'count': len(revisions),
+	        'next': None,
+	        'prev': None,
+	        'results': []
+	    }
+
+	    for revision in revisions:
+	        result = {}
+	        for attr in ('id', 'created_at'):
+	            result[common.to_camel_case(attr)] = revision[attr]
+	        result['count'] = len(revision.pop('documents'))
+	        resp_body['results'].append(result)
+
+	    return resp_body
+
+	def show(self, revision):
+		return {
+			'id': revision.get('id'),
+			'createdAt': revision.get('created_at'),
+			'url': self._gen_url(revision),
+			# TODO: Not yet implemented.
+			'validationPolicies': [],
+		}

deckhand/db/sqlalchemy/api.py

@@ -15,6 +15,8 @@
 
 """Defines interface for DB access."""
 
+import ast
+import copy
 import datetime
 import threading
 
@@ -28,12 +30,15 @@ import six
 from six.moves import range
 import sqlalchemy
 from sqlalchemy.ext.compiler import compiles
+from sqlalchemy import desc
 from sqlalchemy import MetaData, Table
 import sqlalchemy.orm as sa_orm
 from sqlalchemy import sql
 import sqlalchemy.sql as sa_sql
 
 from deckhand.db.sqlalchemy import models
+from deckhand import errors
+from deckhand import utils
 
 sa_logger = None
 LOG = logging.getLogger(__name__)
@@ -112,16 +117,146 @@ def documents_create(documents, session=None):
     return created_docs
 
 
-def document_create(values, session=None):
+def documents_create(values_list, session=None):
-    """Create a document."""
+    """Create a set of documents and associated schema.
-    values = values.copy()
-    values['doc_metadata'] = values.pop('metadata')
-    values['schema_version'] = values.pop('schemaVersion')
 
+    If no changes are detected, a new revision will not be created. This
+    allows services to periodically re-register their schemas without
+    creating unnecessary revisions.
+    """
+    values_list = copy.deepcopy(values_list)
     session = session or get_session()
-    document = models.Document()
+    filters = models.Document.UNIQUE_CONSTRAINTS
-    with session.begin():
-        document.update(values)
-        document.save(session=session)
 
-    return document.to_dict()
+    do_create = False
+    documents_created = []
+
+    def _document_changed(existing_document):
+        # The document has changed if at least one value in ``values`` differs.
+        for key, val in values.items():
+            if val != existing_document[key]:
+                return True
+        return False
+
+    def _document_create(values):
+        document = models.Document()
+        with session.begin():
+            document.update(values)
+            document.save(session=session)
+        return document.to_dict()
+
+    for values in values_list:
+        values['_metadata'] = values.pop('metadata')
+        values['name'] = values['_metadata']['name']
+    
+        try:
+            existing_document = document_get(
+                raw_dict=True,
+                **{c: values[c] for c in filters if c != 'revision_id'})
+        except db_exception.DBError:
+            # Ignore bad data at this point. Allow creation to bubble up the
+            # error related to bad data.
+            existing_document = None
+
+        if not existing_document:
+            do_create = True
+        elif existing_document and _document_changed(existing_document):
+            do_create = True
+
+    if do_create:
+        revision = revision_create()
+
+        for values in values_list:
+            values['revision_id'] = revision['id']
+            doc = _document_create(values)
+            documents_created.append(doc)
+
+    return documents_created
+
+
+def document_get(session=None, raw_dict=False, **filters):
+    session = session or get_session()
+    document = session.query(models.Document).filter_by(**filters).first()
+    return document.to_dict(raw_dict=raw_dict) if document else {}
+
+
+####################
+
+
+def revision_create(session=None):
+    session = session or get_session()
+    revision = models.Revision()
+    with session.begin():
+        revision.save(session=session)
+
+    return revision.to_dict()
+
+
+def revision_get(revision_id, session=None):
+    """Return the specified `revision_id`.
+
+    :raises: RevisionNotFound if the revision was not found.
+    """
+    session = session or get_session()
+    try:
+        revision = session.query(models.Revision).filter_by(
+            id=revision_id).one().to_dict()
+    except sa_orm.exc.NoResultFound:
+        raise errors.RevisionNotFound(revision=revision_id)
+    return revision
+
+
+def revision_get_all(session=None):
+    """Return list of all revisions."""
+    session = session or get_session()
+    revisions = session.query(models.Revision).all()
+    return [r.to_dict() for r in revisions]
+
+
+def revision_get_documents(revision_id, session=None, **filters):
+    """Return the documents that match filters for the specified `revision_id`.
+
+    :raises: RevisionNotFound if the revision was not found.
+    """
+    session = session or get_session()
+    try:
+        revision = session.query(models.Revision).filter_by(
+            id=revision_id).one().to_dict()
+    except sa_orm.exc.NoResultFound:
+        raise errors.RevisionNotFound(revision=revision_id)
+
+    filtered_documents = _filter_revision_documents(
+        revision['documents'], **filters)
+    return filtered_documents
+
+
+def _filter_revision_documents(documents, **filters):
+    """Return the list of documents that match filters.
+
+    :returns: list of documents that match specified filters.
+    """
+    # TODO: Implement this as an sqlalchemy query.
+    filtered_documents = []
+
+    for document in documents:
+        match = True
+
+        for filter_key, filter_val in filters.items():
+            actual_val = utils.multi_getattr(filter_key, document)
+
+            if (isinstance(actual_val, bool)
+                and isinstance(filter_val, six.text_type)):
+                try:
+                    filter_val = ast.literal_eval(filter_val.title())
+                except ValueError:
+                    # If not True/False, set to None to avoid matching
+                    # `actual_val` which is always boolean.
+                    filter_val = None
+
+            if actual_val != filter_val:
+                match = False
+
+        if match:
+            filtered_documents.append(document)
+
+    return filtered_documents

deckhand/db/sqlalchemy/models.py

@@ -15,52 +15,27 @@
 import uuid
 
 from oslo_db.sqlalchemy import models
-from oslo_log import log as logging
+from oslo_db.sqlalchemy import types as oslo_types
-from oslo_serialization import jsonutils as json
 from oslo_utils import timeutils
 from sqlalchemy import Boolean
 from sqlalchemy import Column
 from sqlalchemy import DateTime
 from sqlalchemy.ext import declarative
+from sqlalchemy import ForeignKey
 from sqlalchemy import Integer
 from sqlalchemy import orm
+from sqlalchemy.orm import backref, relationship
 from sqlalchemy import schema
 from sqlalchemy import String
 from sqlalchemy import Text
 from sqlalchemy.types import TypeDecorator
 
-from deckhand.common import timeutils
-
-LOG = logging.getLogger(__name__)
 
 # Declarative base class which maintains a catalog of classes and tables
 # relative to that base.
 BASE = declarative.declarative_base()
 
 
-class JSONEncodedDict(TypeDecorator):
-    """Represents an immutable structure as a json-encoded string.
-
-    Usage::
-
-        JSONEncodedDict(255)
-
-    """
-
-    impl = Text
-
-    def process_bind_param(self, value, dialect):
-        if value is not None:
-            value = json.dumps(value)
-
-        return value
-
-    def process_result_value(self, value, dialect):
-        if value is not None:
-            value = json.loads(value)
-        return value
-
-
 class DeckhandBase(models.ModelBase, models.TimestampMixin):
     """Base class for Deckhand Models."""
 
@@ -101,31 +76,68 @@ class DeckhandBase(models.ModelBase, models.TimestampMixin):
         # CircularReference.
         d.pop("_sa_instance_state")
 
+        if 'deleted_at' not in d:
+            d.setdefault('deleted_at', None)
+
         for k in ["created_at", "updated_at", "deleted_at", "deleted"]:
             if k in d and d[k]:
                 d[k] = d[k].isoformat()
 
         return d
 
+    @staticmethod
+    def gen_unqiue_contraint(self, *fields):
+        constraint_name = 'ix_' + self.__class__.__name__.lower() + '_'
+        for field in fields:
+            constraint_name = constraint_name + '_%s' % field
+        return schema.UniqueConstraint(*fields, name=constraint_name)
 
-class Document(BASE, DeckhandBase):
+
-    __tablename__ = 'document'
+class Revision(BASE, DeckhandBase):
-    __table_args__ = (schema.UniqueConstraint('schema_version', 'kind',
+    __tablename__ = 'revisions'
-                      name='ix_documents_schema_version_kind'),)
 
     id = Column(String(36), primary_key=True,
                 default=lambda: str(uuid.uuid4()))
-    # TODO: the revision_index will be a foreign key to a Revision table.
+    parent_id = Column(Integer, ForeignKey('revisions.id'), nullable=True)
-    revision_index = Column(String(36), nullable=False,
+    child_id = Column(Integer, ForeignKey('revisions.id'), nullable=True)
-                            default=lambda: str(uuid.uuid4()))
+    results = Column(oslo_types.JsonEncodedList(), nullable=True)
-    schema_version = Column(String(64), nullable=False)
+
-    kind = Column(String(64), nullable=False)
+    documents = relationship("Document")
+
+    def to_dict(self):
+        d = super(Revision, self).to_dict()
+        d['documents'] = [doc.to_dict() for doc in self.documents]
+        return d
+
+
+class Document(BASE, DeckhandBase):
+    UNIQUE_CONSTRAINTS = ('schema', 'name', 'revision_id')
+    __tablename__ = 'documents'
+    __table_args__ = (DeckhandBase.gen_unqiue_contraint(*UNIQUE_CONSTRAINTS),)
+
+    id = Column(String(36), primary_key=True,
+                default=lambda: str(uuid.uuid4()))
+    schema = Column(String(64), nullable=False)
+    name = Column(String(64), nullable=False)
     # NOTE: Do not define a maximum length for these JSON data below. However,
     # this approach is not compatible with all database types.
-    # "metadata" is reserved, so use "doc_metadata" instead.
+    # "metadata" is reserved, so use "_metadata" instead.
-    doc_metadata = Column(JSONEncodedDict(), nullable=False)
+    _metadata = Column(oslo_types.JsonEncodedDict(), nullable=False)
-    data = Column(JSONEncodedDict(), nullable=False)
+    data = Column(oslo_types.JsonEncodedDict(), nullable=False)
+    revision_id = Column(Integer, ForeignKey('revisions.id'), nullable=False)
 
+    def to_dict(self, raw_dict=False):
+        """Convert the ``Document`` object into a dictionary format.
+
+        :param raw_dict: if True, returns unmodified data; else returns data
+            expected by users.
+        :returns: dictionary format of ``Document`` object.
+        """
+        d = super(Document, self).to_dict()
+        # ``_metadata`` is used in the DB schema as ``metadata`` is reserved.
+        if not raw_dict:
+            d['metadata'] = d.pop('_metadata')
+        return d
 
 def register_models(engine):
     """Create database tables for all models with the given engine."""

deckhand/engine/document_validation.py

@@ -13,16 +13,20 @@
 # limitations under the License.
 
 import jsonschema
+from oslo_log import log as logging
+import six
 
-from deckhand.engine.schema.v1_0 import default_schema
+from deckhand.engine.schema.v1_0 import default_policy_validation
+from deckhand.engine.schema.v1_0 import default_schema_validation
 from deckhand import errors
 
+LOG = logging.getLogger(__name__)
+
 
 class DocumentValidation(object):
     """Class for document validation logic for YAML files.
 
-    This class is responsible for parsing, validating and retrieving secret
+    This class is responsible for performing built-in validations on Documents.
-    values for values stored in the YAML file.
 
     :param data: YAML data that requires secrets to be validated, merged and
         consolidated.
@@ -30,7 +34,6 @@ class DocumentValidation(object):
 
     def __init__(self, data):
         self.data = data
-        self.pre_validate_data()
 
     class SchemaVersion(object):
         """Class for retrieving correct schema for pre-validation on YAML.
@@ -38,81 +41,59 @@ class DocumentValidation(object):
         Retrieves the schema that corresponds to "apiVersion" in the YAML
         data. This schema is responsible for performing pre-validation on
         YAML data.
+
+        The built-in validation schemas that are always executed include:
+
+          - `deckhand-document-schema-validation`
+          - `deckhand-policy-validation`
         """
 
-        # TODO: Update kind according to requirements.
+        # TODO: Use the correct validation based on the Document's schema.
-        schema_versions_info = [{'version': 'v1', 'kind': 'default',
+        internal_validations = [
-                                 'schema': default_schema}]
+            {'version': 'v1', 'fqn': 'deckhand-document-schema-validation',
+             'schema': default_schema_validation},
+            {'version': 'v1', 'fqn': 'deckhand-policy-validation',
+             'schema': default_policy_validation}]
 
-        def __init__(self, schema_version, kind):
+        def __init__(self, schema_version):
             self.schema_version = schema_version
-            self.kind = kind
 
         @property
         def schema(self):
-            # TODO: return schema based on version and kind.
+            # TODO: return schema based on Document's schema.
-            return [v['schema'] for v in self.schema_versions_info
+            return [v['schema'] for v in self.internal_validations
                     if v['version'] == self.schema_version][0].schema
 
-    def pre_validate_data(self):
+    def pre_validate(self):
         """Pre-validate that the YAML file is correctly formatted."""
         self._validate_with_schema()
 
-        # TODO(fm577c): Query Deckhand API to validate "src" values.
-
-    @property
-    def doc_name(self):
-        return (self.data['schemaVersion'] + self.data['kind'] +
-                self.data['metadata']['name'])
-
     def _validate_with_schema(self):
-        # Validate the document using the schema defined by the document's
+        # Validate the document using the document's ``schema``. Only validate
-        # `schemaVersion` and `kind`.
+        # concrete documents.
         try:
-            schema_version = self.data['schemaVersion'].split('/')[-1]
+            abstract = self.data['metadata']['layeringDefinition'][
-            doc_kind = self.data['kind']
+                'abstract']
-            doc_schema_version = self.SchemaVersion(schema_version, doc_kind)
+            is_abstract = six.text_type(abstract).lower() == 'true'
+        except KeyError as e:
+            raise errors.InvalidFormat(
+                "Could not find 'abstract' property from document.")
+
+        # TODO: This should be done inside a different module.
+        if is_abstract:
+            LOG.info(
+                "Skipping validation for the document because it is abstract")
+            return
+
+        try:
+            schema_version = self.data['schema'].split('/')[-1]
+            doc_schema_version = self.SchemaVersion(schema_version)
         except (AttributeError, IndexError, KeyError) as e:
             raise errors.InvalidFormat(
-                'The provided schemaVersion is invalid or missing. Exception: '
+                'The provided schema is invalid or missing. Exception: '
                 '%s.' % e)
         try:
             jsonschema.validate(self.data, doc_schema_version.schema)
         except jsonschema.exceptions.ValidationError as e:
             raise errors.InvalidFormat('The provided YAML file is invalid. '
                                        'Exception: %s.' % e.message)
-
-    def _multi_getattr(self, multi_key, substitutable_data):
-        """Iteratively check for nested attributes in the YAML data.
-
-        Check for nested attributes included in "dest" attributes in the data
-        section of the YAML file. For example, a "dest" attribute of
-        ".foo.bar.baz" should mean that the YAML data adheres to:
-
-        .. code-block:: yaml
-
-           ---
-           foo:
-               bar:
-                   baz: <data_to_be_substituted_here>
-
-        :param multi_key: A multi-part key that references nested data in the
-            substitutable part of the YAML data, e.g. ".foo.bar.baz".
-        :param substitutable_data: The section of data in the YAML data that
-            is intended to be substituted with secrets.
-        :returns: Tuple where first value is a boolean indicating that the
-            nested attribute was found and the second value is the attribute
-            that was not found, if applicable.
-        """
-        attrs = multi_key.split('.')
-        # Ignore the first attribute if it is "." as that is a self-reference.
-        if attrs[0] == '':
-            attrs = attrs[1:]
-
-        data = substitutable_data
-        for attr in attrs:
-            if attr not in data:
-                return False, attr
-            data = data.get(attr)
-
-        return True, None

deckhand/engine/schema/v1_0/default_policy_validation.py

@@ -0,0 +1,13 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

deckhand/engine/schema/v1_0/default_schema_validation.py

@@ -18,8 +18,7 @@ substitution_schema = {
         'dest': {
             'type': 'object',
             'properties': {
-                'path': {'type': 'string'},
+                'path': {'type': 'string'}
-                'replacePattern': {'type': 'string'}
             },
             'additionalProperties': False,
             # 'replacePattern' is not required.
@@ -28,12 +27,12 @@ substitution_schema = {
         'src': {
             'type': 'object',
             'properties': {
-                'kind': {'type': 'string'},
+                'schema': {'type': 'string'},
                 'name': {'type': 'string'},
                 'path': {'type': 'string'}
             },
             'additionalProperties': False,
-            'required': ['kind', 'name', 'path']
+            'required': ['schema', 'name', 'path']
         }
     },
     'additionalProperties': False,
@@ -43,45 +42,46 @@ substitution_schema = {
 schema = {
     'type': 'object',
     'properties': {
-        'schemaVersion': {
+        'schema': {
             'type': 'string',
-            'pattern': '^([A-Za-z]+\/v[0-9]{1})$'
+            'pattern': '^(.*\/v[0-9]{1})$'
         },
-        # TODO: The kind should be an enum.
-        'kind': {'type': 'string'},
         'metadata': {
             'type': 'object',
             'properties': {
-                'metadataVersion': {
+                'schema': {
                     'type': 'string',
-                    'pattern': '^([A-Za-z]+\/v[0-9]{1})$'
+                    'pattern': '^(.*/v[0-9]{1})$'
                 },
                 'name': {'type': 'string'},
+                'storagePolicy': {'type': 'string'},
                 'labels': {
-                    'type': 'object',
+                    'type': 'object'
-                    'properties': {
-                        'component': {'type': 'string'},
-                        'hostname': {'type': 'string'}
-                    },
-                    'additionalProperties': False,
-                    'required': ['component', 'hostname']
                 },
-                'layerDefinition': {
+                'layeringDefinition': {
                     'type': 'object',
                     'properties': {
-                        'layer': {'enum': ['global', 'region', 'site']},
+                        'layer': {'type': 'string'},
                         'abstract': {'type': 'boolean'},
-                        'childSelector': {
+                        'parentSelector': {
-                            'type': 'object',
+                            'type': 'object'
-                            'properties': {
+                        },
-                                'label': {'type': 'string'}
+                        'actions': {
-                            },
+                            'type': 'array',
-                            'additionalProperties': False,
+                            'items': {
-                            'required': ['label']
+                                'type': 'object',
+                                'properties': {
+                                    'method': {'enum': ['merge', 'delete',
+                                                        'replace']},
+                                    'path': {'type': 'string'}
+                                },
+                                'additionalProperties': False,
+                                'required': ['method', 'path']
+                            }
                         }
                     },
                     'additionalProperties': False,
-                    'required': ['layer', 'abstract', 'childSelector']
+                    'required': ['layer', 'abstract', 'parentSelector']
                 },
                 'substitutions': {
                     'type': 'array',
@@ -89,13 +89,13 @@ schema = {
                 }
             },
             'additionalProperties': False,
-            'required': ['metadataVersion', 'name', 'labels',
+            'required': ['schema', 'name', 'storagePolicy', 'labels',
-                         'layerDefinition', 'substitutions']
+                         'layeringDefinition', 'substitutions']
         },
         'data': {
             'type': 'object'
         }
     },
     'additionalProperties': False,
-    'required': ['schemaVersion', 'kind', 'metadata', 'data']
+    'required': ['schema', 'metadata', 'data']
 }

deckhand/engine/secret_substitution.py

@@ -1,138 +0,0 @@
-# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import yaml
-
-import jsonschema
-
-from deckhand.engine.schema.v1_0 import default_schema
-from deckhand import errors
-
-
-class SecretSubstitution(object):
-    """Class for secret substitution logic for YAML files.
-
-    This class is responsible for parsing, validating and retrieving secret
-    values for values stored in the YAML file. Afterward, secret values will be
-    substituted or "forward-repalced" into the YAML file. The end result is a
-    YAML file containing all necessary secrets to be handed off to other
-    services.
-
-    :param data: YAML data that requires secrets to be validated, merged and
-        consolidated.
-    """
-
-    def __init__(self, data):
-        try:
-            self.data = yaml.safe_load(data)
-        except yaml.YAMLError:
-            raise errors.InvalidFormat(
-                'The provided YAML file cannot be parsed.')
-
-        self.pre_validate_data()
-
-    class SchemaVersion(object):
-        """Class for retrieving correct schema for pre-validation on YAML.
-
-        Retrieves the schema that corresponds to "apiVersion" in the YAML
-        data. This schema is responsible for performing pre-validation on
-        YAML data.
-        """
-
-        # TODO: Update kind according to requirements.
-        schema_versions_info = [{'version': 'v1', 'kind': 'default',
-                                 'schema': default_schema}]
-
-        def __init__(self, schema_version, kind):
-            self.schema_version = schema_version
-            self.kind = kind
-
-        @property
-        def schema(self):
-            # TODO: return schema based on version and kind.
-            return [v['schema'] for v in self.schema_versions_info
-                    if v['version'] == self.schema_version][0].schema
-
-    def pre_validate_data(self):
-        """Pre-validate that the YAML file is correctly formatted."""
-        self._validate_with_schema()
-
-        # Validate that each "dest" field exists in the YAML data.
-        # FIXME(fm577c): Dest fields will be injected if not present - the
-        # validation below needs to be updated or removed.
-        substitutions = self.data['metadata']['substitutions']
-        destinations = [s['dest'] for s in substitutions]
-        sub_data = self.data['data']
-
-        for dest in destinations:
-            result, missing_attr = self._multi_getattr(dest['path'], sub_data)
-            if not result:
-                raise errors.InvalidFormat(
-                    'The attribute "%s" included in the "dest" field "%s" is '
-                    'missing from the YAML data: "%s".' % (
-                        missing_attr, dest, sub_data))
-
-        # TODO(fm577c): Query Deckhand API to validate "src" values.
-
-    def _validate_with_schema(self):
-        # Validate the document using the schema defined by the document's
-        # `schemaVersion` and `kind`.
-        try:
-            schema_version = self.data['schemaVersion'].split('/')[-1]
-            doc_kind = self.data['kind']
-            doc_schema_version = self.SchemaVersion(schema_version, doc_kind)
-        except (AttributeError, IndexError, KeyError) as e:
-            raise errors.InvalidFormat(
-                'The provided schemaVersion is invalid or missing. Exception: '
-                '%s.' % e)
-        try:
-            jsonschema.validate(self.data, doc_schema_version.schema)
-        except jsonschema.exceptions.ValidationError as e:
-            raise errors.InvalidFormat('The provided YAML file is invalid. '
-                                       'Exception: %s.' % e.message)
-
-    def _multi_getattr(self, multi_key, substitutable_data):
-        """Iteratively check for nested attributes in the YAML data.
-
-        Check for nested attributes included in "dest" attributes in the data
-        section of the YAML file. For example, a "dest" attribute of
-        ".foo.bar.baz" should mean that the YAML data adheres to:
-
-        .. code-block:: yaml
-
-           ---
-           foo:
-               bar:
-                   baz: <data_to_be_substituted_here>
-
-        :param multi_key: A multi-part key that references nested data in the
-            substitutable part of the YAML data, e.g. ".foo.bar.baz".
-        :param substitutable_data: The section of data in the YAML data that
-            is intended to be substituted with secrets.
-        :returns: Tuple where first value is a boolean indicating that the
-            nested attribute was found and the second value is the attribute
-            that was not found, if applicable.
-        """
-        attrs = multi_key.split('.')
-        # Ignore the first attribute if it is "." as that is a self-reference.
-        if attrs[0] == '':
-            attrs = attrs[1:]
-
-        data = substitutable_data
-        for attr in attrs:
-            if attr not in data:
-                return False, attr
-            data = data.get(attr)
-
-        return True, None

deckhand/errors.py

@@ -14,7 +14,7 @@
 
 
 class DeckhandException(Exception):
-    """Base Nova Exception
+    """Base Deckhand Exception
     To correctly use this class, inherit from it and define
     a 'msg_fmt' property. That msg_fmt will get printf'd
     with the keyword arguments provided to the constructor.
@@ -57,3 +57,7 @@ class DocumentExists(DeckhandException):
     msg_fmt = ("Document with kind %(kind)s and schemaVersion "
                "%(schema_version)s already exists.")
 
+
+class RevisionNotFound(DeckhandException):
+    msg_fmt = ("The requested revision %(revision)s was not found.")
+    code = 403

deckhand/tests/test_utils.py

@@ -0,0 +1,62 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import random
+import uuid
+
+
+def rand_uuid_hex():
+    """Generate a random UUID hex string
+
+    :return: a random UUID (e.g. '0b98cf96d90447bda4b46f31aeb1508c')
+    :rtype: string
+    """
+    return uuid.uuid4().hex
+
+
+def rand_name(name='', prefix='deckhand'):
+    """Generate a random name that includes a random number
+
+    :param str name: The name that you want to include
+    :param str prefix: The prefix that you want to include
+    :return: a random name. The format is
+             '<prefix>-<name>-<random number>'.
+             (e.g. 'prefixfoo-namebar-154876201')
+    :rtype: string
+    """
+    randbits = str(random.randint(1, 0x7fffffff))
+    rand_name = randbits
+    if name:
+        rand_name = name + '-' + rand_name
+    if prefix:
+        rand_name = prefix + '-' + rand_name
+    return rand_name
+
+
+def rand_bool():
+    """Generate a random boolean value.
+
+    :return: a random boolean value.
+    :rtype: boolean
+    """
+    return random.choice([True, False])
+
+
+def rand_int(min, max):
+    """Generate a random integer value between range (`min`, `max`).
+
+    :return: a random integer between the range(`min`, `max`).
+    :rtype: integer
+    """
+    return random.randint(min, max)

deckhand/tests/unit/base.py

@@ -36,6 +36,9 @@ class DeckhandTestCase(testtools.TestCase):
         CONF.set_override(name, override, group)
         self.addCleanup(CONF.clear_override, name, group)
 
+    def assertEmpty(self, list):
+        self.assertEqual(0, len(list))
+
 
 class DeckhandWithDBTestCase(DeckhandTestCase):
 

deckhand/tests/unit/control/test_api.py

@@ -18,17 +18,27 @@ import testtools
 
 from deckhand.control import api
 from deckhand.control import base as api_base
+from deckhand.control import documents
+from deckhand.control import revision_documents
+from deckhand.control import revisions
+from deckhand.control import secrets
 
 
 class TestApi(testtools.TestCase):
 
+    def setUp(self):
+        super(TestApi, self).setUp()
+        for resource in (documents, revisions, revision_documents, secrets):
+            resource_name = resource.__name__.split('.')[-1]
+            resource_obj = mock.patch.object(
+                resource, '%sResource' % resource_name.title().replace(
+                    '_', '')).start()
+            setattr(self, '%s_resource' % resource_name, resource_obj)
 
     @mock.patch.object(api, 'db_api', autospec=True)
     @mock.patch.object(api, 'config', autospec=True)
-    @mock.patch.object(api, 'secrets', autospec=True)
-    @mock.patch.object(api, 'documents', autospec=True)
     @mock.patch.object(api, 'falcon', autospec=True)
-    def test_start_api(self, mock_falcon, mock_documents, mock_secrets,
+    def test_start_api(self, mock_falcon,
                        mock_config, mock_db_api):
         mock_falcon_api = mock_falcon.API.return_value
 
@@ -38,9 +48,13 @@ class TestApi(testtools.TestCase):
         mock_falcon.API.assert_called_once_with(
             request_type=api_base.DeckhandRequest)
         mock_falcon_api.add_route.assert_has_calls([
-            mock.call(
+            mock.call('/api/v1.0/documents', self.documents_resource()),
-                '/api/v1.0/documents', mock_documents.DocumentsResource()),
+            mock.call('/api/v1.0/revisions', self.revisions_resource()),
-            mock.call('/api/v1.0/secrets', mock_secrets.SecretsResource())
+            mock.call('/api/v1.0/revisions/{revision_id}',
+                      self.revisions_resource()),
+            mock.call('/api/v1.0/revisions/{revision_id}/documents',
+                      self.revision_documents_resource()),
+            mock.call('/api/v1.0/secrets', self.secrets_resource())
         ])
         mock_config.parse_args.assert_called_once_with()
         mock_db_api.setup_db.assert_called_once_with()

deckhand/tests/unit/db/base.py

@@ -0,0 +1,115 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import testtools
+from testtools import matchers
+
+from deckhand.db.sqlalchemy import api as db_api
+from deckhand.tests import test_utils
+from deckhand.tests.unit import base
+
+BASE_EXPECTED_FIELDS = ("created_at", "updated_at", "deleted_at", "deleted")
+DOCUMENT_EXPECTED_FIELDS = BASE_EXPECTED_FIELDS + (
+    "id", "schema", "name", "metadata", "data", "revision_id")
+REVISION_EXPECTED_FIELDS = BASE_EXPECTED_FIELDS + (
+    "id", "child_id", "parent_id", "documents")
+
+
+class DocumentFixture(object):
+
+    @staticmethod
+    def get_minimal_fixture(**kwargs):
+        fixture = {
+            'data': {
+                test_utils.rand_name('key'): test_utils.rand_name('value')
+            },
+            'metadata': {
+                'name': test_utils.rand_name('metadata_data'),
+                'label': test_utils.rand_name('metadata_label'),
+                'layeringDefinition': {
+                    'abstract': test_utils.rand_bool(),
+                    'layer': test_utils.rand_name('layer')
+                }
+            },
+            'schema': test_utils.rand_name('schema')}
+        fixture.update(kwargs)
+        return fixture
+
+    @staticmethod
+    def get_minimal_multi_fixture(count=2, **kwargs):
+        return [DocumentFixture.get_minimal_fixture(**kwargs) 
+                for _ in range(count)]
+
+
+class TestDbBase(base.DeckhandWithDBTestCase):
+
+    def _create_documents(self, payload):
+        if not isinstance(payload, list):
+            payload = [payload]
+
+        docs = db_api.documents_create(payload)
+        for idx, doc in enumerate(docs):
+            self._validate_document(expected=payload[idx], actual=doc)
+        return docs
+
+    def _get_document(self, **fields):
+        doc = db_api.document_get(**fields)
+        self._validate_document(actual=doc)
+        return doc
+
+    def _get_revision(self, revision_id):
+        revision = db_api.revision_get(revision_id)
+        self._validate_revision(revision)
+        return revision
+
+    def _get_revision_documents(self, revision_id, **filters):
+        documents = db_api.revision_get_documents(revision_id, **filters)
+        for document in documents:
+            self._validate_document(document)
+        return documents
+
+    def _list_revisions(self):
+        return db_api.revision_get_all()
+
+    def _validate_object(self, obj):
+        for attr in BASE_EXPECTED_FIELDS:
+            if attr.endswith('_at'):
+                self.assertThat(obj[attr], matchers.MatchesAny(
+                        matchers.Is(None), matchers.IsInstance(str)))
+            else:
+                self.assertIsInstance(obj[attr], bool)
+
+    def _validate_document(self, actual, expected=None, is_deleted=False):
+        self._validate_object(actual)
+
+        # Validate that the document has all expected fields and is a dict.
+        expected_fields = list(DOCUMENT_EXPECTED_FIELDS)
+        if not is_deleted:
+            expected_fields.remove('deleted_at')
+
+        self.assertIsInstance(actual, dict)
+        for field in expected_fields:
+            self.assertIn(field, actual)
+
+        if expected:
+            # Validate that the expected values are equivalent to actual
+            # values.
+            for key, val in expected.items():
+                self.assertEqual(val, actual[key])
+
+    def _validate_revision(self, revision):
+        self._validate_object(revision)
+
+        for attr in REVISION_EXPECTED_FIELDS:
+            self.assertIn(attr, revision)

deckhand/tests/unit/db/test_documents.py

@@ -12,38 +12,76 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import mock
+from deckhand.tests.unit.db import base
-
-import testtools
-
-from deckhand.db.sqlalchemy import api as db_api
-from deckhand.tests.unit import base
 
 
-class DocumentFixture(object):
+class TestDocuments(base.TestDbBase):
 
-    def get_minimal_fixture(self, **kwargs):
+    def test_create_and_get_document(self):
-        fixture = {'data': 'fake document data',
+        payload = base.DocumentFixture.get_minimal_fixture()
-                   'metadata': 'fake meta',
+        documents = self._create_documents(payload)
-                   'kind': 'FakeConfigType',
-                   'schemaVersion': 'deckhand/v1'}
-        fixture.update(kwargs)
-        return fixture
 
+        self.assertIsInstance(documents, list)
+        self.assertEqual(1, len(documents))
 
-class TestDocumentsApi(base.DeckhandWithDBTestCase):
+        for document in documents:
+            retrieved_document = self._get_document(id=document['id'])
+            self.assertEqual(document, retrieved_document)
 
-	def _validate_document(self, expected, actual):
+    def test_create_document_again_with_no_changes(self):
-		expected['doc_metadata'] = expected.pop('metadata')
+        payload = base.DocumentFixture.get_minimal_fixture()
-		expected['schema_version'] = expected.pop('schemaVersion')
+        self._create_documents(payload)
+        documents = self._create_documents(payload)
 
-		# TODO: Validate "status" fields, like created_at.
+        self.assertIsInstance(documents, list)
-		self.assertIsInstance(actual, dict)
+        self.assertEmpty(documents)
-		for key, val in expected.items():
-			self.assertIn(key, actual)
-			self.assertEqual(val, actual[key])
 
-	def test_create_document(self):
+    def test_create_document_and_get_revision(self):
-		fixture = DocumentFixture().get_minimal_fixture()
+        payload = base.DocumentFixture.get_minimal_fixture()
-		document = db_api.document_create(fixture)
+        documents = self._create_documents(payload)
-		self._validate_document(fixture, document)
+
+        self.assertIsInstance(documents, list)
+        self.assertEqual(1, len(documents))
+
+        for document in documents:
+            revision = self._get_revision(document['revision_id'])
+            self._validate_revision(revision)
+            self.assertEqual(document['revision_id'], revision['id'])
+
+    def test_get_documents_by_revision_id(self):
+        payload = base.DocumentFixture.get_minimal_fixture()
+        documents = self._create_documents(payload)
+
+        revision = self._get_revision(documents[0]['revision_id'])
+        self.assertEqual(1, len(revision['documents']))
+        self.assertEqual(documents[0], revision['documents'][0])
+
+    def test_get_multiple_documents_by_revision_id(self):
+        payload = base.DocumentFixture.get_minimal_multi_fixture(count=3)
+        documents = self._create_documents(payload)
+
+        self.assertIsInstance(documents, list)
+        self.assertEqual(3, len(documents))
+
+        for document in documents:
+            revision = self._get_revision(document['revision_id'])
+            self._validate_revision(revision)
+            self.assertEqual(document['revision_id'], revision['id'])
+
+    def test_get_documents_by_revision_id_and_filters(self):
+        payload = base.DocumentFixture.get_minimal_fixture()
+        document = self._create_documents(payload)[0]
+        filters = {
+            'schema': document['schema'],
+            'metadata.name': document['metadata']['name'],
+            'metadata.layeringDefinition.abstract':
+                document['metadata']['layeringDefinition']['abstract'],
+            'metadata.layeringDefinition.layer':
+                document['metadata']['layeringDefinition']['layer'],
+            'metadata.label': document['metadata']['label']
+        }
+
+        documents = self._get_revision_documents(
+            document['revision_id'], **filters)
+        self.assertEqual(1, len(documents))
+        self.assertEqual(document, documents[0])

deckhand/tests/unit/db/test_documents_negative.py

@@ -0,0 +1,39 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from deckhand.tests.unit.db import base
+
+
+class TestDocumentsNegative(base.TestDbBase):
+
+    def test_get_documents_by_revision_id_and_wrong_filters(self):
+        payload = base.DocumentFixture.get_minimal_fixture()
+        document = self._create_documents(payload)[0]
+        filters = {
+            'schema': 'fake_schema',
+            'metadata.name': 'fake_meta_name',
+            'metadata.layeringDefinition.abstract':
+                not document['metadata']['layeringDefinition']['abstract'],
+            'metadata.layeringDefinition.layer': 'fake_layer',
+            'metadata.label': 'fake_label'
+        }
+
+        documents = self._get_revision_documents(
+            document['revision_id'], **filters)
+        self.assertEmpty(documents)
+
+        for filter_key, filter_val in filters.items():
+            documents = self._get_revision_documents(
+                document['revision_id'], filter_key=filter_val)
+            self.assertEmpty(documents)

deckhand/tests/unit/db/test_revisions.py

@@ -0,0 +1,28 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from deckhand.tests.unit.db import base
+
+
+class TestRevisionViews(base.TestDbBase):
+
+    def test_list(self):
+        payload = [base.DocumentFixture.get_minimal_fixture()
+                   for _ in range(4)]
+        self._create_documents(payload)
+
+        revisions = self._list_revisions()
+        self.assertIsInstance(revisions, list)
+        self.assertEqual(1, len(revisions))
+        self.assertEqual(4, len(revisions[0]['documents']))

deckhand/tests/unit/engine/test_document_validation.py

@@ -17,6 +17,7 @@ import os
 import testtools
 import yaml
 
+import mock
 import six
 
 from deckhand.engine import document_validation
@@ -70,19 +71,15 @@ class TestDocumentValidation(testtools.TestCase):
         return corrupted_data
 
     def test_initialization(self):
-        doc_validation = document_validation.DocumentValidation(
+        doc_validation = document_validation.DocumentValidation(self.data)
-            self.data)
+        doc_validation.pre_validate()  # Should not raise any errors.
-        self.assertIsInstance(doc_validation,
-                              document_validation.DocumentValidation)
 
     def test_initialization_missing_sections(self):
         expected_err = ("The provided YAML file is invalid. Exception: '%s' "
                         "is a required property.")
         invalid_data = [
             (self._corrupt_data('data'), 'data'),
-            (self._corrupt_data('metadata'), 'metadata'),
+            (self._corrupt_data('metadata.schema'), 'schema'),
-            (self._corrupt_data('metadata.metadataVersion'),
-                                'metadataVersion'),
             (self._corrupt_data('metadata.name'), 'name'),
             (self._corrupt_data('metadata.substitutions'), 'substitutions'),
             (self._corrupt_data('metadata.substitutions.0.dest'), 'dest'),
@@ -92,4 +89,35 @@ class TestDocumentValidation(testtools.TestCase):
         for invalid_entry, missing_key in invalid_data:
             with six.assertRaisesRegex(self, errors.InvalidFormat,
                                        expected_err % missing_key):
-                document_validation.DocumentValidation(invalid_entry)
+                doc_validation = document_validation.DocumentValidation(
+                    invalid_entry)
+                doc_validation.pre_validate()
+
+    def test_initialization_missing_abstract_section(self):
+        expected_err = ("Could not find 'abstract' property from document.")
+        invalid_data = [
+            self._corrupt_data('metadata'),
+            self._corrupt_data('metadata.layeringDefinition'),
+            self._corrupt_data('metadata.layeringDefinition.abstract'),
+        ]
+
+        for invalid_entry in invalid_data:
+            with six.assertRaisesRegex(self, errors.InvalidFormat,
+                                       expected_err):
+                doc_validation = document_validation.DocumentValidation(
+                    invalid_entry)
+                doc_validation.pre_validate()
+
+    @mock.patch.object(document_validation, 'LOG', autospec=True)
+    def test_initialization_with_abstract_document(self, mock_log):
+        abstract_data = copy.deepcopy(self.data)
+
+        for true_val in (True, 'true', 'True'):
+            abstract_data['metadata']['layeringDefinition']['abstract'] = True
+
+            doc_validation = document_validation.DocumentValidation(
+                abstract_data)
+            doc_validation.pre_validate()
+            mock_log.info.assert_called_once_with(
+                "Skipping validation for the document because it is abstract")
+            mock_log.info.reset_mock()

deckhand/tests/unit/engine/test_secret_substitution.py

@@ -1,123 +0,0 @@
-# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import copy
-import os
-import testtools
-import yaml
-
-import six
-
-from deckhand.engine import secret_substitution
-from deckhand import errors
-
-
-class TestSecretSubtitution(testtools.TestCase):
-
-    def setUp(self):
-        super(TestSecretSubtitution, self).setUp()
-        dir_path = os.path.dirname(os.path.realpath(__file__))
-        test_yaml_path = os.path.abspath(os.path.join(
-            dir_path, os.pardir, 'resources', 'sample.yaml'))
-
-        with open(test_yaml_path, 'r') as yaml_file:
-            yaml_data = yaml_file.read()
-        self.data = yaml.safe_load(yaml_data)
-
-    def _corrupt_data(self, key, data=None):
-        """Corrupt test data to check that pre-validation works.
-
-        Corrupt data by removing a key from the document. Each key must
-        correspond to a value that is a dictionary.
-
-        :param key: The document key to be removed. The key can have the
-            following formats:
-                * 'data' => document.pop('data')
-                * 'metadata.name' => document['metadata'].pop('name')
-                * 'metadata.substitutions.0.dest' =>
-                   document['metadata']['substitutions'][0].pop('dest')
-        :returns: Corrupted YAML data.
-        """
-        if data is None:
-            data = self.data
-        corrupted_data = copy.deepcopy(data)
-
-        if '.' in key:
-            _corrupted_data = corrupted_data
-            nested_keys = key.split('.')
-            for nested_key in nested_keys:
-                if nested_key == nested_keys[-1]:
-                    break
-                if nested_key.isdigit():
-                    _corrupted_data = _corrupted_data[int(nested_key)]
-                else:
-                    _corrupted_data = _corrupted_data[nested_key]
-            _corrupted_data.pop(nested_keys[-1])
-        else:
-            corrupted_data.pop(key)
-
-        return self._format_data(corrupted_data)
-
-    def _format_data(self, data=None):
-        """Re-formats dict data as YAML to pass to ``SecretSubstitution``."""
-        if data is None:
-            data = self.data
-        return yaml.safe_dump(data)
-
-    def test_initialization(self):
-        sub = secret_substitution.SecretSubstitution(self._format_data())
-        self.assertIsInstance(sub, secret_substitution.SecretSubstitution)
-
-    def test_initialization_missing_sections(self):
-        expected_err = ("The provided YAML file is invalid. Exception: '%s' "
-                        "is a required property.")
-        invalid_data = [
-            (self._corrupt_data('data'), 'data'),
-            (self._corrupt_data('metadata'), 'metadata'),
-            (self._corrupt_data('metadata.metadataVersion'), 'metadataVersion'),
-            (self._corrupt_data('metadata.name'), 'name'),
-            (self._corrupt_data('metadata.substitutions'), 'substitutions'),
-            (self._corrupt_data('metadata.substitutions.0.dest'), 'dest'),
-            (self._corrupt_data('metadata.substitutions.0.src'), 'src')
-        ]
-
-        for invalid_entry, missing_key in invalid_data:
-            with six.assertRaisesRegex(self, errors.InvalidFormat,
-                                       expected_err % missing_key):
-                secret_substitution.SecretSubstitution(invalid_entry)
-
-    def test_initialization_bad_substitutions(self):
-        expected_err = ('The attribute "%s" included in the "dest" field "%s" '
-                        'is missing from the YAML data')
-        invalid_data = []
-
-        data = copy.deepcopy(self.data)
-        data['metadata']['substitutions'][0]['dest'] = {'path': 'foo'}
-        invalid_data.append(self._format_data(data))
-
-        data = copy.deepcopy(self.data)
-        data['metadata']['substitutions'][0]['dest'] = {
-            'path': 'tls_endpoint.bar'}
-        invalid_data.append(self._format_data(data))
-
-        def _test(invalid_entry, field, dest):
-            _expected_err = expected_err % (field, dest)
-            with six.assertRaisesRegex(self, errors.InvalidFormat,
-                                       _expected_err):
-                secret_substitution.SecretSubstitution(invalid_entry)
-
-        # Verify that invalid body dest reference is invalid.
-        _test(invalid_data[0], "foo", {'path': 'foo'})
-        # Verify that nested invalid body dest reference is invalid.
-        _test(invalid_data[1], "bar", {'path': 'tls_endpoint.bar'})

deckhand/tests/unit/resources/sample.yaml

@@ -1,33 +1,38 @@
-# Sample YAML file for testing forward replacement.
 ---
-schemaVersion: promenade/v1
+schema: some-service/ResourceType/v1
-kind: SomeConfigType
 metadata:
-  metadataVersion: deckhand/v1
+  schema: metadata/Document/v1
-  name: a-unique-config-name-12345
+  name: unique-name-given-schema
+  storagePolicy: cleartext
   labels:
-    component: apiserver
+    genesis: enabled
-    hostname: server0
+    master: enabled
-  layerDefinition:
+  layeringDefinition:
-      layer: global
+    abstract: false
-      abstract: True
+    layer: region
-      childSelector:
+    parentSelector:
-        label: value
+      required_key_a: required_label_a
+      required_key_b: required_label_b
+    actions:
+      - method: merge
+        path: .path.to.merge.into.parent
+      - method: delete
+        path: .path.to.delete
   substitutions:
     - dest:
-        path: .tls_endpoint.certificate
+        path: .substitution.target
-        replacePattern: 'test.pattern'
       src:
-        kind: Certificate
+        schema: another-service/SourceType/v1
-        name: some-certificate-asdf-1234
+        name: name-of-source-document
-        path: .cert
+        path: .source.path
-    - dest:
-        path: .tls_endpoint.key
-      src:
-        kind: CertificateKey
-        name: some-certificate-asdf-1234
-        path: .key
 data:
-  tls_endpoint:
+  path:
-    certificate: '.cert'
+    to:
-    key: deckhand/v1:some-certificate-asdf-1234
+      merge:
+        into:
+          parent:
+            foo: bar
+          ignored:  # Will not be part of the resultant document after layering.
+            data: here
+  substitution:
+    target: null  # Paths do not need to exist to be specified as substitution destinations.

deckhand/tests/unit/views/test_views.py

@@ -0,0 +1,78 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from deckhand.control.views import revision
+from deckhand.tests.unit.db import base
+from deckhand.tests import test_utils
+
+
+class TestRevisionViews(base.TestDbBase):
+
+    def setUp(self):
+        super(TestRevisionViews, self).setUp()
+        self.view_builder = revision.ViewBuilder()
+
+    def test_list_revisions(self):
+        payload = [base.DocumentFixture.get_minimal_fixture()
+                   for _ in range(4)]
+        self._create_documents(payload)
+        revisions = self._list_revisions()
+        revisions_view = self.view_builder.list(revisions)
+
+        expected_attrs = ('next', 'prev', 'results', 'count')
+        for attr in expected_attrs:
+            self.assertIn(attr, revisions_view)
+        # Validate that only 1 revision was returned.
+        self.assertEqual(1, revisions_view['count'])
+        # Validate that the first revision has 4 documents.
+        self.assertIn('id', revisions_view['results'][0])
+        self.assertIn('count', revisions_view['results'][0])
+        self.assertEqual(4, revisions_view['results'][0]['count'])
+
+    def test_list_many_revisions(self):
+        docs_count = []
+        for _ in range(3):
+            doc_count = test_utils.rand_int(3, 9)
+            docs_count.append(doc_count)
+
+            payload = [base.DocumentFixture.get_minimal_fixture()
+                       for _ in range(doc_count)]
+            self._create_documents(payload)
+            revisions = self._list_revisions()
+        revisions_view = self.view_builder.list(revisions)
+
+        expected_attrs = ('next', 'prev', 'results', 'count')
+        for attr in expected_attrs:
+            self.assertIn(attr, revisions_view)
+        # Validate that only 1 revision was returned.
+        self.assertEqual(3, revisions_view['count'])
+
+        # Validate that each revision has correct number of documents.
+        for idx, doc_count in enumerate(docs_count):
+            self.assertIn('count', revisions_view['results'][idx])
+            self.assertIn('id', revisions_view['results'][idx])
+            self.assertEqual(doc_count, revisions_view['results'][idx][
+                'count'])
+
+    def test_show_revision(self):
+        payload = [base.DocumentFixture.get_minimal_fixture()
+                   for _ in range(4)]
+        documents = self._create_documents(payload)
+        revision = self._get_revision(documents[0]['revision_id'])
+        revision_view = self.view_builder.show(revision)
+
+        expected_attrs = ('id', 'url', 'createdAt', 'validationPolicies')
+        for attr in expected_attrs:
+            self.assertIn(attr, revision_view)
+        self.assertIsInstance(revision_view['validationPolicies'], list)

deckhand/utils.py

@@ -0,0 +1,47 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def multi_getattr(multi_key, dict_data):
+    """Iteratively check for nested attributes in the YAML data.
+
+    Check for nested attributes included in "dest" attributes in the data
+    section of the YAML file. For example, a "dest" attribute of
+    ".foo.bar.baz" should mean that the YAML data adheres to:
+
+    .. code-block:: yaml
+
+       ---
+       foo:
+           bar:
+               baz: <data_to_be_substituted_here>
+
+    :param multi_key: A multi-part key that references nested data in the
+        substitutable part of the YAML data, e.g. ".foo.bar.baz".
+    :param substitutable_data: The section of data in the YAML data that
+        is intended to be substituted with secrets.
+    :returns: nested entry in ``dict_data`` if present; else None.
+    """
+    attrs = multi_key.split('.')
+    # Ignore the first attribute if it is "." as that is a self-reference.
+    if attrs[0] == '':
+        attrs = attrs[1:]
+
+    data = dict_data
+    for attr in attrs:
+        if attr not in data:
+            return None
+        data = data.get(attr)
+
+    return data