airship/deckhand
Code Issues Proposed changes

Remove oslo_versionedobjects

Browse Source 
This commit also updates documentation for launching test app.
This commit is contained in:
Felipe Monteiro 2017-07-21 04:22:09 +01:00
parent 8476370a80
commit bbd0292455
10 changed files with 50 additions and 301 deletions

10
ChangeLog

@ -1,6 +1,16 @@
CHANGES
=======
* Change application/yaml to application/x-yaml
* Cleaned up some logic, added exception handling to document creation
* Add currently necessary oslo namespaces to oslo-config-generator conf file
* Successfully creating document
* Added logic for establishing DB connection
* Refactor database sqlalchemy api/models
* Added oslo\_context-based context for oslo\_db compatibility
* Update database documents schema
* Helper for generating versioned object automatically from dictionary payload
* Update README
* Temporary change - do not commit
* Initial DB API models implementation
* Added control (API) readme

29
README.rst

@ -3,11 +3,36 @@ Deckhand
A foundational python REST YAML processing engine providing data and secrets
management to other platform services.
To run::
To generate a configuration file automatically::
$ tox -e genconfig
Resulting deckhand.conf.sample file is output to
:path:etc/deckhand/deckhand.conf.sample
Copy the config file to a directory discoverably by ``oslo.conf``::
$ cp etc/deckhand/deckhand.conf.sample ~/deckhand.conf
To setup an in-memory database for testing:
.. code-block:: ini
[database]
#
# From oslo.db
#
# The SQLAlchemy connection string to use to connect to the database.
# (string value)
connection = sqlite:///:memory:
To run locally in a development environment::
$ sudo pip install uwsgi
$ virtualenv -p python3 /var/tmp/deckhand
$ . /var/tmp/deckhand/bin/activate
$ sudo pip install .
$ python setup.py install
$ sudo python setup.py install
$ uwsgi --http :9000 -w deckhand.deckhand --callable deckhand_callable --enable-threads -L

2
deckhand/control/README.rst

@ -18,7 +18,7 @@ revision number will be returned.
Testing
-------
Document creation can be tested locally using:
Document creation can be tested locally using (from root deckhand directory):
.. code-block:: console

17
deckhand/control/documents.py

@ -12,16 +12,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import yaml
import falcon
from oslo_db import exception as db_exc
from oslo_log import log as logging
from deckhand.control import base as api_base
from deckhand.db.sqlalchemy import api as db_api
from deckhand.engine import document_validation
from deckhand import errors as deckhand_errors
from deckhand.objects import documents
LOG = logging.getLogger(__name__)
@ -61,17 +63,12 @@ class DocumentsResource(api_base.BaseResource):
return self.return_error(resp, falcon.HTTP_400, message=e)
try:
LOG.debug('Calling Document.create()')
documents.Document().create(document)
db_api.document_create(document)
except db_exc.DBDuplicateEntry as e:
return self.return_error(resp, falcon.HTTP_409, message=e)
except Exception as e:
LOG.exception(e)
raise
return self.return_error(resp, falcon.HTTP_500, message=e)
# Check if a document with the specified name already exists. If so,
# treat this request as an update.
doc_name = doc_validation.doc_name
resp.data = doc_name
resp.status = falcon.HTTP_201
def _check_document_exists(self):

5
deckhand/db/sqlalchemy/api.py

@ -95,9 +95,12 @@ def clear_db_env():
_FACADE = None
def document_create(context, values, session=None):
def document_create(values, session=None):
"""Create a document."""
values = values.copy()
values['doc_metadata'] = values.pop('metadata')
values['schema_version'] = values.pop('schemaVersion')
session = session or get_session()
with session.begin():
document = models.Document()

1
deckhand/db/sqlalchemy/models.py

@ -106,6 +106,7 @@ class Document(BASE, DeckhandBase):
id = Column(String(36), primary_key=True,
default=lambda: str(uuid.uuid4()))
# TODO: the revision_index will be a foreign key to a Revision table.
revision_index = Column(String(36), nullable=False,
default=lambda: str(uuid.uuid4()))
schema_version = Column(String(64), nullable=False)

0
deckhand/objects/__init__.py

165
deckhand/objects/base.py

@ -1,165 +0,0 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
from oslo_log import log as logging
from oslo_versionedobjects import base
from oslo_versionedobjects import exception as ovo_exception
from oslo_versionedobjects import fields as ovo_fields
from deckhand import objects
LOG = logging.getLogger(__name__)
class DeckhandObjectRegistry(base.VersionedObjectRegistry):
# Steal this from Cinder to bring all registered objects
# into the Deckhand_provisioner.objects namespace.
def registration_hook(self, cls, index):
setattr(objects, cls.obj_name(), cls)
class DeckhandObject(base.VersionedObject):
# Version 1.0: Initial version
VERSION = '1.0'
OBJ_PROJECT_NAMESPACE = 'deckhand.objects'
def obj_load_attr(self, attrname):
if attrname in self.fields.keys():
setattr(self, attrname, None)
else:
raise ValueError("Unknown field %s." % (attrname))
def obj_to_simple(self):
"""
Create a simple primitive representation of this object excluding
all the versioning stuff. Used to serialize an object for public
consumption, not intended to be deserialized by OVO.
"""
primitive = dict()
primitive['model_type'] = self.__class__.__name__
primitive['model_version'] = self.VERSION
for name, field in self.fields.items():
if self.obj_attr_is_set(name):
value = getattr(self, name)
if (hasattr(value, 'obj_to_simple') and
callable(value.obj_to_simple)):
primitive[name] = value.obj_to_simple()
else:
value = field.to_primitive(self, name, value)
if value is not None:
primitive[name] = value
return primitive
class DeckhandPersistentObject(base.VersionedObject):
fields = {
'created_at': ovo_fields.DateTimeField(nullable=False),
'created_by': ovo_fields.StringField(nullable=False),
'updated_at': ovo_fields.DateTimeField(nullable=True),
'updated_by': ovo_fields.StringField(nullable=True),
}
def set_create_fields(self, context):
self.created_at = datetime.datetime.now()
self.created_by = context.user
def set_update_fields(self, context):
self.updated_at = datetime.datetime.now()
self.updated_by = context.user
class DeckhandPayloadBase(DeckhandPersistentObject):
"""Base class for the payload of versioned notifications."""
# SCHEMA defines how to populate the payload fields. It is a dictionary
# where every key value pair has the following format:
# <payload_field_name>: (<data_source_name>,
# <field_of_the_data_source>)
# The <payload_field_name> is the name where the data will be stored in the
# payload object, this field has to be defined as a field of the payload.
# The <data_source_name> shall refer to name of the parameter passed as
# kwarg to the payload's populate_schema() call and this object will be
# used as the source of the data. The <field_of_the_data_source> shall be
# a valid field of the passed argument.
# The SCHEMA needs to be applied with the populate_schema() call before the
# notification can be emitted.
# The value of the payload.<payload_field_name> field will be set by the
# <data_source_name>.<field_of_the_data_source> field. The
# <data_source_name> will not be part of the payload object internal or
# external representation.
# Payload fields that are not set by the SCHEMA can be filled in the same
# way as in any versioned object.
SCHEMA = {}
# Version 1.0: Initial version
VERSION = '1.0'
def __init__(self):
super(DeckhandPayloadBase, self).__init__()
self.populated = not self.SCHEMA
def populate_schema(self, **kwargs):
"""Populate the object based on the SCHEMA and the source objects
:param kwargs: A dict contains the source object at the key defined in
the SCHEMA
"""
for key, (obj, field) in self.SCHEMA.items():
source = kwargs[obj]
if isinstance(source, dict):
source = self._dict_to_obj(source)
try:
setattr(self, key, getattr(source, field))
# ObjectActionError - not lazy loadable field
# NotImplementedError - obj_load_attr() is not even defined
# OrphanedObjectError - lazy loadable field but context is None
except (#exception.ObjectActionError,
NotImplementedError,
#exception.OrphanedObjectError,
ovo_exception.OrphanedObjectError) as e:
LOG.debug(("Defaulting the value of the field '%(field)s' "
"to None in %(payload)s due to '%(exception)s'"),
{'field': key,
'payload': self.__class__.__name__,
'exception': e})
# NOTE: This will fail if the payload field is not
# nullable, but that means that either the source object is not
# properly initialized or the payload field needs to be defined
# as nullable
setattr(self, key, None)
self.populated = True
# the schema population will create changed fields but we don't need
# this information in the notification
self.obj_reset_changes(recursive=True)
def _dict_to_obj(self, d):
"""Converts dictionary to object.
:param d: The dictionary to convert into an object.
:returns: The object representation of the dictionary passed in.
"""
class Object:
def __init__(self, **entries):
self.__dict__.update(entries)
return Object(**dict(d.items()))

89
deckhand/objects/documents.py

@ -1,89 +0,0 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Models for deckhand
#
from oslo_db import exception as db_exc
from oslo_log import log as logging
from oslo_versionedobjects import fields as ovo_fields
from deckhand.db.sqlalchemy import api as db_api
from deckhand import errors
from deckhand import objects
from deckhand.objects import base
from deckhand.objects import fields as deckhand_fields
LOG = logging.getLogger(__name__)
class DocumentPayload(base.DeckhandPayloadBase):
SCHEMA = {
'schema_version': ('document', 'schemaVersion'),
'kind': ('document', 'kind'),
'metadata': ('document', 'metadata'),
'data': ('document', 'data')
}
# Version 1.0: Initial version
VERSION = '1.0'
fields = {
'schema_version': ovo_fields.StringField(nullable=False),
'kind': ovo_fields.StringField(nullable=False),
'metadata': ovo_fields.DictOfStringsField(nullable=False),
'data': ovo_fields.DictOfStringsField(nullable=False)
}
def __init__(self, document):
super(DocumentPayload, self).__init__()
self.populate_schema(document=document)
@base.DeckhandObjectRegistry.register
class Document(base.DeckhandPersistentObject, base.DeckhandObject):
# Version 1.0: Initial version
VERSION = '1.0'
fields = {
'id': ovo_fields.IntegerField(nullable=False, read_only=True),
'document': ovo_fields.ObjectField('DocumentPayload', nullable=False),
'revision_index': ovo_fields.NonNegativeIntegerField(nullable=False)
}
def __init__(self, *args, **kwargs):
super(Document, self).__init__(*args, **kwargs)
# Set up defaults.
self.obj_reset_changes()
def create(self, document):
document_obj = DocumentPayload(document)
values = {
'schema_version': document_obj.schema_version,
'kind': document_obj.kind,
'doc_metadata': document_obj.metadata,
'data': document_obj.data
}
try:
db_api.document_create(None, values)
except db_exc.DBDuplicateEntry as e:
raise errors.DocumentExists(
kind=values['kind'], schema_version=values['schema_version'])
except Exception as e:
raise db_exc.DBError(e)

33
deckhand/objects/fields.py

@ -1,33 +0,0 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_versionedobjects import fields
class BaseDeckhandEnum(fields.Enum):
def __init__(self):
super(BaseDeckhandEnum, self).__init__(valid_values=self.__class__.ALL)
class DocumentField(BaseDeckhandEnum):
# A list of potentially valid statuses for a Document to have. This list
# will evolve over time.
active = 'ACTIVE'
conflict_error = 'CONFLICT_ERROR'
merge_error = 'MERGE_ERROR'
substitute_error = 'SUBSTITUTE_ERROR'
warning = 'WARNING'
ALL = (active, conflict_error, merge_error, substitute_error, warning)