airship
/
deckhand
Code Issues Proposed changes

[WIP] Implement documents API 

This commit adds the documents API and adds logic for performing
pre-validation schema checking wherever applicable in the
documents API.

The following endpoints in the documents API have been implemented:
  - POST /documents
This commit is contained in:
Felipe Monteiro 2017-07-13 21:11:49 +01:00
parent 1b31514611
commit 6b88c2b747
8 changed files with 104 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
deckhand/control/api.py
View File

@ -20,6 +20,7 @@ from oslo_log import log as logging
from deckhand.conf import config
from deckhand.control import base as api_base
from deckhand.control import documents
from deckhand.control import secrets
CONF = cfg.CONF
@ -58,6 +59,7 @@ def start_api(state_manager=None):
control_api = falcon.API(request_type=api_base.DeckhandRequest)
v1_0_routes = [
('documents', documents.DocumentsResource()),
('secrets', secrets.SecretsResource())
]

2
deckhand/control/base.py
View File

@ -68,7 +68,7 @@ class BaseResource(object):
raise errors.InvalidFormat("%s: Invalid JSON in body: %s" % (
req.path, jex))
else:
raise errors.InvalidFormat("Requires application/json payload")
raise errors.InvalidFormat("Requires application/json payload.")
def return_error(self, resp, status_code, message="", retry=False):
resp.body = json.dumps(

70
deckhand/control/documents.py Normal file
View File

@ -0,0 +1,70 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import yaml
import falcon
from oslo_log import log as logging
from deckhand.control import base as api_base
from deckhand.engine import document_validation
from deckhand import errors as deckhand_errors
LOG = logging.getLogger(__name__)
class DocumentsResource(api_base.BaseResource):
"""API resource for realizing CRUD endpoints for Documents."""
def __init__(self, **kwargs):
super(DocumentsResource, self).__init__(**kwargs)
self.authorized_roles = ['user']
def on_get(self, req, resp):
pass
def on_head(self, req, resp):
pass
def on_post(self, req, resp):
"""Create a document. Accepts YAML data only."""
if req.content_type != 'application/yaml':
LOG.warning('Requires application/yaml payload.')
document_data = req.stream.read(req.content_length or 0)
try:
document = yaml.safe_load(document_data)
except yaml.YAMLError as e:
error_msg = ("Could not parse the document into YAML data. "
"Details: %s." % e)
LOG.error(error_msg)
return self.return_error(resp, falcon.HTTP_400, message=error_msg)
# Validate the document before doing anything with it.
try:
doc_validation = document_validation.DocumentValidation(document)
except deckhand_errors.InvalidFormat as e:
return self.return_error(resp, falcon.HTTP_400, message=e)
# Check if a document with the specified name already exists. If so,
# treat this request as an update.
doc_name = doc_validation.doc_name
resp.data = doc_name
resp.status = falcon.HTTP_201
def _check_document_exists(self):
pass

4
deckhand/control/secrets.py
View File

@ -40,8 +40,8 @@ class SecretsResource(api_base.BaseResource):
For a list of types, please refer to the following API documentation:
https://docs.openstack.org/barbican/latest/api/reference/secret_types.html
"""
secret_name = req.params.get('name', None)
secret_type = req.params.get('type', None)
secret_name = req.params.get('name')
secret_type = req.params.get('type')
if not secret_name:
resp.status = falcon.HTTP_400

38
deckhand/engine/secret_substitution.py → deckhand/engine/document_validation.py
View File

@ -12,34 +12,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import yaml
import jsonschema
from deckhand.engine.schema.v1_0 import default_schema
from deckhand import errors
class SecretSubstitution(object):
"""Class for secret substitution logic for YAML files.
class DocumentValidation(object):
"""Class for document validation logic for YAML files.
This class is responsible for parsing, validating and retrieving secret
values for values stored in the YAML file. Afterward, secret values will be
substituted or "forward-repalced" into the YAML file. The end result is a
YAML file containing all necessary secrets to be handed off to other
services.
values for values stored in the YAML file.
:param data: YAML data that requires secrets to be validated, merged and
consolidated.
"""
def __init__(self, data):
try:
self.data = yaml.safe_load(data)
except yaml.YAMLError:
raise errors.InvalidFormat(
'The provided YAML file cannot be parsed.')
self.data = data
self.pre_validate_data()
class SchemaVersion(object):
@ -68,23 +58,13 @@ class SecretSubstitution(object):
"""Pre-validate that the YAML file is correctly formatted."""
self._validate_with_schema()
# Validate that each "dest" field exists in the YAML data.
# FIXME(fm577c): Dest fields will be injected if not present - the
# validation below needs to be updated or removed.
substitutions = self.data['metadata']['substitutions']
destinations = [s['dest'] for s in substitutions]
sub_data = self.data['data']
for dest in destinations:
result, missing_attr = self._multi_getattr(dest['path'], sub_data)
if not result:
raise errors.InvalidFormat(
'The attribute "%s" included in the "dest" field "%s" is '
'missing from the YAML data: "%s".' % (
missing_attr, dest, sub_data))
# TODO(fm577c): Query Deckhand API to validate "src" values.
@property
def doc_name(self):
return (self.data['schemaVersion'] + self.data['kind'] +
self.data['metadata']['name'])
def _validate_with_schema(self):
# Validate the document using the schema defined by the document's
# `schemaVersion` and `kind`.

10
deckhand/tests/unit/control/test_api.py
View File

@ -23,8 +23,9 @@ from deckhand.control import base as api_base
class TestApi(testtools.TestCase):
@mock.patch.object(api, 'secrets', autospec=True)
@mock.patch.object(api, 'documents', autospec=True)
@mock.patch.object(api, 'falcon', autospec=True)
def test_start_api(self, mock_falcon, mock_secrets):
def test_start_api(self, mock_falcon, mock_documents, mock_secrets):
mock_falcon_api = mock_falcon.API.return_value
result = api.start_api()
@ -32,5 +33,8 @@ class TestApi(testtools.TestCase):
mock_falcon.API.assert_called_once_with(
request_type=api_base.DeckhandRequest)
mock_falcon_api.add_route.assert_called_once_with(
'/api/v1.0/secrets', mock_secrets.SecretsResource())
mock_falcon_api.add_route.assert_has_calls([
mock.call(
'/api/v1.0/documents', mock_documents.DocumentsResource()),
mock.call('/api/v1.0/secrets', mock_secrets.SecretsResource())
])

52
deckhand/tests/unit/engine/test_secret_substitution.py → deckhand/tests/unit/engine/test_document_validation.py
View File

@ -19,14 +19,14 @@ import yaml
import six
from deckhand.engine import secret_substitution
from deckhand.engine import document_validation
from deckhand import errors
class TestSecretSubtitution(testtools.TestCase):
class TestDocumentValidation(testtools.TestCase):
def setUp(self):
super(TestSecretSubtitution, self).setUp()
super(TestDocumentValidation, self).setUp()
dir_path = os.path.dirname(os.path.realpath(__file__))
test_yaml_path = os.path.abspath(os.path.join(
dir_path, os.pardir, 'resources', 'sample.yaml'))
@ -47,7 +47,7 @@ class TestSecretSubtitution(testtools.TestCase):
* 'metadata.name' => document['metadata'].pop('name')
* 'metadata.substitutions.0.dest' =>
document['metadata']['substitutions'][0].pop('dest')
:returns: Corrupted YAML data.
:returns: Corrupted data.
"""
if data is None:
data = self.data
@ -67,17 +67,13 @@ class TestSecretSubtitution(testtools.TestCase):
else:
corrupted_data.pop(key)
return self._format_data(corrupted_data)
def _format_data(self, data=None):
"""Re-formats dict data as YAML to pass to ``SecretSubstitution``."""
if data is None:
data = self.data
return yaml.safe_dump(data)
return corrupted_data
def test_initialization(self):
sub = secret_substitution.SecretSubstitution(self._format_data())
self.assertIsInstance(sub, secret_substitution.SecretSubstitution)
doc_validation = document_validation.DocumentValidation(
self.data)
self.assertIsInstance(doc_validation,
document_validation.DocumentValidation)
def test_initialization_missing_sections(self):
expected_err = ("The provided YAML file is invalid. Exception: '%s' "
@ -85,7 +81,8 @@ class TestSecretSubtitution(testtools.TestCase):
invalid_data = [
(self._corrupt_data('data'), 'data'),
(self._corrupt_data('metadata'), 'metadata'),
(self._corrupt_data('metadata.metadataVersion'), 'metadataVersion'),
(self._corrupt_data('metadata.metadataVersion'),
'metadataVersion'),
(self._corrupt_data('metadata.name'), 'name'),
(self._corrupt_data('metadata.substitutions'), 'substitutions'),
(self._corrupt_data('metadata.substitutions.0.dest'), 'dest'),
@ -95,29 +92,4 @@ class TestSecretSubtitution(testtools.TestCase):
for invalid_entry, missing_key in invalid_data:
with six.assertRaisesRegex(self, errors.InvalidFormat,
expected_err % missing_key):
secret_substitution.SecretSubstitution(invalid_entry)
def test_initialization_bad_substitutions(self):
expected_err = ('The attribute "%s" included in the "dest" field "%s" '
'is missing from the YAML data')
invalid_data = []
data = copy.deepcopy(self.data)
data['metadata']['substitutions'][0]['dest'] = {'path': 'foo'}
invalid_data.append(self._format_data(data))
data = copy.deepcopy(self.data)
data['metadata']['substitutions'][0]['dest'] = {
'path': 'tls_endpoint.bar'}
invalid_data.append(self._format_data(data))
def _test(invalid_entry, field, dest):
_expected_err = expected_err % (field, dest)
with six.assertRaisesRegex(self, errors.InvalidFormat,
_expected_err):
secret_substitution.SecretSubstitution(invalid_entry)
# Verify that invalid body dest reference is invalid.
_test(invalid_data[0], "foo", {'path': 'foo'})
# Verify that nested invalid body dest reference is invalid.
_test(invalid_data[1], "bar", {'path': 'tls_endpoint.bar'})
document_validation.DocumentValidation(invalid_entry)

2
tox.ini
View File

@ -38,5 +38,5 @@ commands = flake8 {posargs}
[flake8]
# D100-104 deal with docstrings in public functions
# D205, D400, D401 deal with docstring formatting
ignore=E121,E122,E123,E124,E125,E126,E127,E128,E129,E131,E251,H405,D100,D101,D102,D103,D104,D205,D400,D401,I100
ignore=E121,E122,E123,E124,E125,E126,E127,E128,E129,E131,E251,H405,D100,D101,D102,D103,D104,D205,D400,D401,H101,I100
exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,tools/xenserver*,releasenotes