Update schema/db model/db api to align with design document.

This commit is contained in:
Felipe Monteiro 2017-07-24 16:47:58 +01:00
parent dbc80fbfae
commit 7f6788db89
9 changed files with 79 additions and 344 deletions

View File

@ -117,8 +117,9 @@ def documents_create(documents, session=None):
def document_create(values, session=None):
"""Create a document."""
values = values.copy()
values['doc_metadata'] = values.pop('metadata')
values['schema_version'] = values.pop('schemaVersion')
values['_metadata'] = values.pop('metadata')
print(values)
values['name'] = values['_metadata']['name']
session = session or get_session()
filters = models.Document.UNIQUE_CONSTRAINTS

View File

@ -133,18 +133,18 @@ class Revision(BASE, DeckhandBase):
class Document(BASE, DeckhandBase):
UNIQUE_CONSTRAINTS = ('schema_version', 'kind', 'revision_id')
UNIQUE_CONSTRAINTS = ('schema', 'name', 'revision_id')
__tablename__ = 'documents'
__table_args__ = (DeckhandBase.gen_unqiue_contraint(*UNIQUE_CONSTRAINTS),)
id = Column(String(36), primary_key=True,
default=lambda: str(uuid.uuid4()))
schema_version = Column(String(64), nullable=False)
kind = Column(String(64), nullable=False)
schema = Column(String(64), nullable=False)
name = Column(String(64), nullable=False)
# NOTE: Do not define a maximum length for these JSON data below. However,
# this approach is not compatible with all database types.
# "metadata" is reserved, so use "doc_metadata" instead.
doc_metadata = Column(JSONEncodedDict(), nullable=False)
_metadata = Column(JSONEncodedDict(), nullable=False)
data = Column(JSONEncodedDict(), nullable=False)
revision_id = Column(Integer, ForeignKey('revisions.id'), nullable=False)

View File

@ -44,9 +44,8 @@ class DocumentValidation(object):
schema_versions_info = [{'version': 'v1', 'kind': 'default',
'schema': default_schema}]
def __init__(self, schema_version, kind):
def __init__(self, schema_version):
self.schema_version = schema_version
self.kind = kind
@property
def schema(self):
@ -60,21 +59,15 @@ class DocumentValidation(object):
# TODO(fm577c): Query Deckhand API to validate "src" values.
@property
def doc_name(self):
return (self.data['schemaVersion'] + self.data['kind'] +
self.data['metadata']['name'])
def _validate_with_schema(self):
# Validate the document using the schema defined by the document's
# `schemaVersion` and `kind`.
try:
schema_version = self.data['schemaVersion'].split('/')[-1]
doc_kind = self.data['kind']
doc_schema_version = self.SchemaVersion(schema_version, doc_kind)
schema_version = self.data['schema'].split('/')[-1]
doc_schema_version = self.SchemaVersion(schema_version)
except (AttributeError, IndexError, KeyError) as e:
raise errors.InvalidFormat(
'The provided schemaVersion is invalid or missing. Exception: '
'The provided schema is invalid or missing. Exception: '
'%s.' % e)
try:
jsonschema.validate(self.data, doc_schema_version.schema)

View File

@ -18,8 +18,7 @@ substitution_schema = {
'dest': {
'type': 'object',
'properties': {
'path': {'type': 'string'},
'replacePattern': {'type': 'string'}
'path': {'type': 'string'}
},
'additionalProperties': False,
# 'replacePattern' is not required.
@ -28,12 +27,12 @@ substitution_schema = {
'src': {
'type': 'object',
'properties': {
'kind': {'type': 'string'},
'schema': {'type': 'string'},
'name': {'type': 'string'},
'path': {'type': 'string'}
},
'additionalProperties': False,
'required': ['kind', 'name', 'path']
'required': ['schema', 'name', 'path']
}
},
'additionalProperties': False,
@ -43,45 +42,46 @@ substitution_schema = {
schema = {
'type': 'object',
'properties': {
'schemaVersion': {
'schema': {
'type': 'string',
'pattern': '^([A-Za-z]+\/v[0-9]{1})$'
'pattern': '^(.*\/v[0-9]{1})$'
},
# TODO: The kind should be an enum.
'kind': {'type': 'string'},
'metadata': {
'type': 'object',
'properties': {
'metadataVersion': {
'schema': {
'type': 'string',
'pattern': '^([A-Za-z]+\/v[0-9]{1})$'
'pattern': '^(.*/v[0-9]{1})$'
},
'name': {'type': 'string'},
'storagePolicy': {'type': 'string'},
'labels': {
'type': 'object',
'properties': {
'component': {'type': 'string'},
'hostname': {'type': 'string'}
},
'additionalProperties': False,
'required': ['component', 'hostname']
'type': 'object'
},
'layerDefinition': {
'layeringDefinition': {
'type': 'object',
'properties': {
'layer': {'enum': ['global', 'region', 'site']},
'layer': {'type': 'string'},
'abstract': {'type': 'boolean'},
'childSelector': {
'type': 'object',
'properties': {
'label': {'type': 'string'}
},
'additionalProperties': False,
'required': ['label']
'parentSelector': {
'type': 'object'
},
'actions': {
'type': 'array',
'items': {
'type': 'object',
'properties': {
'method': {'enum': ['merge', 'delete',
'replace']},
'path': {'type': 'string'}
},
'additionalProperties': False,
'required': ['method', 'path']
}
}
},
'additionalProperties': False,
'required': ['layer', 'abstract', 'childSelector']
'required': ['layer', 'abstract', 'parentSelector']
},
'substitutions': {
'type': 'array',
@ -89,13 +89,13 @@ schema = {
}
},
'additionalProperties': False,
'required': ['metadataVersion', 'name', 'labels',
'layerDefinition', 'substitutions']
'required': ['schema', 'name', 'storagePolicy', 'labels',
'layeringDefinition', 'substitutions']
},
'data': {
'type': 'object'
}
},
'additionalProperties': False,
'required': ['schemaVersion', 'kind', 'metadata', 'data']
'required': ['schema', 'metadata', 'data']
}

View File

@ -1,138 +0,0 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import yaml
import jsonschema
from deckhand.engine.schema.v1_0 import default_schema
from deckhand import errors
class SecretSubstitution(object):
"""Class for secret substitution logic for YAML files.
This class is responsible for parsing, validating and retrieving secret
values for values stored in the YAML file. Afterward, secret values will be
substituted or "forward-repalced" into the YAML file. The end result is a
YAML file containing all necessary secrets to be handed off to other
services.
:param data: YAML data that requires secrets to be validated, merged and
consolidated.
"""
def __init__(self, data):
try:
self.data = yaml.safe_load(data)
except yaml.YAMLError:
raise errors.InvalidFormat(
'The provided YAML file cannot be parsed.')
self.pre_validate_data()
class SchemaVersion(object):
"""Class for retrieving correct schema for pre-validation on YAML.
Retrieves the schema that corresponds to "apiVersion" in the YAML
data. This schema is responsible for performing pre-validation on
YAML data.
"""
# TODO: Update kind according to requirements.
schema_versions_info = [{'version': 'v1', 'kind': 'default',
'schema': default_schema}]
def __init__(self, schema_version, kind):
self.schema_version = schema_version
self.kind = kind
@property
def schema(self):
# TODO: return schema based on version and kind.
return [v['schema'] for v in self.schema_versions_info
if v['version'] == self.schema_version][0].schema
def pre_validate_data(self):
"""Pre-validate that the YAML file is correctly formatted."""
self._validate_with_schema()
# Validate that each "dest" field exists in the YAML data.
# FIXME(fm577c): Dest fields will be injected if not present - the
# validation below needs to be updated or removed.
substitutions = self.data['metadata']['substitutions']
destinations = [s['dest'] for s in substitutions]
sub_data = self.data['data']
for dest in destinations:
result, missing_attr = self._multi_getattr(dest['path'], sub_data)
if not result:
raise errors.InvalidFormat(
'The attribute "%s" included in the "dest" field "%s" is '
'missing from the YAML data: "%s".' % (
missing_attr, dest, sub_data))
# TODO(fm577c): Query Deckhand API to validate "src" values.
def _validate_with_schema(self):
# Validate the document using the schema defined by the document's
# `schemaVersion` and `kind`.
try:
schema_version = self.data['schemaVersion'].split('/')[-1]
doc_kind = self.data['kind']
doc_schema_version = self.SchemaVersion(schema_version, doc_kind)
except (AttributeError, IndexError, KeyError) as e:
raise errors.InvalidFormat(
'The provided schemaVersion is invalid or missing. Exception: '
'%s.' % e)
try:
jsonschema.validate(self.data, doc_schema_version.schema)
except jsonschema.exceptions.ValidationError as e:
raise errors.InvalidFormat('The provided YAML file is invalid. '
'Exception: %s.' % e.message)
def _multi_getattr(self, multi_key, substitutable_data):
"""Iteratively check for nested attributes in the YAML data.
Check for nested attributes included in "dest" attributes in the data
section of the YAML file. For example, a "dest" attribute of
".foo.bar.baz" should mean that the YAML data adheres to:
.. code-block:: yaml
---
foo:
bar:
baz: <data_to_be_substituted_here>
:param multi_key: A multi-part key that references nested data in the
substitutable part of the YAML data, e.g. ".foo.bar.baz".
:param substitutable_data: The section of data in the YAML data that
is intended to be substituted with secrets.
:returns: Tuple where first value is a boolean indicating that the
nested attribute was found and the second value is the attribute
that was not found, if applicable.
"""
attrs = multi_key.split('.')
# Ignore the first attribute if it is "." as that is a self-reference.
if attrs[0] == '':
attrs = attrs[1:]
data = substitutable_data
for attr in attrs:
if attr not in data:
return False, attr
data = data.get(attr)
return True, None

View File

@ -26,9 +26,8 @@ class DocumentFixture(object):
def get_minimal_fixture(self, **kwargs):
fixture = {'data': 'fake document data',
'metadata': 'fake metadata',
'kind': 'FakeConfigType',
'schemaVersion': 'deckhand/v1'}
'metadata': {'name': 'fake metadata'},
'schema': 'deckhand/v1'}
fixture.update(kwargs)
return fixture
@ -36,8 +35,7 @@ class DocumentFixture(object):
class TestDocumentsApi(base.DeckhandWithDBTestCase):
def _validate_document(self, expected, actual):
expected['doc_metadata'] = expected.pop('metadata')
expected['schema_version'] = expected.pop('schemaVersion')
expected['_metadata'] = expected.pop('metadata')
# TODO: Validate "status" fields, like created_at.
self.assertIsInstance(actual, dict)
@ -74,13 +72,13 @@ class TestDocumentsApi(base.DeckhandWithDBTestCase):
fixture = DocumentFixture().get_minimal_fixture()
child_document = db_api.document_create(fixture)
fixture['metadata'] = 'modified fake metadata'
fixture['metadata'] = {'name': 'modified fake metadata'}
parent_document = db_api.document_create(fixture)
self._validate_document(fixture, parent_document)
# Validate that the new document was created.
self.assertEqual('modified fake metadata',
parent_document['doc_metadata'])
self.assertEqual({'name': 'modified fake metadata'},
parent_document['_metadata'])
self.assertNotEqual(child_document['id'], parent_document['id'])
# Validate that the parent document has a different revision and

View File

@ -81,8 +81,7 @@ class TestDocumentValidation(testtools.TestCase):
invalid_data = [
(self._corrupt_data('data'), 'data'),
(self._corrupt_data('metadata'), 'metadata'),
(self._corrupt_data('metadata.metadataVersion'),
'metadataVersion'),
(self._corrupt_data('metadata.schema'), 'schema'),
(self._corrupt_data('metadata.name'), 'name'),
(self._corrupt_data('metadata.substitutions'), 'substitutions'),
(self._corrupt_data('metadata.substitutions.0.dest'), 'dest'),

View File

@ -1,123 +0,0 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import os
import testtools
import yaml
import six
from deckhand.engine import secret_substitution
from deckhand import errors
class TestSecretSubtitution(testtools.TestCase):
def setUp(self):
super(TestSecretSubtitution, self).setUp()
dir_path = os.path.dirname(os.path.realpath(__file__))
test_yaml_path = os.path.abspath(os.path.join(
dir_path, os.pardir, 'resources', 'sample.yaml'))
with open(test_yaml_path, 'r') as yaml_file:
yaml_data = yaml_file.read()
self.data = yaml.safe_load(yaml_data)
def _corrupt_data(self, key, data=None):
"""Corrupt test data to check that pre-validation works.
Corrupt data by removing a key from the document. Each key must
correspond to a value that is a dictionary.
:param key: The document key to be removed. The key can have the
following formats:
* 'data' => document.pop('data')
* 'metadata.name' => document['metadata'].pop('name')
* 'metadata.substitutions.0.dest' =>
document['metadata']['substitutions'][0].pop('dest')
:returns: Corrupted YAML data.
"""
if data is None:
data = self.data
corrupted_data = copy.deepcopy(data)
if '.' in key:
_corrupted_data = corrupted_data
nested_keys = key.split('.')
for nested_key in nested_keys:
if nested_key == nested_keys[-1]:
break
if nested_key.isdigit():
_corrupted_data = _corrupted_data[int(nested_key)]
else:
_corrupted_data = _corrupted_data[nested_key]
_corrupted_data.pop(nested_keys[-1])
else:
corrupted_data.pop(key)
return self._format_data(corrupted_data)
def _format_data(self, data=None):
"""Re-formats dict data as YAML to pass to ``SecretSubstitution``."""
if data is None:
data = self.data
return yaml.safe_dump(data)
def test_initialization(self):
sub = secret_substitution.SecretSubstitution(self._format_data())
self.assertIsInstance(sub, secret_substitution.SecretSubstitution)
def test_initialization_missing_sections(self):
expected_err = ("The provided YAML file is invalid. Exception: '%s' "
"is a required property.")
invalid_data = [
(self._corrupt_data('data'), 'data'),
(self._corrupt_data('metadata'), 'metadata'),
(self._corrupt_data('metadata.metadataVersion'), 'metadataVersion'),
(self._corrupt_data('metadata.name'), 'name'),
(self._corrupt_data('metadata.substitutions'), 'substitutions'),
(self._corrupt_data('metadata.substitutions.0.dest'), 'dest'),
(self._corrupt_data('metadata.substitutions.0.src'), 'src')
]
for invalid_entry, missing_key in invalid_data:
with six.assertRaisesRegex(self, errors.InvalidFormat,
expected_err % missing_key):
secret_substitution.SecretSubstitution(invalid_entry)
def test_initialization_bad_substitutions(self):
expected_err = ('The attribute "%s" included in the "dest" field "%s" '
'is missing from the YAML data')
invalid_data = []
data = copy.deepcopy(self.data)
data['metadata']['substitutions'][0]['dest'] = {'path': 'foo'}
invalid_data.append(self._format_data(data))
data = copy.deepcopy(self.data)
data['metadata']['substitutions'][0]['dest'] = {
'path': 'tls_endpoint.bar'}
invalid_data.append(self._format_data(data))
def _test(invalid_entry, field, dest):
_expected_err = expected_err % (field, dest)
with six.assertRaisesRegex(self, errors.InvalidFormat,
_expected_err):
secret_substitution.SecretSubstitution(invalid_entry)
# Verify that invalid body dest reference is invalid.
_test(invalid_data[0], "foo", {'path': 'foo'})
# Verify that nested invalid body dest reference is invalid.
_test(invalid_data[1], "bar", {'path': 'tls_endpoint.bar'})

View File

@ -1,33 +1,38 @@
# Sample YAML file for testing forward replacement.
---
schemaVersion: promenade/v1
kind: SomeConfigType
schema: some-service/ResourceType/v1
metadata:
metadataVersion: deckhand/v1
name: a-unique-config-name-12345
schema: metadata/Document/v1
name: unique-name-given-schema
storagePolicy: cleartext
labels:
component: apiserver
hostname: server0
layerDefinition:
layer: global
abstract: True
childSelector:
label: value
genesis: enabled
master: enabled
layeringDefinition:
abstract: true
layer: region
parentSelector:
required_key_a: required_label_a
required_key_b: required_label_b
actions:
- method: merge
path: .path.to.merge.into.parent
- method: delete
path: .path.to.delete
substitutions:
- dest:
path: .tls_endpoint.certificate
replacePattern: 'test.pattern'
path: .substitution.target
src:
kind: Certificate
name: some-certificate-asdf-1234
path: .cert
- dest:
path: .tls_endpoint.key
src:
kind: CertificateKey
name: some-certificate-asdf-1234
path: .key
schema: another-service/SourceType/v1
name: name-of-source-document
path: .source.path
data:
tls_endpoint:
certificate: '.cert'
key: deckhand/v1:some-certificate-asdf-1234
path:
to:
merge:
into:
parent:
foo: bar
ignored: # Will not be part of the resultant document after layering.
data: here
substitution:
target: null # Paths do not need to exist to be specified as substitution destinations.