Initial engine framework

This commit adds the initial engine framework for Deckhand. Included
is the logic for parsing YAML files as well as validating them and
doing forward substitution as specified by the YAML file.

This commit also includes unit tests for the framework changes.
This commit is contained in:
Felipe Monteiro 2017-07-08 23:34:44 +01:00
parent eabd51de97
commit 695ef09f72
18 changed files with 414 additions and 18 deletions

1
.gitignore vendored
View File

@ -46,6 +46,7 @@ nosetests.xml
coverage.xml coverage.xml
*.cover *.cover
.hypothesis/ .hypothesis/
.testrepository/*
# Translations # Translations
*.mo *.mo

7
.testr.conf Normal file
View File

@ -0,0 +1,7 @@
[DEFAULT]
test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \
${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION
test_id_option=--load-list $IDFILE
test_list_option=--list

View File

@ -20,7 +20,6 @@ class BarbicanDriver(object):
def __init__(self): def __init__(self):
self.barbicanclient = client_wrapper.BarbicanClientWrapper() self.barbicanclient = client_wrapper.BarbicanClientWrapper()
def ca_list(self, **kwargs): def create_secret(self, **kwargs):
# FIXME(felipemonteiro): Testing cas.list endpoint. """Create a secret."""
ca_list = self.barbicanclient.call("cas.list", **kwargs) return self.barbicanclient.call("secrets.create", **kwargs)
return ca_list

View File

@ -58,7 +58,7 @@ def start_api(state_manager=None):
control_api = falcon.API(request_type=api_base.DeckhandRequest) control_api = falcon.API(request_type=api_base.DeckhandRequest)
v1_0_routes = [ v1_0_routes = [
('/secrets', secrets.SecretsResource()) ('secrets', secrets.SecretsResource())
] ]
for path, res in v1_0_routes: for path, res in v1_0_routes:

View File

@ -23,8 +23,7 @@ from deckhand.control import base as api_base
class SecretsResource(api_base.BaseResource): class SecretsResource(api_base.BaseResource):
"""API resource for interacting with Barbican. """API resource for interacting with Barbican.
TODO(felipemonteiro): Once Barbican integration is fully implemented, NOTE: Currently only supports Barbican.
implement API endpoints below.
""" """
def __init__(self, **kwargs): def __init__(self, **kwargs):
@ -32,8 +31,25 @@ class SecretsResource(api_base.BaseResource):
self.authorized_roles = ['user'] self.authorized_roles = ['user']
self.barbican_driver = driver.BarbicanDriver() self.barbican_driver = driver.BarbicanDriver()
def on_get(self, req, resp): def on_post(self, req, resp):
# TODO(felipemonteiro): Implement this API endpoint. """Create a secret.
ca_list = self.barbican_driver.ca_list() # Random endpoint to test.
resp.body = json.dumps({'secrets': [c.to_dict() for c in ca_list]}) :param name: The name of the secret. Required.
:param type: The type of the secret. Optional.
For a list of types, please refer to the following API documentation:
https://docs.openstack.org/barbican/latest/api/reference/secret_types.html
"""
secret_name = req.params.get('name', None)
secret_type = req.params.get('type', None)
if not secret_name:
resp.status = falcon.HTTP_400
# Do not allow users to call Barbican with all permitted kwargs.
# Selectively include only what we allow.
kwargs = {'name': secret_name, 'secret_type': secret_type}
secret = self.barbican_driver.create_secret(**kwargs)
resp.body = json.dumps(secret)
resp.status = falcon.HTTP_200 resp.status = falcon.HTTP_200

View File

View File

@ -0,0 +1,134 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import yaml
from deckhand import errors
class SecretSubstitution(object):
"""Initialization of class for secret substitution logic for YAML files.
This class is responsible for parsing, validating and retrieving secret
values for values stored in the YAML file. Afterward, secret values will be
substituted or "forward-repalced" into the YAML file. The end result is a
YAML file containing all necessary secrets to be handed off to other
services.
"""
def __init__(self, data):
try:
self.data = yaml.safe_load(data)
except yaml.YAMLError:
raise errors.InvalidFormat(
'The provided YAML file cannot be parsed.')
self.validate_data()
def validate_data(self):
"""Validate that the YAML file is correctly formatted.
The YAML file must adhere to the following bare minimum format:
.. code-block:: yaml
---
apiVersion: service/v1
kind: ConsumerOfCertificateData
metadata:
substitutions:
- dest: .tls_endpoint.certificate
src:
apiVersion: deckhand/v1
kind: Certificate
name: some-certificate-asdf-1234
# Forward-reference to specific section under "data" below.
- dest: .tls_endpoint.certificateKey
src:
apiVersion: deckhand/v1
kind: CertificateKey
name: some-certificate-key-asdf-1234
data:
tls_endpoint:
certificate: null # Data to be substituted.
certificateKey: null # Data to be substituted.
"""
# Validate that data section exists.
try:
self.data['data']
except (KeyError, TypeError) as e:
raise errors.InvalidFormat(
'The provided YAML file has no data section: %s' % e)
# Validate that substitutions section exists.
try:
substitutions = self.data['metadata']['substitutions']
except (KeyError, TypeError) as e:
raise errors.InvalidFormat(
'The provided YAML file has no metadata/substitutions '
'section: %s' % e)
# Validate that "src" and "dest" fields exist per substitution entry.
error_message = ('The provided YAML file is missing the "%s" field '
'for the %s substition.')
for s in substitutions:
if 'src' not in s:
raise errors.InvalidFormat(error_message % ('src', s))
elif 'dest' not in s:
raise errors.InvalidFormat(error_message % ('dest', s))
# Validate that each "dest" field exists in the YAML data.
destinations = [s['dest'] for s in substitutions]
sub_data = self.data['data']
for dest in destinations:
result, missing_attr = self._multi_getattr(dest, sub_data)
if not result:
raise errors.InvalidFormat(
'The attribute "%s" included in the "dest" field "%s" is '
'missing from the YAML data: "%s".' % (
missing_attr, dest, sub_data))
def _multi_getattr(self, multi_key, substitutable_data):
"""Iteratively check for nested attributes in the YAML data.
Check for nested attributes included in "dest" attributes in the data
section of the YAML file. For example, a "dest" attribute of
".foo.bar.baz" should mean that the YAML data adheres to:
.. code-block:: yaml
---
foo:
bar:
baz: <data_to_be_substituted_here>
:param multi_key: A multi-part key that references nested data in the
substitutable part of the YAML data, e.g. ".foo.bar.baz".
:param substitutable_data: The section of data in the YAML data that
is intended to be substituted with secrets.
:returns: Tuple where first value is a boolean indicating that the
nested attribute was found and the second value is the attribute
that was not found, if applicable.
"""
attrs = multi_key.split('.')
# Ignore the first attribute if it is "." as that is a self-reference.
if attrs[0] == '':
attrs = attrs[1:]
data = substitutable_data
for attr in attrs:
if attr not in data:
return False, attr
data = data.get(attr)
return True, None

View File

@ -18,4 +18,4 @@ class ApiError(Exception):
class InvalidFormat(ApiError): class InvalidFormat(ApiError):
pass """The YAML file is incorrectly formatted and cannot be read."""

View File

View File

View File

View File

@ -0,0 +1,36 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import testtools
from deckhand.control import api
from deckhand.control import base as api_base
class TestApi(testtools.TestCase):
@mock.patch.object(api, 'secrets', autospec=True)
@mock.patch.object(api, 'falcon', autospec=True)
def test_start_api(self, mock_falcon, mock_secrets):
mock_falcon_api = mock_falcon.API.return_value
result = api.start_api()
self.assertEqual(mock_falcon_api, result)
mock_falcon.API.assert_called_once_with(
request_type=api_base.DeckhandRequest)
mock_falcon_api.add_route.assert_called_once_with(
'/api/v1.0/secrets', mock_secrets.SecretsResource())

View File

@ -0,0 +1,40 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import testtools
from deckhand.control import base as api_base
class TestBaseResource(testtools.TestCase):
def setUp(self):
super(TestBaseResource, self).setUp()
self.base_resource = api_base.BaseResource()
def test_on_options(self):
# Override `dir` so that ``dir(self)`` returns `methods`.
expected_methods = ['on_get', 'on_heat', 'on_post', 'on_put',
'on_delete', 'on_patch']
api_base.BaseResource.__dir__ = lambda x: expected_methods
mock_resp = mock.Mock(headers={})
self.base_resource.on_options(None, mock_resp)
self.assertIn('Allow', mock_resp.headers)
self.assertEqual('GET,POST,PUT,DELETE,PATCH',
mock_resp.headers['Allow'])
self.assertEqual('200 OK', mock_resp.status)

View File

View File

@ -0,0 +1,112 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import testtools
from oslo_serialization import jsonutils as json
import six
from deckhand.engine import secret_substitution
from deckhand import errors
class TestSecretSubtitution(testtools.TestCase):
def setUp(self):
super(TestSecretSubtitution, self).setUp()
dir_path = os.path.dirname(os.path.realpath(__file__))
test_yaml_path = os.path.abspath(os.path.join(
dir_path, os.pardir, 'resources', 'sample.yaml'))
with open(test_yaml_path, 'r') as yaml_data:
self.yaml_data = yaml_data.read()
def test_initialization_missing_substitutions_section(self):
expected_err = (
"The provided YAML file has no metadata/substitutions section")
invalid_data = [
{"data": []},
{"data": [], "metadata": None},
{"data": [], "metadata": {"missing_substitutions": None}}
]
for invalid_entry in invalid_data:
invalid_entry = json.dumps(invalid_entry)
with six.assertRaisesRegex(self, errors.InvalidFormat,
expected_err):
secret_substitution.SecretSubstitution(invalid_entry)
expected_err = (
"The provided YAML file has no metadata/substitutions section")
invalid_data = [
{"data": [], "metadata": None},
]
def test_initialization_missing_data_section(self):
expected_err = (
"The provided YAML file has no data section")
invalid_data = '{"metadata": {"substitutions": []}}'
with six.assertRaisesRegex(self, errors.InvalidFormat, expected_err):
secret_substitution.SecretSubstitution(invalid_data)
def test_initialization_missing_src_dest_sections(self):
expected_err = ('The provided YAML file is missing the "%s" field for '
'the %s substition.')
invalid_data = [
{"data": [], "metadata": {"substitutions": [{"dest": "foo"}]}},
{"data": [], "metadata": {"substitutions": [{"src": "bar"}]}},
]
def _test(invalid_entry, field, substitution):
invalid_entry = json.dumps(invalid_entry)
_expected_err = expected_err % (field, substitution)
with six.assertRaisesRegex(self, errors.InvalidFormat,
_expected_err):
secret_substitution.SecretSubstitution(invalid_entry)
_test(invalid_data[0], "src", {"dest": "foo"})
_test(invalid_data[1], "dest", {"src": "bar"})
def test_initialization_bad_substitutions(self):
expected_err = ('The attribute "%s" included in the "dest" field "%s" '
'is missing from the YAML data: "%s".')
invalid_data = [
# Missing attribute.
{"data": {}, "metadata": {"substitutions": [
{"src": "", "dest": "foo"}
]}},
# Missing attribute.
{"data": {"foo": None}, "metadata": {"substitutions": [
{"src": "", "dest": "bar"}
]}},
# Missing nested attribute.
{"data": {"foo": {"baz": None}}, "metadata": {"substitutions": [
{"src": "", "dest": "foo.bar"}
]}},
]
def _test(invalid_entry, field, dest, substitution):
invalid_entry = json.dumps(invalid_entry)
_expected_err = expected_err % (field, dest, substitution)
with six.assertRaisesRegex(self, errors.InvalidFormat,
_expected_err):
secret_substitution.SecretSubstitution(invalid_entry)
_test(invalid_data[0], "foo", "foo", {})
_test(invalid_data[1], "bar", "bar", {"foo": None})
_test(invalid_data[2], "bar", "foo.bar", {'foo': {'baz': None}})

View File

@ -0,0 +1,23 @@
# Sample YAML file for testing forward replacement.
---
apiVersion: service/v1
kind: ConsumerOfCertificateData
metadata:
name: asdf-1234
storage: cleartext
substitutions:
- dest: .tls_endpoint.certificate
src:
apiVersion: deckhand/v1
kind: Certificate
name: some-certificate-asdf-1234
- dest: .tls_endpoint.certificateKey
src:
apiVersion: deckhand/v1
kind: CertificateKey
name: some-certificate-key-asdf-1234
data:
tls_endpoint:
uri: http://localhost:443
certificate: null
certificateKey: null

View File

@ -1 +1,11 @@
falcon==1.1.0 falcon==1.1.0
mock>=2.0
fixtures>=3.0.0 # Apache-2.0/BSD
mock>=2.0 # BSD
mox3!=0.19.0,>=0.7.0 # Apache-2.0
python-subunit>=0.0.18 # Apache-2.0/BSD
oslotest>=1.10.0 # Apache-2.0
os-testr>=0.8.0 # Apache-2.0
testrepository>=0.0.18 # Apache-2.0/BSD
testtools>=1.4.0 # MIT

26
tox.ini
View File

@ -1,15 +1,33 @@
[tox] [tox]
envlist = py35,py27,pep8 envlist = py{35,27},pep8
[testenv] [testenv]
usedevelop = True usedevelop = True
whitelist_externals = bash
find
rm
env
flake8
setenv = VIRTUAL_ENV={envdir} setenv = VIRTUAL_ENV={envdir}
OS_TEST_PATH=./deckhand/tests/unit
LANGUAGE=en_US LANGUAGE=en_US
LC_ALL=en_US.utf-8 LC_ALL=en_US.utf-8
deps= passenv = OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_TEST_TIMEOUT OS_TEST_LOCK_PATH OS_TEST_PATH http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
-r{toxinidir}/requirements.txt deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt -r{toxinidir}/test-requirements.txt
whitelist_externals = flake8 commands =
find . -type f -name "*.pyc" -delete
rm -Rf .testrepository/times.dbm
[testenv:py27]
commands =
{[testenv]commands}
ostestr '{posargs}'
[testenv:py35]
commands =
{[testenv]commands}
ostestr '{posargs}'
[testenv:genconfig] [testenv:genconfig]
commands = oslo-config-generator --config-file=etc/deckhand/config-generator.conf commands = oslo-config-generator --config-file=etc/deckhand/config-generator.conf