Browse Source

Redacts Raw Documents

- If a document has a storage policy of encrypted
 - Redacts (sha256) the data section.
 - Redacts (sha256) the substition paths.
- Uses the same /documents endpoint, adds a new query parameter
  ?cleartext-secrets=true to show the non-redacted values.

Change-Id: I42808901b97c667a1148c00fbb7717a0847c9981
changes/43/610043/14
Aaron Sheffield 3 years ago
parent
commit
349e5600df
12 changed files with 135 additions and 4 deletions
  1. +6
    -0
      deckhand/common/document.py
  2. +25
    -0
      deckhand/common/utils.py
  3. +5
    -1
      deckhand/control/revision_documents.py
  4. +1
    -1
      deckhand/policies/document.py
  5. +2
    -0
      deckhand/tests/integration/gabbits/document-crud-secret.yaml
  6. +1
    -0
      deckhand/tests/integration/gabbits/document-render-secret-edge-cases.yaml
  7. +1
    -0
      deckhand/tests/integration/gabbits/document-substitution-secret-generic.yaml
  8. +1
    -0
      deckhand/tests/integration/gabbits/document-substitution-secret.yaml
  9. +88
    -0
      deckhand/tests/unit/control/test_revision_documents_controller.py
  10. +1
    -1
      deckhand/tests/unit/fixtures.py
  11. +3
    -0
      doc/source/operators/api_ref.rst
  12. +1
    -1
      etc/deckhand/policy.yaml.sample

+ 6
- 0
deckhand/common/document.py View File

@ -15,6 +15,7 @@
import collections
import re
import hashlib
from oslo_serialization import jsonutils as json
from oslo_utils import uuidutils
import six
@ -171,6 +172,11 @@ class DocumentDict(dict):
return [DocumentDict(d) for d in documents]
@classmethod
def redact(cls, input):
return hashlib.sha256(json.dumps(input)
.encode('utf-8')).hexdigest()
def document_dict_representer(dumper, data):
return dumper.represent_mapping('tag:yaml.org,2002:map', dict(data))


+ 25
- 0
deckhand/common/utils.py View File

@ -23,6 +23,7 @@ import jsonpath_ng
from oslo_log import log as logging
import six
from deckhand.common.document import DocumentDict as document_dict
from deckhand.conf import config
from deckhand import errors
@ -381,3 +382,27 @@ def deepfilter(dct, **filters):
return False
return True
def redact_document(document):
d = _to_document(document)
if d.is_encrypted:
document['data'] = document_dict.redact(d.data)
if d.substitutions:
subs = d.substitutions
for s in subs:
s['src']['path'] = document_dict.redact(s['src']['path'])
s['dest']['path'] = document_dict.redact(s['dest']['path'])
document['metadata']['substitutions'] = subs
return document
def redact_documents(documents):
return [redact_document(d) for d in documents]
def _to_document(document):
clazz = document_dict
if not isinstance(document, clazz):
document = clazz(document)
return document

+ 5
- 1
deckhand/control/revision_documents.py View File

@ -39,7 +39,7 @@ class RevisionDocumentsResource(api_base.BaseResource):
@common.sanitize_params([
'schema', 'metadata.name', 'metadata.layeringDefinition.abstract',
'metadata.layeringDefinition.layer', 'metadata.label',
'status.bucket', 'order', 'sort', 'limit'])
'status.bucket', 'order', 'sort', 'limit', 'cleartext-secrets'])
def on_get(self, req, resp, sanitized_params, revision_id):
"""Returns all documents for a `revision_id`.
@ -54,6 +54,7 @@ class RevisionDocumentsResource(api_base.BaseResource):
order_by = sanitized_params.pop('order', None)
sort_by = sanitized_params.pop('sort', None)
limit = sanitized_params.pop('limit', None)
cleartext_secrets = sanitized_params.pop('cleartext-secrets', None)
filters = sanitized_params.copy()
filters['metadata.storagePolicy'] = ['cleartext']
@ -68,6 +69,9 @@ class RevisionDocumentsResource(api_base.BaseResource):
LOG.exception(six.text_type(e))
raise falcon.HTTPNotFound(description=e.format_message())
if cleartext_secrets not in [True, 'true', 'True']:
documents = utils.redact_documents(documents)
# Sorts by creation date by default.
documents = utils.multisort(documents, sort_by, order_by)
if limit is not None:


+ 1
- 1
deckhand/policies/document.py View File

@ -84,7 +84,7 @@ Only enforced after ``list_cleartext_documents`` passes.
Conditionally enforced for the endpoints below if any of the documents in the
request body have a ``metadata.storagePolicy`` of "encrypted". If policy
enforcement fails, encrypted documents are exluded from the response.""",
enforcement fails, encrypted documents are excluded from the response.""",
[
{
'method': 'GET',


+ 2
- 0
deckhand/tests/integration/gabbits/document-crud-secret.yaml View File

@ -75,6 +75,8 @@ tests:
content-type: application/x-yaml
response_headers:
content-type: application/x-yaml
query_parameters:
cleartext-secrets: 'true'
response_multidoc_jsonpaths:
$.`len`: 1
# NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)


+ 1
- 0
deckhand/tests/integration/gabbits/document-render-secret-edge-cases.yaml View File

@ -167,6 +167,7 @@ tests:
content-type: application/x-yaml
query_parameters:
metadata.name: armada-doc
cleartext-secrets: 'true'
response_multidoc_jsonpaths:
$.`len`: 1
$.[0].data.`split(:, 0, 1)` + "://" + $.[0].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL']


+ 1
- 0
deckhand/tests/integration/gabbits/document-substitution-secret-generic.yaml View File

@ -74,6 +74,7 @@ tests:
status: 200
query_parameters:
metadata.name: example-armada-cert
cleartext-secrets: 'true'
response_multidoc_jsonpaths:
$.`len`: 1
# NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)


+ 1
- 0
deckhand/tests/integration/gabbits/document-substitution-secret.yaml View File

@ -180,6 +180,7 @@ tests:
- example-passphrase
- example-private-key
- example-public-key
cleartext-secrets: 'true'
response_multidoc_jsonpaths:
$.`len`: 7
# NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)


+ 88
- 0
deckhand/tests/unit/control/test_revision_documents_controller.py View File

@ -16,6 +16,7 @@ import yaml
import mock
from deckhand.common.document import DocumentDict as document_dict
from deckhand.engine import secrets_manager
from deckhand import factories
from deckhand.tests.unit.control import base as test_base
@ -102,6 +103,93 @@ data:
self.assertEqual(2, len(retrieved_documents))
self.assertEqual(expected_data_section, retrieved_documents[0]['data'])
def _setup_payload(self):
data = '12345'
sub_src = '.source1'
sub_dest = '.destination2'
secrets_factory = factories.DocumentSecretFactory()
payload = [secrets_factory.gen_test('Certificate', 'encrypted')]
payload[0]['data'] = data
sub1 = {'src': {'schema': 'pegleg/SoftwareVersions/v1', 'name': 'sub1',
'path': sub_src}, 'dest': {'path': '.destination1'}}
sub2 = {'src': {'schema': 'pegleg/SoftwareVersions/v1', 'name': 'sub2',
'path': '.source2'}, 'dest': {'path': sub_dest}}
payload[0]['metadata']['substitutions'] = [sub1, sub2]
return payload, data, sub_src, sub_dest
def test_list_encrypted_revision_documents_redacted(self):
rules = {'deckhand:list_cleartext_documents': '@',
'deckhand:list_encrypted_documents': '@',
'deckhand:create_cleartext_documents': '@',
'deckhand:create_encrypted_documents': '@'}
self.policy.set_rules(rules)
# Create a document for a bucket.
payload, data, sub_src, sub_dest = self._setup_payload()
with mock.patch.object(secrets_manager, 'SecretsManager',
autospec=True) as mock_secrets_mgr:
mock_secrets_mgr.create.return_value = payload[0]['data']
resp = self.app.simulate_put(
'/api/v1.0/buckets/mop/documents',
headers={'Content-Type': 'application/x-yaml'},
body=yaml.safe_dump_all(payload))
self.assertEqual(200, resp.status_code)
revision_id = list(yaml.safe_load_all(resp.text))[0]['status'][
'revision']
# Verify that the created document was redacted.
redacted_data = document_dict.redact(data)
redacted_sub_src = document_dict.redact(sub_src)
redacted_sub_dest = document_dict.redact(sub_dest)
resp = self.app.simulate_get(
'/api/v1.0/revisions/%s/documents' % revision_id,
headers={'Content-Type': 'application/x-yaml'},
query_string='cleartext-secrets=false')
self.assertEqual(200, resp.status_code)
self.assertNotEqual(list(yaml.safe_load_all(resp.text)), [])
response_yaml = list(yaml.safe_load_all(resp.text))
self.assertEqual(redacted_data, response_yaml[0]['data'])
subs = response_yaml[0]['metadata']['substitutions']
self.assertEqual(redacted_sub_src, subs[0]['src']['path'])
self.assertEqual(redacted_sub_dest, subs[1]['dest']['path'])
def test_list_encrypted_revision_documents_cleartext_secrets(self):
rules = {'deckhand:list_cleartext_documents': '@',
'deckhand:list_encrypted_documents': '@',
'deckhand:create_cleartext_documents': '@',
'deckhand:create_encrypted_documents': '@'}
self.policy.set_rules(rules)
# Create a document for a bucket.
payload, data, sub_src, sub_dest = self._setup_payload()
with mock.patch.object(secrets_manager, 'SecretsManager',
autospec=True) as mock_secrets_mgr:
mock_secrets_mgr.create.return_value = payload[0]['data']
resp = self.app.simulate_put(
'/api/v1.0/buckets/mop/documents',
headers={'Content-Type': 'application/x-yaml'},
body=yaml.safe_dump_all(payload))
self.assertEqual(200, resp.status_code)
revision_id = list(yaml.safe_load_all(resp.text))[0]['status'][
'revision']
# Verify that the created document was not redacted.
resp = self.app.simulate_get(
'/api/v1.0/revisions/%s/documents' % revision_id,
headers={'Content-Type': 'application/x-yaml'},
query_string='cleartext-secrets=true')
self.assertEqual(200, resp.status_code)
self.assertNotEqual(list(yaml.safe_load_all(resp.text)), [])
response_yaml = list(yaml.safe_load_all(resp.text))
self.assertEqual(data, response_yaml[0]['data'])
subs = response_yaml[0]['metadata']['substitutions']
self.assertEqual(sub_src, subs[0]['src']['path'])
self.assertEqual(sub_dest, subs[1]['dest']['path'])
class TestRevisionDocumentsControllerNegativeRBAC(
test_base.BaseControllerTest):


+ 1
- 1
deckhand/tests/unit/fixtures.py View File

@ -156,7 +156,7 @@ class RealPolicyFixture(fixtures.Fixture):
def enforce_policy_and_remember_actual_rules(
action, *a, **k):
self.actual_policy_actions.append(action)
_do_enforce_rbac(action, *a, **k)
return _do_enforce_rbac(action, *a, **k)
mock_do_enforce_rbac = mock.patch.object(
deckhand.policy, '_do_enforce_rbac', autospec=True).start()


+ 3
- 0
doc/source/operators/api_ref.rst View File

@ -88,6 +88,9 @@ Supported query string parameters:
descending order.
* ``limit`` - int, optional - Controls number of documents returned by this
endpoint.
* ``cleartext-secrets`` - boolean, optional - Determines if data and substitutions
paths should be redacted (sha256) if a user has access to encrypted files.
Default is to redact the values.
GET ``/revisions/{revision_id}/rendered-documents``
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^


+ 1
- 1
etc/deckhand/policy.yaml.sample View File

@ -46,7 +46,7 @@
# documents in the
# request body have a ``metadata.storagePolicy`` of "encrypted". If
# policy
# enforcement fails, encrypted documents are exluded from the
# enforcement fails, encrypted documents are excluded from the
# response.
# GET api/v1.0/revisions/{revision_id}/documents
# GET api/v1.0/revisions/{revision_id}/rendered-documents


Loading…
Cancel
Save