Merge "Redacts Raw Documents"

deckhand/common/document.py

@@ -15,6 +15,7 @@
 import collections
 import re
 
+import hashlib
 from oslo_serialization import jsonutils as json
 from oslo_utils import uuidutils
 import six
@@ -171,6 +172,11 @@ class DocumentDict(dict):
 
         return [DocumentDict(d) for d in documents]
 
+    @classmethod
+    def redact(cls, input):
+        return hashlib.sha256(json.dumps(input)
+                              .encode('utf-8')).hexdigest()
+
 
 def document_dict_representer(dumper, data):
     return dumper.represent_mapping('tag:yaml.org,2002:map', dict(data))

deckhand/common/utils.py

@@ -23,6 +23,7 @@ import jsonpath_ng
 from oslo_log import log as logging
 import six
 
+from deckhand.common.document import DocumentDict as document_dict
 from deckhand.conf import config
 from deckhand import errors
 
@@ -381,3 +382,27 @@ def deepfilter(dct, **filters):
                         return False
 
     return True
+
+
+def redact_document(document):
+    d = _to_document(document)
+    if d.is_encrypted:
+        document['data'] = document_dict.redact(d.data)
+        if d.substitutions:
+            subs = d.substitutions
+            for s in subs:
+                s['src']['path'] = document_dict.redact(s['src']['path'])
+                s['dest']['path'] = document_dict.redact(s['dest']['path'])
+            document['metadata']['substitutions'] = subs
+    return document
+
+
+def redact_documents(documents):
+    return [redact_document(d) for d in documents]
+
+
+def _to_document(document):
+    clazz = document_dict
+    if not isinstance(document, clazz):
+        document = clazz(document)
+    return document

deckhand/control/revision_documents.py

@@ -39,7 +39,7 @@ class RevisionDocumentsResource(api_base.BaseResource):
     @common.sanitize_params([
         'schema', 'metadata.name', 'metadata.layeringDefinition.abstract',
         'metadata.layeringDefinition.layer', 'metadata.label',
-        'status.bucket', 'order', 'sort', 'limit'])
+        'status.bucket', 'order', 'sort', 'limit', 'cleartext-secrets'])
     def on_get(self, req, resp, sanitized_params, revision_id):
         """Returns all documents for a `revision_id`.
 
@@ -54,6 +54,7 @@ class RevisionDocumentsResource(api_base.BaseResource):
         order_by = sanitized_params.pop('order', None)
         sort_by = sanitized_params.pop('sort', None)
         limit = sanitized_params.pop('limit', None)
+        cleartext_secrets = sanitized_params.pop('cleartext-secrets', None)
 
         filters = sanitized_params.copy()
         filters['metadata.storagePolicy'] = ['cleartext']
@@ -68,6 +69,9 @@ class RevisionDocumentsResource(api_base.BaseResource):
             LOG.exception(six.text_type(e))
             raise falcon.HTTPNotFound(description=e.format_message())
 
+        if cleartext_secrets not in [True, 'true', 'True']:
+            documents = utils.redact_documents(documents)
+
         # Sorts by creation date by default.
         documents = utils.multisort(documents, sort_by, order_by)
         if limit is not None:

deckhand/policies/document.py

@@ -84,7 +84,7 @@ Only enforced after ``list_cleartext_documents`` passes.
 
 Conditionally enforced for the endpoints below if any of the documents in the
 request body have a ``metadata.storagePolicy`` of "encrypted". If policy
-enforcement fails, encrypted documents are exluded from the response.""",
+enforcement fails, encrypted documents are excluded from the response.""",
         [
             {
                 'method': 'GET',

deckhand/tests/integration/gabbits/document-crud-secret.yaml

@@ -75,6 +75,8 @@ tests:
       content-type: application/x-yaml
     response_headers:
       content-type: application/x-yaml
+    query_parameters:
+      cleartext-secrets: 'true'
     response_multidoc_jsonpaths:
       $.`len`: 1
       # NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)

deckhand/tests/integration/gabbits/document-render-secret-edge-cases.yaml

@@ -167,6 +167,7 @@ tests:
       content-type: application/x-yaml
     query_parameters:
       metadata.name: armada-doc
+      cleartext-secrets: 'true'
     response_multidoc_jsonpaths:
       $.`len`: 1
       $.[0].data.`split(:, 0, 1)` + "://" + $.[0].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL']

deckhand/tests/integration/gabbits/document-substitution-secret-generic.yaml

@@ -74,6 +74,7 @@ tests:
     status: 200
     query_parameters:
       metadata.name: example-armada-cert
+      cleartext-secrets: 'true'
     response_multidoc_jsonpaths:
       $.`len`: 1
       # NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)

deckhand/tests/integration/gabbits/document-substitution-secret.yaml

@@ -180,6 +180,7 @@ tests:
         - example-passphrase
         - example-private-key
         - example-public-key
+      cleartext-secrets: 'true'
     response_multidoc_jsonpaths:
       $.`len`: 7
       # NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)

deckhand/tests/unit/control/test_revision_documents_controller.py

@@ -16,6 +16,7 @@ import yaml
 
 import mock
 
+from deckhand.common.document import DocumentDict as document_dict
 from deckhand.engine import secrets_manager
 from deckhand import factories
 from deckhand.tests.unit.control import base as test_base
@@ -102,6 +103,93 @@ data:
         self.assertEqual(2, len(retrieved_documents))
         self.assertEqual(expected_data_section, retrieved_documents[0]['data'])
 
+    def _setup_payload(self):
+        data = '12345'
+        sub_src = '.source1'
+        sub_dest = '.destination2'
+        secrets_factory = factories.DocumentSecretFactory()
+        payload = [secrets_factory.gen_test('Certificate', 'encrypted')]
+        payload[0]['data'] = data
+        sub1 = {'src': {'schema': 'pegleg/SoftwareVersions/v1', 'name': 'sub1',
+                'path': sub_src}, 'dest': {'path': '.destination1'}}
+        sub2 = {'src': {'schema': 'pegleg/SoftwareVersions/v1', 'name': 'sub2',
+                'path': '.source2'}, 'dest': {'path': sub_dest}}
+        payload[0]['metadata']['substitutions'] = [sub1, sub2]
+        return payload, data, sub_src, sub_dest
+
+    def test_list_encrypted_revision_documents_redacted(self):
+        rules = {'deckhand:list_cleartext_documents': '@',
+                 'deckhand:list_encrypted_documents': '@',
+                 'deckhand:create_cleartext_documents': '@',
+                 'deckhand:create_encrypted_documents': '@'}
+        self.policy.set_rules(rules)
+
+        # Create a document for a bucket.
+        payload, data, sub_src, sub_dest = self._setup_payload()
+
+        with mock.patch.object(secrets_manager, 'SecretsManager',
+                               autospec=True) as mock_secrets_mgr:
+            mock_secrets_mgr.create.return_value = payload[0]['data']
+            resp = self.app.simulate_put(
+                '/api/v1.0/buckets/mop/documents',
+                headers={'Content-Type': 'application/x-yaml'},
+                body=yaml.safe_dump_all(payload))
+        self.assertEqual(200, resp.status_code)
+        revision_id = list(yaml.safe_load_all(resp.text))[0]['status'][
+            'revision']
+
+        # Verify that the created document was redacted.
+        redacted_data = document_dict.redact(data)
+        redacted_sub_src = document_dict.redact(sub_src)
+        redacted_sub_dest = document_dict.redact(sub_dest)
+        resp = self.app.simulate_get(
+            '/api/v1.0/revisions/%s/documents' % revision_id,
+            headers={'Content-Type': 'application/x-yaml'},
+            query_string='cleartext-secrets=false')
+
+        self.assertEqual(200, resp.status_code)
+        self.assertNotEqual(list(yaml.safe_load_all(resp.text)), [])
+        response_yaml = list(yaml.safe_load_all(resp.text))
+        self.assertEqual(redacted_data, response_yaml[0]['data'])
+        subs = response_yaml[0]['metadata']['substitutions']
+        self.assertEqual(redacted_sub_src, subs[0]['src']['path'])
+        self.assertEqual(redacted_sub_dest, subs[1]['dest']['path'])
+
+    def test_list_encrypted_revision_documents_cleartext_secrets(self):
+        rules = {'deckhand:list_cleartext_documents': '@',
+                 'deckhand:list_encrypted_documents': '@',
+                 'deckhand:create_cleartext_documents': '@',
+                 'deckhand:create_encrypted_documents': '@'}
+        self.policy.set_rules(rules)
+
+        # Create a document for a bucket.
+        payload, data, sub_src, sub_dest = self._setup_payload()
+
+        with mock.patch.object(secrets_manager, 'SecretsManager',
+                               autospec=True) as mock_secrets_mgr:
+            mock_secrets_mgr.create.return_value = payload[0]['data']
+            resp = self.app.simulate_put(
+                '/api/v1.0/buckets/mop/documents',
+                headers={'Content-Type': 'application/x-yaml'},
+                body=yaml.safe_dump_all(payload))
+        self.assertEqual(200, resp.status_code)
+        revision_id = list(yaml.safe_load_all(resp.text))[0]['status'][
+            'revision']
+
+        # Verify that the created document was not redacted.
+        resp = self.app.simulate_get(
+            '/api/v1.0/revisions/%s/documents' % revision_id,
+            headers={'Content-Type': 'application/x-yaml'},
+            query_string='cleartext-secrets=true')
+
+        self.assertEqual(200, resp.status_code)
+        self.assertNotEqual(list(yaml.safe_load_all(resp.text)), [])
+        response_yaml = list(yaml.safe_load_all(resp.text))
+        self.assertEqual(data, response_yaml[0]['data'])
+        subs = response_yaml[0]['metadata']['substitutions']
+        self.assertEqual(sub_src, subs[0]['src']['path'])
+        self.assertEqual(sub_dest, subs[1]['dest']['path'])
+
 
 class TestRevisionDocumentsControllerNegativeRBAC(
         test_base.BaseControllerTest):

deckhand/tests/unit/fixtures.py

@@ -156,7 +156,7 @@ class RealPolicyFixture(fixtures.Fixture):
         def enforce_policy_and_remember_actual_rules(
                 action, *a, **k):
             self.actual_policy_actions.append(action)
-            _do_enforce_rbac(action, *a, **k)
+            return _do_enforce_rbac(action, *a, **k)
 
         mock_do_enforce_rbac = mock.patch.object(
             deckhand.policy, '_do_enforce_rbac', autospec=True).start()

doc/source/operators/api_ref.rst

@@ -88,6 +88,9 @@ Supported query string parameters:
   descending order.
 * ``limit`` - int, optional - Controls number of documents returned by this
    endpoint.
+* ``cleartext-secrets`` - boolean, optional - Determines if data and substitutions
+   paths should be redacted (sha256) if a user has access to encrypted files.
+   Default is to redact the values.
 
 GET ``/revisions/{revision_id}/rendered-documents``
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

etc/deckhand/policy.yaml.sample

@@ -46,7 +46,7 @@
 # documents in the
 # request body have a ``metadata.storagePolicy`` of "encrypted". If
 # policy
-# enforcement fails, encrypted documents are exluded from the
+# enforcement fails, encrypted documents are excluded from the
 # response.
 # GET  api/v1.0/revisions/{revision_id}/documents
 # GET  api/v1.0/revisions/{revision_id}/rendered-documents