Browse Source

Redact rendered Documents

- Uses the rendered-documents endpoint
- Adds a query parameter ?cleartext-secrets
- Adds unit tests, updates integration tests

Change-Id: I02423b9bf7456008d707b3cd91edc4fc281fa5fc
anthony.bellino 6 months ago
parent
commit
7defe473d2

+ 12
- 0
deckhand/common/utils.py View File

@@ -385,9 +385,21 @@ def deepfilter(dct, **filters):
385 385
 
386 386
 
387 387
 def redact_document(document):
388
+    """Redact ``data`` and ``substitutions`` sections for ``document``.
389
+
390
+    :param dict document: Document whose data to redact.
391
+    :returns: Document with redacted data.
392
+    :rtype: dict
393
+    """
388 394
     d = _to_document(document)
389 395
     if d.is_encrypted:
390 396
         document['data'] = document_dict.redact(d.data)
397
+        # FIXME(felipemonteiro): This block should be out-dented by 4 spaces
398
+        # because cleartext documents that substitute from encrypted documents
399
+        # should be subject to this redaction as well. However, doing this
400
+        # will result in substitution failures; the solution is to add a
401
+        # helper to :class:`deckhand.common.DocumentDict` that checks whether
402
+        # its metadata.substitutions is redacted - if so, skips substitution.
391 403
         if d.substitutions:
392 404
             subs = d.substitutions
393 405
             for s in subs:

+ 8
- 2
deckhand/control/common.py View File

@@ -23,6 +23,7 @@ import six
23 23
 
24 24
 from deckhand.barbican import cache as barbican_cache
25 25
 from deckhand.common import document as document_wrapper
26
+from deckhand.common import utils
26 27
 from deckhand.db.sqlalchemy import api as db_api
27 28
 from deckhand import engine
28 29
 from deckhand.engine import cache as engine_cache
@@ -130,7 +131,9 @@ def sanitize_params(allowed_params):
130 131
                         else:
131 132
                             sanitized_params[key] = param_val
132 133
 
133
-            func_args = func_args + (sanitized_params,)
134
+            req.params.clear()
135
+            req.params.update(sanitized_params)
136
+
134 137
             return func(self, req, *func_args, **func_kwargs)
135 138
 
136 139
         return wrapper
@@ -144,10 +147,13 @@ def invalidate_cache_data():
144 147
     engine_cache.invalidate()
145 148
 
146 149
 
147
-def get_rendered_docs(revision_id, **filters):
150
+def get_rendered_docs(revision_id, cleartext_secrets=False, **filters):
148 151
     data = _retrieve_documents_for_rendering(revision_id, **filters)
149 152
     documents = document_wrapper.DocumentDict.from_list(data)
150 153
     encryption_sources = _resolve_encrypted_data(documents)
154
+
155
+    if not cleartext_secrets:
156
+        documents = utils.redact_documents(documents)
151 157
     try:
152 158
         return engine.render(
153 159
             revision_id,

+ 18
- 14
deckhand/control/revision_documents.py View File

@@ -40,7 +40,7 @@ class RevisionDocumentsResource(api_base.BaseResource):
40 40
         'schema', 'metadata.name', 'metadata.layeringDefinition.abstract',
41 41
         'metadata.layeringDefinition.layer', 'metadata.label',
42 42
         'status.bucket', 'order', 'sort', 'limit', 'cleartext-secrets'])
43
-    def on_get(self, req, resp, sanitized_params, revision_id):
43
+    def on_get(self, req, resp, revision_id):
44 44
         """Returns all documents for a `revision_id`.
45 45
 
46 46
         Returns a multi-document YAML response containing all the documents
@@ -51,12 +51,13 @@ class RevisionDocumentsResource(api_base.BaseResource):
51 51
         include_encrypted = policy.conditional_authorize(
52 52
             'deckhand:list_encrypted_documents', req.context, do_raise=False)
53 53
 
54
-        order_by = sanitized_params.pop('order', None)
55
-        sort_by = sanitized_params.pop('sort', None)
56
-        limit = sanitized_params.pop('limit', None)
57
-        cleartext_secrets = sanitized_params.pop('cleartext-secrets', None)
54
+        order_by = req.params.pop('order', None)
55
+        sort_by = req.params.pop('sort', None)
56
+        limit = req.params.pop('limit', None)
57
+        cleartext_secrets = req.get_param_as_bool('cleartext-secrets')
58
+        req.params.pop('cleartext-secrets', None)
58 59
 
59
-        filters = sanitized_params.copy()
60
+        filters = req.params.copy()
60 61
         filters['metadata.storagePolicy'] = ['cleartext']
61 62
         if include_encrypted:
62 63
             filters['metadata.storagePolicy'].append('encrypted')
@@ -69,7 +70,7 @@ class RevisionDocumentsResource(api_base.BaseResource):
69 70
             LOG.exception(six.text_type(e))
70 71
             raise falcon.HTTPNotFound(description=e.format_message())
71 72
 
72
-        if cleartext_secrets not in [True, 'true', 'True']:
73
+        if not cleartext_secrets:
73 74
             documents = utils.redact_documents(documents)
74 75
 
75 76
         # Sorts by creation date by default.
@@ -100,8 +101,9 @@ class RenderedDocumentsResource(api_base.BaseResource):
100 101
     @policy.authorize('deckhand:list_cleartext_documents')
101 102
     @common.sanitize_params([
102 103
         'schema', 'metadata.name', 'metadata.layeringDefinition.layer',
103
-        'metadata.label', 'status.bucket', 'order', 'sort', 'limit'])
104
-    def on_get(self, req, resp, sanitized_params, revision_id):
104
+        'metadata.label', 'status.bucket', 'order', 'sort', 'limit',
105
+        'cleartext-secrets'])
106
+    def on_get(self, req, resp, revision_id):
105 107
         include_encrypted = policy.conditional_authorize(
106 108
             'deckhand:list_encrypted_documents', req.context, do_raise=False)
107 109
         filters = {
@@ -111,8 +113,10 @@ class RenderedDocumentsResource(api_base.BaseResource):
111 113
         if include_encrypted:
112 114
             filters['metadata.storagePolicy'].append('encrypted')
113 115
 
116
+        cleartext_secrets = req.get_param_as_bool('cleartext-secrets')
117
+        req.params.pop('cleartext-secrets', None)
114 118
         rendered_documents, cache_hit = common.get_rendered_docs(
115
-            revision_id, **filters)
119
+            revision_id, cleartext_secrets, **filters)
116 120
 
117 121
         # If the rendered documents result set is cached, then post-validation
118 122
         # for that result set has already been performed successfully, so it
@@ -128,10 +132,10 @@ class RenderedDocumentsResource(api_base.BaseResource):
128 132
         # involved in rendering. User filters can only be applied once all
129 133
         # documents have been rendered. Note that `layering` module only
130 134
         # returns concrete documents, so no filtering for that is needed here.
131
-        order_by = sanitized_params.pop('order', None)
132
-        sort_by = sanitized_params.pop('sort', None)
133
-        limit = sanitized_params.pop('limit', None)
134
-        user_filters = sanitized_params.copy()
135
+        order_by = req.params.pop('order', None)
136
+        sort_by = req.params.pop('sort', None)
137
+        limit = req.params.pop('limit', None)
138
+        user_filters = req.params.copy()
135 139
 
136 140
         rendered_documents = [
137 141
             d for d in rendered_documents if utils.deepfilter(

+ 4
- 4
deckhand/control/revisions.py View File

@@ -64,11 +64,11 @@ class RevisionsResource(api_base.BaseResource):
64 64
 
65 65
     @policy.authorize('deckhand:list_revisions')
66 66
     @common.sanitize_params(['tag', 'order', 'sort'])
67
-    def _list_revisions(self, req, resp, sanitized_params):
68
-        order_by = sanitized_params.pop('order', None)
69
-        sort_by = sanitized_params.pop('sort', None)
67
+    def _list_revisions(self, req, resp):
68
+        order_by = req.params.pop('order', None)
69
+        sort_by = req.params.pop('sort', None)
70 70
 
71
-        revisions = db_api.revision_get_all(**sanitized_params)
71
+        revisions = db_api.revision_get_all(**req.params)
72 72
         if sort_by:
73 73
             revisions = utils.multisort(revisions, sort_by, order_by)
74 74
 

+ 1
- 1
deckhand/engine/layering.py View File

@@ -708,7 +708,7 @@ class DocumentLayering(object):
708 708
             # Otherwise, retrieve the encrypted data for the document if its
709 709
             # data has been encrypted so that future references use the actual
710 710
             # secret payload, rather than the Barbican secret reference.
711
-            elif doc.is_encrypted:
711
+            elif doc.is_encrypted and doc.has_barbican_ref:
712 712
                 encrypted_data = self.secrets_substitution\
713 713
                     .get_unencrypted_data(doc.data, doc, doc)
714 714
                 if not doc.is_abstract:

+ 1
- 0
deckhand/tests/integration/gabbits/document-render-secret-edge-cases.yaml View File

@@ -185,6 +185,7 @@ tests:
185 185
       content-type: application/x-yaml
186 186
     query_parameters:
187 187
       metadata.name: armada-doc
188
+      cleartext-secrets: true
188 189
     response_multidoc_jsonpaths:
189 190
       $.`len`: 1
190 191
       $.[0].data:

+ 1
- 0
deckhand/tests/integration/gabbits/document-render-secret.yaml View File

@@ -52,6 +52,7 @@ tests:
52 52
     GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents
53 53
     status: 200
54 54
     query_parameters:
55
+      cleartext-secrets: true
55 56
       metadata.name: my-passphrase
56 57
     response_multidoc_jsonpaths:
57 58
       $.`len`: 1

+ 1
- 0
deckhand/tests/integration/gabbits/document-substitution-secret-generic.yaml View File

@@ -100,6 +100,7 @@ tests:
100 100
     GET: /api/v1.0/revisions/$HISTORY['encrypt_generic_document_for_secret_substitution'].$RESPONSE['$.[0].status.revision']/rendered-documents
101 101
     status: 200
102 102
     query_parameters:
103
+      cleartext-secrets: true
103 104
       metadata.name:
104 105
         - armada-chart-01
105 106
         - example-armada-cert

+ 1
- 0
deckhand/tests/integration/gabbits/document-substitution-secret.yaml View File

@@ -242,6 +242,7 @@ tests:
242 242
     response_headers:
243 243
       content-type: application/x-yaml
244 244
     query_parameters:
245
+      cleartext-secrets: true
245 246
       sort: 'metadata.name'
246 247
     response_multidoc_jsonpaths:
247 248
       $.`len`: 9

+ 46
- 0
deckhand/tests/unit/common/test_utils.py View File

@@ -12,14 +12,17 @@
12 12
 # See the License for the specific language governing permissions and
13 13
 # limitations under the License.
14 14
 
15
+import hashlib
15 16
 import jsonpath_ng
16 17
 import mock
17 18
 
19
+from oslo_serialization import jsonutils as json
18 20
 from testtools.matchers import Equals
19 21
 from testtools.matchers import MatchesAny
20 22
 
21 23
 from deckhand.common import utils
22 24
 from deckhand import errors
25
+from deckhand import factories
23 26
 from deckhand.tests.unit import base as test_base
24 27
 
25 28
 
@@ -241,3 +244,46 @@ class TestJSONPathUtilsCaching(test_base.DeckhandTestCase):
241 244
         # in case CI jobs clash.)
242 245
         self.assertThat(
243 246
             self.jsonpath_call_count, MatchesAny(Equals(0), Equals(1)))
247
+
248
+
249
+class TestRedactDocuments(test_base.DeckhandTestCase):
250
+    """Validate Redact function works"""
251
+
252
+    def test_redact_rendered_document(self):
253
+
254
+        self.factory = factories.DocumentSecretFactory()
255
+        mapping = {
256
+            "_GLOBAL_DATA_1_": {"data": {"a": {"x": 1, "y": 2}}},
257
+            "_GLOBAL_SUBSTITUTIONS_1_": [{
258
+                "dest": {
259
+                    "path": ".c"
260
+                },
261
+                "src": {
262
+                    "schema": "deckhand/Certificate/v1",
263
+                    "name": "global-cert",
264
+                    "path": "."
265
+                }
266
+            }]
267
+        }
268
+        data = mapping['_GLOBAL_DATA_1_']['data']
269
+        doc_factory = factories.DocumentFactory(1, [1])
270
+        document = doc_factory.gen_test(
271
+            mapping, global_abstract=False)[-1]
272
+        document['metadata']['storagePolicy'] = 'encrypted'
273
+
274
+        with mock.patch.object(hashlib, 'sha256', autospec=True,
275
+                               return_value=mock.sentinel.redacted)\
276
+                as mock_sha256:
277
+            redacted = mock.MagicMock()
278
+            mock_sha256.return_value = redacted
279
+            redacted.hexdigest.return_value = json.dumps(data)
280
+            mock.sentinel.redacted = redacted.hexdigest.return_value
281
+            redacted_doc = utils.redact_document(document)
282
+
283
+        self.assertEqual(mock.sentinel.redacted, redacted_doc['data'])
284
+        self.assertEqual(mock.sentinel.redacted,
285
+                         redacted_doc['metadata']['substitutions'][0]
286
+                         ['src']['path'])
287
+        self.assertEqual(mock.sentinel.redacted,
288
+                         redacted_doc['metadata']['substitutions'][0]
289
+                         ['dest']['path'])

+ 95
- 0
deckhand/tests/unit/control/test_rendered_documents_controller.py View File

@@ -20,6 +20,7 @@ from deckhand.control import revision_documents
20 20
 from deckhand.engine import secrets_manager
21 21
 from deckhand import errors
22 22
 from deckhand import factories
23
+from deckhand.tests import test_utils
23 24
 from deckhand.tests.unit.control import base as test_base
24 25
 from deckhand import types
25 26
 
@@ -196,6 +197,100 @@ class TestRenderedDocumentsController(test_base.BaseControllerTest):
196 197
         self.assertEqual([4, 4], second_revision_ids)
197 198
 
198 199
 
200
+class TestRenderedDocumentsControllerRedaction(test_base.BaseControllerTest):
201
+
202
+    def _test_list_rendered_documents(self, cleartext_secrets):
203
+        rules = {
204
+            'deckhand:list_cleartext_documents': '@',
205
+            'deckhand:list_encrypted_documents': '@',
206
+            'deckhand:create_cleartext_documents': '@',
207
+            'deckhand:create_encrypted_documents': '@'}
208
+
209
+        self.policy.set_rules(rules)
210
+
211
+        doc_factory = factories.DocumentFactory(1, [1])
212
+
213
+        layering_policy = doc_factory.gen_test({})[0]
214
+        layering_policy['data']['layerOrder'] = ['global', 'site']
215
+        certificate_data = 'sample-certificate'
216
+        certificate_ref = ('http://127.0.0.1/key-manager/v1/secrets/%s'
217
+                           % test_utils.rand_uuid_hex())
218
+
219
+        doc1 = {
220
+            'data': certificate_data,
221
+            'schema': 'deckhand/Certificate/v1', 'name': 'example-cert',
222
+            'layer': 'site',
223
+            'metadata': {
224
+                'schema': 'metadata/Document/v1',
225
+                'name': 'example-cert',
226
+                'layeringDefinition': {
227
+                    'abstract': False,
228
+                    'layer': 'site'}, 'storagePolicy': 'encrypted',
229
+                'replacement': False}}
230
+
231
+        doc2 = {'data': {}, 'schema': 'example/Kind/v1',
232
+                'name': 'deckhand-global', 'layer': 'global',
233
+                'metadata': {
234
+                    'labels': {'global': 'global1'},
235
+                    'storagePolicy': 'cleartext',
236
+                    'layeringDefinition': {'abstract': False,
237
+                                           'layer': 'global'},
238
+                    'name': 'deckhand-global',
239
+                    'schema': 'metadata/Document/v1', 'substitutions': [
240
+                        {'dest': {'path': '.'},
241
+                         'src': {'schema': 'deckhand/Certificate/v1',
242
+                                 'name': 'example-cert', 'path': '.'}}],
243
+                    'replacement': False}}
244
+
245
+        payload = [layering_policy, doc1, doc2]
246
+
247
+        # Create both documents and mock out SecretsManager.create to return
248
+        # a fake Barbican ref.
249
+        with mock.patch.object(  # noqa
250
+                secrets_manager.SecretsManager, 'create',
251
+                return_value=certificate_ref):
252
+            resp = self.app.simulate_put(
253
+                '/api/v1.0/buckets/mop/documents',
254
+                headers={'Content-Type': 'application/x-yaml'},
255
+                body=yaml.safe_dump_all(payload))
256
+        self.assertEqual(200, resp.status_code)
257
+        revision_id = list(yaml.safe_load_all(resp.text))[0]['status'][
258
+            'revision']
259
+
260
+        # Retrieve rendered documents and simulate a Barbican lookup by
261
+        # causing the actual certificate data to be returned.
262
+        with mock.patch.object(secrets_manager.SecretsManager, 'get',  # noqa
263
+                               return_value=certificate_data):
264
+            resp = self.app.simulate_get(
265
+                '/api/v1.0/revisions/%s/rendered-documents' % revision_id,
266
+                headers={'Content-Type': 'application/x-yaml'},
267
+                params={
268
+                    'metadata.name': ['example-cert', 'deckhand-global'],
269
+                    'cleartext-secrets': str(cleartext_secrets)
270
+                },
271
+                params_csv=False)
272
+
273
+        self.assertEqual(200, resp.status_code)
274
+        rendered_documents = list(yaml.safe_load_all(resp.text))
275
+        self.assertEqual(2, len(rendered_documents))
276
+
277
+        if cleartext_secrets is True:
278
+            # Expect the cleartext data to be returned.
279
+            self.assertTrue(all(map(lambda x: x['data'] == certificate_data,
280
+                                rendered_documents)))
281
+        else:
282
+            # Expected redacted data for both documents to be returned -
283
+            # because the destination document should receive redacted data.
284
+            self.assertTrue(all(map(lambda x: x['data'] != certificate_data,
285
+                                rendered_documents)))
286
+
287
+    def test_list_rendered_documents_cleartext_secrets_true(self):
288
+        self._test_list_rendered_documents(cleartext_secrets=True)
289
+
290
+    def test_list_rendered_documents_cleartext_secrets_false(self):
291
+        self._test_list_rendered_documents(cleartext_secrets=False)
292
+
293
+
199 294
 class TestRenderedDocumentsControllerNegative(
200 295
         test_base.BaseControllerTest):
201 296
 

Loading…
Cancel
Save