Browse Source

Only collect/parse Deckhand-formatted documents for processing

This patch set changes Pegleg in two similar ways:

1) Ignore certain types of files altogether:
   - those located in hidden folders
   - those prefixed with "." (files like .zuul.yaml)
2) Only read Deckhand-formatted documents for lint/collect/etc.
   commands as Pegleg need not consider other types of documents
   (it separately reads the site-definition.yaml for internal
    processing still).

The tools/ subfolder is also ignored as it can contain
.yaml files which are not Deckhand-formatted documents,
so need not be processed by pegleg.engine.

Change-Id: I8996b5d430cf893122af648ef8e5805b36c1bfd9
Felipe Monteiro 5 months ago
parent
commit
f8d79e119c

+ 1
- 1
pegleg/cli.py View File

@@ -385,7 +385,7 @@ def secrets():
385 385
     'author',
386 386
     required=True,
387 387
     help='Identifier for the program or person who is encrypting the secrets '
388
-         'documents')
388
+    'documents')
389 389
 @click.argument('site_name')
390 390
 def encrypt(*, save_location, author, site_name):
391 391
     engine.repository.process_repositories(site_name)

+ 10
- 11
pegleg/engine/lint.py View File

@@ -18,7 +18,6 @@ import os
18 18
 import pkg_resources
19 19
 import shutil
20 20
 import textwrap
21
-import yaml
22 21
 
23 22
 from prettytable import PrettyTable
24 23
 
@@ -223,16 +222,16 @@ def _verify_single_file(filename, schemas):
223 222
             errors.append((FILE_MISSING_YAML_DOCUMENT_HEADER,
224 223
                            '%s does not begin with YAML beginning of document '
225 224
                            'marker "---".' % filename))
226
-        f.seek(0)
227
-        documents = []
228
-        try:
229
-            documents = list(yaml.safe_load_all(f))
230
-        except Exception as e:
231
-            errors.append((FILE_CONTAINS_INVALID_YAML,
232
-                           '%s is not valid yaml: %s' % (filename, e)))
233
-
234
-        for document in documents:
235
-            errors.extend(_verify_document(document, schemas, filename))
225
+
226
+    documents = []
227
+    try:
228
+        documents = util.files.read(filename)
229
+    except Exception as e:
230
+        errors.append((FILE_CONTAINS_INVALID_YAML,
231
+                       '%s is not valid yaml: %s' % (filename, e)))
232
+
233
+    for document in documents:
234
+        errors.extend(_verify_document(document, schemas, filename))
236 235
 
237 236
     return errors
238 237
 

+ 3
- 5
pegleg/engine/util/definition.py View File

@@ -15,7 +15,6 @@
15 15
 import os
16 16
 
17 17
 import click
18
-import yaml
19 18
 
20 19
 from pegleg import config
21 20
 from pegleg.engine.util import files
@@ -52,6 +51,7 @@ def load_as_params(site_name, *fields, primary_repo_base=None):
52 51
 
53 52
 
54 53
 def path(site_name, primary_repo_base=None):
54
+    """Retrieve path to the site-definition.yaml file for ``site_name``."""
55 55
     if not primary_repo_base:
56 56
         primary_repo_base = config.get_site_repo()
57 57
     return os.path.join(primary_repo_base, 'site', site_name,
@@ -100,8 +100,7 @@ def documents_for_each_site():
100 100
         paths = files.directories_for(**params)
101 101
         filenames = set(files.search(paths))
102 102
         for filename in filenames:
103
-            with open(filename) as f:
104
-                documents[sitename].extend(list(yaml.safe_load_all(f)))
103
+            documents[sitename].extend(files.read(filename))
105 104
 
106 105
     return documents
107 106
 
@@ -122,7 +121,6 @@ def documents_for_site(sitename):
122 121
     paths = files.directories_for(**params)
123 122
     filenames = set(files.search(paths))
124 123
     for filename in filenames:
125
-        with open(filename) as f:
126
-            documents.extend(list(yaml.safe_load_all(f)))
124
+        documents.extend(files.read(filename))
127 125
 
128 126
     return documents

+ 44
- 2
pegleg/engine/util/files.py View File

@@ -18,6 +18,7 @@ import yaml
18 18
 import logging
19 19
 
20 20
 from pegleg import config
21
+from pegleg.engine.util import pegleg_managed_document as md
21 22
 
22 23
 LOG = logging.getLogger(__name__)
23 24
 
@@ -248,9 +249,35 @@ def read(path):
248 249
             '{} not found. Pegleg must be run from the root of a '
249 250
             'configuration repository.'.format(path))
250 251
 
252
+    def is_deckhand_document(document):
253
+        # Deckhand documents only consist of control and application
254
+        # documents.
255
+        valid_schemas = ('metadata/Control', 'metadata/Document')
256
+        if isinstance(document, dict):
257
+            schema = document.get('metadata', {}).get('schema', '')
258
+            # NOTE(felipemonteiro): The Pegleg site-definition.yaml is a
259
+            # Deckhand-formatted document currently but probably shouldn't
260
+            # be, because it has no business being in Deckhand. As such,
261
+            # treat it as a special case.
262
+            if "SiteDefinition" in document.get('schema', ''):
263
+                return False
264
+            if any(schema.startswith(x) for x in valid_schemas):
265
+                return True
266
+            else:
267
+                LOG.debug('Document with schema=%s is not a valid Deckhand '
268
+                          'schema. Ignoring it.', schema)
269
+        return False
270
+
271
+    def is_pegleg_managed_document(document):
272
+        return md.PeglegManagedSecretsDocument.is_pegleg_managed_secret(
273
+            document)
274
+
251 275
     with open(path) as stream:
252 276
         try:
253
-            return list(yaml.safe_load_all(stream))
277
+            return [
278
+                d for d in yaml.safe_load_all(stream)
279
+                if is_deckhand_document(d) or is_pegleg_managed_document(d)
280
+            ]
254 281
         except yaml.YAMLError as e:
255 282
             raise click.ClickException('Failed to parse %s:\n%s' % (path, e))
256 283
 
@@ -296,10 +323,25 @@ def _recurse_subdirs(search_path, depth):
296 323
 
297 324
 
298 325
 def search(search_paths):
326
+    if not isinstance(search_paths, (list, tuple)):
327
+        search_paths = [search_paths]
328
+
299 329
     for search_path in search_paths:
300 330
         LOG.debug("Recursively collecting YAMLs from %s" % search_path)
301
-        for root, _dirs, filenames in os.walk(search_path):
331
+        for root, _, filenames in os.walk(search_path):
332
+
333
+            # Ignore hidden folders like .tox or .git for faster processing.
334
+            if os.path.basename(root).startswith("."):
335
+                continue
336
+            # Skip over anything in tools/ because it will never contain valid
337
+            # Pegleg-owned manifest documents.
338
+            if "tools" in root.split("/"):
339
+                continue
340
+
302 341
             for filename in filenames:
342
+                # Ignore files like .zuul.yaml.
343
+                if filename.startswith("."):
344
+                    continue
303 345
                 if filename.endswith(".yaml"):
304 346
                     yield os.path.join(root, filename)
305 347
 

+ 3
- 5
pegleg/engine/util/pegleg_secret_management.py View File

@@ -44,8 +44,7 @@ class PeglegSecretManagement():
44 44
 
45 45
         if all([file_path, docs]) or \
46 46
             not any([file_path, docs]):
47
-            raise ValueError(
48
-                'Either `file_path` or `docs` must be specified.')
47
+            raise ValueError('Either `file_path` or `docs` must be specified.')
49 48
 
50 49
         self.__check_environment()
51 50
         self.file_path = file_path
@@ -73,7 +72,7 @@ class PeglegSecretManagement():
73 72
         # Verify that passphrase environment variable is defined and is longer
74 73
         # than 24 characters.
75 74
         if not os.environ.get(ENV_PASSPHRASE) or not re.match(
76
-            PASSPHRASE_PATTERN, os.environ.get(ENV_PASSPHRASE)):
75
+                PASSPHRASE_PATTERN, os.environ.get(ENV_PASSPHRASE)):
77 76
             raise click.ClickException(
78 77
                 'Environment variable {} is not defined or '
79 78
                 'is not at least 24-character long.'.format(ENV_PASSPHRASE))
@@ -154,8 +153,7 @@ class PeglegSecretManagement():
154 153
             # do not decrypt already decrypted data
155 154
             if doc.is_encrypted():
156 155
                 doc.set_secret(
157
-                    decrypt(doc.get_secret(),
158
-                            self.passphrase,
156
+                    decrypt(doc.get_secret(), self.passphrase,
159 157
                             self.salt).decode())
160 158
                 doc.set_decrypted()
161 159
             doc_list.append(doc.embedded_document)

+ 35
- 42
tests/unit/engine/test_encryption.py View File

@@ -30,7 +30,6 @@ from pegleg.engine.util.pegleg_secret_management import ENV_SALT
30 30
 from tests.unit.fixtures import temp_path
31 31
 from pegleg.engine.util import files
32 32
 
33
-
34 33
 TEST_DATA = """
35 34
 ---
36 35
 schema: deckhand/Passphrase/v1
@@ -60,22 +59,24 @@ def test_encrypt_and_decrypt():
60 59
 
61 60
 
62 61
 @mock.patch.dict(os.environ, {
63
-    ENV_PASSPHRASE:'aShortPassphrase',
64
-    ENV_SALT: 'MySecretSalt'})
62
+    ENV_PASSPHRASE: 'aShortPassphrase',
63
+    ENV_SALT: 'MySecretSalt'
64
+})
65 65
 def test_short_passphrase():
66
-    with pytest.raises(click.ClickException,
67
-                       match=r'.*is not at least 24-character long.*'):
66
+    with pytest.raises(
67
+            click.ClickException,
68
+            match=r'.*is not at least 24-character long.*'):
68 69
         PeglegSecretManagement('file_path')
69 70
 
70 71
 
71
-def test_PeglegManagedDocument():
72
+def test_pegleg_secret_management_constructor():
72 73
     test_data = yaml.load(TEST_DATA)
73 74
     doc = PeglegManagedSecretsDocument(test_data)
74
-    assert doc.is_storage_policy_encrypted() is True
75
-    assert doc.is_encrypted() is False
75
+    assert doc.is_storage_policy_encrypted()
76
+    assert not doc.is_encrypted()
76 77
 
77 78
 
78
-def test_PeglegSecretManagement():
79
+def test_pegleg_secret_management_constructor_with_invalid_arguments():
79 80
     with pytest.raises(ValueError) as err_info:
80 81
         PeglegSecretManagement(file_path=None, docs=None)
81 82
     assert 'Either `file_path` or `docs` must be specified.' in str(
@@ -87,40 +88,24 @@ def test_PeglegSecretManagement():
87 88
 
88 89
 
89 90
 @mock.patch.dict(os.environ, {
90
-    ENV_PASSPHRASE:'ytrr89erARAiPE34692iwUMvWqqBvC',
91
-    ENV_SALT: 'MySecretSalt'})
92
-def test_encrypt_file():
93
-    # write the test data to temp file
94
-    test_data = yaml.load(TEST_DATA)
95
-    dir = tempfile.mkdtemp()
96
-    file_path = os.path.join(dir, 'secrets_file.yaml')
97
-    save_path = os.path.join(dir, 'encrypted_secrets_file.yaml')
98
-    with open(file_path, 'w') as stream:
99
-        yaml.dump(test_data,
100
-                  stream,
101
-                  explicit_start=True,
102
-                  explicit_end=True,
103
-                  default_flow_style=False)
104
-    # read back the secrets data file and encrypt it
105
-    doc_mgr = PeglegSecretManagement(file_path)
106
-    doc_mgr.encrypt_secrets(save_path, 'test_author')
107
-    doc = doc_mgr.documents[0]
108
-    assert doc.is_encrypted()
109
-    assert doc.data['encrypted']['by'] == 'test_author'
110
-
111
-
112
-@mock.patch.dict(os.environ, {
113
-    ENV_PASSPHRASE:'ytrr89erARAiPE34692iwUMvWqqBvC',
114
-    ENV_SALT: 'MySecretSalt'})
115
-def test_encrypt_decrypt_file(temp_path):
91
+    ENV_PASSPHRASE: 'ytrr89erARAiPE34692iwUMvWqqBvC',
92
+    ENV_SALT: 'MySecretSalt'
93
+})
94
+def test_encrypt_decrypt_using_file_path(temp_path):
116 95
     # write the test data to temp file
117 96
     test_data = list(yaml.safe_load_all(TEST_DATA))
118 97
     file_path = os.path.join(temp_path, 'secrets_file.yaml')
119 98
     files.write(file_path, test_data)
120 99
     save_path = os.path.join(temp_path, 'encrypted_secrets_file.yaml')
100
+
101
+    # encrypt documents and validate that they were encrypted
121 102
     doc_mgr = PeglegSecretManagement(file_path=file_path)
122 103
     doc_mgr.encrypt_secrets(save_path, 'test_author')
123
-    # read back the encrypted file
104
+    doc = doc_mgr.documents[0]
105
+    assert doc.is_encrypted()
106
+    assert doc.data['encrypted']['by'] == 'test_author'
107
+
108
+    # decrypt documents and validate that they were decrypted
124 109
     doc_mgr = PeglegSecretManagement(save_path)
125 110
     decrypted_data = doc_mgr.get_decrypted_secrets()
126 111
     assert test_data[0]['data'] == decrypted_data[0]['data']
@@ -128,23 +113,31 @@ def test_encrypt_decrypt_file(temp_path):
128 113
 
129 114
 
130 115
 @mock.patch.dict(os.environ, {
131
-    ENV_PASSPHRASE:'ytrr89erARAiPE34692iwUMvWqqBvC',
132
-    ENV_SALT: 'MySecretSalt'})
133
-def test_decrypt_document(temp_path):
116
+    ENV_PASSPHRASE: 'ytrr89erARAiPE34692iwUMvWqqBvC',
117
+    ENV_SALT: 'MySecretSalt'
118
+})
119
+def test_encrypt_decrypt_using_docs(temp_path):
134 120
     # write the test data to temp file
135 121
     test_data = list(yaml.safe_load_all(TEST_DATA))
136 122
     save_path = os.path.join(temp_path, 'encrypted_secrets_file.yaml')
123
+
124
+    # encrypt documents and validate that they were encrypted
137 125
     doc_mgr = PeglegSecretManagement(docs=test_data)
138 126
     doc_mgr.encrypt_secrets(save_path, 'test_author')
127
+    doc = doc_mgr.documents[0]
128
+    assert doc.is_encrypted()
129
+    assert doc.data['encrypted']['by'] == 'test_author'
130
+
139 131
     # read back the encrypted file
140 132
     with open(save_path) as stream:
141 133
         encrypted_data = list(yaml.safe_load_all(stream))
142
-    # this time pass a list of dicts to peglegSecretManager
134
+
135
+    # decrypt documents and validate that they were decrypted
143 136
     doc_mgr = PeglegSecretManagement(docs=encrypted_data)
144 137
     decrypted_data = doc_mgr.get_decrypted_secrets()
145 138
     assert test_data[0]['data'] == decrypted_data[0]['data']
146 139
     assert test_data[0]['schema'] == decrypted_data[0]['schema']
147
-    assert test_data[0]['metadata']['name'] == decrypted_data[0][
148
-        'metadata']['name']
140
+    assert test_data[0]['metadata']['name'] == decrypted_data[0]['metadata'][
141
+        'name']
149 142
     assert test_data[0]['metadata']['storagePolicy'] == decrypted_data[0][
150 143
         'metadata']['storagePolicy']

+ 0
- 17
tests/unit/engine/test_lint.py View File

@@ -125,23 +125,6 @@ def test_verify_deckhand_render_site_documents_separately(
125 125
                     'storagePolicy': 'cleartext'
126 126
                 },
127 127
                 'schema': 'deckhand/Passphrase/v1'
128
-            }, {
129
-                'data': {
130
-                    'site_type': sitename,
131
-                    'repositories': {
132
-                        'global': mock.ANY
133
-                    }
134
-                },
135
-                'metadata': {
136
-                    'layeringDefinition': {
137
-                        'abstract': False,
138
-                        'layer': 'site'
139
-                    },
140
-                    'name': sitename,
141
-                    'schema': 'metadata/Document/v1',
142
-                    'storagePolicy': 'cleartext'
143
-                },
144
-                'schema': 'pegleg/SiteDefinition/v1'
145 128
             }]
146 129
             expected_documents.extend(documents)
147 130
 

+ 38
- 0
tests/unit/engine/util/test_files.py View File

@@ -0,0 +1,38 @@
1
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License");
4
+# you may not use this file except in compliance with the License.
5
+# You may obtain a copy of the License at
6
+#
7
+#     http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS,
11
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+# See the License for the specific language governing permissions and
13
+# limitations under the License.
14
+
15
+import os
16
+
17
+from pegleg import config
18
+from pegleg.engine.util import files
19
+from tests.unit.fixtures import create_tmp_deployment_files
20
+
21
+
22
+class TestFileHelpers(object):
23
+    def test_read_compatible_file(self, create_tmp_deployment_files):
24
+        path = os.path.join(config.get_site_repo(), 'site', 'cicd', 'secrets',
25
+                            'passphrases', 'cicd-passphrase.yaml')
26
+        documents = files.read(path)
27
+        assert 1 == len(documents)
28
+
29
+    def test_read_incompatible_file(self, create_tmp_deployment_files):
30
+        # NOTE(felipemonteiro): The Pegleg site-definition.yaml is a
31
+        # Deckhand-formatted document currently but probably shouldn't be,
32
+        # because it has no business being in Deckhand. As such, validate that
33
+        # it is ignored.
34
+        path = os.path.join(config.get_site_repo(), 'site', 'cicd',
35
+                            'site-definition.yaml')
36
+        documents = files.read(path)
37
+        assert not documents, ("Documents returned should be empty for "
38
+                               "site-definition.yaml")

Loading…
Cancel
Save