airship/pegleg
Code Issues Proposed changes

Only collect/parse Deckhand-formatted documents for processing

Browse Source 
This patch set changes Pegleg in two similar ways:

1) Ignore certain types of files altogether:
   - those located in hidden folders
   - those prefixed with "." (files like .zuul.yaml)
2) Only read Deckhand-formatted documents for lint/collect/etc.
   commands as Pegleg need not consider other types of documents
   (it separately reads the site-definition.yaml for internal
    processing still).

The tools/ subfolder is also ignored as it can contain
.yaml files which are not Deckhand-formatted documents,
so need not be processed by pegleg.engine.

Change-Id: I8996b5d430cf893122af648ef8e5805b36c1bfd9
This commit is contained in:
Felipe Monteiro 2018-10-31 15:27:38 -04:00
parent d7740b0f40
commit f8d79e119c
8 changed files with 133 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pegleg/cli.py
View File

@ -385,7 +385,7 @@ def secrets():
'author',
required=True,
help='Identifier for the program or person who is encrypting the secrets '
'documents')
'documents')
@click.argument('site_name')
def encrypt(*, save_location, author, site_name):
engine.repository.process_repositories(site_name)

19
pegleg/engine/lint.py
View File

@ -18,7 +18,6 @@ import os
import pkg_resources
import shutil
import textwrap
import yaml
from prettytable import PrettyTable
@ -223,16 +222,16 @@ def _verify_single_file(filename, schemas):
errors.append((FILE_MISSING_YAML_DOCUMENT_HEADER,
'%s does not begin with YAML beginning of document '
'marker "---".' % filename))
f.seek(0)
documents = []
try:
documents = list(yaml.safe_load_all(f))
except Exception as e:
errors.append((FILE_CONTAINS_INVALID_YAML,
'%s is not valid yaml: %s' % (filename, e)))
for document in documents:
errors.extend(_verify_document(document, schemas, filename))
documents = []
try:
documents = util.files.read(filename)
except Exception as e:
errors.append((FILE_CONTAINS_INVALID_YAML,
'%s is not valid yaml: %s' % (filename, e)))
for document in documents:
errors.extend(_verify_document(document, schemas, filename))
return errors

8
pegleg/engine/util/definition.py
View File

@ -15,7 +15,6 @@
import os
import click
import yaml
from pegleg import config
from pegleg.engine.util import files
@ -52,6 +51,7 @@ def load_as_params(site_name, *fields, primary_repo_base=None):
def path(site_name, primary_repo_base=None):
"""Retrieve path to the site-definition.yaml file for ``site_name``."""
if not primary_repo_base:
primary_repo_base = config.get_site_repo()
return os.path.join(primary_repo_base, 'site', site_name,
@ -100,8 +100,7 @@ def documents_for_each_site():
paths = files.directories_for(**params)
filenames = set(files.search(paths))
for filename in filenames:
with open(filename) as f:
documents[sitename].extend(list(yaml.safe_load_all(f)))
documents[sitename].extend(files.read(filename))
return documents
@ -122,7 +121,6 @@ def documents_for_site(sitename):
paths = files.directories_for(**params)
filenames = set(files.search(paths))
for filename in filenames:
with open(filename) as f:
documents.extend(list(yaml.safe_load_all(f)))
documents.extend(files.read(filename))
return documents

46
pegleg/engine/util/files.py
View File

@ -18,6 +18,7 @@ import yaml
import logging
from pegleg import config
from pegleg.engine.util import pegleg_managed_document as md
LOG = logging.getLogger(__name__)
@ -248,9 +249,35 @@ def read(path):
'{} not found. Pegleg must be run from the root of a '
'configuration repository.'.format(path))
def is_deckhand_document(document):
# Deckhand documents only consist of control and application
# documents.
valid_schemas = ('metadata/Control', 'metadata/Document')
if isinstance(document, dict):
schema = document.get('metadata', {}).get('schema', '')
# NOTE(felipemonteiro): The Pegleg site-definition.yaml is a
# Deckhand-formatted document currently but probably shouldn't
# be, because it has no business being in Deckhand. As such,
# treat it as a special case.
if "SiteDefinition" in document.get('schema', ''):
return False
if any(schema.startswith(x) for x in valid_schemas):
return True
else:
LOG.debug('Document with schema=%s is not a valid Deckhand '
'schema. Ignoring it.', schema)
return False
def is_pegleg_managed_document(document):
return md.PeglegManagedSecretsDocument.is_pegleg_managed_secret(
document)
with open(path) as stream:
try:
return list(yaml.safe_load_all(stream))
return [
d for d in yaml.safe_load_all(stream)
if is_deckhand_document(d) or is_pegleg_managed_document(d)
]
except yaml.YAMLError as e:
raise click.ClickException('Failed to parse %s:\n%s' % (path, e))
@ -296,10 +323,25 @@ def _recurse_subdirs(search_path, depth):
def search(search_paths):
if not isinstance(search_paths, (list, tuple)):
search_paths = [search_paths]
for search_path in search_paths:
LOG.debug("Recursively collecting YAMLs from %s" % search_path)
for root, _dirs, filenames in os.walk(search_path):
for root, _, filenames in os.walk(search_path):
# Ignore hidden folders like .tox or .git for faster processing.
if os.path.basename(root).startswith("."):
continue
# Skip over anything in tools/ because it will never contain valid
# Pegleg-owned manifest documents.
if "tools" in root.split("/"):
continue
for filename in filenames:
# Ignore files like .zuul.yaml.
if filename.startswith("."):
continue
if filename.endswith(".yaml"):
yield os.path.join(root, filename)

8
pegleg/engine/util/pegleg_secret_management.py
View File

@ -44,8 +44,7 @@ class PeglegSecretManagement():
if all([file_path, docs]) or \
not any([file_path, docs]):
raise ValueError(
'Either `file_path` or `docs` must be specified.')
raise ValueError('Either `file_path` or `docs` must be specified.')
self.__check_environment()
self.file_path = file_path
@ -73,7 +72,7 @@ class PeglegSecretManagement():
# Verify that passphrase environment variable is defined and is longer
# than 24 characters.
if not os.environ.get(ENV_PASSPHRASE) or not re.match(
PASSPHRASE_PATTERN, os.environ.get(ENV_PASSPHRASE)):
PASSPHRASE_PATTERN, os.environ.get(ENV_PASSPHRASE)):
raise click.ClickException(
'Environment variable {} is not defined or '
'is not at least 24-character long.'.format(ENV_PASSPHRASE))
@ -154,8 +153,7 @@ class PeglegSecretManagement():
# do not decrypt already decrypted data
if doc.is_encrypted():
doc.set_secret(
decrypt(doc.get_secret(),
self.passphrase,
decrypt(doc.get_secret(), self.passphrase,
self.salt).decode())
doc.set_decrypted()
doc_list.append(doc.embedded_document)

77
tests/unit/engine/test_encryption.py
View File

@ -30,7 +30,6 @@ from pegleg.engine.util.pegleg_secret_management import ENV_SALT
from tests.unit.fixtures import temp_path
from pegleg.engine.util import files
TEST_DATA = """
---
schema: deckhand/Passphrase/v1
@ -60,22 +59,24 @@ def test_encrypt_and_decrypt():
@mock.patch.dict(os.environ, {
ENV_PASSPHRASE:'aShortPassphrase',
ENV_SALT: 'MySecretSalt'})
ENV_PASSPHRASE: 'aShortPassphrase',
ENV_SALT: 'MySecretSalt'
})
def test_short_passphrase():
with pytest.raises(click.ClickException,
match=r'.*is not at least 24-character long.*'):
with pytest.raises(
click.ClickException,
match=r'.*is not at least 24-character long.*'):
PeglegSecretManagement('file_path')
def test_PeglegManagedDocument():
def test_pegleg_secret_management_constructor():
test_data = yaml.load(TEST_DATA)
doc = PeglegManagedSecretsDocument(test_data)
assert doc.is_storage_policy_encrypted() is True
assert doc.is_encrypted() is False
assert doc.is_storage_policy_encrypted()
assert not doc.is_encrypted()
def test_PeglegSecretManagement():
def test_pegleg_secret_management_constructor_with_invalid_arguments():
with pytest.raises(ValueError) as err_info:
PeglegSecretManagement(file_path=None, docs=None)
assert 'Either `file_path` or `docs` must be specified.' in str(
@ -87,40 +88,24 @@ def test_PeglegSecretManagement():
@mock.patch.dict(os.environ, {
ENV_PASSPHRASE:'ytrr89erARAiPE34692iwUMvWqqBvC',
ENV_SALT: 'MySecretSalt'})
def test_encrypt_file():
# write the test data to temp file
test_data = yaml.load(TEST_DATA)
dir = tempfile.mkdtemp()
file_path = os.path.join(dir, 'secrets_file.yaml')
save_path = os.path.join(dir, 'encrypted_secrets_file.yaml')
with open(file_path, 'w') as stream:
yaml.dump(test_data,
stream,
explicit_start=True,
explicit_end=True,
default_flow_style=False)
# read back the secrets data file and encrypt it
doc_mgr = PeglegSecretManagement(file_path)
doc_mgr.encrypt_secrets(save_path, 'test_author')
doc = doc_mgr.documents[0]
assert doc.is_encrypted()
assert doc.data['encrypted']['by'] == 'test_author'
@mock.patch.dict(os.environ, {
ENV_PASSPHRASE:'ytrr89erARAiPE34692iwUMvWqqBvC',
ENV_SALT: 'MySecretSalt'})
def test_encrypt_decrypt_file(temp_path):
ENV_PASSPHRASE: 'ytrr89erARAiPE34692iwUMvWqqBvC',
ENV_SALT: 'MySecretSalt'
})
def test_encrypt_decrypt_using_file_path(temp_path):
# write the test data to temp file
test_data = list(yaml.safe_load_all(TEST_DATA))
file_path = os.path.join(temp_path, 'secrets_file.yaml')
files.write(file_path, test_data)
save_path = os.path.join(temp_path, 'encrypted_secrets_file.yaml')
# encrypt documents and validate that they were encrypted
doc_mgr = PeglegSecretManagement(file_path=file_path)
doc_mgr.encrypt_secrets(save_path, 'test_author')
# read back the encrypted file
doc = doc_mgr.documents[0]
assert doc.is_encrypted()
assert doc.data['encrypted']['by'] == 'test_author'
# decrypt documents and validate that they were decrypted
doc_mgr = PeglegSecretManagement(save_path)
decrypted_data = doc_mgr.get_decrypted_secrets()
assert test_data[0]['data'] == decrypted_data[0]['data']
@ -128,23 +113,31 @@ def test_encrypt_decrypt_file(temp_path):
@mock.patch.dict(os.environ, {
ENV_PASSPHRASE:'ytrr89erARAiPE34692iwUMvWqqBvC',
ENV_SALT: 'MySecretSalt'})
def test_decrypt_document(temp_path):
ENV_PASSPHRASE: 'ytrr89erARAiPE34692iwUMvWqqBvC',
ENV_SALT: 'MySecretSalt'
})
def test_encrypt_decrypt_using_docs(temp_path):
# write the test data to temp file
test_data = list(yaml.safe_load_all(TEST_DATA))
save_path = os.path.join(temp_path, 'encrypted_secrets_file.yaml')
# encrypt documents and validate that they were encrypted
doc_mgr = PeglegSecretManagement(docs=test_data)
doc_mgr.encrypt_secrets(save_path, 'test_author')
doc = doc_mgr.documents[0]
assert doc.is_encrypted()
assert doc.data['encrypted']['by'] == 'test_author'
# read back the encrypted file
with open(save_path) as stream:
encrypted_data = list(yaml.safe_load_all(stream))
# this time pass a list of dicts to peglegSecretManager
# decrypt documents and validate that they were decrypted
doc_mgr = PeglegSecretManagement(docs=encrypted_data)
decrypted_data = doc_mgr.get_decrypted_secrets()
assert test_data[0]['data'] == decrypted_data[0]['data']
assert test_data[0]['schema'] == decrypted_data[0]['schema']
assert test_data[0]['metadata']['name'] == decrypted_data[0][
'metadata']['name']
assert test_data[0]['metadata']['name'] == decrypted_data[0]['metadata'][
'name']
assert test_data[0]['metadata']['storagePolicy'] == decrypted_data[0][
'metadata']['storagePolicy']

17
tests/unit/engine/test_lint.py
View File

@ -125,23 +125,6 @@ def test_verify_deckhand_render_site_documents_separately(
'storagePolicy': 'cleartext'
},
'schema': 'deckhand/Passphrase/v1'
}, {
'data': {
'site_type': sitename,
'repositories': {
'global': mock.ANY
}
},
'metadata': {
'layeringDefinition': {
'abstract': False,
'layer': 'site'
},
'name': sitename,
'schema': 'metadata/Document/v1',
'storagePolicy': 'cleartext'
},
'schema': 'pegleg/SiteDefinition/v1'
}]
expected_documents.extend(documents)

38
tests/unit/engine/util/test_files.py Normal file
View File

@ -0,0 +1,38 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from pegleg import config
from pegleg.engine.util import files
from tests.unit.fixtures import create_tmp_deployment_files
class TestFileHelpers(object):
def test_read_compatible_file(self, create_tmp_deployment_files):
path = os.path.join(config.get_site_repo(), 'site', 'cicd', 'secrets',
'passphrases', 'cicd-passphrase.yaml')
documents = files.read(path)
assert 1 == len(documents)
def test_read_incompatible_file(self, create_tmp_deployment_files):
# NOTE(felipemonteiro): The Pegleg site-definition.yaml is a
# Deckhand-formatted document currently but probably shouldn't be,
# because it has no business being in Deckhand. As such, validate that
# it is ignored.
path = os.path.join(config.get_site_repo(), 'site', 'cicd',
'site-definition.yaml')
documents = files.read(path)
assert not documents, ("Documents returned should be empty for "
"site-definition.yaml")