airship
/
pegleg
Code Issues Proposed changes

Add support for globally encrypted secrets 

This patchset adds support for globally encrypted secrets.
Documents with a "site" layer will be encrypted/decrypted with the
standard PEGLEG_PASSPHRASE and PEGLEG_SALT environment variables.

If any secrets exist for the site with a schema of "global_passphrase"
or "global_salt" their values will be captured and used to decrypt
any secrets that do not belong to "site" layer.  If the global keys
do not exist, Pegleg will default to using site keys.

Expected usage:
1. Set site passphrase/salt environment variables
2. Select a global passphrase and salt
3. Use Pegleg's "wrap" command to wrap and encrypt the global keys
4. Encrypt or wrap documents with "global" layer
5. Provide Pegleg path to decrypt

In the case of (4) and (5) Pegleg will determine the correct keys
to use automatically

Change-Id: I5de6d63573619b346fe011628ae21e053e0711f6
This commit is contained in:
HUGHES, ALEXANDER (ah8742) 2019-07-01 11:14:43 -05:00
parent d39c67046b
commit d888b3e138
9 changed files with 319 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
pegleg/cli.py
View File

@ -477,7 +477,14 @@ def wrap_secret_cli(*, site_name, author, filename, output_path, schema, name,
"""
engine.repository.process_repositories(site_name, overwrite_existing=True)
wrap_secret(author, filename, output_path, schema, name, layer, encrypt)
wrap_secret(author,
filename,
output_path,
schema,
name,
layer,
encrypt,
site_name=site_name)
@site.command('genesis_bundle',
@ -624,7 +631,7 @@ def encrypt(*, save_location, author, site_name):
engine.repository.process_repositories(site_name, overwrite_existing=True)
if save_location is None:
save_location = config.get_site_repo()
engine.secrets.encrypt(save_location, author, site_name)
engine.secrets.encrypt(save_location, author, site_name=site_name)
@secrets.command('decrypt',
@ -654,7 +661,7 @@ def encrypt(*, save_location, author, site_name):
def decrypt(*, path, save_location, overwrite, site_name):
engine.repository.process_repositories(site_name)
decrypted = engine.secrets.decrypt(path)
decrypted = engine.secrets.decrypt(path, site_name=site_name)
if overwrite:
for path, data in decrypted.items():
files.write(path, data)

19
pegleg/config.py
View File

@ -19,6 +19,7 @@
import os
from pegleg.engine import exceptions
from pegleg.engine import secrets
try:
if GLOBAL_CONTEXT:
@ -33,6 +34,8 @@ except NameError:
'type_path': 'type',
'passphrase': None,
'salt': None,
'global_passphrase': None,
'global_salt': None,
'salt_min_length': 24,
'passphrase_min_length': 24,
'default_umask': 0o027
@ -195,3 +198,19 @@ def set_salt():
def get_salt():
"""Get the salt for encryption and decryption."""
return GLOBAL_CONTEXT['salt']
def set_global_enc_keys(site_name):
"""Get the global salt and passphrase for encryption."""
GLOBAL_CONTEXT['global_passphrase'], GLOBAL_CONTEXT['global_salt'] = \
secrets.get_global_creds(site_name)
def get_global_passphrase():
"""Get the global passphrase for encryption and decryption."""
return GLOBAL_CONTEXT['global_passphrase']
def get_global_salt():
"""Get the global salt for encryption and decryption."""
return GLOBAL_CONTEXT['global_salt']

8
pegleg/engine/exceptions.py
View File

@ -139,10 +139,18 @@ class SaltInsufficientLengthException(PeglegBaseException):
message = 'PEGLEG_SALT must be at least 24 characters long.'
class GlobalCredentialsNotFound(PeglegBaseException):
"""Exception raised when global_passphrase or global_salt are not found."""
message = ('global_salt and global_passphrase must either both be '
'defined, or neither can be defined in site documents.')
#
# Shipyard Helper Exceptions
#
class InvalidBufferModeException(PeglegBaseException):
"""Exception raised when invalid buffer mode specified"""

78
pegleg/engine/secrets.py
View File

@ -19,7 +19,9 @@ import os
from prettytable import PrettyTable
import yaml
from pegleg import config
from pegleg.engine.catalog.pki_utility import PKIUtility
from pegleg.engine import exceptions
from pegleg.engine.generators.passphrase_generator import PassphraseGenerator
from pegleg.engine.util.cryptostring import CryptoString
from pegleg.engine.util import definition
@ -60,7 +62,8 @@ def encrypt(save_location, author, site_name):
for repo_base, file_path in definition.site_files_by_repo(site_name):
secrets_found = True
PeglegSecretManagement(file_path=file_path,
author=author).encrypt_secrets(
author=author,
site_name=site_name).encrypt_secrets(
_get_dest_path(repo_base, file_path,
save_location))
if secrets_found:
@ -70,7 +73,7 @@ def encrypt(save_location, author, site_name):
'No secret documents were found for site: {}'.format(site_name))
def decrypt(path):
def decrypt(path, site_name=None):
"""Decrypt one secrets file, and print the decrypted file to standard out.
Search the specified file_path for a file.
@ -93,7 +96,8 @@ def decrypt(path):
return file_dict
if os.path.isfile(path):
file_dict[path] = PeglegSecretManagement(path).decrypt_secrets()
file_dict[path] = PeglegSecretManagement(
path, site_name=site_name).decrypt_secrets()
else:
match = os.path.join(path, '**', '*.yaml')
file_list = glob(match, recursive=True)
@ -121,8 +125,8 @@ def _get_dest_path(repo_base, file_path, save_location):
:rtype: string
"""
if (save_location and save_location != os.path.sep and
save_location.endswith(os.path.sep)):
if (save_location and save_location != os.path.sep
and save_location.endswith(os.path.sep)):
save_location = save_location.rstrip(os.path.sep)
if repo_base and repo_base.endswith(os.path.sep):
repo_base = repo_base.rstrip(os.path.sep)
@ -166,7 +170,14 @@ def generate_crypto_string(length):
return CryptoString().get_crypto_string(length)
def wrap_secret(author, filename, output_path, schema, name, layer, encrypt):
def wrap_secret(author,
filename,
output_path,
schema,
name,
layer,
encrypt,
site_name=None):
"""Wrap a bare secrets file in a YAML and ManagedDocument.
:param author: author for ManagedDocument
@ -199,7 +210,9 @@ def wrap_secret(author, filename, output_path, schema, name, layer, encrypt):
}
managed_secret = PeglegManagedSecret(inner_doc, author=author)
if encrypt:
psm = PeglegSecretManagement(docs=[inner_doc], author=author)
psm = PeglegSecretManagement(docs=[inner_doc],
author=author,
site_name=site_name)
output_doc = psm.get_encrypted_secrets()[0][0]
else:
output_doc = managed_secret.pegleg_document
@ -239,3 +252,54 @@ def check_cert_expiry(site_name, duration=60):
# Return table of cert names and expiration dates that are expiring
return cert_table.get_string()
def get_global_creds(site_name):
"""Determine which credentials to use for global secrets.
If a user desires to encrypt site secrets with one set of credentials but
global secrets with a different set of credentials (in the case of
multiple sites) Pegleg needs a way to handle a two-step encryption or
decryption chain. This is accomplished by storing global credentials at
the site level and encrypting them with site credentials. Pegleg will
attempt to find both the global_salt and global_passphrase, decrypt them
then use these credentials for any global encrypt/decrypt operations.
If both global_salt and global_passphrase are found return both.
If only one global credential is found, raise an error with the assumption
the user wishes to use global credentials but does not have both.
If neither are found, return the site credentials with the assumption
the user wishes to encrypt the global documents with the site credentials.
:param str site_name: The target site
:return: Either the global, or site level - passphrase and salt
"""
log_msg = "Multiple documents containing {} detected. Using latest."
global_passphrase = None
global_salt = None
docs = definition.site_files(site_name)
for doc in docs:
with open(doc, 'r') as f:
results = yaml.safe_load_all(f) # Validate valid YAML.
results = PeglegSecretManagement(
docs=results).get_decrypted_secrets()
for result in results:
if result['schema'] == "deckhand/Passphrase/v1":
if result['metadata']['name'] == 'global_passphrase':
if global_passphrase:
LOG.warn(log_msg.format('global_passphrase'))
global_passphrase = result['data'].encode()
if result['metadata']['name'] == 'global_salt':
if global_salt:
LOG.warn(log_msg.format('global_salt'))
global_salt = result['data'].encode()
# Break out of search if both passphrase and salt are found
if global_passphrase and global_salt:
return (global_passphrase, global_salt)
# End of search, determine if we should use site keys or raise an error
if global_passphrase or global_salt:
raise exceptions.GlobalCredentialsNotFound()
else:
return (config.get_passphrase(), config.get_salt())

12
pegleg/engine/util/pegleg_managed_document.py
View File

@ -32,7 +32,6 @@ class PeglegManagedSecretsDocument(object):
"""Object representing one Pegleg managed secret document."""
def __init__(self, document, generated=False, catalog=None, author=None):
"""
Parse and wrap an externally generated document in a
pegleg managed document.
@ -54,8 +53,8 @@ class PeglegManagedSecretsDocument(object):
if self.is_pegleg_managed_secret(document):
self._pegleg_document = document
else:
self._pegleg_document = self.__wrap(
document, generated, catalog, author)
self._pegleg_document = self.__wrap(document, generated, catalog,
author)
self._embedded_document = \
self._pegleg_document['data']['managedDocument']
@ -160,9 +159,7 @@ class PeglegManagedSecretsDocument(object):
def set_encrypted(self, author=None):
"""Mark the pegleg managed document as encrypted."""
self.data[ENCRYPTED] = {
'at': datetime.utcnow().isoformat()
}
self.data[ENCRYPTED] = {'at': datetime.utcnow().isoformat()}
if author:
self.data[ENCRYPTED]['by'] = author
@ -175,3 +172,6 @@ class PeglegManagedSecretsDocument(object):
def get_secret(self):
return self._embedded_document.get('data')
def get_layer(self):
return self._embedded_document[METADATA]['layeringDefinition']['layer']

37
pegleg/engine/util/pegleg_secret_management.py
View File

@ -35,7 +35,8 @@ class PeglegSecretManagement(object):
docs=None,
generated=False,
catalog=None,
author=None):
author=None,
site_name=None):
"""
Read the source file and the environment data needed to wrap and
process the file documents as pegleg managed document.
@ -43,11 +44,16 @@ class PeglegSecretManagement(object):
provided.
"""
# Set passphrase and salt
config.set_passphrase()
self.passphrase = config.get_passphrase()
config.set_salt()
self.salt = config.get_salt()
# Check if we're working with a specific site, if so determine if the
# global encryption keys already exist. If they don't, set them.
if site_name:
if not (config.get_global_passphrase()
and config.get_global_salt()):
config.set_global_enc_keys(site_name)
if all([file_path, docs]) or not any([file_path, docs]):
raise ValueError('Either `file_path` or `docs` must be '
@ -134,10 +140,19 @@ class PeglegSecretManagement(object):
# policies
doc_list.append(doc.embedded_document)
continue
# Get appropriate encryption keys to use
if doc.get_layer() == 'site':
passphrase = config.get_passphrase()
salt = config.get_salt()
else:
passphrase = config.get_global_passphrase()
salt = config.get_global_salt()
secret_doc = doc.get_secret()
if type(secret_doc) != bytes:
secret_doc = secret_doc.encode()
doc.set_secret(encrypt(secret_doc, self.passphrase, self.salt))
doc.set_secret(encrypt(secret_doc, passphrase, salt))
doc.set_encrypted(self._author)
encrypted_docs = True
doc_list.append(doc.pegleg_document)
@ -170,9 +185,17 @@ class PeglegSecretManagement(object):
for doc in self.documents:
# do not decrypt already decrypted data
if doc.is_encrypted():
# Get appropriate encryption keys to use
if doc.get_layer() == 'site':
passphrase = config.get_passphrase()
salt = config.get_salt()
else:
passphrase = config.get_global_passphrase()
salt = config.get_global_salt()
doc.set_secret(
decrypt(doc.get_secret(), self.passphrase,
self.salt).decode())
decrypt(doc.get_secret(), passphrase, salt).decode())
doc.set_decrypted()
doc_list.append(doc.embedded_document)
return doc_list

173
tests/unit/engine/test_secrets.py
View File

@ -51,6 +51,59 @@ data: 512363f37eab654313991174aef9f867d
...
"""
TEST_GLOBAL_DATA = """
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: osh_addons_keystone_ranger-agent_password
layeringDefinition:
abstract: false
layer: global
storagePolicy: encrypted
data: 512363f37eab654313991174aef9f867d
...
"""
GLOBAL_PASSPHRASE_SALT_DOC = """
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: global_passphrase
layeringDefinition:
abstract: false
layer: site
storagePolicy: encrypted
data: TbKYNtM@3gXpL=AFLAwU?&Ey
...
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: global_salt
layeringDefinition:
abstract: false
layer: site
storagePolicy: encrypted
data: h3=DQ#GNYEuCvybgpfW7ZxAP
...
"""
GLOBAL_SALT_DOC = """
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: global_salt
layeringDefinition:
abstract: false
layer: site
storagePolicy: encrypted
data: h3=DQ#GNYEuCvybgpfW7ZxAP
...
"""
def test_encrypt_and_decrypt():
data = test_utils.rand_name("this is an example of un-encrypted "
@ -314,3 +367,123 @@ def test_check_expiry(create_tmp_deployment_files):
assert cert_info['expired'] is False, \
"%s is expired/expiring on %s" % \
(generated_file.name, cert_info['expiry_date'])
@mock.patch.dict(
os.environ, {
'PEGLEG_PASSPHRASE': 'ytrr89erARAiPE34692iwUMvWqqBvC',
'PEGLEG_SALT': 'MySecretSalt1234567890]['
})
def test_get_global_creds_missing_creds(create_tmp_deployment_files, tmpdir):
# Create site files
site_dir = tmpdir.join("deployment_files", "site", "cicd")
save_location = tmpdir.mkdir("encrypted_site_files")
save_location_str = str(save_location)
# Capture global credentials, verify they are not present and we default
# to site credentials instead.
passphrase, salt = secrets.get_global_creds("cicd")
assert passphrase.decode() == 'ytrr89erARAiPE34692iwUMvWqqBvC'
assert salt.decode() == 'MySecretSalt1234567890]['
@mock.patch.dict(
os.environ, {
'PEGLEG_PASSPHRASE': 'ytrr89erARAiPE34692iwUMvWqqBvC',
'PEGLEG_SALT': 'MySecretSalt1234567890]['
})
def test_get_global_creds_missing_pass(create_tmp_deployment_files, tmpdir):
# Create site files
site_dir = tmpdir.join("deployment_files", "site", "cicd")
# Create global salt file
with open(
os.path.join(str(site_dir), 'secrets', 'passphrases',
'cicd-global-passphrase-encrypted.yaml'),
"w") as outfile:
outfile.write(GLOBAL_SALT_DOC)
save_location = tmpdir.mkdir("encrypted_site_files")
save_location_str = str(save_location)
# Demonstrate that encryption fails when only the global salt or
# only the global passphrase are present among the site files.
with pytest.raises(exceptions.GlobalCredentialsNotFound):
secrets.encrypt(save_location_str, "pytest", "cicd")
@mock.patch.dict(
os.environ, {
'PEGLEG_PASSPHRASE': 'ytrr89erARAiPE34692iwUMvWqqBvC',
'PEGLEG_SALT': 'MySecretSalt1234567890]['
})
def test_get_global_creds(create_tmp_deployment_files, tmpdir):
# Create site files
site_dir = tmpdir.join("deployment_files", "site", "cicd")
# Create global passphrase and salt file
with open(os.path.join(str(site_dir), 'secrets',
'passphrases',
'cicd-global-passphrase-encrypted.yaml'), "w") \
as outfile:
outfile.write(GLOBAL_PASSPHRASE_SALT_DOC)
save_location = tmpdir.mkdir("encrypted_site_files")
save_location_str = str(save_location)
# Encrypt the global passphrase and salt file using site passphrase/salt
secrets.encrypt(save_location_str, "pytest", "cicd")
encrypted_files = listdir(save_location_str)
assert len(encrypted_files) > 0
# Capture global credentials, verify we have the right ones
passphrase, salt = secrets.get_global_creds("cicd")
assert passphrase.decode() == "TbKYNtM@3gXpL=AFLAwU?&Ey"
assert salt.decode() == "h3=DQ#GNYEuCvybgpfW7ZxAP"
@mock.patch.dict(
os.environ, {
'PEGLEG_PASSPHRASE': 'ytrr89erARAiPE34692iwUMvWqqBvC',
'PEGLEG_SALT': 'MySecretSalt1234567890]['
})
def test_global_encrypt_decrypt(create_tmp_deployment_files, tmpdir):
# Create site files
site_dir = tmpdir.join("deployment_files", "site", "cicd")
# Create and encrypt global passphrase and salt file using site keys
with open(os.path.join(str(site_dir), 'secrets',
'passphrases',
'cicd-global-passphrase-encrypted.yaml'), "w") \
as outfile:
outfile.write(GLOBAL_PASSPHRASE_SALT_DOC)
save_location = tmpdir.mkdir("encrypted_site_files")
save_location_str = str(save_location)
# Encrypt the global passphrase and salt file using site passphrase/salt
secrets.encrypt(save_location_str, "pytest", "cicd")
# Create and encrypt a global type document
global_doc_path = os.path.join(str(site_dir), 'secrets', 'passphrases',
'globally_encrypted_doc.yaml')
with open(global_doc_path, "w") as outfile:
outfile.write(TEST_GLOBAL_DATA)
# encrypt documents and validate that they were encrypted
doc_mgr = PeglegSecretManagement(file_path=global_doc_path,
author='pytest',
site_name='cicd')
doc_mgr.encrypt_secrets(global_doc_path)
doc = doc_mgr.documents[0]
assert doc.is_encrypted()
assert doc.data['encrypted']['by'] == 'pytest'
doc_mgr = PeglegSecretManagement(file_path=global_doc_path,
author='pytest',
site_name='cicd')
decrypted_data = doc_mgr.get_decrypted_secrets()
test_data = list(yaml.safe_load_all(TEST_GLOBAL_DATA))
assert test_data[0]['data'] == decrypted_data[0]['data']
assert test_data[0]['schema'] == decrypted_data[0]['schema']

1
tests/unit/test_cli.py
View File

@ -558,6 +558,7 @@ class TestSiteSecretsActions(BaseCLIActionTest):
with open(file_path, "r") as ceph_fsid_fi:
ceph_fsid = yaml.safe_load(ceph_fsid_fi)
ceph_fsid["metadata"]["storagePolicy"] = "encrypted"
ceph_fsid["metadata"]["layeringDefinition"]["layer"] = "site"
with open(file_path, "w") as ceph_fsid_fi:
yaml.dump(ceph_fsid, ceph_fsid_fi)

2
tox.ini
View File

@ -97,6 +97,6 @@ enable-extensions = H106,H201,H904
# [H403] multi line docstrings should end on a new line
# [H404] multi line docstring should start without a leading new line
# [H405] multi line docstring summary not separated with an empty line
ignore = H403,H404,H405
ignore = H403,H404,H405,W503
exclude=.venv,.git,.tox,build,dist,*lib/python*,*egg,tools,*.ini,*.po,*.pot
max-complexity = 24