From d888b3e1386705e57d8034a73db65430343fae13 Mon Sep 17 00:00:00 2001 From: "HUGHES, ALEXANDER (ah8742)" Date: Mon, 1 Jul 2019 11:14:43 -0500 Subject: [PATCH] Add support for globally encrypted secrets This patchset adds support for globally encrypted secrets. Documents with a "site" layer will be encrypted/decrypted with the standard PEGLEG_PASSPHRASE and PEGLEG_SALT environment variables. If any secrets exist for the site with a schema of "global_passphrase" or "global_salt" their values will be captured and used to decrypt any secrets that do not belong to "site" layer. If the global keys do not exist, Pegleg will default to using site keys. Expected usage: 1. Set site passphrase/salt environment variables 2. Select a global passphrase and salt 3. Use Pegleg's "wrap" command to wrap and encrypt the global keys 4. Encrypt or wrap documents with "global" layer 5. Provide Pegleg path to decrypt In the case of (4) and (5) Pegleg will determine the correct keys to use automatically Change-Id: I5de6d63573619b346fe011628ae21e053e0711f6 --- pegleg/cli.py | 13 +- pegleg/config.py | 19 ++ pegleg/engine/exceptions.py | 8 + pegleg/engine/secrets.py | 78 +++++++- pegleg/engine/util/pegleg_managed_document.py | 12 +- .../engine/util/pegleg_secret_management.py | 37 +++- tests/unit/engine/test_secrets.py | 173 ++++++++++++++++++ tests/unit/test_cli.py | 1 + tox.ini | 2 +- 9 files changed, 319 insertions(+), 24 deletions(-) diff --git a/pegleg/cli.py b/pegleg/cli.py index a9070ec0..7e02a4df 100644 --- a/pegleg/cli.py +++ b/pegleg/cli.py @@ -477,7 +477,14 @@ def wrap_secret_cli(*, site_name, author, filename, output_path, schema, name, """ engine.repository.process_repositories(site_name, overwrite_existing=True) - wrap_secret(author, filename, output_path, schema, name, layer, encrypt) + wrap_secret(author, + filename, + output_path, + schema, + name, + layer, + encrypt, + site_name=site_name) @site.command('genesis_bundle', @@ -624,7 +631,7 @@ def encrypt(*, save_location, author, site_name): engine.repository.process_repositories(site_name, overwrite_existing=True) if save_location is None: save_location = config.get_site_repo() - engine.secrets.encrypt(save_location, author, site_name) + engine.secrets.encrypt(save_location, author, site_name=site_name) @secrets.command('decrypt', @@ -654,7 +661,7 @@ def encrypt(*, save_location, author, site_name): def decrypt(*, path, save_location, overwrite, site_name): engine.repository.process_repositories(site_name) - decrypted = engine.secrets.decrypt(path) + decrypted = engine.secrets.decrypt(path, site_name=site_name) if overwrite: for path, data in decrypted.items(): files.write(path, data) diff --git a/pegleg/config.py b/pegleg/config.py index f7791ef2..ce60afbf 100644 --- a/pegleg/config.py +++ b/pegleg/config.py @@ -19,6 +19,7 @@ import os from pegleg.engine import exceptions +from pegleg.engine import secrets try: if GLOBAL_CONTEXT: @@ -33,6 +34,8 @@ except NameError: 'type_path': 'type', 'passphrase': None, 'salt': None, + 'global_passphrase': None, + 'global_salt': None, 'salt_min_length': 24, 'passphrase_min_length': 24, 'default_umask': 0o027 @@ -195,3 +198,19 @@ def set_salt(): def get_salt(): """Get the salt for encryption and decryption.""" return GLOBAL_CONTEXT['salt'] + + +def set_global_enc_keys(site_name): + """Get the global salt and passphrase for encryption.""" + GLOBAL_CONTEXT['global_passphrase'], GLOBAL_CONTEXT['global_salt'] = \ + secrets.get_global_creds(site_name) + + +def get_global_passphrase(): + """Get the global passphrase for encryption and decryption.""" + return GLOBAL_CONTEXT['global_passphrase'] + + +def get_global_salt(): + """Get the global salt for encryption and decryption.""" + return GLOBAL_CONTEXT['global_salt'] diff --git a/pegleg/engine/exceptions.py b/pegleg/engine/exceptions.py index f1d00ec6..829491a5 100644 --- a/pegleg/engine/exceptions.py +++ b/pegleg/engine/exceptions.py @@ -139,10 +139,18 @@ class SaltInsufficientLengthException(PeglegBaseException): message = 'PEGLEG_SALT must be at least 24 characters long.' +class GlobalCredentialsNotFound(PeglegBaseException): + """Exception raised when global_passphrase or global_salt are not found.""" + + message = ('global_salt and global_passphrase must either both be ' + 'defined, or neither can be defined in site documents.') + + # # Shipyard Helper Exceptions # + class InvalidBufferModeException(PeglegBaseException): """Exception raised when invalid buffer mode specified""" diff --git a/pegleg/engine/secrets.py b/pegleg/engine/secrets.py index 6c3e122b..68154cf3 100644 --- a/pegleg/engine/secrets.py +++ b/pegleg/engine/secrets.py @@ -19,7 +19,9 @@ import os from prettytable import PrettyTable import yaml +from pegleg import config from pegleg.engine.catalog.pki_utility import PKIUtility +from pegleg.engine import exceptions from pegleg.engine.generators.passphrase_generator import PassphraseGenerator from pegleg.engine.util.cryptostring import CryptoString from pegleg.engine.util import definition @@ -60,7 +62,8 @@ def encrypt(save_location, author, site_name): for repo_base, file_path in definition.site_files_by_repo(site_name): secrets_found = True PeglegSecretManagement(file_path=file_path, - author=author).encrypt_secrets( + author=author, + site_name=site_name).encrypt_secrets( _get_dest_path(repo_base, file_path, save_location)) if secrets_found: @@ -70,7 +73,7 @@ def encrypt(save_location, author, site_name): 'No secret documents were found for site: {}'.format(site_name)) -def decrypt(path): +def decrypt(path, site_name=None): """Decrypt one secrets file, and print the decrypted file to standard out. Search the specified file_path for a file. @@ -93,7 +96,8 @@ def decrypt(path): return file_dict if os.path.isfile(path): - file_dict[path] = PeglegSecretManagement(path).decrypt_secrets() + file_dict[path] = PeglegSecretManagement( + path, site_name=site_name).decrypt_secrets() else: match = os.path.join(path, '**', '*.yaml') file_list = glob(match, recursive=True) @@ -121,8 +125,8 @@ def _get_dest_path(repo_base, file_path, save_location): :rtype: string """ - if (save_location and save_location != os.path.sep and - save_location.endswith(os.path.sep)): + if (save_location and save_location != os.path.sep + and save_location.endswith(os.path.sep)): save_location = save_location.rstrip(os.path.sep) if repo_base and repo_base.endswith(os.path.sep): repo_base = repo_base.rstrip(os.path.sep) @@ -166,7 +170,14 @@ def generate_crypto_string(length): return CryptoString().get_crypto_string(length) -def wrap_secret(author, filename, output_path, schema, name, layer, encrypt): +def wrap_secret(author, + filename, + output_path, + schema, + name, + layer, + encrypt, + site_name=None): """Wrap a bare secrets file in a YAML and ManagedDocument. :param author: author for ManagedDocument @@ -199,7 +210,9 @@ def wrap_secret(author, filename, output_path, schema, name, layer, encrypt): } managed_secret = PeglegManagedSecret(inner_doc, author=author) if encrypt: - psm = PeglegSecretManagement(docs=[inner_doc], author=author) + psm = PeglegSecretManagement(docs=[inner_doc], + author=author, + site_name=site_name) output_doc = psm.get_encrypted_secrets()[0][0] else: output_doc = managed_secret.pegleg_document @@ -239,3 +252,54 @@ def check_cert_expiry(site_name, duration=60): # Return table of cert names and expiration dates that are expiring return cert_table.get_string() + + +def get_global_creds(site_name): + """Determine which credentials to use for global secrets. + + If a user desires to encrypt site secrets with one set of credentials but + global secrets with a different set of credentials (in the case of + multiple sites) Pegleg needs a way to handle a two-step encryption or + decryption chain. This is accomplished by storing global credentials at + the site level and encrypting them with site credentials. Pegleg will + attempt to find both the global_salt and global_passphrase, decrypt them + then use these credentials for any global encrypt/decrypt operations. + If both global_salt and global_passphrase are found return both. + If only one global credential is found, raise an error with the assumption + the user wishes to use global credentials but does not have both. + If neither are found, return the site credentials with the assumption + the user wishes to encrypt the global documents with the site credentials. + + :param str site_name: The target site + :return: Either the global, or site level - passphrase and salt + """ + + log_msg = "Multiple documents containing {} detected. Using latest." + global_passphrase = None + global_salt = None + docs = definition.site_files(site_name) + for doc in docs: + with open(doc, 'r') as f: + results = yaml.safe_load_all(f) # Validate valid YAML. + results = PeglegSecretManagement( + docs=results).get_decrypted_secrets() + for result in results: + if result['schema'] == "deckhand/Passphrase/v1": + if result['metadata']['name'] == 'global_passphrase': + if global_passphrase: + LOG.warn(log_msg.format('global_passphrase')) + global_passphrase = result['data'].encode() + if result['metadata']['name'] == 'global_salt': + if global_salt: + LOG.warn(log_msg.format('global_salt')) + global_salt = result['data'].encode() + + # Break out of search if both passphrase and salt are found + if global_passphrase and global_salt: + return (global_passphrase, global_salt) + + # End of search, determine if we should use site keys or raise an error + if global_passphrase or global_salt: + raise exceptions.GlobalCredentialsNotFound() + else: + return (config.get_passphrase(), config.get_salt()) diff --git a/pegleg/engine/util/pegleg_managed_document.py b/pegleg/engine/util/pegleg_managed_document.py index 470315cc..454a159f 100644 --- a/pegleg/engine/util/pegleg_managed_document.py +++ b/pegleg/engine/util/pegleg_managed_document.py @@ -32,7 +32,6 @@ class PeglegManagedSecretsDocument(object): """Object representing one Pegleg managed secret document.""" def __init__(self, document, generated=False, catalog=None, author=None): - """ Parse and wrap an externally generated document in a pegleg managed document. @@ -54,8 +53,8 @@ class PeglegManagedSecretsDocument(object): if self.is_pegleg_managed_secret(document): self._pegleg_document = document else: - self._pegleg_document = self.__wrap( - document, generated, catalog, author) + self._pegleg_document = self.__wrap(document, generated, catalog, + author) self._embedded_document = \ self._pegleg_document['data']['managedDocument'] @@ -160,9 +159,7 @@ class PeglegManagedSecretsDocument(object): def set_encrypted(self, author=None): """Mark the pegleg managed document as encrypted.""" - self.data[ENCRYPTED] = { - 'at': datetime.utcnow().isoformat() - } + self.data[ENCRYPTED] = {'at': datetime.utcnow().isoformat()} if author: self.data[ENCRYPTED]['by'] = author @@ -175,3 +172,6 @@ class PeglegManagedSecretsDocument(object): def get_secret(self): return self._embedded_document.get('data') + + def get_layer(self): + return self._embedded_document[METADATA]['layeringDefinition']['layer'] diff --git a/pegleg/engine/util/pegleg_secret_management.py b/pegleg/engine/util/pegleg_secret_management.py index 3ad5f019..7218ed2c 100644 --- a/pegleg/engine/util/pegleg_secret_management.py +++ b/pegleg/engine/util/pegleg_secret_management.py @@ -35,7 +35,8 @@ class PeglegSecretManagement(object): docs=None, generated=False, catalog=None, - author=None): + author=None, + site_name=None): """ Read the source file and the environment data needed to wrap and process the file documents as pegleg managed document. @@ -43,11 +44,16 @@ class PeglegSecretManagement(object): provided. """ + # Set passphrase and salt config.set_passphrase() - self.passphrase = config.get_passphrase() - config.set_salt() - self.salt = config.get_salt() + + # Check if we're working with a specific site, if so determine if the + # global encryption keys already exist. If they don't, set them. + if site_name: + if not (config.get_global_passphrase() + and config.get_global_salt()): + config.set_global_enc_keys(site_name) if all([file_path, docs]) or not any([file_path, docs]): raise ValueError('Either `file_path` or `docs` must be ' @@ -134,10 +140,19 @@ class PeglegSecretManagement(object): # policies doc_list.append(doc.embedded_document) continue + + # Get appropriate encryption keys to use + if doc.get_layer() == 'site': + passphrase = config.get_passphrase() + salt = config.get_salt() + else: + passphrase = config.get_global_passphrase() + salt = config.get_global_salt() + secret_doc = doc.get_secret() if type(secret_doc) != bytes: secret_doc = secret_doc.encode() - doc.set_secret(encrypt(secret_doc, self.passphrase, self.salt)) + doc.set_secret(encrypt(secret_doc, passphrase, salt)) doc.set_encrypted(self._author) encrypted_docs = True doc_list.append(doc.pegleg_document) @@ -170,9 +185,17 @@ class PeglegSecretManagement(object): for doc in self.documents: # do not decrypt already decrypted data if doc.is_encrypted(): + + # Get appropriate encryption keys to use + if doc.get_layer() == 'site': + passphrase = config.get_passphrase() + salt = config.get_salt() + else: + passphrase = config.get_global_passphrase() + salt = config.get_global_salt() + doc.set_secret( - decrypt(doc.get_secret(), self.passphrase, - self.salt).decode()) + decrypt(doc.get_secret(), passphrase, salt).decode()) doc.set_decrypted() doc_list.append(doc.embedded_document) return doc_list diff --git a/tests/unit/engine/test_secrets.py b/tests/unit/engine/test_secrets.py index 1596cb91..f7a0d9da 100644 --- a/tests/unit/engine/test_secrets.py +++ b/tests/unit/engine/test_secrets.py @@ -51,6 +51,59 @@ data: 512363f37eab654313991174aef9f867d ... """ +TEST_GLOBAL_DATA = """ +--- +schema: deckhand/Passphrase/v1 +metadata: + schema: metadata/Document/v1 + name: osh_addons_keystone_ranger-agent_password + layeringDefinition: + abstract: false + layer: global + storagePolicy: encrypted +data: 512363f37eab654313991174aef9f867d +... +""" + +GLOBAL_PASSPHRASE_SALT_DOC = """ +--- +schema: deckhand/Passphrase/v1 +metadata: + schema: metadata/Document/v1 + name: global_passphrase + layeringDefinition: + abstract: false + layer: site + storagePolicy: encrypted +data: TbKYNtM@3gXpL=AFLAwU?&Ey +... +--- +schema: deckhand/Passphrase/v1 +metadata: + schema: metadata/Document/v1 + name: global_salt + layeringDefinition: + abstract: false + layer: site + storagePolicy: encrypted +data: h3=DQ#GNYEuCvybgpfW7ZxAP +... +""" + +GLOBAL_SALT_DOC = """ +--- +schema: deckhand/Passphrase/v1 +metadata: + schema: metadata/Document/v1 + name: global_salt + layeringDefinition: + abstract: false + layer: site + storagePolicy: encrypted +data: h3=DQ#GNYEuCvybgpfW7ZxAP +... +""" + def test_encrypt_and_decrypt(): data = test_utils.rand_name("this is an example of un-encrypted " @@ -314,3 +367,123 @@ def test_check_expiry(create_tmp_deployment_files): assert cert_info['expired'] is False, \ "%s is expired/expiring on %s" % \ (generated_file.name, cert_info['expiry_date']) + + +@mock.patch.dict( + os.environ, { + 'PEGLEG_PASSPHRASE': 'ytrr89erARAiPE34692iwUMvWqqBvC', + 'PEGLEG_SALT': 'MySecretSalt1234567890][' + }) +def test_get_global_creds_missing_creds(create_tmp_deployment_files, tmpdir): + # Create site files + site_dir = tmpdir.join("deployment_files", "site", "cicd") + + save_location = tmpdir.mkdir("encrypted_site_files") + save_location_str = str(save_location) + + # Capture global credentials, verify they are not present and we default + # to site credentials instead. + passphrase, salt = secrets.get_global_creds("cicd") + assert passphrase.decode() == 'ytrr89erARAiPE34692iwUMvWqqBvC' + assert salt.decode() == 'MySecretSalt1234567890][' + + +@mock.patch.dict( + os.environ, { + 'PEGLEG_PASSPHRASE': 'ytrr89erARAiPE34692iwUMvWqqBvC', + 'PEGLEG_SALT': 'MySecretSalt1234567890][' + }) +def test_get_global_creds_missing_pass(create_tmp_deployment_files, tmpdir): + # Create site files + site_dir = tmpdir.join("deployment_files", "site", "cicd") + + # Create global salt file + with open( + os.path.join(str(site_dir), 'secrets', 'passphrases', + 'cicd-global-passphrase-encrypted.yaml'), + "w") as outfile: + outfile.write(GLOBAL_SALT_DOC) + + save_location = tmpdir.mkdir("encrypted_site_files") + save_location_str = str(save_location) + + # Demonstrate that encryption fails when only the global salt or + # only the global passphrase are present among the site files. + with pytest.raises(exceptions.GlobalCredentialsNotFound): + secrets.encrypt(save_location_str, "pytest", "cicd") + + +@mock.patch.dict( + os.environ, { + 'PEGLEG_PASSPHRASE': 'ytrr89erARAiPE34692iwUMvWqqBvC', + 'PEGLEG_SALT': 'MySecretSalt1234567890][' + }) +def test_get_global_creds(create_tmp_deployment_files, tmpdir): + # Create site files + site_dir = tmpdir.join("deployment_files", "site", "cicd") + + # Create global passphrase and salt file + with open(os.path.join(str(site_dir), 'secrets', + 'passphrases', + 'cicd-global-passphrase-encrypted.yaml'), "w") \ + as outfile: + outfile.write(GLOBAL_PASSPHRASE_SALT_DOC) + + save_location = tmpdir.mkdir("encrypted_site_files") + save_location_str = str(save_location) + + # Encrypt the global passphrase and salt file using site passphrase/salt + secrets.encrypt(save_location_str, "pytest", "cicd") + encrypted_files = listdir(save_location_str) + assert len(encrypted_files) > 0 + + # Capture global credentials, verify we have the right ones + passphrase, salt = secrets.get_global_creds("cicd") + assert passphrase.decode() == "TbKYNtM@3gXpL=AFLAwU?&Ey" + assert salt.decode() == "h3=DQ#GNYEuCvybgpfW7ZxAP" + + +@mock.patch.dict( + os.environ, { + 'PEGLEG_PASSPHRASE': 'ytrr89erARAiPE34692iwUMvWqqBvC', + 'PEGLEG_SALT': 'MySecretSalt1234567890][' + }) +def test_global_encrypt_decrypt(create_tmp_deployment_files, tmpdir): + # Create site files + site_dir = tmpdir.join("deployment_files", "site", "cicd") + + # Create and encrypt global passphrase and salt file using site keys + with open(os.path.join(str(site_dir), 'secrets', + 'passphrases', + 'cicd-global-passphrase-encrypted.yaml'), "w") \ + as outfile: + outfile.write(GLOBAL_PASSPHRASE_SALT_DOC) + + save_location = tmpdir.mkdir("encrypted_site_files") + save_location_str = str(save_location) + + # Encrypt the global passphrase and salt file using site passphrase/salt + secrets.encrypt(save_location_str, "pytest", "cicd") + + # Create and encrypt a global type document + global_doc_path = os.path.join(str(site_dir), 'secrets', 'passphrases', + 'globally_encrypted_doc.yaml') + with open(global_doc_path, "w") as outfile: + outfile.write(TEST_GLOBAL_DATA) + + # encrypt documents and validate that they were encrypted + doc_mgr = PeglegSecretManagement(file_path=global_doc_path, + author='pytest', + site_name='cicd') + doc_mgr.encrypt_secrets(global_doc_path) + doc = doc_mgr.documents[0] + assert doc.is_encrypted() + assert doc.data['encrypted']['by'] == 'pytest' + + doc_mgr = PeglegSecretManagement(file_path=global_doc_path, + author='pytest', + site_name='cicd') + decrypted_data = doc_mgr.get_decrypted_secrets() + test_data = list(yaml.safe_load_all(TEST_GLOBAL_DATA)) + assert test_data[0]['data'] == decrypted_data[0]['data'] + assert test_data[0]['schema'] == decrypted_data[0]['schema'] diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 46a02827..8d56f5af 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -558,6 +558,7 @@ class TestSiteSecretsActions(BaseCLIActionTest): with open(file_path, "r") as ceph_fsid_fi: ceph_fsid = yaml.safe_load(ceph_fsid_fi) ceph_fsid["metadata"]["storagePolicy"] = "encrypted" + ceph_fsid["metadata"]["layeringDefinition"]["layer"] = "site" with open(file_path, "w") as ceph_fsid_fi: yaml.dump(ceph_fsid, ceph_fsid_fi) diff --git a/tox.ini b/tox.ini index 5fae3583..c15d4e0f 100644 --- a/tox.ini +++ b/tox.ini @@ -97,6 +97,6 @@ enable-extensions = H106,H201,H904 # [H403] multi line docstrings should end on a new line # [H404] multi line docstring should start without a leading new line # [H405] multi line docstring summary not separated with an empty line -ignore = H403,H404,H405 +ignore = H403,H404,H405,W503 exclude=.venv,.git,.tox,build,dist,*lib/python*,*egg,tools,*.ini,*.po,*.pot max-complexity = 24