diff --git a/pegleg/cli.py b/pegleg/cli.py index a9070ec0..7e02a4df 100644 --- a/pegleg/cli.py +++ b/pegleg/cli.py @@ -477,7 +477,14 @@ def wrap_secret_cli(*, site_name, author, filename, output_path, schema, name, """ engine.repository.process_repositories(site_name, overwrite_existing=True) - wrap_secret(author, filename, output_path, schema, name, layer, encrypt) + wrap_secret(author, + filename, + output_path, + schema, + name, + layer, + encrypt, + site_name=site_name) @site.command('genesis_bundle', @@ -624,7 +631,7 @@ def encrypt(*, save_location, author, site_name): engine.repository.process_repositories(site_name, overwrite_existing=True) if save_location is None: save_location = config.get_site_repo() - engine.secrets.encrypt(save_location, author, site_name) + engine.secrets.encrypt(save_location, author, site_name=site_name) @secrets.command('decrypt', @@ -654,7 +661,7 @@ def encrypt(*, save_location, author, site_name): def decrypt(*, path, save_location, overwrite, site_name): engine.repository.process_repositories(site_name) - decrypted = engine.secrets.decrypt(path) + decrypted = engine.secrets.decrypt(path, site_name=site_name) if overwrite: for path, data in decrypted.items(): files.write(path, data) diff --git a/pegleg/config.py b/pegleg/config.py index f7791ef2..ce60afbf 100644 --- a/pegleg/config.py +++ b/pegleg/config.py @@ -19,6 +19,7 @@ import os from pegleg.engine import exceptions +from pegleg.engine import secrets try: if GLOBAL_CONTEXT: @@ -33,6 +34,8 @@ except NameError: 'type_path': 'type', 'passphrase': None, 'salt': None, + 'global_passphrase': None, + 'global_salt': None, 'salt_min_length': 24, 'passphrase_min_length': 24, 'default_umask': 0o027 @@ -195,3 +198,19 @@ def set_salt(): def get_salt(): """Get the salt for encryption and decryption.""" return GLOBAL_CONTEXT['salt'] + + +def set_global_enc_keys(site_name): + """Get the global salt and passphrase for encryption.""" + GLOBAL_CONTEXT['global_passphrase'], GLOBAL_CONTEXT['global_salt'] = \ + secrets.get_global_creds(site_name) + + +def get_global_passphrase(): + """Get the global passphrase for encryption and decryption.""" + return GLOBAL_CONTEXT['global_passphrase'] + + +def get_global_salt(): + """Get the global salt for encryption and decryption.""" + return GLOBAL_CONTEXT['global_salt'] diff --git a/pegleg/engine/exceptions.py b/pegleg/engine/exceptions.py index f1d00ec6..829491a5 100644 --- a/pegleg/engine/exceptions.py +++ b/pegleg/engine/exceptions.py @@ -139,10 +139,18 @@ class SaltInsufficientLengthException(PeglegBaseException): message = 'PEGLEG_SALT must be at least 24 characters long.' +class GlobalCredentialsNotFound(PeglegBaseException): + """Exception raised when global_passphrase or global_salt are not found.""" + + message = ('global_salt and global_passphrase must either both be ' + 'defined, or neither can be defined in site documents.') + + # # Shipyard Helper Exceptions # + class InvalidBufferModeException(PeglegBaseException): """Exception raised when invalid buffer mode specified""" diff --git a/pegleg/engine/secrets.py b/pegleg/engine/secrets.py index 6c3e122b..68154cf3 100644 --- a/pegleg/engine/secrets.py +++ b/pegleg/engine/secrets.py @@ -19,7 +19,9 @@ import os from prettytable import PrettyTable import yaml +from pegleg import config from pegleg.engine.catalog.pki_utility import PKIUtility +from pegleg.engine import exceptions from pegleg.engine.generators.passphrase_generator import PassphraseGenerator from pegleg.engine.util.cryptostring import CryptoString from pegleg.engine.util import definition @@ -60,7 +62,8 @@ def encrypt(save_location, author, site_name): for repo_base, file_path in definition.site_files_by_repo(site_name): secrets_found = True PeglegSecretManagement(file_path=file_path, - author=author).encrypt_secrets( + author=author, + site_name=site_name).encrypt_secrets( _get_dest_path(repo_base, file_path, save_location)) if secrets_found: @@ -70,7 +73,7 @@ def encrypt(save_location, author, site_name): 'No secret documents were found for site: {}'.format(site_name)) -def decrypt(path): +def decrypt(path, site_name=None): """Decrypt one secrets file, and print the decrypted file to standard out. Search the specified file_path for a file. @@ -93,7 +96,8 @@ def decrypt(path): return file_dict if os.path.isfile(path): - file_dict[path] = PeglegSecretManagement(path).decrypt_secrets() + file_dict[path] = PeglegSecretManagement( + path, site_name=site_name).decrypt_secrets() else: match = os.path.join(path, '**', '*.yaml') file_list = glob(match, recursive=True) @@ -121,8 +125,8 @@ def _get_dest_path(repo_base, file_path, save_location): :rtype: string """ - if (save_location and save_location != os.path.sep and - save_location.endswith(os.path.sep)): + if (save_location and save_location != os.path.sep + and save_location.endswith(os.path.sep)): save_location = save_location.rstrip(os.path.sep) if repo_base and repo_base.endswith(os.path.sep): repo_base = repo_base.rstrip(os.path.sep) @@ -166,7 +170,14 @@ def generate_crypto_string(length): return CryptoString().get_crypto_string(length) -def wrap_secret(author, filename, output_path, schema, name, layer, encrypt): +def wrap_secret(author, + filename, + output_path, + schema, + name, + layer, + encrypt, + site_name=None): """Wrap a bare secrets file in a YAML and ManagedDocument. :param author: author for ManagedDocument @@ -199,7 +210,9 @@ def wrap_secret(author, filename, output_path, schema, name, layer, encrypt): } managed_secret = PeglegManagedSecret(inner_doc, author=author) if encrypt: - psm = PeglegSecretManagement(docs=[inner_doc], author=author) + psm = PeglegSecretManagement(docs=[inner_doc], + author=author, + site_name=site_name) output_doc = psm.get_encrypted_secrets()[0][0] else: output_doc = managed_secret.pegleg_document @@ -239,3 +252,54 @@ def check_cert_expiry(site_name, duration=60): # Return table of cert names and expiration dates that are expiring return cert_table.get_string() + + +def get_global_creds(site_name): + """Determine which credentials to use for global secrets. + + If a user desires to encrypt site secrets with one set of credentials but + global secrets with a different set of credentials (in the case of + multiple sites) Pegleg needs a way to handle a two-step encryption or + decryption chain. This is accomplished by storing global credentials at + the site level and encrypting them with site credentials. Pegleg will + attempt to find both the global_salt and global_passphrase, decrypt them + then use these credentials for any global encrypt/decrypt operations. + If both global_salt and global_passphrase are found return both. + If only one global credential is found, raise an error with the assumption + the user wishes to use global credentials but does not have both. + If neither are found, return the site credentials with the assumption + the user wishes to encrypt the global documents with the site credentials. + + :param str site_name: The target site + :return: Either the global, or site level - passphrase and salt + """ + + log_msg = "Multiple documents containing {} detected. Using latest." + global_passphrase = None + global_salt = None + docs = definition.site_files(site_name) + for doc in docs: + with open(doc, 'r') as f: + results = yaml.safe_load_all(f) # Validate valid YAML. + results = PeglegSecretManagement( + docs=results).get_decrypted_secrets() + for result in results: + if result['schema'] == "deckhand/Passphrase/v1": + if result['metadata']['name'] == 'global_passphrase': + if global_passphrase: + LOG.warn(log_msg.format('global_passphrase')) + global_passphrase = result['data'].encode() + if result['metadata']['name'] == 'global_salt': + if global_salt: + LOG.warn(log_msg.format('global_salt')) + global_salt = result['data'].encode() + + # Break out of search if both passphrase and salt are found + if global_passphrase and global_salt: + return (global_passphrase, global_salt) + + # End of search, determine if we should use site keys or raise an error + if global_passphrase or global_salt: + raise exceptions.GlobalCredentialsNotFound() + else: + return (config.get_passphrase(), config.get_salt()) diff --git a/pegleg/engine/util/pegleg_managed_document.py b/pegleg/engine/util/pegleg_managed_document.py index 470315cc..454a159f 100644 --- a/pegleg/engine/util/pegleg_managed_document.py +++ b/pegleg/engine/util/pegleg_managed_document.py @@ -32,7 +32,6 @@ class PeglegManagedSecretsDocument(object): """Object representing one Pegleg managed secret document.""" def __init__(self, document, generated=False, catalog=None, author=None): - """ Parse and wrap an externally generated document in a pegleg managed document. @@ -54,8 +53,8 @@ class PeglegManagedSecretsDocument(object): if self.is_pegleg_managed_secret(document): self._pegleg_document = document else: - self._pegleg_document = self.__wrap( - document, generated, catalog, author) + self._pegleg_document = self.__wrap(document, generated, catalog, + author) self._embedded_document = \ self._pegleg_document['data']['managedDocument'] @@ -160,9 +159,7 @@ class PeglegManagedSecretsDocument(object): def set_encrypted(self, author=None): """Mark the pegleg managed document as encrypted.""" - self.data[ENCRYPTED] = { - 'at': datetime.utcnow().isoformat() - } + self.data[ENCRYPTED] = {'at': datetime.utcnow().isoformat()} if author: self.data[ENCRYPTED]['by'] = author @@ -175,3 +172,6 @@ class PeglegManagedSecretsDocument(object): def get_secret(self): return self._embedded_document.get('data') + + def get_layer(self): + return self._embedded_document[METADATA]['layeringDefinition']['layer'] diff --git a/pegleg/engine/util/pegleg_secret_management.py b/pegleg/engine/util/pegleg_secret_management.py index 3ad5f019..7218ed2c 100644 --- a/pegleg/engine/util/pegleg_secret_management.py +++ b/pegleg/engine/util/pegleg_secret_management.py @@ -35,7 +35,8 @@ class PeglegSecretManagement(object): docs=None, generated=False, catalog=None, - author=None): + author=None, + site_name=None): """ Read the source file and the environment data needed to wrap and process the file documents as pegleg managed document. @@ -43,11 +44,16 @@ class PeglegSecretManagement(object): provided. """ + # Set passphrase and salt config.set_passphrase() - self.passphrase = config.get_passphrase() - config.set_salt() - self.salt = config.get_salt() + + # Check if we're working with a specific site, if so determine if the + # global encryption keys already exist. If they don't, set them. + if site_name: + if not (config.get_global_passphrase() + and config.get_global_salt()): + config.set_global_enc_keys(site_name) if all([file_path, docs]) or not any([file_path, docs]): raise ValueError('Either `file_path` or `docs` must be ' @@ -134,10 +140,19 @@ class PeglegSecretManagement(object): # policies doc_list.append(doc.embedded_document) continue + + # Get appropriate encryption keys to use + if doc.get_layer() == 'site': + passphrase = config.get_passphrase() + salt = config.get_salt() + else: + passphrase = config.get_global_passphrase() + salt = config.get_global_salt() + secret_doc = doc.get_secret() if type(secret_doc) != bytes: secret_doc = secret_doc.encode() - doc.set_secret(encrypt(secret_doc, self.passphrase, self.salt)) + doc.set_secret(encrypt(secret_doc, passphrase, salt)) doc.set_encrypted(self._author) encrypted_docs = True doc_list.append(doc.pegleg_document) @@ -170,9 +185,17 @@ class PeglegSecretManagement(object): for doc in self.documents: # do not decrypt already decrypted data if doc.is_encrypted(): + + # Get appropriate encryption keys to use + if doc.get_layer() == 'site': + passphrase = config.get_passphrase() + salt = config.get_salt() + else: + passphrase = config.get_global_passphrase() + salt = config.get_global_salt() + doc.set_secret( - decrypt(doc.get_secret(), self.passphrase, - self.salt).decode()) + decrypt(doc.get_secret(), passphrase, salt).decode()) doc.set_decrypted() doc_list.append(doc.embedded_document) return doc_list diff --git a/tests/unit/engine/test_secrets.py b/tests/unit/engine/test_secrets.py index 1596cb91..f7a0d9da 100644 --- a/tests/unit/engine/test_secrets.py +++ b/tests/unit/engine/test_secrets.py @@ -51,6 +51,59 @@ data: 512363f37eab654313991174aef9f867d ... """ +TEST_GLOBAL_DATA = """ +--- +schema: deckhand/Passphrase/v1 +metadata: + schema: metadata/Document/v1 + name: osh_addons_keystone_ranger-agent_password + layeringDefinition: + abstract: false + layer: global + storagePolicy: encrypted +data: 512363f37eab654313991174aef9f867d +... +""" + +GLOBAL_PASSPHRASE_SALT_DOC = """ +--- +schema: deckhand/Passphrase/v1 +metadata: + schema: metadata/Document/v1 + name: global_passphrase + layeringDefinition: + abstract: false + layer: site + storagePolicy: encrypted +data: TbKYNtM@3gXpL=AFLAwU?&Ey +... +--- +schema: deckhand/Passphrase/v1 +metadata: + schema: metadata/Document/v1 + name: global_salt + layeringDefinition: + abstract: false + layer: site + storagePolicy: encrypted +data: h3=DQ#GNYEuCvybgpfW7ZxAP +... +""" + +GLOBAL_SALT_DOC = """ +--- +schema: deckhand/Passphrase/v1 +metadata: + schema: metadata/Document/v1 + name: global_salt + layeringDefinition: + abstract: false + layer: site + storagePolicy: encrypted +data: h3=DQ#GNYEuCvybgpfW7ZxAP +... +""" + def test_encrypt_and_decrypt(): data = test_utils.rand_name("this is an example of un-encrypted " @@ -314,3 +367,123 @@ def test_check_expiry(create_tmp_deployment_files): assert cert_info['expired'] is False, \ "%s is expired/expiring on %s" % \ (generated_file.name, cert_info['expiry_date']) + + +@mock.patch.dict( + os.environ, { + 'PEGLEG_PASSPHRASE': 'ytrr89erARAiPE34692iwUMvWqqBvC', + 'PEGLEG_SALT': 'MySecretSalt1234567890][' + }) +def test_get_global_creds_missing_creds(create_tmp_deployment_files, tmpdir): + # Create site files + site_dir = tmpdir.join("deployment_files", "site", "cicd") + + save_location = tmpdir.mkdir("encrypted_site_files") + save_location_str = str(save_location) + + # Capture global credentials, verify they are not present and we default + # to site credentials instead. + passphrase, salt = secrets.get_global_creds("cicd") + assert passphrase.decode() == 'ytrr89erARAiPE34692iwUMvWqqBvC' + assert salt.decode() == 'MySecretSalt1234567890][' + + +@mock.patch.dict( + os.environ, { + 'PEGLEG_PASSPHRASE': 'ytrr89erARAiPE34692iwUMvWqqBvC', + 'PEGLEG_SALT': 'MySecretSalt1234567890][' + }) +def test_get_global_creds_missing_pass(create_tmp_deployment_files, tmpdir): + # Create site files + site_dir = tmpdir.join("deployment_files", "site", "cicd") + + # Create global salt file + with open( + os.path.join(str(site_dir), 'secrets', 'passphrases', + 'cicd-global-passphrase-encrypted.yaml'), + "w") as outfile: + outfile.write(GLOBAL_SALT_DOC) + + save_location = tmpdir.mkdir("encrypted_site_files") + save_location_str = str(save_location) + + # Demonstrate that encryption fails when only the global salt or + # only the global passphrase are present among the site files. + with pytest.raises(exceptions.GlobalCredentialsNotFound): + secrets.encrypt(save_location_str, "pytest", "cicd") + + +@mock.patch.dict( + os.environ, { + 'PEGLEG_PASSPHRASE': 'ytrr89erARAiPE34692iwUMvWqqBvC', + 'PEGLEG_SALT': 'MySecretSalt1234567890][' + }) +def test_get_global_creds(create_tmp_deployment_files, tmpdir): + # Create site files + site_dir = tmpdir.join("deployment_files", "site", "cicd") + + # Create global passphrase and salt file + with open(os.path.join(str(site_dir), 'secrets', + 'passphrases', + 'cicd-global-passphrase-encrypted.yaml'), "w") \ + as outfile: + outfile.write(GLOBAL_PASSPHRASE_SALT_DOC) + + save_location = tmpdir.mkdir("encrypted_site_files") + save_location_str = str(save_location) + + # Encrypt the global passphrase and salt file using site passphrase/salt + secrets.encrypt(save_location_str, "pytest", "cicd") + encrypted_files = listdir(save_location_str) + assert len(encrypted_files) > 0 + + # Capture global credentials, verify we have the right ones + passphrase, salt = secrets.get_global_creds("cicd") + assert passphrase.decode() == "TbKYNtM@3gXpL=AFLAwU?&Ey" + assert salt.decode() == "h3=DQ#GNYEuCvybgpfW7ZxAP" + + +@mock.patch.dict( + os.environ, { + 'PEGLEG_PASSPHRASE': 'ytrr89erARAiPE34692iwUMvWqqBvC', + 'PEGLEG_SALT': 'MySecretSalt1234567890][' + }) +def test_global_encrypt_decrypt(create_tmp_deployment_files, tmpdir): + # Create site files + site_dir = tmpdir.join("deployment_files", "site", "cicd") + + # Create and encrypt global passphrase and salt file using site keys + with open(os.path.join(str(site_dir), 'secrets', + 'passphrases', + 'cicd-global-passphrase-encrypted.yaml'), "w") \ + as outfile: + outfile.write(GLOBAL_PASSPHRASE_SALT_DOC) + + save_location = tmpdir.mkdir("encrypted_site_files") + save_location_str = str(save_location) + + # Encrypt the global passphrase and salt file using site passphrase/salt + secrets.encrypt(save_location_str, "pytest", "cicd") + + # Create and encrypt a global type document + global_doc_path = os.path.join(str(site_dir), 'secrets', 'passphrases', + 'globally_encrypted_doc.yaml') + with open(global_doc_path, "w") as outfile: + outfile.write(TEST_GLOBAL_DATA) + + # encrypt documents and validate that they were encrypted + doc_mgr = PeglegSecretManagement(file_path=global_doc_path, + author='pytest', + site_name='cicd') + doc_mgr.encrypt_secrets(global_doc_path) + doc = doc_mgr.documents[0] + assert doc.is_encrypted() + assert doc.data['encrypted']['by'] == 'pytest' + + doc_mgr = PeglegSecretManagement(file_path=global_doc_path, + author='pytest', + site_name='cicd') + decrypted_data = doc_mgr.get_decrypted_secrets() + test_data = list(yaml.safe_load_all(TEST_GLOBAL_DATA)) + assert test_data[0]['data'] == decrypted_data[0]['data'] + assert test_data[0]['schema'] == decrypted_data[0]['schema'] diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 46a02827..8d56f5af 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -558,6 +558,7 @@ class TestSiteSecretsActions(BaseCLIActionTest): with open(file_path, "r") as ceph_fsid_fi: ceph_fsid = yaml.safe_load(ceph_fsid_fi) ceph_fsid["metadata"]["storagePolicy"] = "encrypted" + ceph_fsid["metadata"]["layeringDefinition"]["layer"] = "site" with open(file_path, "w") as ceph_fsid_fi: yaml.dump(ceph_fsid, ceph_fsid_fi) diff --git a/tox.ini b/tox.ini index 5fae3583..c15d4e0f 100644 --- a/tox.ini +++ b/tox.ini @@ -97,6 +97,6 @@ enable-extensions = H106,H201,H904 # [H403] multi line docstrings should end on a new line # [H404] multi line docstring should start without a leading new line # [H405] multi line docstring summary not separated with an empty line -ignore = H403,H404,H405 +ignore = H403,H404,H405,W503 exclude=.venv,.git,.tox,build,dist,*lib/python*,*egg,tools,*.ini,*.po,*.pot max-complexity = 24