airship/pegleg
Code Issues Proposed changes

Standardize Pegleg code with YAPF

Browse Source 
This patch addresses inconsistent code style and enforces it with a
gate for future submissions.

Separate work will be done in the future to address several of the
PEP8 ignores for docstrings, and attempt to bring the tests directory
to PEP8 compliance.

This patch:
1. Updates .style.yapf to set the knobs desired for YAPF.
2. Updates tox.ini to allow one of the knobs to work.
3. Removes unused code from several __init__.py files.
4. Updates the YAPF version in test-requirements.txt to latest (this
   is needed for several knobs to work).
5. Stylistic changes to the python codebase in Pegleg.
6. Updates to tox.ini to run YAPF during PEP8 check.

Change-Id: Ieaa0fdef2b601d01c875d64b840986e54df73abf
This commit is contained in:
Alexander Hughes 2019-06-07 21:03:09 -04:00
parent 5c0a3bef03
commit 1c8d92ef6b
45 changed files with 1218 additions and 1071 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.style.yapf
View File

@ -2,4 +2,9 @@
based_on_style = pep8
spaces_before_comment = 2
column_limit = 79
split_before_logical_operator = false
blank_line_before_nested_class_or_def = false
blank_line_before_module_docstring = true
split_before_logical_operator = true
split_before_first_argument = true
allow_split_before_dict_value = false
split_before_arithmetic_operator = true

9
Makefile
View File

@ -63,7 +63,8 @@ lint: py_lint
# Perform auto formatting
.PHONY: format
format: py_format
format:
tox -e fmt
_BASE_IMAGE_ARG := $(if $(BASE_IMAGE),--build-arg FROM="${BASE_IMAGE}" ,)
@ -107,8 +108,4 @@ clean:
.PHONY: py_lint
py_lint:
tox -e pep8
.PHONY: py_format
py_format:
tox -e fmt
tox -e pep8

338
pegleg/cli.py
View File

@ -126,11 +126,12 @@ SITE_REPOSITORY_ARGUMENT = click.argument(
@click.group(context_settings=CONTEXT_SETTINGS)
@click.option('-v',
'--verbose',
is_flag=True,
default=False,
help='Enable debug logging')
@click.option(
'-v',
'--verbose',
is_flag=True,
default=False,
help='Enable debug logging')
def main(*, verbose):
"""Main CLI meta-group, which includes the following groups:
@ -167,19 +168,17 @@ def repo(*, site_repository, clone_path, repo_key, repo_username):
config.set_umask()
def _lint_helper(*,
fail_on_missing_sub_src,
exclude_lint,
warn_lint,
site_name=None):
def _lint_helper(
*, fail_on_missing_sub_src, exclude_lint, warn_lint, site_name=None):
"""Helper for executing lint on specific site or all sites in repo."""
if site_name:
func = functools.partial(engine.lint.site, site_name=site_name)
else:
func = engine.lint.full
warns = func(fail_on_missing_sub_src=fail_on_missing_sub_src,
exclude_lint=exclude_lint,
warn_lint=warn_lint)
warns = func(
fail_on_missing_sub_src=fail_on_missing_sub_src,
exclude_lint=exclude_lint,
warn_lint=warn_lint)
if warns:
click.echo("Linting passed, but produced some warnings.")
for w in warns:
@ -194,9 +193,10 @@ def lint_repo(*, fail_on_missing_sub_src, exclude_lint, warn_lint):
"""Lint all sites using checks defined in :mod:`pegleg.engine.errorcodes`.
"""
engine.repository.process_site_repository(update_config=True)
_lint_helper(fail_on_missing_sub_src=fail_on_missing_sub_src,
exclude_lint=exclude_lint,
warn_lint=warn_lint)
_lint_helper(
fail_on_missing_sub_src=fail_on_missing_sub_src,
exclude_lint=exclude_lint,
warn_lint=warn_lint)
@main.group(help='Commands related to sites')
@ -205,8 +205,9 @@ def lint_repo(*, fail_on_missing_sub_src, exclude_lint, warn_lint):
@EXTRA_REPOSITORY_OPTION
@REPOSITORY_USERNAME_OPTION
@REPOSITORY_KEY_OPTION
def site(*, site_repository, clone_path, extra_repositories, repo_key,
repo_username):
def site(
*, site_repository, clone_path, extra_repositories, repo_key,
repo_username):
"""Group for site-level actions, which include:
* list: list available sites in a manifests repo
@ -225,11 +226,12 @@ def site(*, site_repository, clone_path, extra_repositories, repo_key,
@site.command(help='Output complete config for one site')
@click.option('-s',
'--save-location',
'save_location',
help='Directory to output the complete site definition. Created '
'automatically if it does not already exist.')
@click.option(
'-s',
'--save-location',
'save_location',
help='Directory to output the complete site definition. Created '
'automatically if it does not already exist.')
@click.option(
'--validate/--no-validate',
'validate',
@ -246,11 +248,12 @@ def site(*, site_repository, clone_path, extra_repositories, repo_key,
multiple=True,
help='Excludes specified linting checks. Warnings will still be issued. '
'-w takes priority over -x.')
@click.option('-w',
'--warn',
'warn_lint',
multiple=True,
help='Warn if linting check fails. -w takes priority over -x.')
@click.option(
'-w',
'--warn',
'warn_lint',
multiple=True,
help='Warn if linting check fails. -w takes priority over -x.')
@SITE_REPOSITORY_ARGUMENT
def collect(*, save_location, validate, exclude_lint, warn_lint, site_name):
"""Collects documents into a single site-definition.yaml file, which
@ -265,10 +268,11 @@ def collect(*, save_location, validate, exclude_lint, warn_lint, site_name):
"""
if validate:
# Lint the primary repo prior to document collection.
_lint_helper(site_name=site_name,
fail_on_missing_sub_src=True,
exclude_lint=exclude_lint,
warn_lint=warn_lint)
_lint_helper(
site_name=site_name,
fail_on_missing_sub_src=True,
exclude_lint=exclude_lint,
warn_lint=warn_lint)
engine.site.collect(site_name, save_location)
@ -312,10 +316,11 @@ def lint_site(*, fail_on_missing_sub_src, exclude_lint, warn_lint, site_name):
"""Lint a given site using checks defined in
:mod:`pegleg.engine.errorcodes`.
"""
_lint_helper(site_name=site_name,
fail_on_missing_sub_src=fail_on_missing_sub_src,
exclude_lint=exclude_lint,
warn_lint=warn_lint)
_lint_helper(
site_name=site_name,
fail_on_missing_sub_src=fail_on_missing_sub_src,
exclude_lint=exclude_lint,
warn_lint=warn_lint)
def collection_default_callback(ctx, param, value):
@ -327,14 +332,16 @@ def collection_default_callback(ctx, param, value):
@site.command('upload', help='Upload documents to Shipyard')
# Keystone authentication parameters
@click.option('--os-project-domain-name',
envvar='OS_PROJECT_DOMAIN_NAME',
required=False,
default='default')
@click.option('--os-user-domain-name',
envvar='OS_USER_DOMAIN_NAME',
required=False,
default='default')
@click.option(
'--os-project-domain-name',
envvar='OS_PROJECT_DOMAIN_NAME',
required=False,
default='default')
@click.option(
'--os-user-domain-name',
envvar='OS_USER_DOMAIN_NAME',
required=False,
default='default')
@click.option('--os-project-name', envvar='OS_PROJECT_NAME', required=False)
@click.option('--os-username', envvar='OS_USERNAME', required=False)
@click.option('--os-password', envvar='OS_PASSWORD', required=False)
@ -362,16 +369,18 @@ def collection_default_callback(ctx, param, value):
'collection does not already exist in the Shipyard buffer.\n'
'replace: Clear the Shipyard Buffer before adding the specified '
'collection.\n')
@click.option('--collection',
'collection',
help='Specifies the name to use for the uploaded collection. '
'Defaults to the specified `site_name`.',
callback=collection_default_callback)
@click.option(
'--collection',
'collection',
help='Specifies the name to use for the uploaded collection. '
'Defaults to the specified `site_name`.',
callback=collection_default_callback)
@SITE_REPOSITORY_ARGUMENT
@click.pass_context
def upload(ctx, *, os_project_domain_name, os_user_domain_name,
os_project_name, os_username, os_password, os_auth_url,
os_auth_token, context_marker, site_name, buffer_mode, collection):
def upload(
ctx, *, os_project_domain_name, os_user_domain_name, os_project_name,
os_username, os_password, os_auth_url, os_auth_token, context_marker,
site_name, buffer_mode, collection):
if not ctx.obj:
ctx.obj = {}
@ -415,12 +424,13 @@ def secrets():
'for tracking provenance information in the PeglegManagedDocuments. '
'An attempt is made to automatically determine this value, '
'but should be provided.')
@click.option('-d',
'--days',
'days',
default=365,
show_default=True,
help='Duration in days generated certificates should be valid.')
@click.option(
'-d',
'--days',
'days',
default=365,
show_default=True,
help='Duration in days generated certificates should be valid.')
@click.argument('site_name')
def generate_pki(site_name, author, days):
"""Generate certificates, certificate authorities and keypairs for a given
@ -429,9 +439,8 @@ def generate_pki(site_name, author, days):
"""
engine.repository.process_repositories(site_name, overwrite_existing=True)
pkigenerator = catalog.pki_generator.PKIGenerator(site_name,
author=author,
duration=days)
pkigenerator = catalog.pki_generator.PKIGenerator(
site_name, author=author, duration=days)
output_paths = pkigenerator.generate()
click.echo("Generated PKI files written to:\n%s" % '\n'.join(output_paths))
@ -441,13 +450,12 @@ def generate_pki(site_name, author, days):
'wrap',
help='Wrap bare files (e.g. pem or crt) in a PeglegManagedDocument '
'and encrypt them (by default).')
@click.option('-a',
'--author',
'author',
help='Author for the new wrapped file.')
@click.option('--filename',
'filename',
help='The relative file path for the file to be wrapped.')
@click.option(
'-a', '--author', 'author', help='Author for the new wrapped file.')
@click.option(
'--filename',
'filename',
help='The relative file path for the file to be wrapped.')
@click.option(
'-o',
'--output-path',
@ -455,53 +463,58 @@ def generate_pki(site_name, author, days):
required=False,
help='The output path for the wrapped file. (default: input path with '
'.yaml)')
@click.option('-s',
'--schema',
'schema',
help='The schema for the document to be wrapped, e.g. '
'deckhand/Certificate/v1')
@click.option('-n',
'--name',
'name',
help='The name for the document to be wrapped, e.g. new-cert')
@click.option('-l',
'--layer',
'layer',
help='The layer for the document to be wrapped., e.g. site.')
@click.option('--encrypt/--no-encrypt',
'encrypt',
is_flag=True,
default=True,
show_default=True,
help='Whether to encrypt the wrapped file.')
@click.option(
'-s',
'--schema',
'schema',
help='The schema for the document to be wrapped, e.g. '
'deckhand/Certificate/v1')
@click.option(
'-n',
'--name',
'name',
help='The name for the document to be wrapped, e.g. new-cert')
@click.option(
'-l',
'--layer',
'layer',
help='The layer for the document to be wrapped., e.g. site.')
@click.option(
'--encrypt/--no-encrypt',
'encrypt',
is_flag=True,
default=True,
show_default=True,
help='Whether to encrypt the wrapped file.')
@click.argument('site_name')
def wrap_secret_cli(*, site_name, author, filename, output_path, schema, name,
layer, encrypt):
def wrap_secret_cli(
*, site_name, author, filename, output_path, schema, name, layer,
encrypt):
"""Wrap a bare secrets file in a YAML and ManagedDocument.
"""
engine.repository.process_repositories(site_name, overwrite_existing=True)
wrap_secret(author,
filename,
output_path,
schema,
name,
layer,
encrypt,
site_name=site_name)
wrap_secret(
author,
filename,
output_path,
schema,
name,
layer,
encrypt,
site_name=site_name)
@site.command('genesis_bundle',
help='Construct the genesis deployment bundle.')
@click.option('-b',
'--build-dir',
'build_dir',
type=click.Path(file_okay=False,
dir_okay=True,
resolve_path=True),
required=True,
help='Destination directory to store the genesis bundle.')
@site.command(
'genesis_bundle', help='Construct the genesis deployment bundle.')
@click.option(
'-b',
'--build-dir',
'build_dir',
type=click.Path(file_okay=False, dir_okay=True, resolve_path=True),
required=True,
help='Destination directory to store the genesis bundle.')
@click.option(
'--include-validators',
'validators',
@ -512,8 +525,9 @@ def wrap_secret_cli(*, site_name, author, filename, output_path, schema, name,
@SITE_REPOSITORY_ARGUMENT
def genesis_bundle(*, build_dir, validators, site_name):
encryption_key = os.environ.get("PROMENADE_ENCRYPTION_KEY")
bundle.build_genesis(build_dir, encryption_key, validators,
logging.DEBUG == LOG.getEffectiveLevel(), site_name)
bundle.build_genesis(
build_dir, encryption_key, validators,
logging.DEBUG == LOG.getEffectiveLevel(), site_name)
@secrets.command(
@ -534,8 +548,9 @@ def check_pki_certs(site_name, days):
cert_results = engine.secrets.check_cert_expiry(site_name, duration=days)
click.echo("The following certs will expire within {} days: \n{}".format(
days, cert_results))
click.echo(
"The following certs will expire within {} days: \n{}".format(
days, cert_results))
@main.group(help='Commands related to types')
@ -544,8 +559,9 @@ def check_pki_certs(site_name, days):
@EXTRA_REPOSITORY_OPTION
@REPOSITORY_USERNAME_OPTION
@REPOSITORY_KEY_OPTION
def type(*, site_repository, clone_path, extra_repositories, repo_key,
repo_username):
def type(
*, site_repository, clone_path, extra_repositories, repo_key,
repo_username):
"""Group for repo-level actions, which include:
* list: list all types across the repository
@ -566,8 +582,8 @@ def list_types(*, output_stream):
engine.type.list_types(output_stream)
@secrets.group(name='generate',
help='Command group to generate site secrets documents.')
@secrets.group(
name='generate', help='Command group to generate site secrets documents.')
def generate():
pass
@ -591,12 +607,13 @@ def generate():
required=True,
help='Identifier for the program or person who is generating the secrets '
'documents')
@click.option('-i',
'--interactive',
'interactive',
is_flag=True,
default=False,
help='Generate passphrases interactively, not automatically')
@click.option(
'-i',
'--interactive',
'interactive',
is_flag=True,
default=False,
help='Generate passphrases interactively, not automatically')
@click.option(
'--force-cleartext',
'force_cleartext',
@ -604,17 +621,18 @@ def generate():
default=False,
show_default=True,
help='Force cleartext generation of passphrases. This is not recommended.')
def generate_passphrases(*, site_name, save_location, author, interactive,
force_cleartext):
def generate_passphrases(
*, site_name, save_location, author, interactive, force_cleartext):
engine.repository.process_repositories(site_name)
engine.secrets.generate_passphrases(site_name, save_location, author,
interactive, force_cleartext)
engine.secrets.generate_passphrases(
site_name, save_location, author, interactive, force_cleartext)
@secrets.command('encrypt',
help='Command to encrypt and wrap site secrets '
'documents with metadata.storagePolicy set '
'to encrypted, in pegleg managed documents.')
@secrets.command(
'encrypt',
help='Command to encrypt and wrap site secrets '
'documents with metadata.storagePolicy set '
'to encrypted, in pegleg managed documents.')
@click.option(
'-s',
'--save-location',
@ -639,14 +657,16 @@ def encrypt(*, save_location, author, site_name):
engine.secrets.encrypt(save_location, author, site_name=site_name)
@secrets.command('decrypt',
help='Command to unwrap and decrypt one site '
'secrets document and print it to stdout.')
@click.option('--path',
'path',
type=click.Path(exists=True, readable=True),
required=True,
help='The file or directory path to decrypt.')
@secrets.command(
'decrypt',
help='Command to unwrap and decrypt one site '
'secrets document and print it to stdout.')
@click.option(
'--path',
'path',
type=click.Path(exists=True, readable=True),
required=True,
help='The file or directory path to decrypt.')
@click.option(
'-s',
'--save-location',
@ -688,27 +708,31 @@ def generate():
@generate.command(
'passphrase',
help='Command to generate a passphrase and print out to stdout')
@click.option('-l',
'--length',
'length',
default=24,
show_default=True,
help='Generate a passphrase of the given length. '
'Length is >= 24, no maximum length.')
@click.option(
'-l',
'--length',
'length',
default=24,
show_default=True,
help='Generate a passphrase of the given length. '
'Length is >= 24, no maximum length.')
def generate_passphrase(length):
click.echo('Generated Passhprase: {}'.format(
engine.secrets.generate_crypto_string(length)))
click.echo(
'Generated Passhprase: {}'.format(
engine.secrets.generate_crypto_string(length)))
@generate.command('salt',
help='Command to generate a salt and print out to stdout')
@click.option('-l',
'--length',
'length',
default=24,
show_default=True,
help='Generate a passphrase of the given length. '
'Length is >= 24, no maximum length.')
@generate.command(
'salt', help='Command to generate a salt and print out to stdout')
@click.option(
'-l',
'--length',
'length',
default=24,
show_default=True,
help='Generate a passphrase of the given length. '
'Length is >= 24, no maximum length.')
def generate_salt(length):
click.echo("Generated Salt: {}".format(
engine.secrets.generate_crypto_string(length)))
click.echo(
"Generated Salt: {}".format(
engine.secrets.generate_crypto_string(length)))

11
pegleg/engine/bundle.py
View File

@ -16,16 +16,15 @@ import logging
import os
import click
from promenade.builder import Builder
from promenade.config import Configuration
from promenade import exceptions
from pegleg.engine.exceptions import GenesisBundleEncryptionException
from pegleg.engine.exceptions import GenesisBundleGenerateException
from pegleg.engine import util
from pegleg.engine.util.pegleg_secret_management import PeglegSecretManagement
from promenade.builder import Builder
from promenade.config import Configuration
from promenade import exceptions
LOG = logging.getLogger(__name__)
__all__ = [
@ -80,8 +79,8 @@ def build_genesis(build_path, encryption_key, validators, debug, site_name):
raise GenesisBundleEncryptionException()
except exceptions.PromenadeException as e:
LOG.error('Build genesis bundle failed! {}.'.format(
e.display(debug=debug)))
LOG.error(
'Build genesis bundle failed! {}.'.format(e.display(debug=debug)))
raise GenesisBundleGenerateException()
LOG.info('=== Done! ===')

54
pegleg/engine/catalog/pki_generator.py
View File

@ -42,8 +42,8 @@ class PKIGenerator(object):
"""
def __init__(self, sitename, block_strings=True, author=None,
duration=365):
def __init__(
self, sitename, block_strings=True, author=None, duration=365):
"""Constructor for ``PKIGenerator``.
:param int duration: Duration in days that generated certificates
@ -61,8 +61,8 @@ class PKIGenerator(object):
self._documents = util.definition.documents_for_site(sitename)
self._author = author
self.keys = pki_utility.PKIUtility(block_strings=block_strings,
duration=duration)
self.keys = pki_utility.PKIUtility(
block_strings=block_strings, duration=duration)
self.outputs = collections.defaultdict(dict)
# Maps certificates to CAs in order to derive certificate paths.
@ -119,10 +119,8 @@ class PKIGenerator(object):
def gen_cert(self, document_name, *, ca_cert, ca_key, **kwargs):
ca_cert_data = ca_cert['data']['managedDocument']['data']
ca_key_data = ca_key['data']['managedDocument']['data']
return self.keys.generate_certificate(document_name,
ca_cert=ca_cert_data,
ca_key=ca_key_data,
**kwargs)
return self.keys.generate_certificate(
document_name, ca_cert=ca_cert_data, ca_key=ca_key_data, **kwargs)
def gen_keypair(self, document_name):
return self.keys.generate_keypair(document_name)
@ -149,30 +147,31 @@ class PKIGenerator(object):
docs = self._find_among_collected(schemas, document_name)
if docs:
if len(docs) == len(kinds):
LOG.debug('Found docs in input config named %s, kinds: %s',
document_name, kinds)
LOG.debug(
'Found docs in input config named %s, kinds: %s',
document_name, kinds)
return docs
else:
raise exceptions.IncompletePKIPairError(kinds=kinds,
name=document_name)
raise exceptions.IncompletePKIPairError(
kinds=kinds, name=document_name)
else:
docs = self._find_among_outputs(schemas, document_name)
if docs:
LOG.debug('Found docs in current outputs named %s, kinds: %s',
document_name, kinds)
LOG.debug(
'Found docs in current outputs named %s, kinds: %s',
document_name, kinds)
return docs
# TODO(felipemonteiro): Should this be a critical error?
LOG.debug('No docs existing docs named %s, kinds: %s', document_name,
kinds)
LOG.debug(
'No docs existing docs named %s, kinds: %s', document_name, kinds)
return []
def _find_among_collected(self, schemas, document_name):
result = []
for schema in schemas:
doc = _find_document_by(self._documents,
schema=schema,
name=document_name)
doc = _find_document_by(
self._documents, schema=schema, name=document_name)
# If the document wasn't found, then means it needs to be
# generated.
if doc:
@ -224,20 +223,21 @@ class PKIGenerator(object):
document = PeglegSecretManagement(
docs=[document]).get_encrypted_secrets()[0][0]
util.files.dump(document,
output_path,
flag='a',
default_flow_style=False,
explicit_start=True,
indent=2)
util.files.dump(
document,
output_path,
flag='a',
default_flow_style=False,
explicit_start=True,
indent=2)
output_paths.add(output_path)
return output_paths
def get_documents(self):
return list(
itertools.chain.from_iterable(v.values()
for v in self.outputs.values()))
itertools.chain.from_iterable(
v.values() for v in self.outputs.values()))
def get_host_list(service_names):

142
pegleg/engine/catalog/pki_utility.py
View File

@ -69,18 +69,18 @@ class PKIUtility(object):
raise exceptions.PKICertificateInvalidDuration()
if not self._ca_config_string:
self._ca_config_string = json.dumps({
'signing': {
'default': {
'expiry':
str(24 * self.duration) + 'h',
'usages': [
'signing', 'key encipherment', 'server auth',
'client auth'
],
self._ca_config_string = json.dumps(
{
'signing': {
'default': {
'expiry': str(24 * self.duration) + 'h',
'usages': [
'signing', 'key encipherment', 'server auth',
'client auth'
],
},
},
},
})
})
return self._ca_config_string
def generate_ca(self, ca_name):
@ -92,11 +92,13 @@ class PKIUtility(object):
"""
result = self._cfssl(['gencert', '-initca', 'csr.json'],
files={'csr.json': self.csr(name=ca_name)})
result = self._cfssl(
['gencert', '-initca', 'csr.json'],
files={'csr.json': self.csr(name=ca_name)})
return (self._wrap_ca(ca_name, result['cert']),
self._wrap_ca_key(ca_name, result['key']))
return (
self._wrap_ca(ca_name, result['cert']),
self._wrap_ca_key(ca_name, result['key']))
def generate_keypair(self, name):
"""Generate keypair.
@ -114,17 +116,12 @@ class PKIUtility(object):
'priv.pem': priv_result['priv.pem'],
})
return (self._wrap_pub_key(name, pub_result['pub.pem']),
self._wrap_priv_key(name, priv_result['priv.pem']))
return (
self._wrap_pub_key(name, pub_result['pub.pem']),
self._wrap_priv_key(name, priv_result['priv.pem']))
def generate_certificate(self,
name,
*,
ca_cert,
ca_key,
cn,
groups=None,
hosts=None):
def generate_certificate(
self, name, *, ca_cert, ca_key, cn, groups=None, hosts=None):
"""Generate certificate and associated key given CA cert and key.
:param str name: Name of certificate in wrapped document.
@ -155,10 +152,12 @@ class PKIUtility(object):
'csr.json': self.csr(name=cn, groups=groups, hosts=hosts),
})
return (self._wrap_cert(name, result['cert']),
self._wrap_cert_key(name, result['key']))
return (
self._wrap_cert(name, result['cert']),
self._wrap_cert_key(name, result['key']))
def csr(self,
def csr(
self,
*,
name,
groups=None,
@ -172,14 +171,15 @@ class PKIUtility(object):
if hosts is None:
hosts = []
return json.dumps({
'CN': name,
'key': key,
'hosts': hosts,
'names': [{
'O': g
} for g in groups],
})
return json.dumps(
{
'CN': name,
'key': key,
'hosts': hosts,
'names': [{
'O': g
} for g in groups],
})
def cert_info(self, cert):
"""Retrieve certificate info via ``cfssl``.
@ -190,8 +190,8 @@ class PKIUtility(object):
"""
return self._cfssl(['certinfo', '-cert', 'cert.pem'],
files={'cert.pem': cert})
return self._cfssl(
['certinfo', '-cert', 'cert.pem'], files={'cert.pem': cert})
def check_expiry(self, cert):
"""Chek whether a given certificate is expired.
@ -223,8 +223,8 @@ class PKIUtility(object):
files = {}
with tempfile.TemporaryDirectory() as tmp:
for filename, data in files.items():
util.files.write(decode_bytes(data),
os.path.join(tmp, filename))
util.files.write(
decode_bytes(data), os.path.join(tmp, filename))
# Ignore bandit false positive:
# B603:subprocess_without_shell_equals_true
@ -241,8 +241,8 @@ class PKIUtility(object):
with tempfile.TemporaryDirectory() as tmp:
for filename, data in files.items():
util.files.write(decode_bytes(data),
os.path.join(tmp, filename))
util.files.write(
decode_bytes(data), os.path.join(tmp, filename))
# Ignore bandit false positive:
# B603:subprocess_without_shell_equals_true
@ -261,40 +261,46 @@ class PKIUtility(object):
return result
def _wrap_ca(self, name, data):
return self.wrap_document(kind='CertificateAuthority',
name=name,
data=data,
block_strings=self.block_strings)
return self.wrap_document(
kind='CertificateAuthority',
name=name,
data=data,
block_strings=self.block_strings)
def _wrap_ca_key(self, name, data):
return self.wrap_document(kind='CertificateAuthorityKey',
name=name,
data=data,
block_strings=self.block_strings)
return self.wrap_document(
kind='CertificateAuthorityKey',
name=name,
data=data,
block_strings=self.block_strings)
def _wrap_cert(self, name, data):
return self.wrap_document(kind='Certificate',
name=name,
data=data,
block_strings=self.block_strings)
return self.wrap_document(
kind='Certificate',
name=name,
data=data,
block_strings=self.block_strings)
def _wrap_cert_key(self, name, data):
return self.wrap_document(kind='CertificateKey',
name=name,
data=data,
block_strings=self.block_strings)
return self.wrap_document(
kind='CertificateKey',
name=name,
data=data,
block_strings=self.block_strings)
def _wrap_priv_key(self, name, data):
return self.wrap_document(kind='PrivateKey',
name=name,
data=data,
block_strings=self.block_strings)
return self.wrap_document(
kind='PrivateKey',
name=name,
data=data,
block_strings=self.block_strings)
def _wrap_pub_key(self, name, data):
return self.wrap_document(kind='PublicKey',
name=name,
data=data,
block_strings=self.block_strings)
return self.wrap_document(
kind='PublicKey',
name=name,
data=data,
block_strings=self.block_strings)
@staticmethod
def wrap_document(kind, name, data, block_strings=True):
@ -319,8 +325,8 @@ class PKIUtility(object):
},
'storagePolicy': 'cleartext'
}
wrapped_data = PKIUtility._block_literal(data,
block_strings=block_strings)
wrapped_data = PKIUtility._block_literal(
data, block_strings=block_strings)
document = {
"schema": wrapped_schema,

10
pegleg/engine/catalogs/base_catalog.py
View File

@ -50,8 +50,9 @@ class BaseCatalog(ABC):
if schema == 'pegleg/%s/v1' % kind:
self._catalog_docs.append(document)
elif schema == 'promenade/%s/v1' % kind:
LOG.warning('The schema promenade/%s/v1 is deprecated. Use '
'pegleg/%s/v1 instead.', kind, kind)
LOG.warning(
'The schema promenade/%s/v1 is deprecated. Use '
'pegleg/%s/v1 instead.', kind, kind)
self._catalog_docs.append(document)
@property
@ -73,8 +74,9 @@ class BaseCatalog(ABC):
if not self._catalog_path:
# Cound not find the Catalog for this generated passphrase
# raise an exception.
LOG.error('Catalog path: {} was not found in repo: {}'.format(
catalog_name, repo_name))
LOG.error(
'Catalog path: {} was not found in repo: {}'.format(
catalog_name, repo_name))
raise PassphraseCatalogNotFoundException()
def _get_document_name(self, name):

6
pegleg/engine/catalogs/passphrase_catalog.py
View File

@ -57,9 +57,9 @@ class PassphraseCatalog(BaseCatalog):
@property
def get_passphrase_names(self):
"""Return the list of passphrases in the catalog."""
return (passphrase[P_DOCUMENT_NAME]
for catalog in self._catalog_docs
for passphrase in catalog['data']['passphrases'])
return (
passphrase[P_DOCUMENT_NAME] for catalog in self._catalog_docs
for passphrase in catalog['data']['passphrases'])
def get_length(self, passphrase_name):
"""

32
pegleg/engine/exceptions.py
View File

@ -14,9 +14,10 @@
import logging
__all__ = ('PeglegBaseException', 'GitException', 'GitAuthException',
'GitProxyException', 'GitSSHException', 'GitConfigException',
'GitInvalidRepoException')
__all__ = (
'PeglegBaseException', 'GitException', 'GitAuthException',
'GitProxyException', 'GitSSHException', 'GitConfigException',
'GitInvalidRepoException')
LOG = logging.getLogger(__name__)
@ -37,14 +38,16 @@ class PeglegBaseException(Exception):
class GitException(PeglegBaseException):
"""Exception when an error occurs cloning a Git repository."""
message = ('Git exception occurred: [%(location)s] may not be a valid '
'git repository. Details: %(details)s')
message = (
'Git exception occurred: [%(location)s] may not be a valid '
'git repository. Details: %(details)s')
class GitAuthException(PeglegBaseException):
"""Exception that occurs when authentication fails for cloning a repo."""
message = ('Failed to authenticate for repo %(repo_url)s with ssh-key '
'at path %(ssh_key_path)s')
message = (
'Failed to authenticate for repo %(repo_url)s with ssh-key '
'at path %(ssh_key_path)s')
class GitProxyException(PeglegBaseException):
@ -84,8 +87,9 @@ class IncompletePKIPairError(PeglegBaseException):
class PassphraseCatalogNotFoundException(PeglegBaseException):
"""Failed to find Catalog for Passphrases generation."""
message = ('Could not find the Passphrase Catalog to generate '
'the site Passphrases!')
message = (
'Could not find the Passphrase Catalog to generate '
'the site Passphrases!')
class GenesisBundleEncryptionException(PeglegBaseException):
@ -106,8 +110,9 @@ class GenesisBundleGenerateException(PeglegBaseException):
class PKICertificateInvalidDuration(PeglegBaseException):
"""Exception for invalid duration of PKI Certificate."""
message = ('Provided duration is invalid. Certificate durations must be '
'a positive integer.')
message = (
'Provided duration is invalid. Certificate durations must be '
'a positive integer.')
#
@ -142,8 +147,9 @@ class SaltInsufficientLengthException(PeglegBaseException):
class GlobalCredentialsNotFound(PeglegBaseException):
"""Exception raised when global_passphrase or global_salt are not found."""
message = ('global_salt and global_passphrase must either both be '
'defined, or neither can be defined in site documents.')
message = (
'global_salt and global_passphrase must either both be '
'defined, or neither can be defined in site documents.')
#

10
pegleg/engine/generators/base_generator.py
View File

@ -71,9 +71,7 @@ class BaseGenerator(ABC):
def get_save_path(self, passphrase_name):
"""Calculate and return the save path of the ``passphrase_name``."""
return os.path.abspath(os.path.join(self._save_location,
'site',
self._sitename,
'secrets',
self.kind_path,
'{}.yaml'.format(passphrase_name)))
return os.path.abspath(
os.path.join(
self._save_location, 'site', self._sitename, 'secrets',
self.kind_path, '{}.yaml'.format(passphrase_name)))

13
pegleg/engine/generators/passphrase_generator.py
View File

@ -46,10 +46,10 @@ class PassphraseGenerator(BaseGenerator):
certificates.
"""
super(PassphraseGenerator, self).__init__(sitename, save_location,
author)
self._catalog = PassphraseCatalog(self._sitename,
documents=self._documents)
super(PassphraseGenerator,
self).__init__(sitename, save_location, author)
self._catalog = PassphraseCatalog(
self._sitename, documents=self._documents)
self._pass_util = CryptoString()
def generate(self, interactive=False, force_cleartext=False):
@ -81,8 +81,9 @@ class PassphraseGenerator(BaseGenerator):
docs = list()
if force_cleartext:
storage_policy = passphrase_catalog.P_CLEARTEXT
LOG.warning("Passphrases for {} will be "
"generated in clear text.".format(p_name))
LOG.warning(
"Passphrases for {} will be "
"generated in clear text.".format(p_name))
else:
storage_policy = self._catalog.get_storage_policy(p_name)

103
pegleg/engine/lint.py
View File

@ -12,13 +12,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import click
import logging
import os
import pkg_resources
import shutil
import textwrap
import click
import pkg_resources
from prettytable import PrettyTable
from pegleg import config
@ -84,10 +84,11 @@ def full(fail_on_missing_sub_src=False, exclude_lint=None, warn_lint=None):
messages=messages, exclude_lint=exclude_lint, warn_lint=warn_lint)
def site(site_name,
fail_on_missing_sub_src=False,
exclude_lint=None,
warn_lint=None):
def site(
site_name,
fail_on_missing_sub_src=False,
exclude_lint=None,
warn_lint=None):
"""Lint ``site_name``.
:param str site_name: Name of site to lint.
@ -133,10 +134,8 @@ def site(site_name,
messages=messages, exclude_lint=exclude_lint, warn_lint=warn_lint)
def _filter_messages_by_warn_and_error_lint(*,
messages=None,
exclude_lint=None,
warn_lint=None):
def _filter_messages_by_warn_and_error_lint(
*, messages=None, exclude_lint=None, warn_lint=None):
"""Helper that only filters messages depending on whether or not they
are present in ``exclude_lint`` or ``warn_lint``.
@ -171,8 +170,8 @@ def _filter_messages_by_warn_and_error_lint(*,
if errors:
raise click.ClickException(
'Linting failed:\n' + errors_table.get_string() +
'\nLinting warnings:\n' + warnings_table.get_string())
'Linting failed:\n' + errors_table.get_string()
+ '\nLinting warnings:\n' + warnings_table.get_string())
return warns
@ -189,14 +188,18 @@ def _verify_no_unexpected_files(*, sitenames=None):
errors = []
for unused_dir in sorted(found_directories - expected_directories):
errors.append((REPOS_MISSING_DIRECTORIES_FLAG,
'%s exists, but is unused' % unused_dir))
errors.append(
(
REPOS_MISSING_DIRECTORIES_FLAG,
'%s exists, but is unused' % unused_dir))
for missing_dir in sorted(expected_directories - found_directories):
if not missing_dir.endswith('common'):
errors.append(
(REPOS_MISSING_DIRECTORIES_FLAG,
'%s was not found, but expected by manifest' % missing_dir))
(
REPOS_MISSING_DIRECTORIES_FLAG,
'%s was not found, but expected by manifest'
% missing_dir))
return errors
@ -219,16 +222,20 @@ def _verify_single_file(filename, schemas):
LOG.debug("Validating file %s.", filename)
with open(filename, 'r') as f:
if not f.read(4) == '---\n':
errors.append((FILE_MISSING_YAML_DOCUMENT_HEADER,
'%s does not begin with YAML beginning of document '
'marker "---".' % filename))
errors.append(
(
FILE_MISSING_YAML_DOCUMENT_HEADER,
'%s does not begin with YAML beginning of document '
'marker "---".' % filename))
documents = []
try:
documents = util.files.read(filename)
except Exception as e:
errors.append((FILE_CONTAINS_INVALID_YAML,
'%s is not valid yaml: %s' % (filename, e)))
errors.append(
(
FILE_CONTAINS_INVALID_YAML, '%s is not valid yaml: %s' %
(filename, e)))
for document in documents:
errors.extend(_verify_document(document, schemas, filename))
@ -245,18 +252,20 @@ MANDATORY_ENCRYPTED_TYPES = {
def _verify_document(document, schemas, filename):
name = ':'.join([
document.get('schema', ''),
document.get('metadata', {}).get('name', '')
])
name = ':'.join(
[
document.get('schema', ''),
document.get('metadata', {}).get('name', '')
])
errors = []
layer = _layer(document)
if layer is not None and layer != _expected_layer(filename):
errors.append(
(DOCUMENT_LAYER_MISMATCH,
'%s (document %s) had unexpected layer "%s", expected "%s"' %
(filename, name, layer, _expected_layer(filename))))
(
DOCUMENT_LAYER_MISMATCH,
'%s (document %s) had unexpected layer "%s", expected "%s"' %
(filename, name, layer, _expected_layer(filename))))
# secrets must live in the appropriate directory, and must be
# "storagePolicy: encrypted".
@ -264,16 +273,19 @@ def _verify_document(document, schemas, filename):
storage_policy = document.get('metadata', {}).get('storagePolicy')
if (storage_policy != 'encrypted'):
errors.append((SCHEMA_STORAGE_POLICY_MISMATCH_FLAG,
'%s (document %s) is a secret, but has unexpected '
'storagePolicy: "%s"' % (filename, name,
storage_policy)))
errors.append(
(
SCHEMA_STORAGE_POLICY_MISMATCH_FLAG,
'%s (document %s) is a secret, but has unexpected '
'storagePolicy: "%s"' % (filename, name, storage_policy)))
# Check if the file is in a secrets directory
if not util.files.file_in_subdir(filename, 'secrets'):
errors.append((SECRET_NOT_ENCRYPTED_POLICY,
'%s (document %s) is a secret, is not stored in a'
' secrets path' % (filename, name)))
errors.append(
(
SECRET_NOT_ENCRYPTED_POLICY,
'%s (document %s) is a secret, is not stored in a'
' secrets path' % (filename, name)))
return errors
@ -303,8 +315,10 @@ def _verify_deckhand_render(*, sitename=None, fail_on_missing_sub_src=False):
all_errors = []
if sitename:
documents_to_render = [_handle_managed_document(doc) for doc in
util.definition.documents_for_site(sitename)]
documents_to_render = [
_handle_managed_document(doc)
for doc in util.definition.documents_for_site(sitename)
]
LOG.debug('Rendering documents for site: %s.', sitename)
_, errors = util.deckhand.deckhand_render(
@ -312,23 +326,26 @@ def _verify_deckhand_render(*, sitename=None, fail_on_missing_sub_src=False):
fail_on_missing_sub_src=fail_on_missing_sub_src,
validate=True,
)
LOG.debug('Generated %d rendering errors for site: %s.', len(errors),
sitename)
LOG.debug(
'Generated %d rendering errors for site: %s.', len(errors),
sitename)
all_errors.extend(errors)
else:
documents_to_render = util.definition.documents_for_each_site()
for site_name, documents in documents_to_render.items():
clean_documents = [_handle_managed_document(doc) for doc
in documents]
clean_documents = [
_handle_managed_document(doc) for doc in documents
]
LOG.debug('Rendering documents for site: %s.', site_name)
_, errors = util.deckhand.deckhand_render(
documents=clean_documents,
fail_on_missing_sub_src=fail_on_missing_sub_src,
validate=True,
)
LOG.debug('Generated %d rendering errors for site: %s.',
len(errors), site_name)
LOG.debug(
'Generated %d rendering errors for site: %s.', len(errors),
site_name)
all_errors.extend(errors)
return list(set(all_errors))

83
pegleg/engine/repository.py
View File

@ -28,8 +28,9 @@ from pegleg.engine import util
__all__ = ('process_repositories', 'process_site_repository')
__REPO_FOLDERS = {}
_INVALID_FORMAT_MSG = ("The repository %s must be in the form of "
"name=repoUrl[@revision]")
_INVALID_FORMAT_MSG = (
"The repository %s must be in the form of "
"name=repoUrl[@revision]")
LOG = logging.getLogger(__name__)
@ -64,8 +65,9 @@ def process_repositories(site_name, overwrite_existing=False):
# Dict mapping repository names to associated URL/revision info for clone.
repo_overrides = _process_repository_overrides(site_def_repos)
if not site_def_repos:
LOG.info('No repositories found in site-definition.yaml for site: %s. '
'Defaulting to specified repository overrides.', site_name)
LOG.info(
'No repositories found in site-definition.yaml for site: %s. '