# Copyright 2018 AT&T Intellectual Property. All other rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from collections import OrderedDict import logging import os import click import git from prettytable import PrettyTable import yaml from yaml.constructor import SafeConstructor from pegleg import config from pegleg.engine import util from pegleg.engine.util import files from pegleg.engine.util.files import add_representer_ordered_dict from pegleg.engine.util.git import TEMP_PEGLEG_COMMIT_MSG __all__ = ('collect', 'list_', 'show', 'render') LOG = logging.getLogger(__name__) def _read_and_format_yaml(filename): with open(filename, 'r') as f: lines_to_write = f.readlines() if lines_to_write[0] != '---\n': lines_to_write = ['---\n'] + lines_to_write if lines_to_write[-1] != '...\n': lines_to_write.append('...\n') return lines_to_write or [] def _collect_to_stdout(site_name): """Collects all documents related to ``site_name`` and outputs them to stdout via ``output_stream``. """ try: for repo_base, filename in util.definition.site_files_by_repo( site_name): for line in _read_and_format_yaml(filename): # This code is a pattern to convert \r\n to \n. click.echo("\n".join(line.splitlines())) add_representer_ordered_dict() res = yaml.safe_dump( get_deployment_data_doc(site_name), explicit_start=True, explicit_end=True, default_flow_style=False) # Click isn't splitting these lines correctly, so do it manually for line in res.split('\n'): click.echo(line) except Exception as ex: raise click.ClickException("Error printing output: %s" % str(ex)) def _collect_to_file(site_name, save_location): """Collects all documents related to ``site_name`` and outputs them to the file denoted by ``save_location``. """ files.check_file_save_location(save_location) save_files = dict() curr_site_repo = files.path_leaf(config.get_site_repo()) try: for repo_base, filename in util.definition.site_files_by_repo( site_name): repo_name = os.path.normpath(repo_base).split(os.sep)[-1] save_file = os.path.join(save_location, repo_name + '.yaml') if repo_name not in save_files: save_files[repo_name] = open(save_file, 'w') LOG.debug("Collecting file %s to file %s", filename, save_file) save_files[repo_name].writelines(_read_and_format_yaml(filename)) add_representer_ordered_dict() save_files[curr_site_repo].writelines( yaml.safe_dump( get_deployment_data_doc(site_name), default_flow_style=False, explicit_start=True, explicit_end=True)) except Exception as ex: raise click.ClickException("Error saving output: %s" % str(ex)) finally: for f in save_files.values(): f.close() def collect(site_name, save_location): if save_location: _collect_to_file(site_name, save_location) else: _collect_to_stdout(site_name) def render(site_name, output_stream, validate): rendered_documents = get_rendered_docs(site_name, validate=validate) rendered_documents.append(get_deployment_data_doc(site_name)) if output_stream: files.dump_all( rendered_documents, output_stream, default_flow_style=False, explicit_start=True, explicit_end=True) else: add_representer_ordered_dict() click.echo( yaml.dump_all( rendered_documents, default_flow_style=False, explicit_start=True, explicit_end=True)) def get_rendered_docs(site_name, validate=True): documents = [] # Ignore YAML tags, only construct dicts SafeConstructor.add_multi_constructor( '', lambda loader, suffix, node: None) for filename in util.definition.site_files(site_name): with open(filename, 'r') as f: docs = yaml.safe_load_all(f) for doc in docs: # Managed documents may be encrypted, and require slight # alteration for rendering without decrypting. if doc['schema'] == 'pegleg/PeglegManagedDocument/v1': # Do not decrypt secret, but convert it from bytes to # string to pass schema validation. if 'encrypted' in doc['data'].keys(): doc['data']['managedDocument']['data'] = doc['data'][ 'managedDocument']['data'].decode() # Append the document if it was encrypted using the # encrypted string. If not, using original value. documents.append(doc['data']['managedDocument']) # File was not Pegleg managed, so it can be added directly. else: documents.append(doc) rendered_documents, errors = util.deckhand.deckhand_render( documents=documents, validate=validate) if errors: err_msg = '' for err in errors: if isinstance(err, tuple) and len(err) > 1: err_msg += ': '.join(err) + '\n' else: err_msg += str(err) + '\n' raise click.ClickException(err_msg) return rendered_documents def list_(output_stream): """List site names for a given repository.""" # Create a table to output site information for all sites for a given repo site_table = PrettyTable() field_names = ['site_name', 'site_type'] site_table.field_names = field_names for site_name in util.files.list_sites(): params = util.definition.load_as_params(site_name, *field_names) site_table.add_row(list(map(lambda k: params[k], field_names))) # Write table to specified output_stream msg = site_table.get_string() if output_stream: files.write(msg + "\n", output_stream) else: click.echo(msg) def show(site_name, output_stream): data = util.definition.load_as_params(site_name) data['files'] = list(util.definition.site_files(site_name)) # Create a table to output site information for specific site site_table = PrettyTable() site_table.field_names = ['revision', 'site_name', 'site_type', 'files'] # TODO(felipemonteiro): Drop support for 'revision' once manifest # repositories have removed it altogether. if 'revision' in data.keys(): for file in data['files']: site_table.add_row( [data['revision'], data['site_name'], data['site_type'], file]) else: for file in data['files']: site_table.add_row( ["", data['site_name'], data['site_type'], file]) # Write tables to specified output_stream msg = site_table.get_string() if output_stream: files.write(msg + "\n", output_stream) else: click.echo(msg) def get_deployment_data_doc(site_name): stanzas = { files.path_leaf(repo): _get_repo_deployment_data_stanza(repo) for repo in config.all_repos() } basedeployment_data = OrderedDict( [ ("schema", "pegleg/DeploymentData/v1"), ( "metadata", OrderedDict( [ ("schema", "metadata/Document/v1"), ("name", "deployment-version"), ( "layeringDefinition", OrderedDict( [("abstract", False), ("layer", "global")])), ("storagePolicy", "cleartext"), ])), ("data", OrderedDict([("documents", stanzas)])) ]) try: data = util.definition.load(site_name) basedeployment_data['data'].update( {'site_type': data['data']['site_type']}) except Exception as ex: LOG.debug( "Unable to get the site definition data for" " site: %s, Exception :%s", site_name, ex) try: basedeployment_data['data'].update( {'version': data['data']['repositories']['global']['revision']}) except Exception as ex: LOG.debug( "Unable to get the site revision data for global in" " site: %s, Exception :%s", site_name, ex) return basedeployment_data def _get_repo_deployment_data_stanza(repo_path): try: repo = git.Repo(repo_path) commit = repo.commit() contains_pegleg_commit = TEMP_PEGLEG_COMMIT_MSG in commit.message # The repo may not appear dirty if Pegleg has made a temporary commit # on top of changed/untracked files, but we know if that temporary # commit happened the repo is indeed dirty dirty = (repo.is_dirty() or contains_pegleg_commit) if contains_pegleg_commit: # The commit grabbed above isn't really what we want this data to # reflect, because it was a commit made by Pegleg itself. # Grab the commit before Pegleg made its temporary commit(s) while (TEMP_PEGLEG_COMMIT_MSG in commit.message and len(commit.parents) > 0): commit = commit.parents[0] # If we're at a particular tag, reference it tag = [tag.name for tag in repo.tags if tag.commit == commit] if tag: tag = ", ".join(tag) else: # Otherwise just use the branch name try: tag = repo.active_branch.name except TypeError as e: if "HEAD is a detached symbolic reference" in str(e): tag = "Detached HEAD" else: raise e return {"commit": commit.hexsha, "tag": tag, "dirty": dirty} except git.InvalidGitRepositoryError: return {"commit": "None", "tag": "None", "dirty": "None"}