283 lines
10 KiB
Python
283 lines
10 KiB
Python
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from collections import OrderedDict
|
|
import logging
|
|
import os
|
|
|
|
import click
|
|
import git
|
|
from prettytable import PrettyTable
|
|
import yaml
|
|
from yaml.constructor import SafeConstructor
|
|
|
|
from pegleg import config
|
|
from pegleg.engine import util
|
|
from pegleg.engine.util import files
|
|
from pegleg.engine.util.files import add_representer_ordered_dict
|
|
from pegleg.engine.util.git import TEMP_PEGLEG_COMMIT_MSG
|
|
|
|
__all__ = ('collect', 'list_', 'show', 'render')
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
def _read_and_format_yaml(filename):
|
|
with open(filename, 'r') as f:
|
|
lines_to_write = f.readlines()
|
|
if lines_to_write[0] != '---\n':
|
|
lines_to_write = ['---\n'] + lines_to_write
|
|
if lines_to_write[-1] != '...\n':
|
|
lines_to_write.append('...\n')
|
|
return lines_to_write or []
|
|
|
|
|
|
def _collect_to_stdout(site_name):
|
|
"""Collects all documents related to ``site_name`` and outputs them to
|
|
stdout via ``output_stream``.
|
|
"""
|
|
try:
|
|
for repo_base, filename in util.definition.site_files_by_repo(
|
|
site_name):
|
|
for line in _read_and_format_yaml(filename):
|
|
# This code is a pattern to convert \r\n to \n.
|
|
click.echo("\n".join(line.splitlines()))
|
|
add_representer_ordered_dict()
|
|
res = yaml.safe_dump(
|
|
get_deployment_data_doc(site_name),
|
|
explicit_start=True,
|
|
explicit_end=True,
|
|
default_flow_style=False)
|
|
# Click isn't splitting these lines correctly, so do it manually
|
|
for line in res.split('\n'):
|
|
click.echo(line)
|
|
except Exception as ex:
|
|
raise click.ClickException("Error printing output: %s" % str(ex))
|
|
|
|
|
|
def _collect_to_file(site_name, save_location):
|
|
"""Collects all documents related to ``site_name`` and outputs them to
|
|
the file denoted by ``save_location``.
|
|
"""
|
|
|
|
files.check_file_save_location(save_location)
|
|
|
|
save_files = dict()
|
|
curr_site_repo = files.path_leaf(config.get_site_repo())
|
|
|
|
try:
|
|
for repo_base, filename in util.definition.site_files_by_repo(
|
|
site_name):
|
|
repo_name = os.path.normpath(repo_base).split(os.sep)[-1]
|
|
save_file = os.path.join(save_location, repo_name + '.yaml')
|
|
if repo_name not in save_files:
|
|
save_files[repo_name] = open(save_file, 'w')
|
|
LOG.debug("Collecting file %s to file %s", filename, save_file)
|
|
save_files[repo_name].writelines(_read_and_format_yaml(filename))
|
|
add_representer_ordered_dict()
|
|
save_files[curr_site_repo].writelines(
|
|
yaml.safe_dump(
|
|
get_deployment_data_doc(site_name),
|
|
default_flow_style=False,
|
|
explicit_start=True,
|
|
explicit_end=True))
|
|
except Exception as ex:
|
|
raise click.ClickException("Error saving output: %s" % str(ex))
|
|
finally:
|
|
for f in save_files.values():
|
|
f.close()
|
|
|
|
|
|
def collect(site_name, save_location):
|
|
if save_location:
|
|
_collect_to_file(site_name, save_location)
|
|
else:
|
|
_collect_to_stdout(site_name)
|
|
|
|
|
|
def render(site_name, output_stream, validate):
|
|
rendered_documents = get_rendered_docs(site_name, validate=validate)
|
|
rendered_documents.append(get_deployment_data_doc(site_name))
|
|
if output_stream:
|
|
files.dump_all(
|
|
rendered_documents,
|
|
output_stream,
|
|
default_flow_style=False,
|
|
explicit_start=True,
|
|
explicit_end=True)
|
|
else:
|
|
add_representer_ordered_dict()
|
|
click.echo(
|
|
yaml.dump_all(
|
|
rendered_documents,
|
|
default_flow_style=False,
|
|
explicit_start=True,
|
|
explicit_end=True))
|
|
|
|
|
|
def get_rendered_docs(site_name, validate=True):
|
|
documents = []
|
|
# Ignore YAML tags, only construct dicts
|
|
SafeConstructor.add_multi_constructor(
|
|
'', lambda loader, suffix, node: None)
|
|
for filename in util.definition.site_files(site_name):
|
|
with open(filename, 'r') as f:
|
|
docs = yaml.safe_load_all(f)
|
|
|
|
for doc in docs:
|
|
|
|
# Managed documents may be encrypted, and require slight
|
|
# alteration for rendering without decrypting.
|
|
if doc['schema'] == 'pegleg/PeglegManagedDocument/v1':
|
|
|
|
# Do not decrypt secret, but convert it from bytes to
|
|
# string to pass schema validation.
|
|
if 'encrypted' in doc['data'].keys():
|
|
doc['data']['managedDocument']['data'] = doc['data'][
|
|
'managedDocument']['data'].decode()
|
|
|
|
# Append the document if it was encrypted using the
|
|
# encrypted string. If not, using original value.
|
|
documents.append(doc['data']['managedDocument'])
|
|
|
|
# File was not Pegleg managed, so it can be added directly.
|
|
else:
|
|
documents.append(doc)
|
|
|
|
rendered_documents, errors = util.deckhand.deckhand_render(
|
|
documents=documents, validate=validate)
|
|
|
|
if errors:
|
|
err_msg = ''
|
|
for err in errors:
|
|
if isinstance(err, tuple) and len(err) > 1:
|
|
err_msg += ': '.join(err) + '\n'
|
|
else:
|
|
err_msg += str(err) + '\n'
|
|
raise click.ClickException(err_msg)
|
|
|
|
return rendered_documents
|
|
|
|
|
|
def list_(output_stream):
|
|
"""List site names for a given repository."""
|
|
|
|
# Create a table to output site information for all sites for a given repo
|
|
site_table = PrettyTable()
|
|
field_names = ['site_name', 'site_type']
|
|
site_table.field_names = field_names
|
|
|
|
for site_name in util.files.list_sites():
|
|
params = util.definition.load_as_params(site_name, *field_names)
|
|
site_table.add_row(list(map(lambda k: params[k], field_names)))
|
|
# Write table to specified output_stream
|
|
msg = site_table.get_string()
|
|
if output_stream:
|
|
files.write(msg + "\n", output_stream)
|
|
else:
|
|
click.echo(msg)
|
|
|
|
|
|
def show(site_name, output_stream):
|
|
data = util.definition.load_as_params(site_name)
|
|
data['files'] = list(util.definition.site_files(site_name))
|
|
# Create a table to output site information for specific site
|
|
site_table = PrettyTable()
|
|
site_table.field_names = ['revision', 'site_name', 'site_type', 'files']
|
|
# TODO(felipemonteiro): Drop support for 'revision' once manifest
|
|
# repositories have removed it altogether.
|
|
if 'revision' in data.keys():
|
|
for file in data['files']:
|
|
site_table.add_row(
|
|
[data['revision'], data['site_name'], data['site_type'], file])
|
|
else:
|
|
for file in data['files']:
|
|
site_table.add_row(
|
|
["", data['site_name'], data['site_type'], file])
|
|
# Write tables to specified output_stream
|
|
msg = site_table.get_string()
|
|
if output_stream:
|
|
files.write(msg + "\n", output_stream)
|
|
else:
|
|
click.echo(msg)
|
|
|
|
|
|
def get_deployment_data_doc(site_name):
|
|
stanzas = {
|
|
files.path_leaf(repo): _get_repo_deployment_data_stanza(repo)
|
|
for repo in config.all_repos()
|
|
}
|
|
basedeployment_data = OrderedDict(
|
|
[
|
|
("schema", "pegleg/DeploymentData/v1"),
|
|
(
|
|
"metadata",
|
|
OrderedDict(
|
|
[
|
|
("schema", "metadata/Document/v1"),
|
|
("name", "deployment-version"),
|
|
(
|
|
"layeringDefinition",
|
|
OrderedDict(
|
|
[("abstract", False), ("layer", "global")])),
|
|
("storagePolicy", "cleartext"),
|
|
])), ("data", OrderedDict([("documents", stanzas)]))
|
|
])
|
|
try:
|
|
data = util.definition.load_as_params(site_name)
|
|
basedeployment_data['data'].update({'site_type': data['site_type']})
|
|
return basedeployment_data
|
|
except Exception as ex:
|
|
LOG.debug(
|
|
"Unable to get the site definition data for"
|
|
" site: %s, Exception :%s", site_name, ex)
|
|
return basedeployment_data
|
|
|
|
|
|
def _get_repo_deployment_data_stanza(repo_path):
|
|
try:
|
|
repo = git.Repo(repo_path)
|
|
commit = repo.commit()
|
|
contains_pegleg_commit = TEMP_PEGLEG_COMMIT_MSG in commit.message
|
|
|
|
# The repo may not appear dirty if Pegleg has made a temporary commit
|
|
# on top of changed/untracked files, but we know if that temporary
|
|
# commit happened the repo is indeed dirty
|
|
dirty = (repo.is_dirty() or contains_pegleg_commit)
|
|
|
|
if contains_pegleg_commit:
|
|
# The commit grabbed above isn't really what we want this data to
|
|
# reflect, because it was a commit made by Pegleg itself.
|
|
# Grab the commit before Pegleg made its temporary commit(s)
|
|
while (TEMP_PEGLEG_COMMIT_MSG in commit.message
|
|
and len(commit.parents) > 0):
|
|
commit = commit.parents[0]
|
|
|
|
# If we're at a particular tag, reference it
|
|
tag = [tag.name for tag in repo.tags if tag.commit == commit]
|
|
if tag:
|
|
tag == ", ".join(tag)
|
|
else:
|
|
# Otherwise just use the branch name
|
|
try:
|
|
tag = repo.active_branch.name
|
|
except TypeError as e:
|
|
if "HEAD is a detached symbolic reference" in str(e):
|
|
tag = "Detached HEAD"
|
|
else:
|
|
raise e
|
|
return {"commit": commit.hexsha, "tag": tag, "dirty": dirty}
|
|
except git.InvalidGitRepositoryError:
|
|
return {"commit": "None", "tag": "None", "dirty": "None"}
|