Standardize Pegleg directory structure

This patch set standardizes the Pegleg directory
structure because of the following reasons:

1) src/bin/pegleg is not necessary and only makes
building (e.g. documentation building) and running
of tox targets unnecessarily difficult.
2) src/bin/pegleg is a Java-like standard that
bears no relevance to Python.

Change-Id: I37d39d3d6186b92f8fbfe234221c9e44da48cf10

.zuul.yaml

@@ -66,7 +66,7 @@
     timeout: 300
     nodeset: airship-pegleg-single-node
     irrelevant-files:
-      - ^src/bin/pegleg/tests/.*$
+      - ^tests/.*$
       - ^setup.cfg$
 
 - job:

Makefile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-PEGLEG_BUILD_CTX  ?= src/bin/pegleg
+PEGLEG_BUILD_CTX  ?= pegleg
 IMAGE_NAME        ?= pegleg
 IMAGE_PREFIX      ?= airshipit
 DOCKER_REGISTRY   ?= quay.io
@@ -45,12 +45,12 @@ tests: run_tests
 
 .PHONY: security
 security:
-	tox -c src/bin/pegleg/tox.ini -e bandit
+	tox -e bandit
 
-# Run all unit tests under src/bin/pegleg
+# Run all unit tests under pegleg
 .PHONY: run_tests
 run_tests:
-	tox -c src/bin/pegleg/tox.ini -e py35
+	tox -e py35
 
 # Perform Linting
 .PHONY: lint
@@ -83,7 +83,7 @@ endif
 
 .PHONY: docs
 docs: clean
-	tox -edocs
+	tox -e docs
 
 .PHONY: clean
 clean:
@@ -91,8 +91,8 @@ clean:
 
 .PHONY: py_lint
 py_lint:
-	cd src/bin/pegleg;tox -e pep8
+	cd pegleg;tox -e pep8
 
 .PHONY: py_format
 py_format:
-	cd src/bin/pegleg;tox -e fmt
+	cd pegleg;tox -e fmt

doc/source/conf.py

@@ -18,7 +18,8 @@
 #
 import os
 import sys
-sys.path.insert(0, os.path.abspath('../../src/bin/pegleg'))
+sys.path.insert(0, os.path.abspath('.'))
+sys.path.insert(0, os.path.abspath('../../'))
 import sphinx_rtd_theme
 
 
@@ -105,7 +106,7 @@ html_static_path = []
 # -- Options for HTMLHelp output ------------------------------------------
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'ucpintdoc'
+htmlhelp_basename = 'peglegdoc'
 
 
 # -- Options for LaTeX output ---------------------------------------------
@@ -128,3 +129,32 @@ latex_elements = {
     # 'figure_align': 'htbp',
 }
 
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'pegleg.tex', u'Pegleg Documentation',
+     u'Pegleg Authors', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'pegleg', u'Pegleg Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'pegleg', u'Pegleg Documentation',
+     author, 'pegleg', 'Document-based aggregation and linting service.',
+     'Miscellaneous'),
+]

doc/source/developer-overview.rst

@@ -72,7 +72,7 @@ Pegleg strives to conform to the `Airship coding conventions`_.
 Python
 ------
 
-The Pegleg code base lives under ``/src/bin/pegleg``. Pegleg supports py35
+The Pegleg code base lives under ``pegleg``. Pegleg supports py35 and py36
 interpreters.
 
 Docker
@@ -91,7 +91,7 @@ target: ``run_pegleg``.
 Testing
 =======
 
-All Pegleg tests are nested under ``/src/bin/pegleg/tests``.
+All Pegleg tests are nested under ``tests``.
 
 Pegleg comes equipped with a number of `tox`_ targets for running unit tests,
 as well as `pep8`_ and `Bandit`_ checks.

images/pegleg/Dockerfile

@@ -4,10 +4,8 @@ FROM ${FROM}
 VOLUME /var/pegleg
 WORKDIR /var/pegleg
 
-ARG ctx_base=src/bin/pegleg
+COPY requirements.txt /opt/pegleg/requirements.txt
-
-COPY ${ctx_base}/requirements.txt /opt/pegleg/requirements.txt
 RUN pip3 install --no-cache-dir -r /opt/pegleg/requirements.txt
 
-COPY ${ctx_base} /opt/pegleg
+COPY . /opt/pegleg
 RUN pip3 install -e /opt/pegleg

pegleg/engine/site.py

@@ -0,0 +1,135 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import csv
+import json
+import logging
+import os
+
+import click
+import yaml
+
+from pegleg.engine import util
+
+__all__ = ('collect', 'list_', 'show', 'render')
+
+LOG = logging.getLogger(__name__)
+
+
+def _read_and_format_yaml(filename):
+    with open(filename) as f:
+        lines_to_write = f.readlines()
+        if lines_to_write[0] != '---\n':
+            lines_to_write = ['---\n'] + lines_to_write
+        if lines_to_write[-1] != '...\n':
+            lines_to_write.append('...\n')
+    return lines_to_write or []
+
+
+def _collect_to_stdout(site_name):
+    """Collects all documents related to ``site_name`` and outputs them to
+    stdout via ``output_stream``.
+    """
+    try:
+        for repo_base, filename in util.definition.site_files_by_repo(
+                site_name):
+            for line in _read_and_format_yaml(filename):
+                # This code is a pattern to convert \r\n to \n.
+                click.echo("\n".join(line.splitlines()))
+    except Exception as ex:
+        raise click.ClickException("Error printing output: %s" % str(ex))
+
+
+def _collect_to_file(site_name, save_location):
+    """Collects all documents related to ``site_name`` and outputs them to
+    the file denoted by ``save_location``.
+    """
+    if not os.path.exists(save_location):
+        LOG.debug("Collection save location %s does not exist. Creating "
+                  "automatically.", save_location)
+        os.makedirs(save_location)
+    # In case save_location already exists and isn't a directory.
+    if not os.path.isdir(save_location):
+        raise click.ClickException('save_location %s already exists, but must '
+                                   'be a directory' % save_location)
+
+    save_files = dict()
+    try:
+        for repo_base, filename in util.definition.site_files_by_repo(
+                site_name):
+            repo_name = os.path.normpath(repo_base).split(os.sep)[-1]
+            save_file = os.path.join(save_location, repo_name + '.yaml')
+            if repo_name not in save_files:
+                save_files[repo_name] = open(save_file, "w")
+            LOG.debug("Collecting file %s to file %s" % (filename, save_file))
+            save_files[repo_name].writelines(_read_and_format_yaml(filename))
+    except Exception as ex:
+        raise click.ClickException("Error saving output: %s" % str(ex))
+    finally:
+        for f in save_files.values():
+            f.close()
+
+
+def collect(site_name, save_location):
+    if save_location:
+        _collect_to_file(site_name, save_location)
+    else:
+        _collect_to_stdout(site_name)
+
+
+def render(site_name, output_stream):
+    documents = []
+    for filename in util.definition.site_files(site_name):
+        with open(filename) as f:
+            documents.extend(list(yaml.safe_load_all(f)))
+
+    rendered_documents, errors = util.deckhand.deckhand_render(
+        documents=documents)
+    err_msg = ''
+    if errors:
+        for err in errors:
+            if isinstance(err, tuple) and len(err) > 1:
+                err_msg += ': '.join(err) + '\n'
+            else:
+                err_msg += str(err) + '\n'
+        raise click.ClickException(err_msg)
+    yaml.dump_all(rendered_documents, output_stream, default_flow_style=False)
+
+
+def list_(output_stream):
+    """List site names for a given repository."""
+
+    # TODO(felipemonteiro): This should output a formatted table, not rows of
+    # data without delimited columns.
+    fieldnames = ['site_name', 'site_type', 'repositories']
+    writer = csv.DictWriter(
+        output_stream, fieldnames=fieldnames, delimiter=' ')
+    for site_name in util.files.list_sites():
+        params = util.definition.load_as_params(site_name)
+        # TODO(felipemonteiro): This is a temporary hack around legacy manifest
+        # repositories containing the name of a directory that symbolizes a
+        # repository. Once all these manifest repositories migrate over to Git
+        # references instead, remove this hack.
+        # NOTE(felipemonteiro): The 'revision' information can instead be
+        # computed using :func:`process_site_repository` and storing into
+        # a configuration via a "set_site_revision" function, for example.
+        if 'revision' in params:
+            params.pop('revision')
+        writer.writerow(params)
+
+
+def show(site_name, output_stream):
+    data = util.definition.load_as_params(site_name)
+    data['files'] = list(util.definition.site_files(site_name))
+    json.dump(data, output_stream, indent=2, sort_keys=True)

src/bin/pegleg/tox.ini

@@ -1,37 +0,0 @@
-[tox]
-envlist = py35, py36, pep8
-skipsdist = True
-
-[testenv]
-deps =
-  -r{toxinidir}/requirements.txt
-  -r{toxinidir}/test-requirements.txt
-basepython=python3
-whitelist_externals =
-  find
-commands =
-  find . -type f -name "*.pyc" -delete
-  pytest \
-    {posargs}
-
-[testenv:fmt]
-basepython=python3
-deps =
-    -r{toxinidir}/test-requirements.txt
-commands =
-    yapf --style=pep8 -ir {toxinidir}/pegleg {toxinidir}/tests
-
-[testenv:pep8]
-basepython=python3
-deps =
-    -r{toxinidir}/test-requirements.txt
-commands =
-    flake8 {toxinidir}/pegleg
-    bandit -r pegleg -n 5
-
-[testenv:bandit]
-basepython=python3
-commands = bandit -r pegleg -n 5
-
-[flake8]
-ignore = E125,E251,W503

tox.ini

@@ -10,40 +10,46 @@ setenv = VIRTUAL_ENV={envdir}
          LANGUAGE=en_US
          LC_ALL=en_US.utf-8
 passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
-
+deps =
-[testenv:venv]
+  -r{toxinidir}/requirements.txt
-commands = {posargs}
+  -r{toxinidir}/test-requirements.txt
-
+basepython=python3
-[testenv:py35]
+whitelist_externals =
-basepython = python3.5
+  find
 commands =
-  # Run all unit tests under src/bin/pegleg
+  find . -type f -name "*.pyc" -delete
-  tox -c src/bin/pegleg/tox.ini -e py35
+  pytest \
-whitelist_externals = tox
+    {posargs}
-
-[testenv:py36]
-basepython = python3.6
-commands =
-  # Run all unit tests under src/bin/pegleg
-  tox -c src/bin/pegleg/tox.ini -e py36
-whitelist_externals = tox
 
 [testenv:fmt]
-basepython = python3
+basepython=python3
+deps =
+    -r{toxinidir}/test-requirements.txt
 commands =
-  tox -c src/bin/pegleg/tox.ini -e fmt
+    yapf --style=pep8 -ir {toxinidir}/pegleg {toxinidir}/tests
-whitelist_externals = tox
 
 [testenv:pep8]
-basepython = python3
+basepython=python3
+deps =
+    -r{toxinidir}/test-requirements.txt
 commands =
-  tox -c src/bin/pegleg/tox.ini -e pep8
+    flake8 {toxinidir}/pegleg
-whitelist_externals = tox
+    bandit -r pegleg -n 5
 
 [testenv:docs]
 basepython = python3
 deps =
-  -r{toxinidir}/src/bin/pegleg/requirements.txt
+  -r{toxinidir}/requirements.txt
   -r{toxinidir}/doc/requirements.txt
 commands =
   {toxinidir}/tools/gate/build-docs.sh
+
+[testenv:bandit]
+basepython=python3
+commands = bandit -r pegleg -n 5
+
+[testenv:venv]
+commands = {posargs}
+
+[flake8]
+ignore = E125,E251,W503