Browse Source

Standardize Pegleg directory structure

This patch set standardizes the Pegleg directory
structure because of the following reasons:

1) src/bin/pegleg is not necessary and only makes
building (e.g. documentation building) and running
of tox targets unnecessarily difficult.
2) src/bin/pegleg is a Java-like standard that
bears no relevance to Python.

Change-Id: I37d39d3d6186b92f8fbfe234221c9e44da48cf10
changes/57/604157/7
Felipe Monteiro 1 year ago
parent
commit
893ea9f4bb

+ 1
- 1
.zuul.yaml View File

@@ -66,7 +66,7 @@
timeout: 300
nodeset: airship-pegleg-single-node
irrelevant-files:
- ^src/bin/pegleg/tests/.*$
- ^tests/.*$
- ^setup.cfg$

- job:

+ 7
- 7
Makefile View File

@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

PEGLEG_BUILD_CTX ?= src/bin/pegleg
PEGLEG_BUILD_CTX ?= pegleg
IMAGE_NAME ?= pegleg
IMAGE_PREFIX ?= airshipit
DOCKER_REGISTRY ?= quay.io
@@ -45,12 +45,12 @@ tests: run_tests

.PHONY: security
security:
tox -c src/bin/pegleg/tox.ini -e bandit
tox -e bandit

# Run all unit tests under src/bin/pegleg
# Run all unit tests under pegleg
.PHONY: run_tests
run_tests:
tox -c src/bin/pegleg/tox.ini -e py35
tox -e py35

# Perform Linting
.PHONY: lint
@@ -83,7 +83,7 @@ endif

.PHONY: docs
docs: clean
tox -edocs
tox -e docs

.PHONY: clean
clean:
@@ -91,8 +91,8 @@ clean:

.PHONY: py_lint
py_lint:
cd src/bin/pegleg;tox -e pep8
cd pegleg;tox -e pep8

.PHONY: py_format
py_format:
cd src/bin/pegleg;tox -e fmt
cd pegleg;tox -e fmt

+ 32
- 2
doc/source/conf.py View File

@@ -18,7 +18,8 @@
#
import os
import sys
sys.path.insert(0, os.path.abspath('../../src/bin/pegleg'))
sys.path.insert(0, os.path.abspath('.'))
sys.path.insert(0, os.path.abspath('../../'))
import sphinx_rtd_theme


@@ -105,7 +106,7 @@ html_static_path = []
# -- Options for HTMLHelp output ------------------------------------------

# Output file base name for HTML help builder.
htmlhelp_basename = 'ucpintdoc'
htmlhelp_basename = 'peglegdoc'


# -- Options for LaTeX output ---------------------------------------------
@@ -128,3 +129,32 @@ latex_elements = {
# 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'pegleg.tex', u'Pegleg Documentation',
u'Pegleg Authors', 'manual'),
]


# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'pegleg', u'Pegleg Documentation',
[author], 1)
]


# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'pegleg', u'Pegleg Documentation',
author, 'pegleg', 'Document-based aggregation and linting service.',
'Miscellaneous'),
]

+ 2
- 2
doc/source/developer-overview.rst View File

@@ -72,7 +72,7 @@ Pegleg strives to conform to the `Airship coding conventions`_.
Python
------

The Pegleg code base lives under ``/src/bin/pegleg``. Pegleg supports py35
The Pegleg code base lives under ``pegleg``. Pegleg supports py35 and py36
interpreters.

Docker
@@ -91,7 +91,7 @@ target: ``run_pegleg``.
Testing
=======

All Pegleg tests are nested under ``/src/bin/pegleg/tests``.
All Pegleg tests are nested under ``tests``.

Pegleg comes equipped with a number of `tox`_ targets for running unit tests,
as well as `pep8`_ and `Bandit`_ checks.

+ 2
- 4
images/pegleg/Dockerfile View File

@@ -4,10 +4,8 @@ FROM ${FROM}
VOLUME /var/pegleg
WORKDIR /var/pegleg

ARG ctx_base=src/bin/pegleg

COPY ${ctx_base}/requirements.txt /opt/pegleg/requirements.txt
COPY requirements.txt /opt/pegleg/requirements.txt
RUN pip3 install --no-cache-dir -r /opt/pegleg/requirements.txt

COPY ${ctx_base} /opt/pegleg
COPY . /opt/pegleg
RUN pip3 install -e /opt/pegleg

src/bin/pegleg/pegleg/__init__.py → pegleg/__init__.py View File


src/bin/pegleg/pegleg/cli.py → pegleg/cli.py View File


src/bin/pegleg/pegleg/config.py → pegleg/config.py View File


src/bin/pegleg/pegleg/engine/__init__.py → pegleg/engine/__init__.py View File


src/bin/pegleg/pegleg/engine/errorcodes.py → pegleg/engine/errorcodes.py View File


src/bin/pegleg/pegleg/engine/exceptions.py → pegleg/engine/exceptions.py View File


src/bin/pegleg/pegleg/engine/lint.py → pegleg/engine/lint.py View File


src/bin/pegleg/pegleg/engine/repository.py → pegleg/engine/repository.py View File


+ 135
- 0
pegleg/engine/site.py View File

@@ -0,0 +1,135 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import csv
import json
import logging
import os

import click
import yaml

from pegleg.engine import util

__all__ = ('collect', 'list_', 'show', 'render')

LOG = logging.getLogger(__name__)


def _read_and_format_yaml(filename):
with open(filename) as f:
lines_to_write = f.readlines()
if lines_to_write[0] != '---\n':
lines_to_write = ['---\n'] + lines_to_write
if lines_to_write[-1] != '...\n':
lines_to_write.append('...\n')
return lines_to_write or []


def _collect_to_stdout(site_name):
"""Collects all documents related to ``site_name`` and outputs them to
stdout via ``output_stream``.
"""
try:
for repo_base, filename in util.definition.site_files_by_repo(
site_name):
for line in _read_and_format_yaml(filename):
# This code is a pattern to convert \r\n to \n.
click.echo("\n".join(line.splitlines()))
except Exception as ex:
raise click.ClickException("Error printing output: %s" % str(ex))


def _collect_to_file(site_name, save_location):
"""Collects all documents related to ``site_name`` and outputs them to
the file denoted by ``save_location``.
"""
if not os.path.exists(save_location):
LOG.debug("Collection save location %s does not exist. Creating "
"automatically.", save_location)
os.makedirs(save_location)
# In case save_location already exists and isn't a directory.
if not os.path.isdir(save_location):
raise click.ClickException('save_location %s already exists, but must '
'be a directory' % save_location)

save_files = dict()
try:
for repo_base, filename in util.definition.site_files_by_repo(
site_name):
repo_name = os.path.normpath(repo_base).split(os.sep)[-1]
save_file = os.path.join(save_location, repo_name + '.yaml')
if repo_name not in save_files:
save_files[repo_name] = open(save_file, "w")
LOG.debug("Collecting file %s to file %s" % (filename, save_file))
save_files[repo_name].writelines(_read_and_format_yaml(filename))
except Exception as ex:
raise click.ClickException("Error saving output: %s" % str(ex))
finally:
for f in save_files.values():
f.close()


def collect(site_name, save_location):
if save_location:
_collect_to_file(site_name, save_location)
else:
_collect_to_stdout(site_name)


def render(site_name, output_stream):
documents = []
for filename in util.definition.site_files(site_name):
with open(filename) as f:
documents.extend(list(yaml.safe_load_all(f)))

rendered_documents, errors = util.deckhand.deckhand_render(
documents=documents)
err_msg = ''
if errors:
for err in errors:
if isinstance(err, tuple) and len(err) > 1:
err_msg += ': '.join(err) + '\n'
else:
err_msg += str(err) + '\n'
raise click.ClickException(err_msg)
yaml.dump_all(rendered_documents, output_stream, default_flow_style=False)


def list_(output_stream):
"""List site names for a given repository."""

# TODO(felipemonteiro): This should output a formatted table, not rows of
# data without delimited columns.
fieldnames = ['site_name', 'site_type', 'repositories']
writer = csv.DictWriter(
output_stream, fieldnames=fieldnames, delimiter=' ')
for site_name in util.files.list_sites():
params = util.definition.load_as_params(site_name)
# TODO(felipemonteiro): This is a temporary hack around legacy manifest
# repositories containing the name of a directory that symbolizes a
# repository. Once all these manifest repositories migrate over to Git
# references instead, remove this hack.
# NOTE(felipemonteiro): The 'revision' information can instead be
# computed using :func:`process_site_repository` and storing into
# a configuration via a "set_site_revision" function, for example.
if 'revision' in params:
params.pop('revision')
writer.writerow(params)


def show(site_name, output_stream):
data = util.definition.load_as_params(site_name)
data['files'] = list(util.definition.site_files(site_name))
json.dump(data, output_stream, indent=2, sort_keys=True)

src/bin/pegleg/pegleg/engine/stub.py → pegleg/engine/stub.py View File


src/bin/pegleg/pegleg/engine/util/__init__.py → pegleg/engine/util/__init__.py View File


src/bin/pegleg/pegleg/engine/util/deckhand.py → pegleg/engine/util/deckhand.py View File


src/bin/pegleg/pegleg/engine/util/definition.py → pegleg/engine/util/definition.py View File


src/bin/pegleg/pegleg/engine/util/files.py → pegleg/engine/util/files.py View File


src/bin/pegleg/pegleg/engine/util/git.py → pegleg/engine/util/git.py View File


src/bin/pegleg/pegleg/schemas/deckhand-metadata-control.yaml → pegleg/schemas/deckhand-metadata-control.yaml View File


src/bin/pegleg/pegleg/schemas/deckhand-metadata-document.yaml → pegleg/schemas/deckhand-metadata-document.yaml View File


src/bin/pegleg/pegleg/schemas/deckhand-root.yaml → pegleg/schemas/deckhand-root.yaml View File


src/bin/pegleg/pegleg/schemas/manifest.yaml → pegleg/schemas/manifest.yaml View File


src/bin/pegleg/requirements.txt → requirements.txt View File


src/bin/pegleg/setup.py → setup.py View File


+ 0
- 37
src/bin/pegleg/tox.ini View File

@@ -1,37 +0,0 @@
[tox]
envlist = py35, py36, pep8
skipsdist = True

[testenv]
deps =
-r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
basepython=python3
whitelist_externals =
find
commands =
find . -type f -name "*.pyc" -delete
pytest \
{posargs}

[testenv:fmt]
basepython=python3
deps =
-r{toxinidir}/test-requirements.txt
commands =
yapf --style=pep8 -ir {toxinidir}/pegleg {toxinidir}/tests

[testenv:pep8]
basepython=python3
deps =
-r{toxinidir}/test-requirements.txt
commands =
flake8 {toxinidir}/pegleg
bandit -r pegleg -n 5

[testenv:bandit]
basepython=python3
commands = bandit -r pegleg -n 5

[flake8]
ignore = E125,E251,W503

src/bin/pegleg/test-requirements.txt → test-requirements.txt View File


src/bin/pegleg/tests/__init__.py → tests/__init__.py View File


src/bin/pegleg/tests/unit/__init__.py → tests/unit/__init__.py View File


src/bin/pegleg/tests/unit/conftest.py → tests/unit/conftest.py View File


src/bin/pegleg/tests/unit/engine/__init__.py → tests/unit/engine/__init__.py View File


src/bin/pegleg/tests/unit/engine/test_lint.py → tests/unit/engine/test_lint.py View File


src/bin/pegleg/tests/unit/engine/test_selectable_linting.py → tests/unit/engine/test_selectable_linting.py View File


src/bin/pegleg/tests/unit/engine/test_site_collect.py → tests/unit/engine/test_site_collect.py View File


src/bin/pegleg/tests/unit/engine/test_site_repository.py → tests/unit/engine/test_site_repository.py View File


src/bin/pegleg/tests/unit/engine/test_util_files.py → tests/unit/engine/test_util_files.py View File


src/bin/pegleg/tests/unit/engine/util/__init__.py → tests/unit/engine/util/__init__.py View File


src/bin/pegleg/tests/unit/engine/util/test_git.py → tests/unit/engine/util/test_git.py View File


src/bin/pegleg/tests/unit/fixtures.py → tests/unit/fixtures.py View File


src/bin/pegleg/tests/unit/test_utils.py → tests/unit/test_utils.py View File


+ 29
- 23
tox.ini View File

@@ -10,40 +10,46 @@ setenv = VIRTUAL_ENV={envdir}
LANGUAGE=en_US
LC_ALL=en_US.utf-8
passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY

[testenv:venv]
commands = {posargs}

[testenv:py35]
basepython = python3.5
commands =
# Run all unit tests under src/bin/pegleg
tox -c src/bin/pegleg/tox.ini -e py35
whitelist_externals = tox

[testenv:py36]
basepython = python3.6
deps =
-r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
basepython=python3
whitelist_externals =
find
commands =
# Run all unit tests under src/bin/pegleg
tox -c src/bin/pegleg/tox.ini -e py36
whitelist_externals = tox
find . -type f -name "*.pyc" -delete
pytest \
{posargs}

[testenv:fmt]
basepython = python3
basepython=python3
deps =
-r{toxinidir}/test-requirements.txt
commands =
tox -c src/bin/pegleg/tox.ini -e fmt
whitelist_externals = tox
yapf --style=pep8 -ir {toxinidir}/pegleg {toxinidir}/tests

[testenv:pep8]
basepython = python3
basepython=python3
deps =
-r{toxinidir}/test-requirements.txt
commands =
tox -c src/bin/pegleg/tox.ini -e pep8
whitelist_externals = tox
flake8 {toxinidir}/pegleg
bandit -r pegleg -n 5

[testenv:docs]
basepython = python3
deps =
-r{toxinidir}/src/bin/pegleg/requirements.txt
-r{toxinidir}/requirements.txt
-r{toxinidir}/doc/requirements.txt
commands =
{toxinidir}/tools/gate/build-docs.sh

[testenv:bandit]
basepython=python3
commands = bandit -r pegleg -n 5

[testenv:venv]
commands = {posargs}

[flake8]
ignore = E125,E251,W503

Loading…
Cancel
Save