Alexander Hughes
1c8d92ef6b
This patch addresses inconsistent code style and enforces it with a gate for future submissions. Separate work will be done in the future to address several of the PEP8 ignores for docstrings, and attempt to bring the tests directory to PEP8 compliance. This patch: 1. Updates .style.yapf to set the knobs desired for YAPF. 2. Updates tox.ini to allow one of the knobs to work. 3. Removes unused code from several __init__.py files. 4. Updates the YAPF version in test-requirements.txt to latest (this is needed for several knobs to work). 5. Stylistic changes to the python codebase in Pegleg. 6. Updates to tox.ini to run YAPF during PEP8 check. Change-Id: Ieaa0fdef2b601d01c875d64b840986e54df73abf
366 lines
14 KiB
Python
366 lines
14 KiB
Python
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import atexit
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import tempfile
|
|
|
|
import click
|
|
|
|
from pegleg import config
|
|
from pegleg.engine import exceptions
|
|
from pegleg.engine import util
|
|
|
|
__all__ = ('process_repositories', 'process_site_repository')
|
|
|
|
__REPO_FOLDERS = {}
|
|
_INVALID_FORMAT_MSG = (
|
|
"The repository %s must be in the form of "
|
|
"name=repoUrl[@revision]")
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
@atexit.register
|
|
def _clean_temp_folders():
|
|
global __REPO_FOLDERS
|
|
|
|
for r in __REPO_FOLDERS.values():
|
|
shutil.rmtree(r, ignore_errors=True)
|
|
|
|
|
|
def process_repositories(site_name, overwrite_existing=False):
|
|
"""Process and setup all repositories including ensuring we are at the
|
|
right revision based on the site's own site-definition.yaml file.
|
|
|
|
:param site_name: Site name for which to clone relevant repos.
|
|
:param overwrite_existing: Whether to overwrite an existing directory
|
|
|
|
"""
|
|
|
|
# Only tracks extra repositories - not the site (primary) repository.
|
|
extra_repos = []
|
|
|
|
site_repo = process_site_repository(overwrite_existing=overwrite_existing)
|
|
|
|
# Retrieve extra repo data from site-definition.yaml files.
|
|
site_data = util.definition.load_as_params(
|
|
site_name, 'site_type', 'repositories', primary_repo_base=site_repo)
|
|
site_def_repos = _get_and_validate_site_repositories(site_name, site_data)
|
|
|
|
# Dict mapping repository names to associated URL/revision info for clone.
|
|
repo_overrides = _process_repository_overrides(site_def_repos)
|
|
if not site_def_repos:
|
|
LOG.info(
|
|
'No repositories found in site-definition.yaml for site: %s. '
|
|
'Defaulting to specified repository overrides.', site_name)
|
|
site_def_repos = repo_overrides
|
|
|
|
# Extract user/key that we will use for all repositories.
|
|
repo_key = config.get_repo_key()
|
|
repo_user = config.get_repo_username()
|
|
|
|
for repo_alias in site_def_repos.keys():
|
|
if repo_alias == "site":
|
|
LOG.warning(
|
|
"The primary site repository path must be specified "
|
|
"via the -r flag. Ignoring the provided "
|
|
"site-definition entry: %s", site_def_repos[repo_alias])
|
|
continue
|
|
|
|
# Extract URL and revision, prioritizing overrides over the defaults in
|
|
# the site-definition.yaml.
|
|
if repo_alias in repo_overrides:
|
|
repo_url_or_path = repo_overrides[repo_alias]['url']
|
|
repo_revision = repo_overrides[repo_alias]['revision']
|
|
else:
|
|
repo_url_or_path = site_def_repos[repo_alias]['url']
|
|
repo_revision = site_def_repos[repo_alias]['revision']
|
|
|
|
repo_url_or_path = _format_url_with_repo_username(repo_url_or_path)
|
|
|
|
LOG.info(
|
|
"Processing repository %s with url=%s, repo_key=%s, "
|
|
"repo_username=%s, revision=%s", repo_alias, repo_url_or_path,
|
|
repo_key, repo_user, repo_revision)
|
|
|
|
temp_extra_repo = _process_repository(
|
|
repo_url_or_path,
|
|
repo_revision,
|
|
overwrite_existing=overwrite_existing)
|
|
extra_repos.append(temp_extra_repo)
|
|
|
|
# Overwrite the site repo and extra repos in the config because further
|
|
# processing will fail if they contain revision info in their paths.
|
|
LOG.debug(
|
|
"Updating site_repo=%s extra_repo_list=%s in config", site_repo,
|
|
extra_repos)
|
|
config.set_site_repo(site_repo)
|
|
config.set_extra_repo_list(extra_repos)
|
|
|
|
|
|
def process_site_repository(update_config=False, overwrite_existing=False):
|
|
"""Process and setup site repository including ensuring we are at the right
|
|
revision based on the site's own site-definition.yaml file.
|
|
|
|
:param bool update_config: Whether to update Pegleg config with computed
|
|
site repo path.
|
|
:param overwrite_existing: Whether to overwrite an existing directory
|
|
|
|
"""
|
|
|
|
# Retrieve the main site repository and validate it.
|
|
site_repo_or_path = config.get_site_repo()
|
|
if not site_repo_or_path:
|
|
raise ValueError(
|
|
"Site repository directory (%s) must be specified"
|
|
% site_repo_or_path)
|
|
|
|
repo_url_or_path, repo_revision = _extract_repo_url_and_revision(
|
|
site_repo_or_path)
|
|
config.set_site_rev(repo_revision)
|
|
repo_url_or_path = _format_url_with_repo_username(repo_url_or_path)
|
|
new_repo_path = _process_repository(
|
|
repo_url_or_path, repo_revision, overwrite_existing=overwrite_existing)
|
|
|
|
if update_config:
|
|
# Overwrite the site repo in the config because further processing will
|
|
# fail if they contain revision info in their paths.
|
|
LOG.debug("Updating site_repo=%s in config", new_repo_path)
|
|
config.set_site_repo(new_repo_path)
|
|
|
|
return new_repo_path
|
|
|
|
|
|
def _process_repository(
|
|
repo_url_or_path, repo_revision, overwrite_existing=False):
|
|
"""Process a repository located at ``repo_url_or_path``.
|
|
|
|
:param str repo_url_or_path: Path to local repo or URL of remote URL.
|
|
:param str repo_revision: branch, commit or ref in the repo to checkout.
|
|
:param overwrite_existing: Whether to overwrite an existing directory
|
|
|
|
"""
|
|
|
|
global __REPO_FOLDERS
|
|
|
|
if os.path.exists(repo_url_or_path) and not overwrite_existing:
|
|
repo_name = util.git.repo_name(repo_url_or_path)
|
|
parent_temp_path = tempfile.mkdtemp()
|
|
__REPO_FOLDERS.setdefault(repo_name, parent_temp_path)
|
|
new_temp_path = os.path.join(parent_temp_path, repo_name)
|
|
norm_path, sub_path = util.git.normalize_repo_path(repo_url_or_path)
|
|
shutil.copytree(src=norm_path, dst=new_temp_path, symlinks=True)
|
|
__REPO_FOLDERS.setdefault(repo_name, new_temp_path)
|
|
git_repo_path = _process_site_repository(new_temp_path, repo_revision)
|
|
return os.path.join(git_repo_path, sub_path)
|
|
else:
|
|
repo_url, sub_path = util.git.normalize_repo_path(repo_url_or_path)
|
|
git_repo_path = _process_site_repository(repo_url, repo_revision)
|
|
return os.path.join(git_repo_path, sub_path)
|
|
|
|
|
|
def _process_site_repository(repo_url_or_path, repo_revision):
|
|
"""Process the primary or site repository located at ``repo_url_or_path``.
|
|
|
|
Also validate that the provided ``repo_url_or_path`` is a valid Git
|
|
repository. If ``repo_url_or_path`` doesn't already exist, clone it.
|
|
If it does, extract the appropriate revision and check it out.
|
|
|
|
:param repo_url_or_path: Repo path or URL and associated auth information.
|
|
If URL, examples include:
|
|
|
|
* ssh://REPO_USERNAME@<GIT_URL>:29418/global-manifests.git@<ref>
|
|
* https://<GIT_URL>/global-manifests.git@<ref>
|
|
* http://<GIT_URL>/global-manifests.git@<ref>
|
|
* <LOCAL_REPO_PATH>@<ref>
|
|
* same values as above without @<ref>
|
|
:param str repo_revision: branch, commit or ref in the repo to checkout.
|
|
|
|
"""
|
|
|
|
repo_alias = 'site' # We are processing the site repo necessarily.
|
|
repo_key = config.get_repo_key()
|
|
repo_user = config.get_repo_username()
|
|
|
|
LOG.info(
|
|
"Processing repository %s with url=%s, repo_key=%s, "
|
|
"repo_username=%s, revision=%s", repo_alias, repo_url_or_path,
|
|
repo_key, repo_user, repo_revision)
|
|
return _handle_repository(
|
|
repo_url_or_path, ref=repo_revision, auth_key=repo_key)
|
|
|
|
|
|
def _get_and_validate_site_repositories(site_name, site_data):
|
|
"""Validate that repositories entry exists in ``site_data``."""
|
|
if 'repositories' not in site_data:
|
|
LOG.info(
|
|
"The repository for site_name: %s does not contain a "
|
|
"site-definition.yaml with a 'repositories' key. Ensure "
|
|
"your repository is self-contained and doesn't require "
|
|
"extra repositories for correct rendering.", site_name)
|
|
return site_data.get('repositories', {})
|
|
|
|
|
|
def _process_repository_overrides(site_def_repos):
|
|
"""Process specified repository overrides via the CLI ``-e`` flag.
|
|
|
|
This will resolve ``-e`` site CLI arguments and override the corresponding
|
|
values in the relevant :file:`site-definition.yaml`, if applicable.
|
|
|
|
For example, given CLI override of::
|
|
|
|
-e global=/opt/global@foo
|
|
|
|
And site-definition.yaml ``repositories`` value of::
|
|
|
|
repositories:
|
|
global:
|
|
revision: bar
|
|
url: /opt/global
|
|
|
|
Then the resulting dictionary that is returned will be::
|
|
|
|
{"global": {"url": "/opt/global", "revision": "foo"}}
|
|
|
|
:param site_def_repos: Dictionary of ``repositories`` field included
|
|
in relevant :file:`site-definition.yaml`.
|
|
:returns: Dictionary with above format.
|
|
|
|
"""
|
|
|
|
# Extra repositories to process.
|
|
provided_repo_overrides = config.get_extra_repo_overrides()
|
|
# Map repository names to the associated URL/revision for cloning.
|
|
repo_overrides = {}
|
|
|
|
for repo_override in provided_repo_overrides:
|
|
# break apart global=repoUrl
|
|
try:
|
|
repo_alias, repo_url_or_path = repo_override.split('=', 1)
|
|
except ValueError:
|
|
# TODO(felipemonteiro): Use internal exceptions for this.
|
|
raise click.ClickException(_INVALID_FORMAT_MSG % repo_override)
|
|
|
|
if repo_alias == "site":
|
|
LOG.warning(
|
|
"The primary site repository path must be specified "
|
|
"via the -r flag. Ignoring the provided override: %s",
|
|
repo_override)
|
|
continue
|
|
|
|
if repo_alias not in site_def_repos:
|
|
# If we are overriding a value that doesn't exist in the
|
|
# site-definition.yaml make a note of it in case the override
|
|
# is something bogus, but we won't make this a hard requirement,
|
|
# so just log the discrepancy.
|
|
LOG.debug(
|
|
"Repo override: %s not found under `repositories` for "
|
|
"site-definition.yaml. Site def repositories: %s",
|
|
repo_override, ", ".join(site_def_repos.keys()))
|
|
|
|
repo_url, revision = _extract_repo_url_and_revision(repo_url_or_path)
|
|
|
|
# store what we've learned
|
|
repo_overrides.setdefault(repo_alias, {})
|
|
repo_overrides[repo_alias]['url'] = repo_url
|
|
repo_overrides[repo_alias]['revision'] = revision
|
|
|
|
return repo_overrides
|
|
|
|
|
|
def _extract_repo_url_and_revision(repo_url_or_path):
|
|
"""Break up repository path/url into the repo URL and revision.
|
|
|
|
:param repo_url_or_path: Repo URL and associated auth information. E.g.:
|
|
|
|
* ssh://REPO_USERNAME@<GIT_URL>:29418/global-manifests.git@<ref>
|
|
* https://<GIT_URL>/global-manifests.git@<ref>
|
|
* http://<GIT_URL>/global-manifests.git@<ref>
|
|
* <LOCAL_REPO_PATH>@<ref>
|
|
* same values as above without @<ref>
|
|
|
|
"""
|
|
|
|
ssh_username_pattern = re.compile(r'ssh:\/\/.+@.+\/.+')
|
|
|
|
def has_revision(repo_url_or_path):
|
|
if repo_url_or_path.lower().startswith('ssh'):
|
|
if ssh_username_pattern.match(repo_url_or_path):
|
|
return repo_url_or_path.count('@') == 2
|
|
return repo_url_or_path.count('@') == 1
|
|
else:
|
|
return '@' in repo_url_or_path
|
|
|
|
# they've forced a revision using @revision - careful not to confuse
|
|
# this with auth
|
|
revision = None
|
|
try:
|
|
if has_revision(repo_url_or_path):
|
|
repo_url_or_path, revision = repo_url_or_path.rsplit('@', 1)
|
|
revision = revision.rstrip('/')
|
|
if revision.endswith(".git"):
|
|
revision = revision[:-4]
|
|
if repo_url_or_path.endswith(".git"):
|
|
repo_url_or_path = repo_url_or_path[:-4]
|
|
else:
|
|
repo_url_or_path = repo_url_or_path
|
|
except Exception:
|
|
# TODO(felipemonteiro): Use internal exceptions for this.
|
|
raise click.ClickException(_INVALID_FORMAT_MSG % repo_url_or_path)
|
|
|
|
return repo_url_or_path, revision
|
|
|
|
|
|
def _format_url_with_repo_username(repo_url_or_path):
|
|
# If a repo user is provided, do the necessary replacements.
|
|
repo_user = config.get_repo_username()
|
|
if repo_user:
|
|
if "REPO_USERNAME" not in repo_url_or_path:
|
|
LOG.warning(
|
|
"A repository username was specified but no REPO_USERNAME "
|
|
"string found in repository url %s", repo_url_or_path)
|
|
else:
|
|
repo_url_or_path = repo_url_or_path.replace(
|
|
'REPO_USERNAME', repo_user)
|
|
|
|
elif "REPO_USERNAME" in repo_url_or_path:
|
|
raise exceptions.GitMissingUserException(url=repo_url_or_path)
|
|
|
|
return repo_url_or_path
|
|
|
|
|
|
def _handle_repository(repo_url_or_path, *args, **kwargs):
|
|
"""Clone remote remote (if ``repo_url_or_path`` is a remote URL) and
|
|
checkout specified reference .
|
|
|
|
"""
|
|
# Retrieve the clone path where the repo will be cloned
|
|
clone_path = config.get_clone_path()
|
|
|
|
try:
|
|
return util.git.git_handler(
|
|
repo_url_or_path, clone_path=clone_path, *args, **kwargs)
|
|
except exceptions.GitException as e:
|
|
raise click.ClickException(e)
|
|
except Exception as e:
|
|
LOG.exception(
|
|
'Unknown exception was raised during git clone/checkout:'
|
|
' %s', e)
|
|
# TODO(felipemonteiro): Use internal exceptions for this.
|
|
raise click.ClickException(e)
|