Add git and branch revision support to pegleg

* Add support for URLs and directories including git clone support
* Add support for http://, https://, and ssh:// git cloning
* Add support for cloning behind proxy
* Add support for checking out references of cloned repos
* Add support for checking out references of local repos
* Add support for Pegleg Git exceptions

This patch set also adds support for including Pegleg
source code in documentation and adds exceptions
documentation.

Change-Id: I417a62c815f97a70f3abc432cc342707e8ce1f54

doc/source/conf.py

@@ -16,9 +16,9 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
+import os
+import sys
+sys.path.insert(0, os.path.abspath('../../src/bin/pegleg'))
 import sphinx_rtd_theme
 
 
@@ -52,7 +52,7 @@ master_doc = 'index'
 # General information about the project.
 project = u'pegleg'
 copyright = u'2018 AT&T Intellectual Property.'
-author = u'pegleg Authors'
+author = u'Pegleg Authors'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

doc/source/exceptions.rst

@@ -0,0 +1,67 @@
+..
+  Copyright 2018 AT&T Intellectual Property.
+  All Rights Reserved.
+
+  Licensed under the Apache License, Version 2.0 (the "License"); you may
+  not use this file except in compliance with the License. You may obtain
+  a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  License for the specific language governing permissions and limitations
+  under the License.
+
+Pegleg  Exceptions
+==================
+
+Base Exceptions
+---------------
+
+.. list-table::
+  :widths: 5 50
+  :header-rows: 1
+
+  * - Exception Name
+    - Description
+  * - PeglegBaseException
+    - .. autoexception:: pegleg.engine.exceptions.PeglegBaseException
+         :members:
+         :undoc-members:
+
+Git Exceptions
+--------------
+
+.. list-table::
+  :widths: 5 50
+  :header-rows: 1
+
+  * - Exception Name
+    - Description
+  * - BaseGitException
+    - .. autoexception:: pegleg.engine.exceptions.BaseGitException
+         :members:
+         :show-inheritance:
+         :undoc-members:
+  * - GitException
+    - .. autoexception:: pegleg.engine.exceptions.GitException
+         :members:
+         :show-inheritance:
+         :undoc-members:
+  * - GitAuthException
+    - .. autoexception:: pegleg.engine.exceptions.GitAuthException
+         :members:
+         :show-inheritance:
+         :undoc-members:
+  * - GitProxyException
+    - .. autoexception:: pegleg.engine.exceptions.GitProxyException
+         :members:
+         :show-inheritance:
+         :undoc-members:
+  * - GitSSHException
+    - .. autoexception:: pegleg.engine.exceptions.GitSSHException
+         :members:
+         :show-inheritance:
+         :undoc-members:

doc/source/index.rst

@@ -14,26 +14,32 @@
       License for the specific language governing permissions and limitations
       under the License.
 
-.. tip::
+====================
+Pegleg Documentation
+====================
 
-  The Undercloud Platform is part of the AIC CP (AT&T Integrated Cloud
-  Containerized Platform). More details may be found by using the `Treasuremap`_
-
-Building this Documentation
----------------------------
-
-Use of ``tox -e docs`` will build an HTML version of this documentation that
-can be viewed using a browser at docs/build/index.html on the local filesystem.
-
-Conventions and Standards
--------------------------
+Overview
+--------
 
 .. toctree::
    :maxdepth: 2
 
    getting_started
-   authoring_strategy
-   artifacts
-   cli
 
-.. _Treasuremap: https://github.com/att-comdev/treasuremap
+Design
+------
+
+.. toctree::
+   :maxdepth: 2
+
+   artifacts
+   authoring_strategy
+
+Operator's Guide
+----------------
+
+.. toctree::
+   :maxdepth: 2
+
+   cli
+   exceptions

src/bin/pegleg/pegleg/engine/exceptions.py

@@ -0,0 +1,79 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class PeglegBaseException(Exception):
+    """Base class for Pegleg exception and error handling."""
+
+    def __init__(self, message=None, **kwargs):
+        self.message = message or self.message
+        try:  # nosec
+            self.message = self.message % kwargs
+        except Exception:
+            pass
+        super(PeglegBaseException, self).__init__(self.message)
+
+
+class BaseGitException(PeglegBaseException):
+    """Base class for Git exceptions and error handling."""
+
+    message = 'An unknown error occurred while accessing a chart source.'
+
+
+class GitException(BaseGitException):
+    """Exception when an error occurs cloning a Git repository."""
+
+    def __init__(self, location, details=None):
+        self._message = ('Git exception occurred: [%s] may not be a valid git '
+                         'repository' % location)
+        if details:
+            self._message += '. Details: %s' % details
+
+        super(GitException, self).__init__(self._message)
+
+
+class GitAuthException(BaseGitException):
+    """Exception that occurs when authentication fails for cloning a repo."""
+
+    def __init__(self, repo_url, ssh_key_path):
+        self._repo_url = repo_url
+        self._ssh_key_path = ssh_key_path
+
+        self._message = ('Failed to authenticate for repo %s with ssh-key at '
+                         'path %s.' % (self._repo_url, self._ssh_key_path))
+
+        super(GitAuthException, self).__init__(self._message)
+
+
+class GitProxyException(BaseGitException):
+    """Exception when an error occurs cloning a Git repository
+       through a proxy."""
+
+    def __init__(self, location):
+        self._location = location
+        self._message = ('Could not resolve proxy [%s].' % self._location)
+
+        super(GitProxyException, self).__init__(self._message)
+
+
+class GitSSHException(BaseGitException):
+    """Exception that occurs when an SSH key could not be found."""
+
+    def __init__(self, ssh_key_path):
+        self._ssh_key_path = ssh_key_path
+
+        self._message = ('Failed to find specified SSH key: %s.' %
+                         (self._ssh_key_path))
+
+        super(GitSSHException, self).__init__(self._message)

src/bin/pegleg/pegleg/engine/util/__init__.py

@@ -16,3 +16,4 @@
 from . import definition
 from . import files
 from . import deckhand
+from . import git

src/bin/pegleg/pegleg/engine/util/git.py

@@ -0,0 +1,286 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import tempfile
+from urllib.parse import urlparse
+
+from git import exc as git_exc
+from git import Git
+from git import Repo
+
+from pegleg.engine import exceptions
+
+LOG = logging.getLogger(__name__)
+
+__all__ = [
+    'git_handler',
+]
+
+
+def git_handler(repo_url, ref, proxy_server=None, auth_key=None):
+    """Handle directories that are Git repositories.
+
+    If ``repo_url`` is a valid URL for which a local repository doesn't
+    exist, then clone ``repo_url`` and checkout the given ``ref``. Otherwise,
+    treat ``repo_url`` as an already-cloned repository and checkout the given
+    ``ref``.
+
+    Supported ``ref`` formats include:
+
+    * branch name (e.g. 'master')
+    * refpath (e.g. 'refs/changes/54/457754/73')
+    * hexsha (e.g. 'ff5496b9c781918fdc49d79f927323eeef2f5320')
+
+    :param repo_url: URL of remote Git repo or path to local Git repo. If no
+        local copy exists, clone it. Afterward, check out ``ref`` in the repo.
+    :param ref: branch, commit or reference in the repo to clone.
+    :param proxy_server: optional, HTTP proxy to use while cloning the repo.
+    :param auth_key: If supplied results in using SSH to clone the repository
+        with the specified key.  If the value is None, SSH is not used.
+    :returns: Path to the cloned repo if a repo was cloned, else absolute
+        path to ``repo_url``.
+    :raises ValueError: If ``repo_url`` isn't a valid URL or doesn't begin
+        with a valid protocol (http, https or ssh) for cloning.
+    :raises NotADirectoryError: If ``repo_url`` isn't a valid directory path.
+
+    """
+
+    supported_clone_protocols = ('http', 'https', 'ssh')
+
+    try:
+        parsed_url = urlparse(repo_url)
+    except Exception as e:
+        raise ValueError('repo_url=%s is invalid. Details: %s' % (repo_url, e))
+
+    if not ref:
+        raise ValueError('ref=%s must be a non-empty, valid Git ref' % ref)
+
+    if not os.path.exists(repo_url):
+        # we need to clone the repo_url first since it doesn't exist and then
+        # checkout the appropriate reference - and return the tmpdir
+        if parsed_url.scheme in supported_clone_protocols:
+            return _try_git_clone(repo_url, ref, proxy_server, auth_key)
+        else:
+            raise ValueError('repo_url=%s must use one of the following '
+                             'protocols: %s' %
+                             (repo_url, ', '.join(supported_clone_protocols)))
+
+    # otherwise, we're dealing with a local directory so although
+    # we do not need to clone, we may need to process the reference
+    # by checking that out and returning the directory they passed in
+    else:
+        LOG.debug('Treating repo_url=%s as an already-cloned repository. '
+                  'Attempting to checkout ref=%s', repo_url, ref)
+        try:
+            # get absolute path of what is probably a directory
+            repo_url = os.path.abspath(repo_url)
+        except Exception:
+            msg = "The repo_url=%s is not a valid directory" % repo_url
+            LOG.error(msg)
+            raise NotADirectoryError(msg)
+
+        repo = Repo(repo_url)
+        if repo.is_dirty():
+            LOG.warning('The locally cloned repo_url=%s is dirty. '
+                        'Cleaning up untracked files.', repo_url)
+            # Reset the index and working tree to match current ref.
+            repo.head.reset(index=True, working_tree=True)
+
+        try:
+            # Check whether the ref exists locally.
+            LOG.info('Attempting to checkout ref=%s from repo_url=%s locally',
+                     ref, repo_url)
+            _try_git_checkout(repo, repo_url, ref, fetch=False)
+        except exceptions.GitException:
+            # Otherwise, attempt to fetch and checkout the missing ref.
+            LOG.info('ref=%s not found locally for repo_url=%s, fetching from '
+                     'remote', ref, repo_url)
+            # Allow any errors to bubble up.
+            _try_git_checkout(repo, repo_url, ref, fetch=True)
+
+        return repo_url
+
+
+def _try_git_clone(repo_url, ref='master', proxy_server=None, auth_key=None):
+    """Try cloning Git repo from ``repo_url`` using the reference ``ref``.
+
+    :param repo_url: URL of remote Git repo or path to local Git repo.
+    :param ref: branch, commit or reference in the repo to clone. Default is
+        'master'.
+    :param proxy_server: optional, HTTP proxy to use while cloning the repo.
+    :param auth_key: If supplied results in using SSH to clone the repository
+        with the specified key.  If the value is None, SSH is not used.
+    :returns: Path to the cloned repo.
+    :rtype: str
+    :raises GitException: If ``repo_url`` is invalid or could not be found.
+    :raises GitAuthException: If authentication with the Git repository failed.
+    :raises GitProxyException: If the repo could not be cloned due to a proxy
+        issue.
+
+    """
+
+    # the name here is important as it bubbles back up to the output filename
+    # and ensure we handle url/foo.git/ cases. prefix is 'tmp' by default.
+    temp_dir = tempfile.mkdtemp(suffix=repo_url.rstrip('/').split('/')[-1])
+    env_vars = _get_clone_env_vars(repo_url, ref, auth_key)
+    ssh_cmd = env_vars.get('GIT_SSH_COMMAND')
+
+    try:
+        if proxy_server:
+            LOG.debug('Cloning [%s] with proxy [%s]', repo_url, proxy_server)
+            # TODO(felipemonteiro): proxy_server can be finicky. Need a config
+            # option to retry up to N times.
+            repo = Repo.clone_from(
+                repo_url,
+                temp_dir,
+                config='http.proxy=%s' % proxy_server,
+                env=env_vars)
+        else:
+            LOG.debug('Cloning [%s]', repo_url)
+            repo = Repo.clone_from(repo_url, temp_dir, env=env_vars)
+    except git_exc.GitCommandError as e:
+        LOG.exception('Failed to clone repo_url=%s using ref=%s.', repo_url,
+                      ref)
+        if (ssh_cmd and ssh_cmd in e.stderr
+                or 'permission denied' in e.stderr.lower()):
+            raise exceptions.GitAuthException(repo_url, auth_key)
+        elif 'could not resolve proxy' in e.stderr.lower():
+            raise exceptions.GitProxyException(proxy_server)
+        else:
+            raise exceptions.GitException(repo_url, details=e)
+    except Exception as e:
+        msg = 'Encountered unknown Exception during clone of %s' % repo_url
+        LOG.exception(msg)
+        raise exceptions.GitException(repo_url, details=e)
+
+    _try_git_checkout(repo=repo, repo_url=repo_url, ref=ref)
+
+    return temp_dir
+
+
+def _get_clone_env_vars(repo_url, ref, auth_key):
+    """Generate environment variables include SSH command for Git clone.
+
+    :param repo_url: URL of remote Git repo or path to local Git repo.
+    :param ref: branch, commit or reference in the repo to clone. Default is
+        'master'.
+    :param auth_key: If supplied results in using SSH to clone the repository
+        with the specified key.  If the value is None, SSH is not used.
+    :returns: Dictionary of key-value pairs for Git clone.
+    :rtype: dict
+    :raises GitSSHException: If the SSH key specified by ``CONF.ssh_key_path``
+        could not be found and ``auth_method`` is "SSH".
+
+    """
+    ssh_cmd = None
+    env_vars = {'GIT_TERMINAL_PROMPT': '0'}
+
+    if auth_key:
+        if os.path.exists(auth_key):
+            LOG.debug('Attempting to clone the repo at %s using reference %s '
+                      'with SSH authentication.', repo_url, ref)
+            # Ensure that host checking is ignored, to avoid unnecessary
+            # required CLI input.
+            ssh_cmd = (
+                'ssh -i {} -o ConnectionAttempts=20 -o ConnectTimeout=10 -o '
+                'StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'
+                .format(os.path.expanduser(auth_key)))
+            env_vars.update({'GIT_SSH_COMMAND': ssh_cmd})
+        else:
+            msg = "The auth_key path '%s' was not found" % auth_key
+            LOG.error(msg)
+            raise exceptions.GitSSHException(auth_key)
+    return env_vars
+
+
+def _try_git_checkout(repo, repo_url, ref, fetch=True):
+    """Try to checkout a ``ref`` from ``repo``.
+
+    Local branches are created for multiple variations of the ``ref``,
+    including its refpath and hexpath (i.e. commit ID).
+
+    This is to locally "memoize" references that would otherwise require
+    resolution upstream. We increase performance by creating local branches
+    for these other ``ref`` formats when the ``ref`` is fetched remotely for
+    the first time only.
+
+    :param repo: Git Repo object.
+    :param repo_url: URL of remote Git repo or path to local Git repo.
+    :param ref: branch, commit or reference in the repo to clone. Default is
+        'master'.
+    :param fetch: Whether to fetch the ``ref`` from remote before checkout or
+        to use the already-cloned local repo.
+    :raises GitException: If ``ref`` could not be checked out.
+
+    """
+    try:
+        g = Git(repo.working_dir)
+        branches = [b.name for b in repo.branches]
+        LOG.debug('Available branches for repo_url=%s: %s', repo_url, branches)
+
+        if fetch:
+            LOG.debug('Fetching ref=%s from remote repo_url=%s', ref, repo_url)
+            # fetch_info is guaranteed to be populated if ref resolves, else
+            # a GitCommandError is raised.
+            fetch_info = repo.remotes.origin.fetch(ref)
+            hexsha = fetch_info[0].commit.hexsha.strip()
+            ref_path = fetch_info[0].remote_ref_path.strip()
+
+            # If ``ref`` doesn't match the hexsha/refpath then create a branch
+            # for each so that future checkouts can be performed using either
+            # format. This way, no future processing is required to figure
+            # out whether a refpath/hexsha exists within the repo.
+            _create_local_ref(
+                g, branches, ref=ref, newref=hexsha, reftype='hexsha')
+            _create_local_ref(
+                g, branches, ref=ref, newref=ref_path, reftype='refpath')
+            _create_or_checkout_local_ref(g, branches, ref=ref)
+        else:
+            LOG.debug('Checking out ref=%s from local repo_url=%s', ref,
+                      repo_url)
+            # Expect the reference to exist if checking out locally.
+            g.checkout(ref)
+
+        LOG.debug('Successfully checked out ref=%s for repo_url=%s', ref,
+                  repo_url)
+    except git_exc.GitCommandError as e:
+        LOG.exception('Failed to checkout ref=%s from repo_url=%s.', ref,
+                      repo_url)
+        raise exceptions.GitException(repo_url, details=e)
+    except Exception as e:
+        msg = ('Encountered unknown Exception during checkout of ref=%s for '
+               'repo_url=%s' % (ref, repo_url))
+        LOG.exception(msg)
+        raise exceptions.GitException(repo_url, details=e)
+
+
+def _create_or_checkout_local_ref(g, branches, ref):
+    if ref not in branches:
+        LOG.debug('Creating local branch for ref=%s', ref)
+        g.checkout('FETCH_HEAD', b=ref)
+        branches.append(ref)
+    else:
+        LOG.debug('Checking out ref=%s from local repo', ref)
+        g.checkout('FETCH_HEAD')
+
+
+def _create_local_ref(g, branches, ref, newref, reftype=None):
+    if newref not in branches:
+        if newref and ref != newref:
+            LOG.debug('Creating local branch for ref=%s (%s for %s)', newref,
+                      reftype, ref)
+            g.checkout('FETCH_HEAD', b=newref)
+            branches.append(newref)

src/bin/pegleg/requirements.txt

@@ -1,3 +1,4 @@
+gitpython
 click==6.7
 jsonschema==2.6.0
 pyyaml==3.12

src/bin/pegleg/tests/unit/engine/util/test_git.py

@@ -0,0 +1,432 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import socket
+import requests
+
+import fixtures
+import mock
+import pytest
+
+from pegleg.engine import exceptions
+from pegleg.engine.util import git
+from tests.unit import test_utils
+
+_REPO_DIR = None
+_PROXY_SERVERS = {
+    'http':
+    os.getenv('HTTP_PROXY',
+              os.getenv('http_proxy', 'http://one.proxy.att.com:8888')),
+    'https':
+    os.getenv('HTTPS_PROXY',
+              os.getenv('https_proxy', 'https://one.proxy.att.com:8888'))
+}
+
+
+def is_connected():
+    """Verifies whether network connectivity is up.
+
+    :returns: True if connected else False.
+    """
+    try:
+        r = requests.get("http://www.github.com/", proxies={})
+        return r.ok
+    except requests.exceptions.RequestException:
+        return False
+
+
+def is_connected_behind_proxy():
+    """Verifies whether network connectivity is up behind given proxy.
+
+    :returns: True if connected else False.
+    """
+    try:
+        r = requests.get("http://www.github.com/", proxies=_PROXY_SERVERS)
+        return r.ok
+    except requests.exceptions.RequestException:
+        return False
+
+
+@pytest.fixture()
+def clean_git_repo():
+    global _REPO_DIR
+
+    try:
+        yield
+    finally:
+        if _REPO_DIR and os.path.exists(_REPO_DIR):
+            shutil.rmtree(_REPO_DIR)
+            _REPO_DIR = None
+
+
+def _validate_git_clone(repo_dir, fetched_ref=None, checked_out_ref=None):
+    """Validate that git clone/checkout work.
+
+    :param repo_dir: Path to local Git repo.
+    :param fetched_ref: Reference that is stored in FETCH_HEAD following a
+        remote fetch.
+    :param checked_out_ref: Reference that is stored in HEAD following a local
+        ref checkout.
+    """
+    global _REPO_DIR
+    _REPO_DIR = repo_dir
+
+    assert os.path.isdir(repo_dir)
+    # Assert that the directory is a Git repo.
+    assert os.path.isdir(os.path.join(repo_dir, '.git'))
+    if fetched_ref:
+        # Assert the FETCH_HEAD is at the fetched_ref ref.
+        with open(os.path.join(repo_dir, '.git', 'FETCH_HEAD'), 'r') \
+                as git_file:
+            assert fetched_ref in git_file.read()
+    if checked_out_ref:
+        # Assert the HEAD is at the checked_out_ref.
+        with open(os.path.join(repo_dir, '.git', 'HEAD'), 'r') \
+                as git_file:
+            assert checked_out_ref in git_file.read()
+
+
+def _assert_repo_url_was_cloned(mock_log, git_dir):
+    expected_msg = ('Treating repo_url=%s as an already-cloned repository')
+    assert mock_log.debug.called
+    mock_calls = mock_log.debug.mock_calls
+    assert any(m[1][0].startswith(expected_msg) for m in mock_calls)
+    assert any(m[1][1] == git_dir for m in mock_calls)
+    mock_log.debug.reset_mock()
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+def test_git_clone_valid_url_http_protocol(clean_git_repo):
+    url = 'http://github.com/openstack/airship-armada'
+    git_dir = git.git_handler(url, ref='master')
+    _validate_git_clone(git_dir)
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+def test_git_clone_valid_url_https_protocol(clean_git_repo):
+    url = 'https://github.com/openstack/airship-armada'
+    git_dir = git.git_handler(url, ref='master')
+    _validate_git_clone(git_dir)
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+def test_git_clone_with_commit_reference(clean_git_repo):
+    url = 'https://github.com/openstack/airship-armada'
+    commit = 'cba78d1d03e4910f6ab1691bae633c5bddce893d'
+    git_dir = git.git_handler(url, commit)
+    _validate_git_clone(git_dir, commit)
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+def test_git_clone_with_patch_ref(clean_git_repo):
+    ref = 'refs/changes/54/457754/73'
+    git_dir = git.git_handler('https://github.com/openstack/openstack-helm',
+                              ref)
+    _validate_git_clone(git_dir, ref)
+
+
+@pytest.mark.skipif(
+    not is_connected_behind_proxy(),
+    reason='git clone requires proxy connectivity.')
+@mock.patch.object(git, 'LOG', autospec=True)
+def test_git_clone_behind_proxy(mock_log, clean_git_repo):
+    url = 'https://github.com/openstack/airship-armada'
+    commit = 'cba78d1d03e4910f6ab1691bae633c5bddce893d'
+
+    for proxy_server in _PROXY_SERVERS.values():
+        git_dir = git.git_handler(url, commit, proxy_server=proxy_server)
+        _validate_git_clone(git_dir, commit)
+
+        mock_log.debug.assert_any_call('Cloning [%s] with proxy [%s]', url,
+                                       proxy_server)
+        mock_log.debug.reset_mock()
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+@mock.patch.object(git, 'LOG', autospec=True)
+def test_git_clone_existing_directory_checks_out_earlier_ref_from_local(
+        mock_log, clean_git_repo):
+    """Validate Git checks out an earlier patch or ref that should exist
+    locally (as a later ref was already fetched which should contain that
+    revision history).
+    """
+    # Clone the openstack-helm repo and automatically checkout patch 34.
+    ref = 'refs/changes/15/536215/35'
+    repo_url = 'https://github.com/openstack/openstack-helm'
+    git_dir = git.git_handler(repo_url, ref)
+    _validate_git_clone(git_dir, fetched_ref=ref)
+
+    # Checkout ref='master' now that the repo already exists locally.
+    ref = 'refs/changes/15/536215/34'
+    git_dir = git.git_handler(git_dir, ref)
+    _validate_git_clone(git_dir, checked_out_ref=ref)
+    _assert_repo_url_was_cloned(mock_log, git_dir)
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+@mock.patch.object(git, 'LOG', autospec=True)
+def test_git_clone_existing_directory_checks_out_master_from_local(
+        mock_log, clean_git_repo):
+    """Validate Git checks out the ref of an already cloned repo that exists
+    locally.
+    """
+    # Clone the openstack-helm repo and automatically checkout patch 34.
+    ref = 'refs/changes/15/536215/34'
+    repo_url = 'https://github.com/openstack/openstack-helm'
+    git_dir = git.git_handler(repo_url, ref)
+    _validate_git_clone(git_dir, fetched_ref=ref)
+
+    # Checkout ref='master' now that the repo already exists locally.
+    ref = 'master'
+    git_dir = git.git_handler(git_dir, ref)
+    _validate_git_clone(git_dir, checked_out_ref=ref)
+    _assert_repo_url_was_cloned(mock_log, git_dir)
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+@mock.patch.object(git, 'LOG', autospec=True)
+def test_git_clone_checkout_refpath_saves_references_locally(
+        mock_log, clean_git_repo):
+    """Validate that refpath/hexsha branches are created in the local repo
+    following clone of the repo using a refpath during initial checkout.
+    """
+    # Clone the openstack-helm repo and automatically checkout patch 34.
+    ref = 'refs/changes/15/536215/34'
+    repo_url = 'https://github.com/openstack/openstack-helm'
+    git_dir = git.git_handler(repo_url, ref)
+    _validate_git_clone(git_dir, fetched_ref=ref)
+
+    # Now checkout patch 34 again to ensure it's still there.
+    ref = 'refs/changes/15/536215/34'
+    git_dir = git.git_handler(git_dir, ref)
+    _validate_git_clone(git_dir, checked_out_ref=ref)
+    _assert_repo_url_was_cloned(mock_log, git_dir)
+
+    # Verify that passing in the hexsha variation of refpath
+    # 'refs/changes/15/536215/34' also works.
+    hexref = '276102a115dac3c0a6e91f9047d8b086bc8d2ff0'
+    git_dir = git.git_handler(git_dir, hexref)
+    _validate_git_clone(git_dir, checked_out_ref=hexref)
+    _assert_repo_url_was_cloned(mock_log, git_dir)
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+@mock.patch.object(git, 'LOG', autospec=True)
+def test_git_clone_checkout_hexsha_saves_references_locally(
+        mock_log, clean_git_repo):
+    """Validate that refpath/hexsha branches are created in the local repo
+    following clone of the repo using a hexsha during initial checkout.
+    """
+    # Clone the openstack-helm repo and automatically checkout patch using
+    # hexsha.
+    # NOTE(felipemonteiro): We have to use the commit ID (hexsha) corresponding
+    # to the last patch as that is what gets pushed to github. In this case,
+    # this corresponds to patch 'refs/changes/15/536215/35'.
+    ref = 'bf126f46b1c175a8038949a87dafb0a716e3b9b6'
+    repo_url = 'https://github.com/openstack/openstack-helm'
+    git_dir = git.git_handler(repo_url, ref)
+    _validate_git_clone(git_dir, fetched_ref=ref)
+
+    # Now checkout patch using hexsha again to ensure it's still there.
+    ref = 'bf126f46b1c175a8038949a87dafb0a716e3b9b6'
+    git_dir = git.git_handler(git_dir, ref)
+    _validate_git_clone(git_dir, checked_out_ref=ref)
+    _assert_repo_url_was_cloned(mock_log, git_dir)
+
+    # Verify that passing in the refpath variation of hexsha also works.
+    hexref = 'refs/changes/15/536215/35'
+    git_dir = git.git_handler(git_dir, hexref)
+    _validate_git_clone(git_dir, checked_out_ref=hexref)
+    _assert_repo_url_was_cloned(mock_log, git_dir)
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+@mock.patch.object(git, 'LOG', autospec=True)
+def test_git_clone_existing_directory_checks_out_next_local_ref(
+        mock_log, clean_git_repo):
+    """Validate Git fetches the newer ref upstream that doesn't exist locally
+    in the cloned repo.
+    """
+    # Clone the openstack-helm repo and automatically checkout patch 73.
+    ref = 'refs/changes/54/457754/73'
+    repo_url = 'https://github.com/openstack/openstack-helm'
+    git_dir = git.git_handler(repo_url, ref)
+    _validate_git_clone(git_dir, ref)
+
+    # Attempt to checkout patch 74 which requires a remote fetch even though
+    # the repo has already been cloned.
+    ref = 'refs/changes/54/457754/74'
+    git_dir = git.git_handler(git_dir, ref)
+    _validate_git_clone(git_dir, ref)
+    _assert_repo_url_was_cloned(mock_log, git_dir)
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+@mock.patch.object(git, 'LOG', autospec=True)
+def test_git_clone_delete_repo_and_reclone(mock_log, clean_git_repo):
+    """Validate that cloning a repo, then deleting it, then recloning it works.
+    """
+    # Clone the openstack-helm repo and automatically checkout patch 73.
+    ref = 'refs/changes/54/457754/73'
+    repo_url = 'https://github.com/openstack/openstack-helm'
+    first_git_dir = git.git_handler(repo_url, ref)
+    _validate_git_clone(first_git_dir, ref)
+
+    # Validate that the repo was cloned.
+    assert mock_log.debug.called
+    mock_log.debug.assert_any_call('Cloning [%s]', repo_url)
+    mock_log.debug.reset_mock()
+
+    # Delete the just-cloned repo.
+    shutil.rmtree(first_git_dir)
+
+    # Verify that checking out the same ref results in a re-clone.
+    second_git_dir = git.git_handler(repo_url, ref)
+    _validate_git_clone(second_git_dir, ref)
+
+    # Validate that the repo was cloned.
+    assert first_git_dir != second_git_dir
+    assert mock_log.debug.called
+    mock_log.debug.assert_any_call('Cloning [%s]', repo_url)
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+@mock.patch.object(git, 'LOG', autospec=True)
+def test_git_clone_clean_dirty_local_repo(mock_log, clean_git_repo):
+    """Validate that a dirty repo is cleaned before a ref is checked out."""
+    ref = 'refs/changes/54/457754/73'
+    repo_url = 'https://github.com/openstack/openstack-helm'
+    git_dir = git.git_handler(repo_url, ref)
+    _validate_git_clone(git_dir, ref)
+
+    file_to_rename = os.path.join(git_dir, os.listdir(git_dir)[0])
+    os.rename(file_to_rename, file_to_rename + '-renamed')
+
+    git_dir = git.git_handler(git_dir, ref)
+    _validate_git_clone(git_dir, ref)
+
+    assert mock_log.warning.called
+    mock_log.warning.assert_any_call(
+        'The locally cloned repo_url=%s is dirty. Cleaning up untracked '
+        'files.', git_dir)
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+@mock.patch.object(git, 'LOG', autospec=True)
+def test_git_clone_existing_directory_raises_exc_for_invalid_ref(
+        mock_log, clean_git_repo):
+    """Validate Git throws an error for an invalid ref when trying to checkout
+    a ref for an already-cloned repo.
+    """
+    # Clone the openstack-helm repo and automatically checkout patch 73.
+    ref = 'refs/changes/54/457754/73'
+    repo_url = 'https://github.com/openstack/openstack-helm'
+    git_dir = git.git_handler(repo_url, ref)
+    _validate_git_clone(git_dir, ref)
+
+    # Attempt to checkout patch 9000 now that the repo already exists locally.
+    ref = 'refs/changes/54/457754/9000'
+    with pytest.raises(exceptions.GitException):
+        git_dir = git.git_handler(git_dir, ref)
+    _assert_repo_url_was_cloned(mock_log, git_dir)
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+def test_git_clone_empty_url_raises_value_error(clean_git_repo):
+    url = ''
+    with pytest.raises(ValueError):
+        git.git_handler(url, ref='master')
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+def test_git_clone_invalid_url_type_raises_value_error(clean_git_repo):
+    url = 5
+    with pytest.raises(ValueError):
+        git.git_handler(url, ref='master')
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+def test_git_clone_invalid_local_repo_url_raises_notadirectory_error(
+        clean_git_repo):
+    url = False
+    with pytest.raises(NotADirectoryError):
+        git.git_handler(url, ref='master')
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+def test_git_clone_invalid_remote_url(clean_git_repo):
+    url = 'https://github.com/dummy/armada'
+    with pytest.raises(exceptions.GitException):
+        git.git_handler(url, ref='master')
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+def test_git_clone_invalid_remote_url_protocol(clean_git_repo):
+    url = 'ftp://foo.bar'
+    with pytest.raises(ValueError):
+        git.git_handler(url, ref='master')
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+def test_git_clone_fake_proxy(clean_git_repo):
+    url = 'https://github.com/openstack/airship-armada'
+    proxy_url = test_utils.rand_name(
+        'not.a.proxy.that.works.and.never.will', prefix='http://') + ":8080"
+
+    with pytest.raises(exceptions.GitProxyException):
+        git.git_handler(url, ref='master', proxy_server=proxy_url)
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+@mock.patch('os.path.exists', return_value=True, autospec=True)
+def test_git_clone_ssh_auth_method_fails_auth(_, clean_git_repo):
+    fake_user = test_utils.rand_name('fake_user')
+    url = ('ssh://%s@review.openstack.org:29418/openstack/airship-armada' %
+           fake_user)
+    with pytest.raises(exceptions.GitAuthException):
+        git._try_git_clone(
+            url, ref='refs/changes/17/388517/5', auth_key='/home/user/.ssh/')
+
+
+@pytest.mark.skipif(
+    not is_connected(), reason='git clone requires network connectivity.')
+@mock.patch('os.path.exists', return_value=False, autospec=True)
+def test_git_clone_ssh_auth_method_missing_ssh_key(_, clean_git_repo):
+    fake_user = test_utils.rand_name('fake_user')
+    url = ('ssh://%s@review.openstack.org:29418/openstack/airship-armada' %
+           fake_user)
+    with pytest.raises(exceptions.GitSSHException):
+        git.git_handler(
+            url, ref='refs/changes/17/388517/5', auth_key='/home/user/.ssh/')

src/bin/pegleg/tests/unit/fixtures.py

@@ -47,6 +47,9 @@ def _gen_document(**kwargs):
 def create_tmp_deployment_files(tmpdir):
     """Fixture that creates a temporary directory structure."""
     sitenames = ['cicd', 'lab']
+    # Used for ensuring the original global context is reset in memory
+    # following each test execution.
+    original_global_context = copy.deepcopy(config.GLOBAL_CONTEXT)
 
     SITE_TEST_STRUCTURE = {
         'directories': {
@@ -152,8 +155,5 @@ schema: pegleg/SiteDefinition/v1
 
     yield
 
-    config.GLOBAL_CONTEXT = {
-        'primary_repo': './',
-        'aux_repos': [],
-        'site_path': 'site'
-    }
+    # Restore the global context back to blank slate status.
+    config.GLOBAL_CONTEXT = original_global_context

src/bin/pegleg/tests/unit/test_utils.py

@@ -0,0 +1,39 @@
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2015 Hewlett-Packard Development Company, L.P.
+# Copyright 2017 AT&T Intellectual Property.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import random
+import uuid
+
+
+def rand_name(name='', prefix='armada'):
+    """Generate a random name that includes a random number
+
+    :param str name: The name that you want to include
+    :param str prefix: The prefix that you want to include
+    :return: a random name. The format is
+             '<prefix>-<name>-<random number>'.
+             (e.g. 'prefixfoo-namebar-154876201')
+    :rtype: string
+    """
+    randbits = str(random.randint(1, 0x7fffffff))
+    rand_name = randbits
+    if name:
+        rand_name = name + '-' + rand_name
+    if prefix:
+        rand_name = prefix + '-' + rand_name
+    return rand_name

tox.ini

@@ -33,7 +33,10 @@ commands =
 whitelist_externals = tox
 
 [testenv:docs]
-deps = -r{toxinidir}/doc/requirements.txt
+basepython = python3
+deps =
+    -r{toxinidir}/src/bin/pegleg/requirements.txt
+    -r{toxinidir}/doc/requirements.txt
 commands =
     rm -rf doc/build
     sphinx-build -b html doc/source doc/build -n -W -v