pegleg/tests/unit/engine/test_site_repository.py

# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
from unittest import mock

import click
import pytest

from pegleg import config
from pegleg.engine import exceptions
from pegleg.engine import repository
from pegleg.engine import util

REPO_USERNAME = "test_username"

TEST_REPOSITORIES = {
    'repositories': {
        'global': {
            'revision': '843d1a50106e1f17f3f722e2ef1634ae442fe68f',
            'url': 'ssh://REPO_USERNAME@gerrit:29418/aic-clcp-manifests.git'
        },
        'secrets': {
            'revision': 'master',
            'url': (
                'ssh://REPO_USERNAME@gerrit:29418/aic-clcp-security-'
                'manifests.git')
        }
    }
}

FORMATTED_REPOSITORIES = {
    'repositories': {
        'global': {
            'revision': '843d1a50106e1f17f3f722e2ef1634ae442fe68f',
            'url': 'ssh://{}@gerrit:29418/aic-clcp-manifests.git'.format(
                REPO_USERNAME)
        },
        'secrets': {
            'revision': 'master',
            'url': (
                'ssh://{}@gerrit:29418/aic-clcp-security-'
                'manifests.git'.format(REPO_USERNAME))
        }
    }
}

config.set_repo_username(REPO_USERNAME)


@pytest.fixture(autouse=True)
def clean_temp_folders():
    try:
        yield
    finally:
        repository._clean_temp_folders()


@pytest.fixture(autouse=True)
def stub_out_misc_functionality():
    try:
        # Stub out copy functionality.
        mock.patch(
            'pegleg.engine.repository.shutil.copytree', autospec=True).start()
        # Stub out problematic Git functions with these unit tests.
        mock.patch.object(
            util.git,
            'repo_name',
            side_effect=lambda *a, **k: 'test',
            autospec=True).start()
        yield
    finally:
        mock.patch.stopall()


def _repo_name(repo_url):
    repo_name = repo_url.split('/')[-1]
    if repo_name.endswith('.git'):
        return repo_name[:-4]
    return repo_name


def _test_process_repositories_inner(
        site_name="test_site", expected_extra_repos=None):
    repository.process_repositories(site_name)
    actual_repo_list = config.get_extra_repo_list()
    expected_repos = expected_extra_repos.get('repositories', {})

    assert len(expected_repos) == len(actual_repo_list)
    for repo in expected_repos.values():
        repo_name = _repo_name(repo['url'])
        assert any(repo_name in r for r in actual_repo_list)


def _test_process_repositories(
        site_repo=None,
        repo_username=None,
        repo_overrides=None,
        expected_repo_url=None,
        expected_repo_revision=None,
        expected_repo_overrides=None):
    """Validate :func:`repository.process_repositories`.

    :param site_repo: Primary site repository.
    :param repo_username: Auth username that replaces REPO_USERNAME.
    :param dict repo_overrides: Overrides with format: -e global=/opt/global,
        keyed with name of override, e.g. global.

    All params above are mutually exclusive. Can only test one at a time.

    """
    @mock.patch.object(
        util.definition,
        'load_as_params',
        autospec=True,
        return_value=TEST_REPOSITORIES)
    @mock.patch.object(
        util.git, 'is_repository', autospec=True, return_value=True)
    @mock.patch.object(
        repository,
        '_handle_repository',
        autospec=True,
        side_effect=lambda repo_url, *a, **k: _repo_name(repo_url))
    def do_test(m_clone_repo, *_):
        _test_process_repositories_inner(
            expected_extra_repos=TEST_REPOSITORIES)

        if site_repo:
            # Validate that the primary site repository is cloned, in addition
            # to the extra repositories.
            mock_calls = [
                mock.call(
                    expected_repo_url,
                    ref=expected_repo_revision,
                    auth_key=None)
            ]
            mock_calls.extend(
                [
                    mock.call(r['url'], ref=r['revision'], auth_key=None)
                    for r in FORMATTED_REPOSITORIES['repositories'].values()
                ])
            m_clone_repo.assert_has_calls(mock_calls)
        elif repo_username:
            # Validate that the REPO_USERNAME placeholder is replaced by
            # repo_username.
            m_clone_repo.assert_has_calls(
                [
                    mock.call(
                        r['url'].replace('REPO_USERNAME', repo_username),
                        ref=r['revision'],
                        auth_key=None)
                    for r in FORMATTED_REPOSITORIES['repositories'].values()
                ])
        elif repo_overrides:
            # This is computed from: len(cloned extra repos) +
            # len(cloned primary repo), which is len(cloned extra repos) + 1
            expected_call_count = len(TEST_REPOSITORIES['repositories']) + 1
            assert (expected_call_count == m_clone_repo.call_count)

            for x, r in FORMATTED_REPOSITORIES['repositories'].items():
                if x in expected_repo_overrides:
                    repo_url = expected_repo_overrides[x]['url']
                    ref = expected_repo_overrides[x]['revision']
                else:
                    # Handles default values in TEST_REPOSITORIES -- which
                    # represents defaults in site-definition.yaml.
                    repo_url = r['url']
                    ref = r['revision']
                m_clone_repo.assert_any_call(repo_url, ref=ref, auth_key=None)
        else:
            m_clone_repo.assert_has_calls(
                [
                    mock.call(r['url'], ref=r['revision'], auth_key=None)
                    for r in FORMATTED_REPOSITORIES['repositories'].values()
                ])

    if site_repo:
        # Set a test site repo, call the test and clean up.
        with mock.patch.object(config, 'get_site_repo', autospec=True,
                               return_value=site_repo):
            do_test()
    elif repo_username:
        # Set a test repo username, call the test and clean up.
        with mock.patch.object(config, 'get_repo_username', autospec=True,
                               return_value=repo_username):
            do_test()
    elif repo_overrides:
        with mock.patch.object(config, 'get_extra_repo_overrides',
                               autospec=True,
                               return_value=list(repo_overrides.values())):
            do_test()
    else:
        do_test()


def test_process_repositories():
    _test_process_repositories()


def test_process_repositories_with_site_repo_url():
    """Test process_repository when site repo is a remote URL."""
    # Without REPO_USERNAME.
    site_repo = 'ssh://{}:29418/aic-clcp-site-manifests.git@333'.format(
        REPO_USERNAME)
    _test_process_repositories(
        site_repo=site_repo,
        expected_repo_url="ssh://{}:29418/aic-clcp-site-manifests".format(
            REPO_USERNAME),
        expected_repo_revision="333")


def test_process_repositories_handles_local_site_repo_path():
    site_repo = '/opt/aic-clcp-site-manifests'
    _test_process_repositories(
        site_repo=site_repo,
        expected_repo_url='/opt/aic-clcp-site-manifests',
        expected_repo_revision=None)


def test_process_repositories_handles_local_site_repo_path_with_revision():
    site_repo = '/opt/aic-clcp-site-manifests@333'
    _test_process_repositories(
        site_repo=site_repo,
        expected_repo_url="/opt/aic-clcp-site-manifests",
        expected_repo_revision="333")


@mock.patch.object(
    util.definition,
    'load_as_params',
    autospec=True,
    return_value=TEST_REPOSITORIES)
@mock.patch('os.path.exists', autospec=True, return_value=True)
@mock.patch.object(
    util.git, 'is_repository', autospec=True, return_value=False)
def test_process_repositories_with_local_site_path_exists_not_repo(*_):
    """Validate that when the site repo already exists but isn't a repository
    that an error is raised.
    """
    with pytest.raises(exceptions.GitInvalidRepoException) as exc:
        _test_process_repositories_inner(
            expected_extra_repos=TEST_REPOSITORIES)
    assert "The repository path or URL is invalid" in str(exc.value)


def test_process_repositories_with_repo_username():
    _test_process_repositories(repo_username='test_username')


def test_process_repositories_with_repo_overrides_remote_urls():
    # Same URL, different revision (than TEST_REPOSITORIES).
    overrides = {
        'global': 'global=ssh://REPO_USERNAME@gerrit:29418/aic-clcp-manifests.git@12345'
    }
    expected_repo_overrides = {
        'global': {
            'url': 'ssh://{}@gerrit:29418/aic-clcp-manifests'.format(
                REPO_USERNAME),
            'revision': '12345'
        },
    }
    _test_process_repositories(
        repo_overrides=overrides,
        expected_repo_overrides=expected_repo_overrides)

    # Different URL, different revision (than TEST_REPOSITORIES).
    overrides = {
        'global': 'global=https://gerrit/aic-clcp-manifests.git@12345'
    }
    expected_repo_overrides = {
        'global': {
            'url': 'https://gerrit/aic-clcp-manifests',
            'revision': '12345'
        },
    }
    _test_process_repositories(
        repo_overrides=overrides,
        expected_repo_overrides=expected_repo_overrides)


def test_process_repositories_with_repo_overrides_local_paths():
    # Local path without revision.
    overrides = {'global': 'global=/opt/aic-clcp-manifests'}
    expected_repo_overrides = {
        'global': {
            'url': '/opt/aic-clcp-manifests',
            'revision': None
        },
    }
    _test_process_repositories(
        repo_overrides=overrides,
        expected_repo_overrides=expected_repo_overrides)

    # Local path with revision.
    overrides = {'global': 'global=/opt/aic-clcp-manifests@12345'}
    expected_repo_overrides = {
        'global': {
            'url': '/opt/aic-clcp-manifests',
            'revision': '12345'
        },
    }
    _test_process_repositories(
        repo_overrides=overrides,
        expected_repo_overrides=expected_repo_overrides)


def test_process_repositories_with_multiple_repo_overrides_remote_urls():
    overrides = {
        'global': 'global=ssh://gerrit:29418/aic-clcp-manifests.git@12345',
        'secrets': 'secrets=ssh://gerrit:29418/aic-clcp-security-manifests.git@54321'
    }
    expected_repo_overrides = {
        'global': {
            'url': 'ssh://gerrit:29418/aic-clcp-manifests',
            'revision': '12345'
        },
        'secrets': {
            'url': 'ssh://gerrit:29418/aic-clcp-security-manifests',
            'revision': '54321'
        },
    }
    _test_process_repositories(
        repo_overrides=overrides,
        expected_repo_overrides=expected_repo_overrides)


def test_process_repositories_with_multiple_repo_overrides_local_paths():
    overrides = {
        'global': 'global=/opt/aic-clcp-manifests@12345',
        'secrets': 'secrets=/opt/aic-clcp-security-manifests.git@54321'
    }
    expected_repo_overrides = {
        'global': {
            'url': '/opt/aic-clcp-manifests',
            'revision': '12345'
        },
        'secrets': {
            'url': '/opt/aic-clcp-security-manifests',
            'revision': '54321'
        },
    }
    _test_process_repositories(
        repo_overrides=overrides,
        expected_repo_overrides=expected_repo_overrides)


@mock.patch.object(
    util.definition,
    'load_as_params',
    autospec=True,
    return_value=TEST_REPOSITORIES)
@mock.patch.object(util.git, 'is_repository', autospec=True, return_value=True)
@mock.patch.object(
    repository,
    '_handle_repository',
    autospec=True,
    side_effect=lambda repo_url, *a, **k: _repo_name(repo_url))
@mock.patch.object(repository, 'LOG', autospec=True)
def test_process_repositiories_extraneous_user_repo_value(m_log, *_):
    repo_overrides = ['global=ssh://gerrit:29418/aic-clcp-manifests.git']

    # Provide a repo user value.
    with mock.patch.object(config, 'get_repo_username', autospec=True,
                           return_value='test_username'):
        # Get rid of REPO_USERNAME through an override.
        with mock.patch.object(config, 'get_extra_repo_overrides',
                               autospec=True, return_value=repo_overrides):
            _test_process_repositories_inner(
                expected_extra_repos=TEST_REPOSITORIES)

    msg = (
        "A repository username was specified but no REPO_USERNAME "
        "string found in repository url %s", repo_overrides[0].split('=')[-1])
    m_log.warning.assert_any_call(*msg)


@mock.patch.object(
    util.definition, 'load_as_params', autospec=True,
    return_value={})  # No repositories in site definition.
@mock.patch.object(util.git, 'is_repository', autospec=True, return_value=True)
@mock.patch.object(
    repository,
    '_handle_repository',
    autospec=True,
    side_effect=lambda repo_url, *a, **k: _repo_name(repo_url))
def test_process_repositiories_no_site_def_repos(*_):
    _test_process_repositories_inner(expected_extra_repos={})


@mock.patch.object(
    util.definition, 'load_as_params', autospec=True,
    return_value={})  # No repositories in site definition.
@mock.patch.object(util.git, 'is_repository', autospec=True, return_value=True)
@mock.patch.object(
    repository,
    '_handle_repository',
    autospec=True,
    side_effect=lambda repo_url, *a, **k: _repo_name(repo_url))
@mock.patch.object(repository, 'LOG', autospec=True)
def test_process_repositiories_no_site_def_repos_with_extraneous_overrides(
        m_log, *_):
    """Validate that overrides that don't match site-definition entries are
    ignored.
    """
    site_name = mock.sentinel.site
    repo_overrides = ['global=ssh://gerrit:29418/aic-clcp-manifests.git']
    expected_overrides = {
        'repositories': {
            'global': {
                'revision': '843d1a50106e1f17f3f722e2ef1634ae442fe68f',
                'url': repo_overrides[0]
            }
        }
    }

    # Provide repo overrides.
    with mock.patch.object(config, 'get_extra_repo_overrides', autospec=True,
                           return_value=repo_overrides):
        _test_process_repositories_inner(
            site_name=site_name, expected_extra_repos=expected_overrides)

    debug_msg = (
        "Repo override: %s not found under `repositories` for "
        "site-definition.yaml. Site def repositories: %s", repo_overrides[0],
        "")
    info_msg = (
        "No repositories found in site-definition.yaml for site: %s. "
        "Defaulting to specified repository overrides.", site_name)
    m_log.debug.assert_any_call(*debug_msg)
    m_log.info.assert_any_call(*info_msg)


@mock.patch.object(
    util.definition, 'load_as_params', autospec=True,
    return_value={})  # No repositories in site definition.
@mock.patch.object(util.git, 'is_repository', autospec=True, return_value=True)
@mock.patch.object(repository, 'LOG', autospec=True)
def test_process_repositories_without_repositories_key_in_site_definition(
        m_log, *_):
    # Stub this out since default config site repo is '.' and local repo might
    # be dirty.
    with mock.patch.object(repository, '_handle_repository', autospec=True,
                           return_value=''):
        _test_process_repositories_inner(
            site_name=mock.sentinel.site, expected_extra_repos={})
    msg = (
        "The repository for site_name: %s does not contain a "
        "site-definition.yaml with a 'repositories' key")
    assert any(msg in x[1][0] for x in m_log.info.mock_calls)


@mock.patch.object(
    util.definition,
    'load_as_params',
    autospec=True,
    return_value=TEST_REPOSITORIES)
@mock.patch.object(util.git, 'is_repository', autospec=True, return_value=True)
@mock.patch.object(config, 'get_extra_repo_overrides', autospec=True)
def test_process_extra_repositories_malformed_format_raises_exception(
        m_get_extra_repo_overrides, *_):
    # Will fail since it doesn't contain "=".
    broken_repo_url = 'broken_url'
    m_get_extra_repo_overrides.return_value = [broken_repo_url]
    error = (
        "The repository %s must be in the form of "
        "name=repoUrl[@revision]" % broken_repo_url)

    # Stub this out since default config site repo is '.' and local repo might
    # be dirty.
    with mock.patch.object(repository, '_handle_repository', autospec=True,
                           return_value=''):
        with pytest.raises(click.ClickException) as exc:
            repository.process_repositories(mock.sentinel.site)
        assert error == str(exc.value)


@mock.patch.object(util.git, 'is_repository', autospec=True, return_value=True)
def test_process_site_repository(_):
    def _do_test(site_repo, expected):
        config.set_site_repo(site_repo)

        with mock.patch.object(repository, '_handle_repository', autospec=True,
                               side_effect=lambda x, *a, **k: x):
            result = repository.process_site_repository()
        assert os.path.normpath(expected) == os.path.normpath(result)

    # Ensure that the reference is always pruned.
    _do_test(
        'https://opendev.org/airship/treasuremap@master',
        expected='https://opendev.org/airship/treasuremap')
    _do_test(
        'https://opendev.org/airship/treasuremap',
        expected='https://opendev.org/airship/treasuremap')
    _do_test(
        'https://opendev.org/airship/treasuremap@master',
        expected='https://opendev.org/airship/treasuremap')
    _do_test(
        'https://opendev.org/airship/treasuremap',
        expected='https://opendev.org/airship/treasuremap')
    _do_test(
        'ssh://foo@opendev.org/airship/treasuremap:12345@master',
        expected='ssh://foo@opendev.org/airship/treasuremap:12345')
    _do_test(
        'ssh://foo@opendev.org/airship/treasuremap:12345',
        expected='ssh://foo@opendev.org/airship/treasuremap:12345')


def test_format_url_with_repo_username():
    TEST_URL = 'ssh://REPO_USERNAME@gerrit:29418/airship/pegleg'

    with mock.patch.object(config, 'get_repo_username', autospec=True,
                           return_value=REPO_USERNAME):
        res = repository._format_url_with_repo_username(TEST_URL)
        assert res == 'ssh://{}@gerrit:29418/airship/pegleg'.format(
            REPO_USERNAME)

    with mock.patch.object(config, 'get_repo_username', autospec=True,
                           return_value=''):
        pytest.raises(
            exceptions.GitMissingUserException,
            repository._format_url_with_repo_username, TEST_URL)