zuul/zuul/connection/__init__.py

# Copyright 2014 Rackspace Australia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import abc
import logging

from zuul.lib.logutil import get_annotated_logger
from zuul import model


class ReadOnlyBranchCacheError(RuntimeError):
    pass


class BaseConnection(object, metaclass=abc.ABCMeta):
    """Base class for connections.

    A connection is a shared object that sources, triggers and reporters can
    use to speak with a remote API without needing to establish a new
    connection each time or without having to authenticate each time.

    Multiple instances of the same connection may exist with different
    credentials, for example, thus allowing for different pipelines to operate
    on different Gerrit installations or post back as a different user etc.

    Connections can implement their own public methods. Required connection
    methods are validated by the {trigger, source, reporter} they are loaded
    into. For example, a trigger will likely require some kind of query method
    while a reporter may need a review method."""

    log = logging.getLogger('zuul.BaseConnection')

    def __init__(self, driver, connection_name, connection_config):
        # connection_name is the name given to this connection in zuul.ini
        # connection_config is a dictionary of config_section from zuul.ini for
        # this connection.
        # __init__ shouldn't make the actual connection in case this connection
        # isn't used in the layout.
        self.driver = driver
        self.connection_name = connection_name
        self.connection_config = connection_config
        self.sched = None

    def logEvent(self, event):
        log = get_annotated_logger(self.log, event.zuul_event_id)
        log.debug('Scheduling event from {connection}: {event}'.format(
            connection=self.connection_name,
            event=event))
        try:
            if self.sched.statsd:
                self.sched.statsd.incr(
                    'zuul.event.{driver}.{event}'.format(
                        driver=self.driver.name, event=event.type))
                self.sched.statsd.incr(
                    'zuul.event.{driver}.{connection}.{event}'.format(
                        driver=self.driver.name,
                        connection=self.connection_name,
                        event=event.type))
        except Exception:
            self.log.exception("Exception reporting event stats")

    def onLoad(self, zk_client, component_registry):
        pass

    def onStop(self):
        pass

    def registerScheduler(self, sched) -> None:
        self.sched = sched

    def cleanupCache(self):
        """Clean up the connection cache.

        This allows a connection to perform periodic cleanup actions of
        the cache, e.g. garbage collection.
        """
        pass

    def maintainCache(self, relevant, max_age):
        """Remove stale changes from the cache.

        This lets the user supply a list of change cache keys that are
        still in use.  Anything in our cache that isn't in the supplied
        list and is older than the given max. age (in seconds) should
        be safe to remove from the cache."""
        pass

    def getWebController(self, zuul_web):
        """Return a cherrypy web controller to register with zuul-web.

        :param zuul.web.ZuulWeb zuul_web:
            Zuul Web instance.
        :returns: A `zuul.web.handler.BaseWebController` instance.
        """
        return None

    def getEventQueue(self):
        """Return the event queue for this connection.

        :returns: A `zuul.zk.event_queues.ConnectionEventQueue` instance
                  or None.
        """
        return None

    def validateWebConfig(self, config, connections):
        """Validate web config.

        If there is a fatal error, the method should raise an exception.

        :param config:
           The parsed config object.
        :param zuul.lib.connections.ConnectionRegistry connections:
           Registry of all configured connections.
        """
        return False

    def toDict(self):
        """Return public information about the connection
        """
        return {
            "name": self.connection_name,
            "driver": self.driver.name,
        }


class ZKBranchCacheMixin:
    # Expected to be defined by the connection and to be an instance
    # of BranchCache
    _branch_cache = None
    read_only = False

    @abc.abstractmethod
    def isBranchProtected(self, project_name, branch_name,
                          zuul_event_id):
        """Return if the branch is protected or None if the branch is unknown.

        :param str project_name:
            The name of the project.
        :param str branch_name:
            The name of the branch.
        :param zuul_event_id:
            Unique id associated to the handled event.
        """
        pass

    @abc.abstractmethod
    def _fetchProjectBranches(self, project, exclude_unprotected):
        """Perform a remote query to determine the project's branches.

        Connection subclasses should implement this method.

        :param model.Project project:
            The project.
        :param bool exclude_unprotected:
            Whether the query should exclude unprotected branches from
            the response.

        :returns: A list of branch names.
        """

    def _fetchProjectMergeModes(self, project):
        """Perform a remote query to determine the project's supported merge
           modes.

        Connection subclasses should implement this method if they are
        able to determine which merge modes apply for a project.  The
        default implemantion returns that all merge modes are valid.

        :param model.Project project:
            The project.

        :returns: A list of merge modes as model IDs.

        """
        return model.ALL_MERGE_MODES

    def _fetchProjectDefaultBranch(self, project):
        """Perform a remote query to determine the project's default branch.

        Connection subclasses should implement this method if they are
        able to determine the upstream default branch for a project.  The
        default implemantion returns 'master' for now and will likely change
        to return something else if and when the git default changes.

        :param model.Project project:
            The project.

        :returns: The name of the default branch.

        """
        return 'master'

    def clearConnectionCacheOnBranchEvent(self, event):
        """Update event and clear connection cache if needed.

        This checks whether the event created or deleted a branch so
        that Zuul may know to perform a reconfiguration on the
        project. Drivers must call this method when a branch event is
        received.

        :param event:
            The event, inherit from `zuul.model.TriggerEvent` class.
        """
        if event.oldrev == '0' * 40:
            event.branch_created = True
        elif event.newrev == '0' * 40:
            event.branch_deleted = True
        else:
            event.branch_updated = True

        project = self.source.getProject(event.project_name)
        if event.branch:
            if event.branch_deleted:
                # We currently cannot determine if a deleted branch was
                # protected so we need to assume it was. GitHub/GitLab don't
                # allow deletion of protected branches but we don't get a
                # notification about branch protection settings. Thus we don't
                # know if branch protection has been disabled before deletion
                # of the branch.
                # FIXME(tobiash): Find a way to handle that case
                self.updateProjectBranches(project)
            elif event.branch_created:
                # In GitHub, a new branch never can be protected
                # because that needs to be configured after it has
                # been created.  Other drivers could optimize this,
                # but for the moment, implement the lowest common
                # denominator and clear the cache so that we query.
                self.updateProjectBranches(project)
            event.branch_cache_ltime = self._branch_cache.ltime
        return event

    def updateProjectBranches(self, project):
        """Update the branch cache for the project.

        :param zuul.model.Project project:
            The project for which the branches are returned.
        """
        # Figure out which queries we have a cache for
        protected_branches = self._branch_cache.getProjectBranches(
            project.name, True, default=None)
        all_branches = self._branch_cache.getProjectBranches(
            project.name, False, default=None)

        # Update them if we have them
        if protected_branches is not None:
            protected_branches = self._fetchProjectBranches(project, True)
            self._branch_cache.setProjectBranches(
                project.name, True, protected_branches)

        if all_branches is not None:
            all_branches = self._fetchProjectBranches(project, False)
            self._branch_cache.setProjectBranches(
                project.name, False, all_branches)

        merge_modes = self._fetchProjectMergeModes(project)
        self._branch_cache.setProjectMergeModes(
            project.name, merge_modes)

        default_branch = self._fetchProjectDefaultBranch(project)
        self._branch_cache.setProjectDefaultBranch(
            project.name, default_branch)
        self.log.info("Got branches for %s" % project.name)

    def getProjectBranches(self, project, tenant, min_ltime=-1):
        """Get the branch names for the given project.

        :param zuul.model.Project project:
            The project for which the branches are returned.
        :param zuul.model.Tenant tenant:
            The related tenant.
        :param int min_ltime:
            The minimum ltime to determine if we need to refresh the cache.

        :returns: The list of branch names.
        """
        exclude_unprotected = tenant.getExcludeUnprotectedBranches(project)
        branches = None

        if self._branch_cache:
            try:
                branches = self._branch_cache.getProjectBranches(
                    project.name, exclude_unprotected, min_ltime)
            except LookupError:
                if self.read_only:
                    # A scheduler hasn't attempted to fetch them yet
                    raise ReadOnlyBranchCacheError(
                        "Will not fetch project branches as read-only is set")
                else:
                    branches = None

        if branches is not None:
            return sorted(branches)
        elif self.read_only:
            # A scheduler has previously attempted a fetch, but got
            # the None due to an error; we can't retry since we're
            # read-only.
            raise RuntimeError(
                "Will not fetch project branches as read-only is set")

        # We need to perform a query
        try:
            branches = self._fetchProjectBranches(project, exclude_unprotected)
        except Exception:
            # We weren't able to get the branches.  We need to tell
            # future schedulers to try again but tell zuul-web that we
            # tried and failed.  Set the branches to None to indicate
            # that we have performed a fetch and retrieved no data.  Any
            # time we encounter None in the cache, we will try again.
            if self._branch_cache:
                self._branch_cache.setProjectBranches(
                    project.name, exclude_unprotected, None)
            raise
        self.log.info("Got branches for %s" % project.name)

        if self._branch_cache:
            self._branch_cache.setProjectBranches(
                project.name, exclude_unprotected, branches)

        return sorted(branches)

    def getProjectMergeModes(self, project, tenant, min_ltime=-1):
        """Get the merge modes for the given project.

        :param zuul.model.Project project:
            The project for which the merge modes are returned.
        :param zuul.model.Tenant tenant:
            The related tenant.
        :param int min_ltime:
            The minimum ltime to determine if we need to refresh the cache.

        :returns: The list of merge modes by model id.
        """
        merge_modes = None

        if self._branch_cache:
            try:
                merge_modes = self._branch_cache.getProjectMergeModes(
                    project.name, min_ltime)
            except LookupError:
                if self.read_only:
                    # A scheduler hasn't attempted to fetch them yet
                    raise ReadOnlyBranchCacheError(
                        "Will not fetch merge modes as read-only is set")
                else:
                    merge_modes = None

        if merge_modes is not None:
            return merge_modes
        elif self.read_only:
            # A scheduler has previously attempted a fetch, but got
            # the None due to an error; we can't retry since we're
            # read-only.
            raise RuntimeError(
                "Will not fetch merge_modes as read-only is set")

        # We need to perform a query
        try:
            merge_modes = self._fetchProjectMergeModes(project)
        except Exception:
            # We weren't able to get the merge modes.  We need to tell
            # future schedulers to try again but tell zuul-web that we
            # tried and failed.  Set the merge modes to None to indicate
            # that we have performed a fetch and retrieved no data.  Any
            # time we encounter None in the cache, we will try again.
            if self._branch_cache:
                self._branch_cache.setProjectMergeModes(
                    project.name, None)
            raise
        self.log.info("Got merge modes for %s" % project.name)

        if self._branch_cache:
            self._branch_cache.setProjectMergeModes(
                project.name, merge_modes)

        return merge_modes

    def getProjectDefaultBranch(self, project, tenant, min_ltime=-1):
        """Get the default branch for the given project.

        :param zuul.model.Project project:
            The project for which the default branch is returned.
        :param zuul.model.Tenant tenant:
            The related tenant.
        :param int min_ltime:
            The minimum ltime to determine if we need to refresh the cache.

        :returns: The name of the default branch.
        """
        default_branch = None

        if self._branch_cache:
            try:
                default_branch = self._branch_cache.getProjectDefaultBranch(
                    project.name, min_ltime)
            except LookupError:
                if self.read_only:
                    # A scheduler hasn't attempted to fetch it yet
                    raise ReadOnlyBranchCacheError(
                        "Will not fetch default branch as read-only is set")
                else:
                    default_branch = None

        if default_branch is not None:
            return default_branch
        elif self.read_only:
            # A scheduler has previously attempted a fetch, but got
            # the None due to an error; we can't retry since we're
            # read-only.
            raise RuntimeError(
                "Will not fetch default branch as read-only is set")

        # We need to perform a query
        try:
            default_branch = self._fetchProjectDefaultBranch(project)
        except Exception:
            # We weren't able to get the default branch.  We need to tell
            # future schedulers to try again but tell zuul-web that we
            # tried and failed.  Set the default branch to None to indicate
            # that we have performed a fetch and retrieved no data.  Any
            # time we encounter None in the cache, we will try again.
            if self._branch_cache:
                self._branch_cache.setProjectDefaultBranch(
                    project.name, None)
            raise
        self.log.info("Got default branch for %s: %s", project.name,
                      default_branch)

        if self._branch_cache:
            self._branch_cache.setProjectDefaultBranch(
                project.name, default_branch)

        return default_branch

    def checkBranchCache(self, project_name: str, event,
                         protected: bool = None) -> None:
        """Clear the cache for a project when a branch event is processed

        This method must be called when a branch event is processed: if the
        event references a branch and the unprotected branches are excluded,
        the branch protection status could have been changed.

        :params str project_name:
            The project name.
        :params event:
            The event, inherit from `zuul.model.TriggerEvent` class.
        :params protected:
            If defined the caller already knows if the branch is protected
            so the query can be skipped.
        """
        if protected is None:
            protected = self.isBranchProtected(project_name, event.branch,
                                               zuul_event_id=event)
        if protected is not None:

            # If the branch appears in the exclude_unprotected cache but
            # is unprotected, clear the exclude cache.

            # If the branch does not appear in the exclude_unprotected
            # cache but is protected, clear the exclude cache.

            # All branches should always appear in the include_unprotected
            # cache, so we never clear it.

            branches = self._branch_cache.getProjectBranches(
                project_name, True, default=None)

            if not branches:
                branches = []

            update = False
            if (event.branch in branches) and (not protected):
                update = True
            if (event.branch not in branches) and (protected):
                update = True
            if update:
                self.log.info("Project %s branch %s protected state "
                              "changed to %s",
                              project_name, event.branch, protected)
                self._branch_cache.setProtected(project_name, event.branch,
                                                protected)
                event.branch_cache_ltime = self._branch_cache.ltime

            event.branch_protected = protected
        else:
            # This can happen if the branch was deleted in GitHub/GitLab.
            # In this case we assume that the branch COULD have
            # been protected before. The cache update is handled by
            # the push event, so we don't touch the cache here
            # again.
            event.branch_protected = True

    def clearBranchCache(self, projects=None):
        """Clear the branch cache

        In case the branch cache gets out of sync with the source,
        this method can be called to clear it and force querying the
        source the next time the cache is used.
        """
        self._branch_cache.clear(projects)


class ZKChangeCacheMixin:
    # Expected to be defined by the connection and to be an instance
    # that implements the AbstractChangeCache API.
    _change_cache = None

    def cleanupCache(self):
        self._change_cache.cleanup()

    def maintainCache(self, relevant, max_age):
        self._change_cache.prune(relevant, max_age)

    def updateChangeAttributes(self, change, **attrs):
        def _update_attrs(c):
            for name, value in attrs.items():
                setattr(c, name, value)
        self._change_cache.updateChangeWithRetry(change.cache_stat.key,
                                                 change, _update_attrs)

    def estimateCacheDataSize(self):
        return self._change_cache.estimateDataSize()