add 'repos clone' command

Add a command for checking out all of the repositories from a given
team at one time.

Change-Id: I2a43b67e0390453bfc6051c613b94780057a3187
Signed-off-by: Doug Hellmann <doug@doughellmann.com>

goal_tools/governance.py

@@ -63,6 +63,18 @@ class Governance:
 
         return team_data
 
+    def get_repos_for_team(self, team_name):
+        try:
+            team = self._team_data[team_name]
+        except KeyError:
+            try:
+                team = self._team_data[team_name.lower()]
+            except KeyError:
+                raise ValueError('No data for team {!r}'.format(team_name))
+        for deliv in team['deliverables'].values():
+            for repo in deliv['repos']:
+                yield repo
+
     @functools.lru_cache()
     def get_repo_owner(self, repo_name):
         "Return the name of the team that owns the repository."

goal_tools/python3_first/repos.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+import logging
+import os.path
+import subprocess
+
+from goal_tools import governance
+
+from cliff import command
+
+LOG = logging.getLogger(__name__)
+
+_TOOLS_DIR = os.path.realpath(
+    os.path.join(
+        os.path.basename(__file__),
+        '..',
+        'tools',
+    )
+)
+
+
+class ReposClone(command.Command):
+    "clone the repositories for a team"
+
+    def get_parser(self, prog_name):
+        parser = super().get_parser(prog_name)
+        parser.add_argument(
+            '--project-list',
+            default=governance.PROJECTS_LIST,
+            help='URL for projects.yaml',
+        )
+        parser.add_argument(
+            'workdir',
+            help='directory where the cloned repos should go',
+        )
+        parser.add_argument(
+            'team',
+            help='the team name',
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        clone_script = os.path.join(_TOOLS_DIR, 'clone_repo.sh')
+        if not os.path.exists(parsed_args.workdir):
+            LOG.info('creating working directory %s', parsed_args.workdir)
+            os.makedirs(parsed_args.workdir)
+        gov_dat = governance.Governance(url=parsed_args.project_list)
+        try:
+            for repo in gov_dat.get_repos_for_team(parsed_args.team):
+                LOG.info('\ncloning %s', repo)
+                subprocess.run(
+                    [clone_script,
+                     '--workspace', parsed_args.workdir,
+                     repo],
+                    check=True,
+                )
+        except ValueError as err:
+            print(err)
+            return 1

setup.cfg

@@ -47,6 +47,7 @@ python3_first =
     jobs extract = goal_tools.python3_first.jobs:JobsExtract
     jobs retain = goal_tools.python3_first.jobs:JobsRetain
     jobs update = goal_tools.python3_first.jobs:JobsUpdate
+    repos clone = goal_tools.python3_first.repos:ReposClone
 
 [wheel]
 universal = 1

tools/clone_repo.sh

@@ -0,0 +1,188 @@
+#!/bin/bash -x
+#
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# This script is used to clone a repository, possibly starting from a
+# local cache, and then update from the global remote. It replaces v2
+# of zuul-cloner, and is a bash script instead of being included in
+# openstack_releases/gitutils.py because we use it from jobs that
+# cannot run pip or tox for security reasons.
+#
+
+BINDIR=$(dirname $0)
+
+function print_help {
+    cat <<EOF
+USAGE:
+
+    clone_repo.sh -h
+
+    clone_repo.sh [--workspace WORK_DIR] [--cache-dir CACHE]
+                  [--branch BRANCH] [--ref REF]
+                  [--upstream URL] repo-name
+
+Arguments:
+
+  repo-name -- The full repository name, such as
+              "openstack/oslo.config". This name is also used as the
+              output directory.
+
+Options:
+
+  -h -- Print help.
+
+  -v -- Verbose output (turn on set -x).
+
+  --workspace -- The name of the parent directory where the cloned
+                 repo should be put.
+
+  --cache-dir -- A location where a local copy of a repo exists and
+                 can be used to seed the clone, which will still be
+                 updated from the remote. Defaults to $ZUUL_CACHE_DIR
+                 or /opt/git.
+
+  --branch -- The branch to check out. Defaults to "master".
+
+  --ref -- The git reference to check out. Defaults to HEAD.
+
+  --upstream -- The upstream server URL, without the git repo
+                part. Defaults to git://git.openstack.org
+
+EOF
+}
+
+# Defaults
+WORKSPACE="."
+CACHE_DIR="${ZUUL_CACHE_DIR:-/opt/git}"
+BRANCH="master"
+REF=""
+UPSTREAM="git://git.openstack.org"
+
+if [[ $(uname) != "Darwin" ]]; then
+    OPTS=`getopt -o hv --long branch:,cache-dir:,ref:,upstream:,workspace: -n $0 -- "$@"`
+    if [ $? != 0 ] ; then
+        echo "Failed parsing options." >&2
+        print_help
+        exit 1
+    fi
+    eval set -- "$OPTS"
+fi
+
+while true; do
+    case "$1" in
+        -h)
+            print_help
+            exit 0
+            ;;
+        -v)
+            set -x
+            ;;
+        --branch)
+            BRANCH="$2"
+            shift
+            shift
+            ;;
+        --cache-dir)
+            CACHE_DIR="$2"
+            shift
+            shift
+            ;;
+        --ref)
+            REF="$2"
+            shift
+            shift
+            ;;
+        --upstream)
+            UPSTREAM="$2"
+            shift
+            shift
+            ;;
+        --workspace)
+            WORKSPACE="$2"
+            shift
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+        *)
+            # Under macOS we don't get -- because getopt doesn't work
+            # the same so we aren't using it. If we see an
+            # unrecognized argument, that's the REPO name, so break
+            # out of the loop.
+            break
+            ;;
+    esac
+done
+
+REPO="$1"
+shift
+
+if [ -z "$REPO" ]; then
+    print_help
+    echo "ERROR: No repository given."
+    exit 1
+fi
+
+if [ ! -d "$WORKSPACE" ]; then
+    echo "ERROR: Workspace $WORKSPACE does not exist."
+    exit 1
+fi
+
+set -e
+
+cache_remote="$CACHE_DIR/$REPO"
+if [ ! -d "$cache_remote" ]; then
+    echo "WARNING: Cache directory $cache_remote does not exist, ignoring."
+    cache_remote=""
+fi
+
+upstream_remote="$UPSTREAM/$REPO"
+local_dir="$WORKSPACE/$REPO"
+
+# Clone the repository.
+if [ -d $local_dir ]; then
+    echo "Already have a local copy of $REPO in $local_dir"
+
+elif [ ! -z "$cache_remote" ]; then
+    # Clone from the cache then update the origin remote to point
+    # upstream so we can pull down more recent changes.
+    (cd $WORKSPACE &&
+            git clone $cache_remote $REPO &&
+            cd $REPO &&
+            git remote set-url origin "$upstream_remote"
+    )
+
+else
+    (cd $WORKSPACE && git clone $upstream_remote $REPO)
+fi
+
+# Make sure it is up to date compared to the upstream remote.
+(cd $local_dir &&
+        git fetch origin --tags
+)
+
+if [ ! -z "$REF" ]; then
+    # Check out the specified reference.
+    (cd $local_dir && git checkout "$REF")
+else
+    # Check out the expected branch (master is the default, but if the
+    # directory already existed we might have checked out something else
+    # before so just do it again).
+    (cd $local_dir &&
+            (git checkout $BRANCH || git checkout master) &&
+            git pull --ff-only)
+fi

tox.ini

@@ -11,6 +11,8 @@ deps =
   -r{toxinidir}/test-requirements.txt
 setenv =
     PYTHON=coverage run --source goal_tools --parallel-mode
+passenv =
+    ZUUL_CACHE_DIR
 commands =
     stestr run {posargs}
     stestr slowest