zuul/zuul
Code Issues Proposed changes

Remove zuul-cloner command

Browse Source 
With the release of zuul 3.0.0 is command is no longer needed. Jobs
are setup to push the known repo state on to the remove nodes.

Change-Id: I0df6e41dc05276e648d393ec62329a85f1b8c415
Signed-off-by: Paul Belanger <pabelanger@redhat.com>
This commit is contained in:
Paul Belanger 2018-04-12 11:49:13 -04:00
parent f1580877cc
commit fc120688f8
No known key found for this signature in database
GPG Key ID: 611A80832067AF38
7 changed files with 0 additions and 636 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
etc/clonemap.yaml-sample
View File

@ -1,16 +0,0 @@
# vim: ft=yaml
#
# Example clone map for Zuul cloner
#
# By default it would clone projects under the directory specified by its
# option --basepath, but you can override this behavior by definining per
# project destinations.
clonemap:
# Clone project 'mediawiki/core' directly in {basepath}
- name: 'mediawiki/core'
dest: '.'
# Clone projects below mediawiki/extensions to {basepath}/extensions/
- name: 'mediawiki/extensions/(.*)'
dest: 'extensions/\1'

1
setup.cfg
View File

@ -27,7 +27,6 @@ console_scripts =
zuul-scheduler = zuul.cmd.scheduler:main
zuul-merger = zuul.cmd.merger:main
zuul = zuul.cmd.client:main
zuul-cloner = zuul.cmd.cloner:main
zuul-executor = zuul.cmd.executor:main
zuul-bwrap = zuul.driver.bubblewrap:main
zuul-web = zuul.cmd.web:main

79
tests/unit/test_clonemapper.py
View File

@ -1,79 +0,0 @@
# Copyright 2014 Antoine "hashar" Musso
# Copyright 2014 Wikimedia Foundation Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import testtools
from zuul.lib.clonemapper import CloneMapper
class TestCloneMapper(testtools.TestCase):
def test_empty_mapper(self):
"""Given an empty map, the slashes in project names are directory
separators"""
cmap = CloneMapper(
{},
[
'project1',
'plugins/plugin1'
])
self.assertEqual(
{'project1': '/basepath/project1',
'plugins/plugin1': '/basepath/plugins/plugin1'},
cmap.expand('/basepath')
)
def test_map_to_a_dot_dir(self):
"""Verify we normalize path, hence '.' refers to the basepath"""
cmap = CloneMapper(
[{'name': 'mediawiki/core', 'dest': '.'}],
['mediawiki/core'])
self.assertEqual(
{'mediawiki/core': '/basepath'},
cmap.expand('/basepath'))
def test_map_using_regex(self):
"""One can use regex in maps and use \\1 to forge the directory"""
cmap = CloneMapper(
[{'name': 'plugins/(.*)', 'dest': 'project/plugins/\\1'}],
['plugins/PluginFirst'])
self.assertEqual(
{'plugins/PluginFirst': '/basepath/project/plugins/PluginFirst'},
cmap.expand('/basepath'))
def test_map_discarding_regex_group(self):
cmap = CloneMapper(
[{'name': 'plugins/(.*)', 'dest': 'project/'}],
['plugins/Plugin_1'])
self.assertEqual(
{'plugins/Plugin_1': '/basepath/project'},
cmap.expand('/basepath'))
def test_cant_dupe_destinations(self):
"""We cant clone multiple projects in the same directory"""
cmap = CloneMapper(
[{'name': 'plugins/(.*)', 'dest': 'catchall/'}],
['plugins/plugin1', 'plugins/plugin2']
)
self.assertRaises(Exception, cmap.expand, '/basepath')
def test_map_with_dot_and_regex(self):
"""Combining relative path and regex"""
cmap = CloneMapper(
[{'name': 'plugins/(.*)', 'dest': './\\1'}],
['plugins/PluginInBasePath'])
self.assertEqual(
{'plugins/PluginInBasePath': '/basepath/PluginInBasePath'},
cmap.expand('/basepath'))

51
tests/unit/test_cloner_cmd.py
View File

@ -1,51 +0,0 @@
#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import testtools
import zuul.cmd.cloner
class TestClonerCmdArguments(testtools.TestCase):
def setUp(self):
super(TestClonerCmdArguments, self).setUp()
self.app = zuul.cmd.cloner.Cloner()
def test_default_cache_dir_empty(self):
self.app.parse_arguments(['base', 'repo'])
self.assertIsNone(self.app.args.cache_dir)
def test_default_cache_dir_environ(self):
try:
os.environ['ZUUL_CACHE_DIR'] = 'fromenviron'
self.app.parse_arguments(['base', 'repo'])
self.assertEqual('fromenviron', self.app.args.cache_dir)
finally:
del os.environ['ZUUL_CACHE_DIR']
def test_default_cache_dir_override_environ(self):
try:
os.environ['ZUUL_CACHE_DIR'] = 'fromenviron'
self.app.parse_arguments(['--cache-dir', 'argument',
'base', 'repo'])
self.assertEqual('argument', self.app.args.cache_dir)
finally:
del os.environ['ZUUL_CACHE_DIR']
def test_default_cache_dir_argument(self):
self.app.parse_arguments(['--cache-dir', 'argument',
'base', 'repo'])
self.assertEqual('argument', self.app.args.cache_dir)

167
zuul/cmd/cloner.py
View File

@ -1,167 +0,0 @@
#!/usr/bin/env python
#
# Copyright 2014 Antoine "hashar" Musso
# Copyright 2014 Wikimedia Foundation Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import logging
import os
import sys
import zuul.cmd
import zuul.lib.cloner
ZUUL_ENV_SUFFIXES = (
'branch',
'ref',
'url',
'project',
'newrev',
)
class Cloner(zuul.cmd.ZuulApp):
log = logging.getLogger("zuul.Cloner")
def parse_arguments(self, args=sys.argv[1:]):
"""Parse command line arguments and returns argparse structure"""
parser = argparse.ArgumentParser(
description='Zuul Project Gating System Cloner.')
parser.add_argument('-m', '--map', dest='clone_map_file',
help='specifiy clone map file')
parser.add_argument('--workspace', dest='workspace',
default=os.getcwd(),
help='where to clone repositories too')
parser.add_argument('-v', '--verbose', dest='verbose',
action='store_true',
help='verbose output')
parser.add_argument('--color', dest='color', action='store_true',
help='use color output')
parser.add_argument('--version', dest='version', action='version',
version=self._get_version(),
help='show zuul version')
parser.add_argument('--cache-dir', dest='cache_dir',
default=os.environ.get('ZUUL_CACHE_DIR'),
help=('a directory that holds cached copies of '
'repos from which to make an initial clone. '
'Can also be set via ZUUL_CACHE_DIR '
'environment variable.'
))
parser.add_argument('git_base_url',
help='reference repo to clone from')
parser.add_argument('projects', nargs='+',
help='list of Gerrit projects to clone')
project_env = parser.add_argument_group(
'project tuning'
)
project_env.add_argument(
'--branch',
help=('branch to checkout instead of Zuul selected branch, '
'for example to specify an alternate branch to test '
'client library compatibility.')
)
project_env.add_argument(
'--project-branch', nargs=1, action='append',
metavar='PROJECT=BRANCH',
help=('project-specific branch to checkout which takes precedence '
'over --branch if it is provided; may be specified multiple '
'times.')
)
zuul_env = parser.add_argument_group(
'zuul environment',
'Let you override $ZUUL_* environment variables.'
)
for zuul_suffix in ZUUL_ENV_SUFFIXES:
env_name = 'ZUUL_%s' % zuul_suffix.upper()
zuul_env.add_argument(
'--zuul-%s' % zuul_suffix, metavar='$' + env_name,
default=os.environ.get(env_name)
)
args = parser.parse_args(args)
# Validate ZUUL_* arguments. If ref is provided then URL is required.
zuul_args = [zuul_opt for zuul_opt, val in vars(args).items()
if zuul_opt.startswith('zuul') and val is not None]
if 'zuul_ref' in zuul_args and 'zuul_url' not in zuul_args:
parser.error("Specifying a Zuul ref requires a Zuul url. "
"Define Zuul arguments either via environment "
"variables or using options above.")
if 'zuul_newrev' in zuul_args and 'zuul_project' not in zuul_args:
parser.error("ZUUL_NEWREV has been specified without "
"ZUUL_PROJECT. Please define a ZUUL_PROJECT or do "
"not set ZUUL_NEWREV.")
self.args = args
def setup_logging(self, color=False, verbose=False):
"""Cloner logging does not rely on conf file"""
if verbose:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
if color:
# Color codes http://www.tldp.org/HOWTO/Bash-Prompt-HOWTO/x329.html
logging.addLevelName( # cyan
logging.DEBUG, "\033[36m%s\033[0m" %
logging.getLevelName(logging.DEBUG))
logging.addLevelName( # green
logging.INFO, "\033[32m%s\033[0m" %
logging.getLevelName(logging.INFO))
logging.addLevelName( # yellow
logging.WARNING, "\033[33m%s\033[0m" %
logging.getLevelName(logging.WARNING))
logging.addLevelName( # red
logging.ERROR, "\033[31m%s\033[0m" %
logging.getLevelName(logging.ERROR))
logging.addLevelName( # red background
logging.CRITICAL, "\033[41m%s\033[0m" %
logging.getLevelName(logging.CRITICAL))
def main(self):
self.parse_arguments()
self.setup_logging(color=self.args.color, verbose=self.args.verbose)
project_branches = {}
if self.args.project_branch:
for x in self.args.project_branch:
project, branch = x[0].split('=')
project_branches[project] = branch
cloner = zuul.lib.cloner.Cloner(
git_base_url=self.args.git_base_url,
projects=self.args.projects,
workspace=self.args.workspace,
zuul_branch=self.args.zuul_branch,
zuul_ref=self.args.zuul_ref,
zuul_url=self.args.zuul_url,
branch=self.args.branch,
clone_map_file=self.args.clone_map_file,
project_branches=project_branches,
cache_dir=self.args.cache_dir,
zuul_newrev=self.args.zuul_newrev,
zuul_project=self.args.zuul_project,
)
cloner.execute()
def main():
cloner = Cloner()
cloner.main()
if __name__ == "__main__":
sys.path.insert(0, '.')
main()

75
zuul/lib/clonemapper.py
View File

@ -1,75 +0,0 @@
# Copyright 2014 Antoine "hashar" Musso
# Copyright 2014 Wikimedia Foundation Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from collections import defaultdict
from collections import OrderedDict
import logging
import os
import re
class CloneMapper(object):
log = logging.getLogger("zuul.CloneMapper")
def __init__(self, clonemap, projects):
self.clonemap = clonemap
self.projects = projects
def expand(self, workspace):
self.log.info("Workspace path set to: %s", workspace)
is_valid = True
ret = OrderedDict()
for project in self.projects:
dests = []
for mapping in self.clonemap:
if re.match(r'^%s$' % mapping['name'],
project):
# Might be matched more than one time
dests.append(
re.sub(mapping['name'], mapping['dest'], project))
if len(dests) > 1:
self.log.error("Duplicate destinations for %s: %s.",
project, dests)
is_valid = False
elif len(dests) == 0:
self.log.debug("Using %s as destination (unmatched)",
project)
ret[project] = [project]
else:
ret[project] = dests
if not is_valid:
raise Exception("Expansion error. Check error messages above")
self.log.info("Mapping projects to workspace...")
for project, dest in ret.items():
dest = os.path.normpath(os.path.join(workspace, dest[0]))
ret[project] = dest
self.log.info(" %s -> %s", project, dest)
self.log.debug("Checking overlap in destination directories...")
check = defaultdict(list)
for project, dest in ret.items():
check[dest].append(project)
dupes = dict((d, p) for (d, p) in check.items() if len(p) > 1)
if dupes:
raise Exception("Some projects share the same destination: %s",
dupes)
self.log.info("Expansion completed.")
return ret

247
zuul/lib/cloner.py
View File

@ -1,247 +0,0 @@
# Copyright 2014 Antoine "hashar" Musso
# Copyright 2014 Wikimedia Foundation Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import git
import logging
import os
import re
from git import GitCommandError
from zuul import exceptions
from zuul.lib.clonemapper import CloneMapper
from zuul.lib import yamlutil as yaml
from zuul.merger.merger import Repo
class Cloner(object):
log = logging.getLogger("zuul.Cloner")
def __init__(self, git_base_url, projects, workspace, zuul_branch,
zuul_ref, zuul_url, branch=None, clone_map_file=None,
project_branches=None, cache_dir=None, zuul_newrev=None,
zuul_project=None):
self.clone_map = []
self.dests = None
self.branch = branch
self.git_url = git_base_url
self.cache_dir = cache_dir
self.projects = projects
self.workspace = workspace
self.zuul_branch = zuul_branch or ''
self.zuul_ref = zuul_ref or ''
self.zuul_url = zuul_url
self.zuul_project = zuul_project
self.project_branches = project_branches or {}
self.project_revisions = {}
if zuul_newrev and zuul_project:
self.project_revisions[zuul_project] = zuul_newrev
if clone_map_file:
self.readCloneMap(clone_map_file)
def readCloneMap(self, clone_map_file):
clone_map_file = os.path.expanduser(clone_map_file)
if not os.path.exists(clone_map_file):
raise Exception("Unable to read clone map file at %s." %
clone_map_file)
clone_map_file = open(clone_map_file)
self.clone_map = yaml.safe_load(clone_map_file).get('clonemap')
self.log.info("Loaded map containing %s rules", len(self.clone_map))
return self.clone_map
def execute(self):
mapper = CloneMapper(self.clone_map, self.projects)
dests = mapper.expand(workspace=self.workspace)
self.log.info("Preparing %s repositories", len(dests))
for project, dest in dests.items():
self.prepareRepo(project, dest)
self.log.info("Prepared all repositories")
def cloneUpstream(self, project, dest):
# Check for a cached git repo first
git_cache = '%s/%s' % (self.cache_dir, project)
# Then, if we are cloning the repo for the zuul_project, then
# set its origin to be the zuul merger, as it is guaranteed to
# be correct and up to date even if mirrors haven't updated
# yet. Otherwise, we can not be sure about the state of the
# project, so our best chance to get the most current state is
# by setting origin to the git_url.
if (self.zuul_url and project == self.zuul_project):
git_upstream = '%s/%s' % (self.zuul_url, project)
else:
git_upstream = '%s/%s' % (self.git_url, project)
repo_is_cloned = os.path.exists(os.path.join(dest, '.git'))
if (self.cache_dir and
os.path.exists(git_cache) and
not repo_is_cloned):
# file:// tells git not to hard-link across repos
git_cache = 'file://%s' % git_cache
self.log.info("Creating repo %s from cache %s",
project, git_cache)
new_repo = git.Repo.clone_from(git_cache, dest)
self.log.info("Updating origin remote in repo %s to %s",
project, git_upstream)
new_repo.remotes.origin.config_writer.set('url',
git_upstream).release()
else:
self.log.info("Creating repo %s from upstream %s",
project, git_upstream)
repo = Repo(
remote=git_upstream,
local=dest,
email=None,
username=None)
if not repo.isInitialized():
raise Exception("Error cloning %s to %s" % (git_upstream, dest))
return repo
def fetchRef(self, repo, project, ref):
# If we are fetching a zuul ref, the only place to get it is
# from the zuul merger (and it is guaranteed to be correct).
# Otherwise, the only way we can be certain that the ref
# (which, since it is not a zuul ref, is a branch or tag) is
# correct is in the case that it matches zuul_project. If
# neither of those two conditions are met, we are most likely
# to get the correct state from the git_url.
if (ref.startswith('refs/zuul') or
project == self.zuul_project):
remote = '%s/%s' % (self.zuul_url, project)
else:
remote = '%s/%s' % (self.git_url, project)
try:
repo.fetchFrom(remote, ref)
self.log.debug("Fetched ref %s from %s", ref, remote)
return True
except ValueError:
self.log.debug("Repo %s does not have ref %s",
remote, ref)
return False
except GitCommandError as error:
# Bail out if fetch fails due to infrastructure reasons
if error.stderr.startswith('fatal: unable to access'):
raise
self.log.debug("Repo %s does not have ref %s",
remote, ref)
return False
def prepareRepo(self, project, dest):
"""Clone a repository for project at dest and apply a reference
suitable for testing. The reference lookup is attempted in this order:
1) The indicated revision for specific project
2) Zuul reference for the indicated branch
3) Zuul reference for the master branch
4) The tip of the indicated branch
5) The tip of the master branch
If an "indicated revision" is specified for this project, and we are
unable to meet this requirement, we stop attempting to check this
repo out and raise a zuul.exceptions.RevNotFound exception.
The "indicated branch" is one of the following:
A) The project-specific override branch (from project_branches arg)
B) The user specified branch (from the branch arg)
C) ZUUL_BRANCH (from the zuul_branch arg)
"""
repo = self.cloneUpstream(project, dest)
# Ensure that we don't have stale remotes around
repo.prune()
# We must reset after pruning because reseting sets HEAD to point
# at refs/remotes/origin/master, but `git branch` which prune runs
# explodes if HEAD does not point at something in refs/heads.
# Later with repo.checkout() we set HEAD to something that
# `git branch` is happy with.
repo.reset()
indicated_revision = None
if project in self.project_revisions:
indicated_revision = self.project_revisions[project]
indicated_branch = self.branch or self.zuul_branch
if project in self.project_branches:
indicated_branch = self.project_branches[project]
if indicated_branch:
override_zuul_ref = re.sub(self.zuul_branch, indicated_branch,
self.zuul_ref)
else:
override_zuul_ref = None
if indicated_branch and repo.hasBranch(indicated_branch):
self.log.info("upstream repo has branch %s", indicated_branch)
fallback_branch = indicated_branch
else:
if indicated_branch:
self.log.info("upstream repo is missing branch %s",
indicated_branch)
# FIXME should be origin HEAD branch which might not be 'master'
fallback_branch = 'master'
if self.zuul_branch:
fallback_zuul_ref = re.sub(self.zuul_branch, fallback_branch,
self.zuul_ref)
else:
fallback_zuul_ref = None
# If the user has requested an explicit revision to be checked out,
# we use it above all else, and if we cannot satisfy this requirement
# we raise an error and do not attempt to continue.
if indicated_revision:
self.log.info("Attempting to check out revision %s for "
"project %s", indicated_revision, project)
try:
self.fetchRef(repo, project, self.zuul_ref)
commit = repo.checkout(indicated_revision)
except (ValueError, GitCommandError):
raise exceptions.RevNotFound(project, indicated_revision)
self.log.info("Prepared '%s' repo at revision '%s'", project,
indicated_revision)
# If we have a non empty zuul_ref to use, use it. Otherwise we fall
# back to checking out the branch.
elif ((override_zuul_ref and
self.fetchRef(repo, project, override_zuul_ref)) or
(fallback_zuul_ref and
fallback_zuul_ref != override_zuul_ref and
self.fetchRef(repo, project, fallback_zuul_ref))):
# Work around a bug in GitPython which can not parse FETCH_HEAD
gitcmd = git.Git(dest)
fetch_head = gitcmd.rev_parse('FETCH_HEAD')
repo.checkout(fetch_head)
self.log.info("Prepared %s repo with commit %s",
project, fetch_head)
else:
# Checkout branch
self.log.info("Falling back to branch %s", fallback_branch)
try:
commit = repo.checkout('remotes/origin/%s' % fallback_branch)
except (ValueError, GitCommandError):
self.log.exception("Fallback branch not found: %s",
fallback_branch)
self.log.info("Prepared %s repo with branch %s at commit %s",
project, fallback_branch, commit)