diff --git a/doc/source/cloner.rst b/doc/source/cloner.rst new file mode 100644 index 0000000000..bb33f828ef --- /dev/null +++ b/doc/source/cloner.rst @@ -0,0 +1,77 @@ +:title: Zuul Cloner + +Zuul Cloner +=========== + +Zuul includes a simple command line client that may be used to clone +repositories with Zuul references applied. + +Configuration +------------- + +Clone map +''''''''' + +By default, Zuul cloner will clone the project under ``basepath`` which +would create sub directories whenever a project name contains slashes. Since +you might want to finely tweak the final destination, a clone map lets you +change the destination on a per project basis. The configuration is done using +a YAML file passed with ``-m``. + +With a project hierarchy such as:: + + project + thirdparty/plugins/plugin1 + +You might want to get ``project`` straight in the base path, the clone map would be:: + + clonemap: + - name: 'project' + dest: '.' + +Then to strip out ``thirdparty`` such that the plugins land under the +``/plugins`` directory of the basepath, you can use regex and capturing +groups:: + + clonemap: + - name: 'project' + dest: '.' + - name: 'thirdparty/(plugins/.*)' + dest: '\1' + +The resulting workspace will contains:: + + project -> ./ + thirdparty/plugins/plugin1 -> ./plugins/plugin1 + + +Zuul parameters +''''''''''''''' + +The Zuul cloner reuses Zuul parameters such as ZUUL_BRANCH, ZUUL_REF or +ZUUL_PROJECT. It will attempt to load them from the environment variables or +you can pass them as parameters (in which case it will override the +environment variable if it is set). The matching command line parameters use +the ``zuul`` prefix hence ZUUL_REF can be passed to the cloner using +``--zuul-ref``. + +Usage +----- +The general options that apply are: + +.. program-output:: zuul-cloner --help + +Clone order +----------- + +When cloning repositories, the destination folder should not exist or +``git clone`` will complain. This happens whenever cloning a sub project +before its parent project. For example:: + + zuul-cloner project/plugins/plugin1 project + +Will create the directory ``project`` when cloning the plugin. The +cloner processes the clones in the order supplied, so you should swap the +projects:: + + zuul-cloner project project/plugins/plugin1 diff --git a/doc/source/index.rst b/doc/source/index.rst index fcc0d458e9..abe80893a7 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -20,6 +20,7 @@ Contents: reporters zuul client + cloner statsd Indices and tables diff --git a/etc/clonemap.yaml-sample b/etc/clonemap.yaml-sample new file mode 100644 index 0000000000..9695d9d174 --- /dev/null +++ b/etc/clonemap.yaml-sample @@ -0,0 +1,16 @@ +# vim: ft=yaml +# +# Example clone map for Zuul cloner +# +# By default it would clone projects under the directory specified by its +# option --basepath, but you can override this behavior by definining per +# project destinations. +clonemap: + + # Clone project 'mediawiki/core' directly in {basepath} + - name: 'mediawiki/core' + dest: '.' + + # Clone projects below mediawiki/extensions to {basepath}/extensions/ + - name: 'mediawiki/extensions/(.*)' + dest: 'extensions/\1' diff --git a/setup.cfg b/setup.cfg index 21b1199b10..a4deb2f235 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,6 +24,7 @@ console_scripts = zuul-server = zuul.cmd.server:main zuul-merger = zuul.cmd.merger:main zuul = zuul.cmd.client:main + zuul-cloner = zuul.cmd.cloner:main [build_sphinx] source-dir = doc/source diff --git a/tests/fixtures/clonemap.yaml b/tests/fixtures/clonemap.yaml new file mode 100644 index 0000000000..0f9e084eaa --- /dev/null +++ b/tests/fixtures/clonemap.yaml @@ -0,0 +1,5 @@ +clonemap: + - name: 'mediawiki/core' + dest: '.' + - name: 'mediawiki/extensions/(.*)' + dest: '\1' diff --git a/tests/fixtures/layout-gating.yaml b/tests/fixtures/layout-gating.yaml new file mode 100644 index 0000000000..a544a80f89 --- /dev/null +++ b/tests/fixtures/layout-gating.yaml @@ -0,0 +1,29 @@ +pipelines: + - name: gate + manager: DependentPipelineManager + failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures + trigger: + gerrit: + - event: comment-added + approval: + - approved: 1 + start: + gerrit: + verified: 0 + success: + gerrit: + verified: 2 + submit: true + failure: + gerrit: + verified: -2 + +projects: + + - name: org/project1 + gate: + - project1-project2-integration + + - name: org/project2 + gate: + - project1-project2-integration diff --git a/tests/test_clonemapper.py b/tests/test_clonemapper.py new file mode 100644 index 0000000000..b7814f82ef --- /dev/null +++ b/tests/test_clonemapper.py @@ -0,0 +1,84 @@ +# Copyright 2014 Antoine "hashar" Musso +# Copyright 2014 Wikimedia Foundation Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import testtools +from zuul.lib.clonemapper import CloneMapper + +logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s %(name)-17s ' + '%(levelname)-8s %(message)s') + + +class TestCloneMapper(testtools.TestCase): + + def test_empty_mapper(self): + """Given an empty map, the slashes in project names are directory + separators""" + cmap = CloneMapper( + {}, + [ + 'project1', + 'plugins/plugin1' + ]) + + self.assertEqual( + {'project1': '/basepath/project1', + 'plugins/plugin1': '/basepath/plugins/plugin1'}, + cmap.expand('/basepath') + ) + + def test_map_to_a_dot_dir(self): + """Verify we normalize path, hence '.' refers to the basepath""" + cmap = CloneMapper( + [{'name': 'mediawiki/core', 'dest': '.'}], + ['mediawiki/core']) + self.assertEqual( + {'mediawiki/core': '/basepath'}, + cmap.expand('/basepath')) + + def test_map_using_regex(self): + """One can use regex in maps and use \\1 to forge the directory""" + cmap = CloneMapper( + [{'name': 'plugins/(.*)', 'dest': 'project/plugins/\\1'}], + ['plugins/PluginFirst']) + self.assertEqual( + {'plugins/PluginFirst': '/basepath/project/plugins/PluginFirst'}, + cmap.expand('/basepath')) + + def test_map_discarding_regex_group(self): + cmap = CloneMapper( + [{'name': 'plugins/(.*)', 'dest': 'project/'}], + ['plugins/Plugin_1']) + self.assertEqual( + {'plugins/Plugin_1': '/basepath/project'}, + cmap.expand('/basepath')) + + def test_cant_dupe_destinations(self): + """We cant clone multiple projects in the same directory""" + cmap = CloneMapper( + [{'name': 'plugins/(.*)', 'dest': 'catchall/'}], + ['plugins/plugin1', 'plugins/plugin2'] + ) + self.assertRaises(Exception, cmap.expand, '/basepath') + + def test_map_with_dot_and_regex(self): + """Combining relative path and regex""" + cmap = CloneMapper( + [{'name': 'plugins/(.*)', 'dest': './\\1'}], + ['plugins/PluginInBasePath']) + self.assertEqual( + {'plugins/PluginInBasePath': '/basepath/PluginInBasePath'}, + cmap.expand('/basepath')) diff --git a/tests/test_cloner.py b/tests/test_cloner.py new file mode 100644 index 0000000000..bb9d91f35e --- /dev/null +++ b/tests/test_cloner.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python + +# Copyright 2012 Hewlett-Packard Development Company, L.P. +# Copyright 2014 Wikimedia Foundation Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import shutil + +import git + +import zuul.lib.cloner + +from tests.base import ZuulTestCase +from tests.base import FIXTURE_DIR + +logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s %(name)-32s ' + '%(levelname)-8s %(message)s') + + +class TestCloner(ZuulTestCase): + + log = logging.getLogger("zuul.test.cloner") + workspace_root = None + + def setUp(self): + super(TestCloner, self).setUp() + self.workspace_root = os.path.join(self.test_root, 'workspace') + + self.config.set('zuul', 'layout_config', + 'tests/fixtures/layout-gating.yaml') + self.sched.reconfigure(self.config) + self.registerJobs() + + def test_cloner(self): + self.worker.hold_jobs_in_build = True + + A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') + B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') + + A.addPatchset(['project_one.txt']) + B.addPatchset(['project_two.txt']) + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) + + A.addApproval('CRVW', 2) + B.addApproval('CRVW', 2) + self.fake_gerrit.addEvent(A.addApproval('APRV', 1)) + self.fake_gerrit.addEvent(B.addApproval('APRV', 1)) + + self.waitUntilSettled() + + self.assertEquals(2, len(self.builds), "Two builds are running") + + a_zuul_ref = b_zuul_ref = None + for build in self.builds: + self.log.debug("Build parameters: %s", build.parameters) + if build.parameters['ZUUL_CHANGE'] == '1': + a_zuul_ref = build.parameters['ZUUL_REF'] + a_zuul_commit = build.parameters['ZUUL_COMMIT'] + if build.parameters['ZUUL_CHANGE'] == '2': + b_zuul_ref = build.parameters['ZUUL_REF'] + b_zuul_commit = build.parameters['ZUUL_COMMIT'] + + self.worker.hold_jobs_in_build = False + self.worker.release() + self.waitUntilSettled() + + # Repos setup, now test the cloner + for zuul_ref in [a_zuul_ref, b_zuul_ref]: + cloner = zuul.lib.cloner.Cloner( + git_base_url=self.upstream_root, + projects=['org/project1', 'org/project2'], + workspace=self.workspace_root, + zuul_branch='master', + zuul_ref=zuul_ref, + zuul_url=self.git_root, + branch='master', + clone_map_file=os.path.join(FIXTURE_DIR, 'clonemap.yaml') + ) + cloner.execute() + work_repo1 = git.Repo(os.path.join(self.workspace_root, + 'org/project1')) + self.assertEquals(a_zuul_commit, str(work_repo1.commit('HEAD'))) + + work_repo2 = git.Repo(os.path.join(self.workspace_root, + 'org/project2')) + self.assertEquals(b_zuul_commit, str(work_repo2.commit('HEAD'))) + + shutil.rmtree(self.workspace_root) diff --git a/zuul/cmd/cloner.py b/zuul/cmd/cloner.py new file mode 100755 index 0000000000..1310c1690d --- /dev/null +++ b/zuul/cmd/cloner.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python +# +# Copyright 2014 Antoine "hashar" Musso +# Copyright 2014 Wikimedia Foundation Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import logging +import os +import sys + +import zuul.cmd +import zuul.lib.cloner + +ZUUL_ENV_SUFFIXES = ( + 'branch', + 'change', + 'patchset', + 'pipeline', + 'project', + 'ref', + 'url', +) + + +class Cloner(zuul.cmd.ZuulApp): + log = logging.getLogger("zuul.Cloner") + + def parse_arguments(self): + """Parse command line arguments and returns argparse structure""" + parser = argparse.ArgumentParser( + description='Zuul Project Gating System Cloner.') + parser.add_argument('-m', '--map', dest='clone_map_file', + help='specifiy clone map file') + parser.add_argument('--workspace', dest='workspace', + default=os.getcwd(), + help='where to clone repositories too') + parser.add_argument('-v', '--verbose', dest='verbose', + action='store_true', + help='verbose output') + parser.add_argument('--color', dest='color', action='store_true', + help='use color output') + parser.add_argument('--version', dest='version', action='version', + version=self._get_version(), + help='show zuul version') + parser.add_argument('git_base_url', + help='reference repo to clone from') + parser.add_argument('projects', nargs='+', + help='list of Gerrit projects to clone') + + project_env = parser.add_argument_group( + 'project tuning' + ) + project_env.add_argument( + '--branch', + help=('branch to checkout instead of Zuul selected branch, ' + 'for example to specify an alternate branch to test ' + 'client library compatibility.') + ) + + zuul_env = parser.add_argument_group( + 'zuul environnement', + 'Let you override $ZUUL_* environnement variables.' + ) + for zuul_suffix in ZUUL_ENV_SUFFIXES: + env_name = 'ZUUL_%s' % zuul_suffix.upper() + zuul_env.add_argument( + '--zuul-%s' % zuul_suffix, metavar='$' + env_name, + default=os.environ.get(env_name) + ) + + args = parser.parse_args() + + # Validate ZUUL_* arguments + zuul_missing = [zuul_opt for zuul_opt, val in vars(args).items() + if zuul_opt.startswith('zuul') and val is None] + if zuul_missing: + parser.error(("Some Zuul parameters are not properly set:\n\t%s\n" + "Define them either via environment variables or " + "using options above." % + "\n\t".join(sorted(zuul_missing)))) + self.args = args + + def setup_logging(self, color=False, verbose=False): + """Cloner logging does not rely on conf file""" + if verbose: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + if color: + # Color codes http://www.tldp.org/HOWTO/Bash-Prompt-HOWTO/x329.html + logging.addLevelName( # cyan + logging.DEBUG, "\033[36m%s\033[0m" % + logging.getLevelName(logging.DEBUG)) + logging.addLevelName( # green + logging.INFO, "\033[32m%s\033[0m" % + logging.getLevelName(logging.INFO)) + logging.addLevelName( # yellow + logging.WARNING, "\033[33m%s\033[0m" % + logging.getLevelName(logging.WARNING)) + logging.addLevelName( # red + logging.ERROR, "\033[31m%s\033[0m" % + logging.getLevelName(logging.ERROR)) + logging.addLevelName( # red background + logging.CRITICAL, "\033[41m%s\033[0m" % + logging.getLevelName(logging.CRITICAL)) + + def main(self): + self.parse_arguments() + self.setup_logging(color=self.args.color, verbose=self.args.verbose) + cloner = zuul.lib.cloner.Cloner( + git_base_url=self.args.git_base_url, + projects=self.args.projects, + workspace=self.args.workspace, + zuul_branch=self.args.zuul_branch, + zuul_ref=self.args.zuul_ref, + zuul_url=self.args.zuul_url, + branch=self.args.branch, + clone_map_file=self.args.clone_map_file + ) + cloner.execute() + + +def main(): + cloner = Cloner() + cloner.main() + + +if __name__ == "__main__": + sys.path.insert(0, '.') + main() diff --git a/zuul/lib/clonemapper.py b/zuul/lib/clonemapper.py new file mode 100644 index 0000000000..ae558cd8d0 --- /dev/null +++ b/zuul/lib/clonemapper.py @@ -0,0 +1,78 @@ +# Copyright 2014 Antoine "hashar" Musso +# Copyright 2014 Wikimedia Foundation Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from collections import defaultdict +import extras +import logging +import os +import re + +OrderedDict = extras.try_imports(['collections.OrderedDict', + 'ordereddict.OrderedDict']) + + +class CloneMapper(object): + log = logging.getLogger("zuul.CloneMapper") + + def __init__(self, clonemap, projects): + self.clonemap = clonemap + self.projects = projects + + def expand(self, workspace): + self.log.info("Workspace path set to: %s", workspace) + + is_valid = True + ret = OrderedDict() + for project in self.projects: + dests = [] + for mapping in self.clonemap: + if re.match(r'^%s$' % mapping['name'], + project): + # Might be matched more than one time + dests.append( + re.sub(mapping['name'], mapping['dest'], project)) + + if len(dests) > 1: + self.log.error("Duplicate destinations for %s: %s.", + project, dests) + is_valid = False + elif len(dests) == 0: + self.log.debug("Using %s as destination (unmatched)", + project) + ret[project] = [project] + else: + ret[project] = dests + + if not is_valid: + raise Exception("Expansion error. Check error messages above") + + self.log.info("Mapping projects to workspace...") + for project, dest in ret.iteritems(): + dest = os.path.normpath(os.path.join(workspace, dest[0])) + ret[project] = dest + self.log.info(" %s -> %s", project, dest) + + self.log.debug("Checking overlap in destination directories...") + check = defaultdict(list) + for project, dest in ret.iteritems(): + check[dest].append(project) + + dupes = dict((d, p) for (d, p) in check.iteritems() if len(p) > 1) + if dupes: + raise Exception("Some projects share the same destination: %s", + dupes) + + self.log.info("Expansion completed.") + return ret diff --git a/zuul/lib/cloner.py b/zuul/lib/cloner.py new file mode 100644 index 0000000000..0961eb44d5 --- /dev/null +++ b/zuul/lib/cloner.py @@ -0,0 +1,145 @@ +# Copyright 2014 Antoine "hashar" Musso +# Copyright 2014 Wikimedia Foundation Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import git +import logging +import os +import re +import yaml + +from git import GitCommandError +from zuul.lib.clonemapper import CloneMapper +from zuul.merger.merger import Repo + + +class Cloner(object): + log = logging.getLogger("zuul.Cloner") + + def __init__(self, git_base_url, projects, workspace, zuul_branch, + zuul_ref, zuul_url, branch=None, clone_map_file=None): + + self.clone_map = [] + self.dests = None + + self.branch = branch + self.git_url = git_base_url + self.projects = projects + self.workspace = workspace + self.zuul_branch = zuul_branch + self.zuul_ref = zuul_ref + self.zuul_url = zuul_url + + if clone_map_file: + self.readCloneMap(clone_map_file) + + def readCloneMap(self, clone_map_file): + clone_map_file = os.path.expanduser(clone_map_file) + if not os.path.exists(clone_map_file): + raise Exception("Unable to read clone map file at %s." % + clone_map_file) + clone_map_file = open(clone_map_file) + self.clone_map = yaml.load(clone_map_file).get('clonemap') + self.log.info("Loaded map containing %s rules", len(self.clone_map)) + return self.clone_map + + def execute(self): + mapper = CloneMapper(self.clone_map, self.projects) + dests = mapper.expand(workspace=self.workspace) + + self.log.info("Preparing %s repositories", len(dests)) + for project, dest in dests.iteritems(): + self.prepareRepo(project, dest) + self.log.info("Prepared all repositories") + + def cloneUpstream(self, project, dest): + git_upstream = '%s/%s' % (self.git_url, project) + self.log.info("Creating repo %s from upstream %s", + project, git_upstream) + repo = Repo( + remote=git_upstream, + local=dest, + email=None, + username=None) + + if not repo.isInitialized(): + raise Exception("Error cloning %s to %s" % (git_upstream, dest)) + + return repo + + def fetchFromZuul(self, repo, project, ref): + zuul_remote = '%s/%s' % (self.zuul_url, project) + + try: + repo.fetchFrom(zuul_remote, ref) + self.log.debug("Fetched ref %s from %s", ref, project) + return True + except (ValueError, GitCommandError): + self.log.debug("Project %s in Zuul does not have ref %s", + project, ref) + return False + + def prepareRepo(self, project, dest): + """Clone a repository for project at dest and apply a reference + suitable for testing. The reference lookup is attempted in this order: + + 1) Zuul reference for the indicated branch + 2) Zuul reference for the master branch + 3) The tip of the indicated branch + 4) The tip of the master branch + """ + + repo = self.cloneUpstream(project, dest) + + repo.update() + # Ensure that we don't have stale remotes around + repo.prune() + + override_zuul_ref = self.zuul_ref + # FIXME should be origin HEAD branch which might not be 'master' + fallback_branch = 'master' + fallback_zuul_ref = re.sub(self.zuul_branch, fallback_branch, + self.zuul_ref) + + if self.branch: + override_zuul_ref = re.sub(self.zuul_branch, self.branch, + self.zuul_ref) + if repo.hasBranch(self.branch): + self.log.debug("upstream repo has branch %s", self.branch) + fallback_branch = self.branch + fallback_zuul_ref = self.zuul_ref + else: + self.log.exception("upstream repo is missing branch %s", + self.branch) + + if (self.fetchFromZuul(repo, project, override_zuul_ref) + or (fallback_zuul_ref != override_zuul_ref and + self.fetchFromZuul(repo, project, fallback_zuul_ref)) + ): + # Work around a bug in GitPython which can not parse FETCH_HEAD + gitcmd = git.Git(dest) + fetch_head = gitcmd.rev_parse('FETCH_HEAD') + repo.checkout(fetch_head) + self.log.info("Prepared %s repo with commit %s", + project, fetch_head) + else: + # Checkout branch + self.log.debug("Falling back to branch %s", fallback_branch) + try: + repo.checkout('remotes/origin/%s' % fallback_branch) + except (ValueError, GitCommandError): + self.log.exception("Fallback branch not found: %s", + fallback_branch) + self.log.info("Prepared %s repo with branch %s", + project, fallback_branch) diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py index 52cbc338eb..922c67ef36 100644 --- a/zuul/merger/merger.py +++ b/zuul/merger/merger.py @@ -146,6 +146,10 @@ class Repo(object): except AssertionError: origin.fetch(ref) + def fetchFrom(self, repository, refspec): + repo = self.createRepoObject() + repo.git.fetch(repository, refspec) + def createZuulRef(self, ref, commit='HEAD'): repo = self.createRepoObject() self.log.debug("CreateZuulRef %s at %s" % (ref, commit))