cloner to easily clone dependent repositories
The intent is to replace the devstack shell script with a python utility which would be easy to reuse. Change-Id: I3c8748e2544af80e72fdd0b2e3627b4fc2212c01
This commit is contained in:
parent
a6529c64eb
commit
45dd2cb40e
77
doc/source/cloner.rst
Normal file
77
doc/source/cloner.rst
Normal file
@ -0,0 +1,77 @@
|
||||
:title: Zuul Cloner
|
||||
|
||||
Zuul Cloner
|
||||
===========
|
||||
|
||||
Zuul includes a simple command line client that may be used to clone
|
||||
repositories with Zuul references applied.
|
||||
|
||||
Configuration
|
||||
-------------
|
||||
|
||||
Clone map
|
||||
'''''''''
|
||||
|
||||
By default, Zuul cloner will clone the project under ``basepath`` which
|
||||
would create sub directories whenever a project name contains slashes. Since
|
||||
you might want to finely tweak the final destination, a clone map lets you
|
||||
change the destination on a per project basis. The configuration is done using
|
||||
a YAML file passed with ``-m``.
|
||||
|
||||
With a project hierarchy such as::
|
||||
|
||||
project
|
||||
thirdparty/plugins/plugin1
|
||||
|
||||
You might want to get ``project`` straight in the base path, the clone map would be::
|
||||
|
||||
clonemap:
|
||||
- name: 'project'
|
||||
dest: '.'
|
||||
|
||||
Then to strip out ``thirdparty`` such that the plugins land under the
|
||||
``/plugins`` directory of the basepath, you can use regex and capturing
|
||||
groups::
|
||||
|
||||
clonemap:
|
||||
- name: 'project'
|
||||
dest: '.'
|
||||
- name: 'thirdparty/(plugins/.*)'
|
||||
dest: '\1'
|
||||
|
||||
The resulting workspace will contains::
|
||||
|
||||
project -> ./
|
||||
thirdparty/plugins/plugin1 -> ./plugins/plugin1
|
||||
|
||||
|
||||
Zuul parameters
|
||||
'''''''''''''''
|
||||
|
||||
The Zuul cloner reuses Zuul parameters such as ZUUL_BRANCH, ZUUL_REF or
|
||||
ZUUL_PROJECT. It will attempt to load them from the environment variables or
|
||||
you can pass them as parameters (in which case it will override the
|
||||
environment variable if it is set). The matching command line parameters use
|
||||
the ``zuul`` prefix hence ZUUL_REF can be passed to the cloner using
|
||||
``--zuul-ref``.
|
||||
|
||||
Usage
|
||||
-----
|
||||
The general options that apply are:
|
||||
|
||||
.. program-output:: zuul-cloner --help
|
||||
|
||||
Clone order
|
||||
-----------
|
||||
|
||||
When cloning repositories, the destination folder should not exist or
|
||||
``git clone`` will complain. This happens whenever cloning a sub project
|
||||
before its parent project. For example::
|
||||
|
||||
zuul-cloner project/plugins/plugin1 project
|
||||
|
||||
Will create the directory ``project`` when cloning the plugin. The
|
||||
cloner processes the clones in the order supplied, so you should swap the
|
||||
projects::
|
||||
|
||||
zuul-cloner project project/plugins/plugin1
|
@ -20,6 +20,7 @@ Contents:
|
||||
reporters
|
||||
zuul
|
||||
client
|
||||
cloner
|
||||
statsd
|
||||
|
||||
Indices and tables
|
||||
|
16
etc/clonemap.yaml-sample
Normal file
16
etc/clonemap.yaml-sample
Normal file
@ -0,0 +1,16 @@
|
||||
# vim: ft=yaml
|
||||
#
|
||||
# Example clone map for Zuul cloner
|
||||
#
|
||||
# By default it would clone projects under the directory specified by its
|
||||
# option --basepath, but you can override this behavior by definining per
|
||||
# project destinations.
|
||||
clonemap:
|
||||
|
||||
# Clone project 'mediawiki/core' directly in {basepath}
|
||||
- name: 'mediawiki/core'
|
||||
dest: '.'
|
||||
|
||||
# Clone projects below mediawiki/extensions to {basepath}/extensions/
|
||||
- name: 'mediawiki/extensions/(.*)'
|
||||
dest: 'extensions/\1'
|
@ -24,6 +24,7 @@ console_scripts =
|
||||
zuul-server = zuul.cmd.server:main
|
||||
zuul-merger = zuul.cmd.merger:main
|
||||
zuul = zuul.cmd.client:main
|
||||
zuul-cloner = zuul.cmd.cloner:main
|
||||
|
||||
[build_sphinx]
|
||||
source-dir = doc/source
|
||||
|
5
tests/fixtures/clonemap.yaml
vendored
Normal file
5
tests/fixtures/clonemap.yaml
vendored
Normal file
@ -0,0 +1,5 @@
|
||||
clonemap:
|
||||
- name: 'mediawiki/core'
|
||||
dest: '.'
|
||||
- name: 'mediawiki/extensions/(.*)'
|
||||
dest: '\1'
|
29
tests/fixtures/layout-gating.yaml
vendored
Normal file
29
tests/fixtures/layout-gating.yaml
vendored
Normal file
@ -0,0 +1,29 @@
|
||||
pipelines:
|
||||
- name: gate
|
||||
manager: DependentPipelineManager
|
||||
failure-message: Build failed. For information on how to proceed, see http://wiki.example.org/Test_Failures
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: comment-added
|
||||
approval:
|
||||
- approved: 1
|
||||
start:
|
||||
gerrit:
|
||||
verified: 0
|
||||
success:
|
||||
gerrit:
|
||||
verified: 2
|
||||
submit: true
|
||||
failure:
|
||||
gerrit:
|
||||
verified: -2
|
||||
|
||||
projects:
|
||||
|
||||
- name: org/project1
|
||||
gate:
|
||||
- project1-project2-integration
|
||||
|
||||
- name: org/project2
|
||||
gate:
|
||||
- project1-project2-integration
|
84
tests/test_clonemapper.py
Normal file
84
tests/test_clonemapper.py
Normal file
@ -0,0 +1,84 @@
|
||||
# Copyright 2014 Antoine "hashar" Musso
|
||||
# Copyright 2014 Wikimedia Foundation Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging
|
||||
import testtools
|
||||
from zuul.lib.clonemapper import CloneMapper
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG,
|
||||
format='%(asctime)s %(name)-17s '
|
||||
'%(levelname)-8s %(message)s')
|
||||
|
||||
|
||||
class TestCloneMapper(testtools.TestCase):
|
||||
|
||||
def test_empty_mapper(self):
|
||||
"""Given an empty map, the slashes in project names are directory
|
||||
separators"""
|
||||
cmap = CloneMapper(
|
||||
{},
|
||||
[
|
||||
'project1',
|
||||
'plugins/plugin1'
|
||||
])
|
||||
|
||||
self.assertEqual(
|
||||
{'project1': '/basepath/project1',
|
||||
'plugins/plugin1': '/basepath/plugins/plugin1'},
|
||||
cmap.expand('/basepath')
|
||||
)
|
||||
|
||||
def test_map_to_a_dot_dir(self):
|
||||
"""Verify we normalize path, hence '.' refers to the basepath"""
|
||||
cmap = CloneMapper(
|
||||
[{'name': 'mediawiki/core', 'dest': '.'}],
|
||||
['mediawiki/core'])
|
||||
self.assertEqual(
|
||||
{'mediawiki/core': '/basepath'},
|
||||
cmap.expand('/basepath'))
|
||||
|
||||
def test_map_using_regex(self):
|
||||
"""One can use regex in maps and use \\1 to forge the directory"""
|
||||
cmap = CloneMapper(
|
||||
[{'name': 'plugins/(.*)', 'dest': 'project/plugins/\\1'}],
|
||||
['plugins/PluginFirst'])
|
||||
self.assertEqual(
|
||||
{'plugins/PluginFirst': '/basepath/project/plugins/PluginFirst'},
|
||||
cmap.expand('/basepath'))
|
||||
|
||||
def test_map_discarding_regex_group(self):
|
||||
cmap = CloneMapper(
|
||||
[{'name': 'plugins/(.*)', 'dest': 'project/'}],
|
||||
['plugins/Plugin_1'])
|
||||
self.assertEqual(
|
||||
{'plugins/Plugin_1': '/basepath/project'},
|
||||
cmap.expand('/basepath'))
|
||||
|
||||
def test_cant_dupe_destinations(self):
|
||||
"""We cant clone multiple projects in the same directory"""
|
||||
cmap = CloneMapper(
|
||||
[{'name': 'plugins/(.*)', 'dest': 'catchall/'}],
|
||||
['plugins/plugin1', 'plugins/plugin2']
|
||||
)
|
||||
self.assertRaises(Exception, cmap.expand, '/basepath')
|
||||
|
||||
def test_map_with_dot_and_regex(self):
|
||||
"""Combining relative path and regex"""
|
||||
cmap = CloneMapper(
|
||||
[{'name': 'plugins/(.*)', 'dest': './\\1'}],
|
||||
['plugins/PluginInBasePath'])
|
||||
self.assertEqual(
|
||||
{'plugins/PluginInBasePath': '/basepath/PluginInBasePath'},
|
||||
cmap.expand('/basepath'))
|
103
tests/test_cloner.py
Normal file
103
tests/test_cloner.py
Normal file
@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2012 Hewlett-Packard Development Company, L.P.
|
||||
# Copyright 2014 Wikimedia Foundation Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
|
||||
import git
|
||||
|
||||
import zuul.lib.cloner
|
||||
|
||||
from tests.base import ZuulTestCase
|
||||
from tests.base import FIXTURE_DIR
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG,
|
||||
format='%(asctime)s %(name)-32s '
|
||||
'%(levelname)-8s %(message)s')
|
||||
|
||||
|
||||
class TestCloner(ZuulTestCase):
|
||||
|
||||
log = logging.getLogger("zuul.test.cloner")
|
||||
workspace_root = None
|
||||
|
||||
def setUp(self):
|
||||
super(TestCloner, self).setUp()
|
||||
self.workspace_root = os.path.join(self.test_root, 'workspace')
|
||||
|
||||
self.config.set('zuul', 'layout_config',
|
||||
'tests/fixtures/layout-gating.yaml')
|
||||
self.sched.reconfigure(self.config)
|
||||
self.registerJobs()
|
||||
|
||||
def test_cloner(self):
|
||||
self.worker.hold_jobs_in_build = True
|
||||
|
||||
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
|
||||
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
|
||||
|
||||
A.addPatchset(['project_one.txt'])
|
||||
B.addPatchset(['project_two.txt'])
|
||||
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
||||
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
|
||||
|
||||
A.addApproval('CRVW', 2)
|
||||
B.addApproval('CRVW', 2)
|
||||
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
|
||||
self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
|
||||
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.assertEquals(2, len(self.builds), "Two builds are running")
|
||||
|
||||
a_zuul_ref = b_zuul_ref = None
|
||||
for build in self.builds:
|
||||
self.log.debug("Build parameters: %s", build.parameters)
|
||||
if build.parameters['ZUUL_CHANGE'] == '1':
|
||||
a_zuul_ref = build.parameters['ZUUL_REF']
|
||||
a_zuul_commit = build.parameters['ZUUL_COMMIT']
|
||||
if build.parameters['ZUUL_CHANGE'] == '2':
|
||||
b_zuul_ref = build.parameters['ZUUL_REF']
|
||||
b_zuul_commit = build.parameters['ZUUL_COMMIT']
|
||||
|
||||
self.worker.hold_jobs_in_build = False
|
||||
self.worker.release()
|
||||
self.waitUntilSettled()
|
||||
|
||||
# Repos setup, now test the cloner
|
||||
for zuul_ref in [a_zuul_ref, b_zuul_ref]:
|
||||
cloner = zuul.lib.cloner.Cloner(
|
||||
git_base_url=self.upstream_root,
|
||||
projects=['org/project1', 'org/project2'],
|
||||
workspace=self.workspace_root,
|
||||
zuul_branch='master',
|
||||
zuul_ref=zuul_ref,
|
||||
zuul_url=self.git_root,
|
||||
branch='master',
|
||||
clone_map_file=os.path.join(FIXTURE_DIR, 'clonemap.yaml')
|
||||
)
|
||||
cloner.execute()
|
||||
work_repo1 = git.Repo(os.path.join(self.workspace_root,
|
||||
'org/project1'))
|
||||
self.assertEquals(a_zuul_commit, str(work_repo1.commit('HEAD')))
|
||||
|
||||
work_repo2 = git.Repo(os.path.join(self.workspace_root,
|
||||
'org/project2'))
|
||||
self.assertEquals(b_zuul_commit, str(work_repo2.commit('HEAD')))
|
||||
|
||||
shutil.rmtree(self.workspace_root)
|
143
zuul/cmd/cloner.py
Executable file
143
zuul/cmd/cloner.py
Executable file
@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright 2014 Antoine "hashar" Musso
|
||||
# Copyright 2014 Wikimedia Foundation Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
import zuul.cmd
|
||||
import zuul.lib.cloner
|
||||
|
||||
ZUUL_ENV_SUFFIXES = (
|
||||
'branch',
|
||||
'change',
|
||||
'patchset',
|
||||
'pipeline',
|
||||
'project',
|
||||
'ref',
|
||||
'url',
|
||||
)
|
||||
|
||||
|
||||
class Cloner(zuul.cmd.ZuulApp):
|
||||
log = logging.getLogger("zuul.Cloner")
|
||||
|
||||
def parse_arguments(self):
|
||||
"""Parse command line arguments and returns argparse structure"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Zuul Project Gating System Cloner.')
|
||||
parser.add_argument('-m', '--map', dest='clone_map_file',
|
||||
help='specifiy clone map file')
|
||||
parser.add_argument('--workspace', dest='workspace',
|
||||
default=os.getcwd(),
|
||||
help='where to clone repositories too')
|
||||
parser.add_argument('-v', '--verbose', dest='verbose',
|
||||
action='store_true',
|
||||
help='verbose output')
|
||||
parser.add_argument('--color', dest='color', action='store_true',
|
||||
help='use color output')
|
||||
parser.add_argument('--version', dest='version', action='version',
|
||||
version=self._get_version(),
|
||||
help='show zuul version')
|
||||
parser.add_argument('git_base_url',
|
||||
help='reference repo to clone from')
|
||||
parser.add_argument('projects', nargs='+',
|
||||
help='list of Gerrit projects to clone')
|
||||
|
||||
project_env = parser.add_argument_group(
|
||||
'project tuning'
|
||||
)
|
||||
project_env.add_argument(
|
||||
'--branch',
|
||||
help=('branch to checkout instead of Zuul selected branch, '
|
||||
'for example to specify an alternate branch to test '
|
||||
'client library compatibility.')
|
||||
)
|
||||
|
||||
zuul_env = parser.add_argument_group(
|
||||
'zuul environnement',
|
||||
'Let you override $ZUUL_* environnement variables.'
|
||||
)
|
||||
for zuul_suffix in ZUUL_ENV_SUFFIXES:
|
||||
env_name = 'ZUUL_%s' % zuul_suffix.upper()
|
||||
zuul_env.add_argument(
|
||||
'--zuul-%s' % zuul_suffix, metavar='$' + env_name,
|
||||
default=os.environ.get(env_name)
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Validate ZUUL_* arguments
|
||||
zuul_missing = [zuul_opt for zuul_opt, val in vars(args).items()
|
||||
if zuul_opt.startswith('zuul') and val is None]
|
||||
if zuul_missing:
|
||||
parser.error(("Some Zuul parameters are not properly set:\n\t%s\n"
|
||||
"Define them either via environment variables or "
|
||||
"using options above." %
|
||||
"\n\t".join(sorted(zuul_missing))))
|
||||
self.args = args
|
||||
|
||||
def setup_logging(self, color=False, verbose=False):
|
||||
"""Cloner logging does not rely on conf file"""
|
||||
if verbose:
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
else:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
if color:
|
||||
# Color codes http://www.tldp.org/HOWTO/Bash-Prompt-HOWTO/x329.html
|
||||
logging.addLevelName( # cyan
|
||||
logging.DEBUG, "\033[36m%s\033[0m" %
|
||||
logging.getLevelName(logging.DEBUG))
|
||||
logging.addLevelName( # green
|
||||
logging.INFO, "\033[32m%s\033[0m" %
|
||||
logging.getLevelName(logging.INFO))
|
||||
logging.addLevelName( # yellow
|
||||
logging.WARNING, "\033[33m%s\033[0m" %
|
||||
logging.getLevelName(logging.WARNING))
|
||||
logging.addLevelName( # red
|
||||
logging.ERROR, "\033[31m%s\033[0m" %
|
||||
logging.getLevelName(logging.ERROR))
|
||||
logging.addLevelName( # red background
|
||||
logging.CRITICAL, "\033[41m%s\033[0m" %
|
||||
logging.getLevelName(logging.CRITICAL))
|
||||
|
||||
def main(self):
|
||||
self.parse_arguments()
|
||||
self.setup_logging(color=self.args.color, verbose=self.args.verbose)
|
||||
cloner = zuul.lib.cloner.Cloner(
|
||||
git_base_url=self.args.git_base_url,
|
||||
projects=self.args.projects,
|
||||
workspace=self.args.workspace,
|
||||
zuul_branch=self.args.zuul_branch,
|
||||
zuul_ref=self.args.zuul_ref,
|
||||
zuul_url=self.args.zuul_url,
|
||||
branch=self.args.branch,
|
||||
clone_map_file=self.args.clone_map_file
|
||||
)
|
||||
cloner.execute()
|
||||
|
||||
|
||||
def main():
|
||||
cloner = Cloner()
|
||||
cloner.main()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.path.insert(0, '.')
|
||||
main()
|
78
zuul/lib/clonemapper.py
Normal file
78
zuul/lib/clonemapper.py
Normal file
@ -0,0 +1,78 @@
|
||||
# Copyright 2014 Antoine "hashar" Musso
|
||||
# Copyright 2014 Wikimedia Foundation Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from collections import defaultdict
|
||||
import extras
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
||||
OrderedDict = extras.try_imports(['collections.OrderedDict',
|
||||
'ordereddict.OrderedDict'])
|
||||
|
||||
|
||||
class CloneMapper(object):
|
||||
log = logging.getLogger("zuul.CloneMapper")
|
||||
|
||||
def __init__(self, clonemap, projects):
|
||||
self.clonemap = clonemap
|
||||
self.projects = projects
|
||||
|
||||
def expand(self, workspace):
|
||||
self.log.info("Workspace path set to: %s", workspace)
|
||||
|
||||
is_valid = True
|
||||
ret = OrderedDict()
|
||||
for project in self.projects:
|
||||
dests = []
|
||||
for mapping in self.clonemap:
|
||||
if re.match(r'^%s$' % mapping['name'],
|
||||
project):
|
||||
# Might be matched more than one time
|
||||
dests.append(
|
||||
re.sub(mapping['name'], mapping['dest'], project))
|
||||
|
||||
if len(dests) > 1:
|
||||
self.log.error("Duplicate destinations for %s: %s.",
|
||||
project, dests)
|
||||
is_valid = False
|
||||
elif len(dests) == 0:
|
||||
self.log.debug("Using %s as destination (unmatched)",
|
||||
project)
|
||||
ret[project] = [project]
|
||||
else:
|
||||
ret[project] = dests
|
||||
|
||||
if not is_valid:
|
||||
raise Exception("Expansion error. Check error messages above")
|
||||
|
||||
self.log.info("Mapping projects to workspace...")
|
||||
for project, dest in ret.iteritems():
|
||||
dest = os.path.normpath(os.path.join(workspace, dest[0]))
|
||||
ret[project] = dest
|
||||
self.log.info(" %s -> %s", project, dest)
|
||||
|
||||
self.log.debug("Checking overlap in destination directories...")
|
||||
check = defaultdict(list)
|
||||
for project, dest in ret.iteritems():
|
||||
check[dest].append(project)
|
||||
|
||||
dupes = dict((d, p) for (d, p) in check.iteritems() if len(p) > 1)
|
||||
if dupes:
|
||||
raise Exception("Some projects share the same destination: %s",
|
||||
dupes)
|
||||
|
||||
self.log.info("Expansion completed.")
|
||||
return ret
|
145
zuul/lib/cloner.py
Normal file
145
zuul/lib/cloner.py
Normal file
@ -0,0 +1,145 @@
|
||||
# Copyright 2014 Antoine "hashar" Musso
|
||||
# Copyright 2014 Wikimedia Foundation Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import git
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import yaml
|
||||
|
||||
from git import GitCommandError
|
||||
from zuul.lib.clonemapper import CloneMapper
|
||||
from zuul.merger.merger import Repo
|
||||
|
||||
|
||||
class Cloner(object):
|
||||
log = logging.getLogger("zuul.Cloner")
|
||||
|
||||
def __init__(self, git_base_url, projects, workspace, zuul_branch,
|
||||
zuul_ref, zuul_url, branch=None, clone_map_file=None):
|
||||
|
||||
self.clone_map = []
|
||||
self.dests = None
|
||||
|
||||
self.branch = branch
|
||||
self.git_url = git_base_url
|
||||
self.projects = projects
|
||||
self.workspace = workspace
|
||||
self.zuul_branch = zuul_branch
|
||||
self.zuul_ref = zuul_ref
|
||||
self.zuul_url = zuul_url
|
||||
|
||||
if clone_map_file:
|
||||
self.readCloneMap(clone_map_file)
|
||||
|
||||
def readCloneMap(self, clone_map_file):
|
||||
clone_map_file = os.path.expanduser(clone_map_file)
|
||||
if not os.path.exists(clone_map_file):
|
||||
raise Exception("Unable to read clone map file at %s." %
|
||||
clone_map_file)
|
||||
clone_map_file = open(clone_map_file)
|
||||
self.clone_map = yaml.load(clone_map_file).get('clonemap')
|
||||
self.log.info("Loaded map containing %s rules", len(self.clone_map))
|
||||
return self.clone_map
|
||||
|
||||
def execute(self):
|
||||
mapper = CloneMapper(self.clone_map, self.projects)
|
||||
dests = mapper.expand(workspace=self.workspace)
|
||||
|
||||
self.log.info("Preparing %s repositories", len(dests))
|
||||
for project, dest in dests.iteritems():
|
||||
self.prepareRepo(project, dest)
|
||||
self.log.info("Prepared all repositories")
|
||||
|
||||
def cloneUpstream(self, project, dest):
|
||||
git_upstream = '%s/%s' % (self.git_url, project)
|
||||
self.log.info("Creating repo %s from upstream %s",
|
||||
project, git_upstream)
|
||||
repo = Repo(
|
||||
remote=git_upstream,
|
||||
local=dest,
|
||||
email=None,
|
||||
username=None)
|
||||
|
||||
if not repo.isInitialized():
|
||||
raise Exception("Error cloning %s to %s" % (git_upstream, dest))
|
||||
|
||||
return repo
|
||||
|
||||
def fetchFromZuul(self, repo, project, ref):
|
||||
zuul_remote = '%s/%s' % (self.zuul_url, project)
|
||||
|
||||
try:
|
||||
repo.fetchFrom(zuul_remote, ref)
|
||||
self.log.debug("Fetched ref %s from %s", ref, project)
|
||||
return True
|
||||
except (ValueError, GitCommandError):
|
||||
self.log.debug("Project %s in Zuul does not have ref %s",
|
||||
project, ref)
|
||||
return False
|
||||
|
||||
def prepareRepo(self, project, dest):
|
||||
"""Clone a repository for project at dest and apply a reference
|
||||
suitable for testing. The reference lookup is attempted in this order:
|
||||
|
||||
1) Zuul reference for the indicated branch
|
||||
2) Zuul reference for the master branch
|
||||
3) The tip of the indicated branch
|
||||
4) The tip of the master branch
|
||||
"""
|
||||
|
||||
repo = self.cloneUpstream(project, dest)
|
||||
|
||||
repo.update()
|
||||
# Ensure that we don't have stale remotes around
|
||||
repo.prune()
|
||||
|
||||
override_zuul_ref = self.zuul_ref
|
||||
# FIXME should be origin HEAD branch which might not be 'master'
|
||||
fallback_branch = 'master'
|
||||
fallback_zuul_ref = re.sub(self.zuul_branch, fallback_branch,
|
||||
self.zuul_ref)
|
||||
|
||||
if self.branch:
|
||||
override_zuul_ref = re.sub(self.zuul_branch, self.branch,
|
||||
self.zuul_ref)
|
||||
if repo.hasBranch(self.branch):
|
||||
self.log.debug("upstream repo has branch %s", self.branch)
|
||||
fallback_branch = self.branch
|
||||
fallback_zuul_ref = self.zuul_ref
|
||||
else:
|
||||
self.log.exception("upstream repo is missing branch %s",
|
||||
self.branch)
|
||||
|
||||
if (self.fetchFromZuul(repo, project, override_zuul_ref)
|
||||
or (fallback_zuul_ref != override_zuul_ref and
|
||||
self.fetchFromZuul(repo, project, fallback_zuul_ref))
|
||||
):
|
||||
# Work around a bug in GitPython which can not parse FETCH_HEAD
|
||||
gitcmd = git.Git(dest)
|
||||
fetch_head = gitcmd.rev_parse('FETCH_HEAD')
|
||||
repo.checkout(fetch_head)
|
||||
self.log.info("Prepared %s repo with commit %s",
|
||||
project, fetch_head)
|
||||
else:
|
||||
# Checkout branch
|
||||
self.log.debug("Falling back to branch %s", fallback_branch)
|
||||
try:
|
||||
repo.checkout('remotes/origin/%s' % fallback_branch)
|
||||
except (ValueError, GitCommandError):
|
||||
self.log.exception("Fallback branch not found: %s",
|
||||
fallback_branch)
|
||||
self.log.info("Prepared %s repo with branch %s",
|
||||
project, fallback_branch)
|
@ -146,6 +146,10 @@ class Repo(object):
|
||||
except AssertionError:
|
||||
origin.fetch(ref)
|
||||
|
||||
def fetchFrom(self, repository, refspec):
|
||||
repo = self.createRepoObject()
|
||||
repo.git.fetch(repository, refspec)
|
||||
|
||||
def createZuulRef(self, ref, commit='HEAD'):
|
||||
repo = self.createRepoObject()
|
||||
self.log.debug("CreateZuulRef %s at %s" % (ref, commit))
|
||||
|
Loading…
Reference in New Issue
Block a user