Browse Source

Add cache-dir argument to cloner

Clone repos from cache-dir if they exist, then reset their origins
to be the git base url and update from there.

Change-Id: Ic42961aa5da9e4a66524ec14758d421944403f91
changes/40/116740/6
James E. Blair 7 years ago
parent
commit
48ff4d5fe3
  1. 12
      doc/source/cloner.rst
  2. 58
      tests/test_cloner.py
  3. 5
      zuul/cmd/cloner.py
  4. 22
      zuul/lib/cloner.py

12
doc/source/cloner.rst

@ -75,3 +75,15 @@ cloner processes the clones in the order supplied, so you should swap the
projects::
zuul-cloner project project/plugins/plugin1
Cached repositories
-------------------
The ``--cache-dir`` option can be used to reduce network traffic by
cloning from a local repository which may not be up to date.
If the ``--cache-dir`` option is supplied, zuul-cloner will start by
cloning any projects it processes from those found in that directory.
The URL of origin remote of the resulting clone will be reset to use
the ``git_base_url`` and then the remote will be updated so that the
repository has all the information in the upstream repository.

58
tests/test_cloner.py

@ -59,6 +59,64 @@ class TestCloner(ZuulTestCase):
os.path.join(self.upstream_root, project))
return repos
def test_cache_dir(self):
projects = ['org/project1', 'org/project2']
cache_root = os.path.join(self.test_root, "cache")
for project in projects:
upstream_repo_path = os.path.join(self.upstream_root, project)
cache_repo_path = os.path.join(cache_root, project)
git.Repo.clone_from(upstream_repo_path, cache_repo_path)
self.worker.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
A.addApproval('CRVW', 2)
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
self.waitUntilSettled()
self.assertEquals(1, len(self.builds), "One build is running")
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
B.setMerged()
upstream = self.getUpstreamRepos(projects)
states = [
{'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
'org/project2': str(upstream['org/project2'].commit('master')),
},
]
for number, build in enumerate(self.builds):
self.log.debug("Build parameters: %s", build.parameters)
cloner = zuul.lib.cloner.Cloner(
git_base_url=self.upstream_root,
projects=projects,
workspace=self.workspace_root,
zuul_branch=build.parameters['ZUUL_BRANCH'],
zuul_ref=build.parameters['ZUUL_REF'],
zuul_url=self.git_root,
cache_dir=cache_root,
)
cloner.execute()
work = self.getWorkspaceRepos(projects)
state = states[number]
for project in projects:
self.assertEquals(state[project],
str(work[project].commit('HEAD')),
'Project %s commit for build %s should '
'be correct' % (project, number))
work = self.getWorkspaceRepos(projects)
upstream_repo_path = os.path.join(self.upstream_root, 'org/project1')
self.assertEquals(work['org/project1'].remotes.origin.url,
upstream_repo_path,
'workspace repo origin should be upstream, not cache')
self.worker.hold_jobs_in_build = False
self.worker.release()
self.waitUntilSettled()
def test_one_branch(self):
self.worker.hold_jobs_in_build = True

5
zuul/cmd/cloner.py

@ -54,6 +54,10 @@ class Cloner(zuul.cmd.ZuulApp):
parser.add_argument('--version', dest='version', action='version',
version=self._get_version(),
help='show zuul version')
parser.add_argument('--cache-dir', dest='cache_dir',
help=('a directory that holds cached copies of '
'repos from which to make an initial clone.'
))
parser.add_argument('git_base_url',
help='reference repo to clone from')
parser.add_argument('projects', nargs='+',
@ -142,6 +146,7 @@ class Cloner(zuul.cmd.ZuulApp):
branch=self.args.branch,
clone_map_file=self.args.clone_map_file,
project_branches=project_branches,
cache_dir=self.args.cache_dir,
)
cloner.execute()

22
zuul/lib/cloner.py

@ -29,13 +29,14 @@ class Cloner(object):
def __init__(self, git_base_url, projects, workspace, zuul_branch,
zuul_ref, zuul_url, branch=None, clone_map_file=None,
project_branches=None):
project_branches=None, cache_dir=None):
self.clone_map = []
self.dests = None
self.branch = branch
self.git_url = git_base_url
self.cache_dir = cache_dir
self.projects = projects
self.workspace = workspace
self.zuul_branch = zuul_branch
@ -66,9 +67,24 @@ class Cloner(object):
self.log.info("Prepared all repositories")
def cloneUpstream(self, project, dest):
# Check for a cached git repo first
git_cache = '%s/%s' % (self.cache_dir, project)
git_upstream = '%s/%s' % (self.git_url, project)
self.log.info("Creating repo %s from upstream %s",
project, git_upstream)
if (self.cache_dir and
os.path.exists(git_cache) and
not os.path.exists(dest)):
# file:// tells git not to hard-link across repos
git_cache = 'file://%s' % git_cache
self.log.info("Creating repo %s from cache %s",
project, git_cache)
new_repo = git.Repo.clone_from(git_cache, dest)
self.log.info("Updating origin remote in repo %s to %s",
project, git_upstream)
origin = new_repo.remotes.origin.config_writer.set(
'url', git_upstream)
else:
self.log.info("Creating repo %s from upstream %s",
project, git_upstream)
repo = Repo(
remote=git_upstream,
local=dest,

Loading…
Cancel
Save