zuul
/
zuul
Code Issues Proposed changes

Use the executor cached repos more often 

Currently, we clone the job's working repos from the repo cache that
the executor maintains.  However, some playbook and role repos do
not use that cache.

To facilitate more operations using the merger class, add the concept
of a repo cache to it, and use that in all job related git repo operations
on the executor.

Change-Id: I747b6602540458506e1f6d480a95b80c6543c5a8
This commit is contained in:
James E. Blair 2017-05-24 13:58:42 -07:00
parent 57c51a1cac
commit f327c5771a
4 changed files with 58 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tests/base.py
View File

@ -1189,10 +1189,10 @@ class RecordingExecutorServer(zuul.executor.server.ExecutorServer):
class RecordingAnsibleJob(zuul.executor.server.AnsibleJob):
def doMergeChanges(self, items, repo_state):
def doMergeChanges(self, merger, items, repo_state):
# Get a merger in order to update the repos involved in this job.
commit = super(RecordingAnsibleJob, self).doMergeChanges(
items, repo_state)
merger, items, repo_state)
if not commit: # merge conflict
self.recordResult('MERGER_FAILURE')
return commit

1
zuul/executor/client.py
View File

@ -298,6 +298,7 @@ class ExecutorClient(object):
connection = project.source.connection
return dict(connection=connection.connection_name,
name=project.name,
canonical_name=project.canonical_name,
override_branch=override_branch,
default_branch=project_default_branch)

39
zuul/executor/server.py
View File

@ -27,7 +27,6 @@ import traceback
from zuul.lib.yamlutil import yaml
import gear
import git
from six.moves import shlex_quote
import zuul.merger.merger
@ -304,8 +303,13 @@ class ExecutorServer(object):
self.job_workers = {}
def _getMerger(self, root):
if root != self.merge_root:
cache_root = self.merge_root
else:
cache_root = None
return zuul.merger.merger.Merger(root, self.connections,
self.merge_email, self.merge_name)
self.merge_email, self.merge_name,
cache_root)
def start(self):
self._running = True
@ -566,36 +570,27 @@ class AnsibleJob(object):
task.wait()
self.log.debug("Job %s: git updates complete" % (self.job.unique,))
repos = []
merger = self.executor_server._getMerger(self.jobdir.src_root)
repos = {}
for project in args['projects']:
self.log.debug("Cloning %s/%s" % (project['connection'],
project['name'],))
source = self.executor_server.connections.getSource(
project['connection'])
project_object = source.getProject(project['name'])
url = source.getGitUrl(project_object)
repo = git.Repo.clone_from(
os.path.join(self.executor_server.merge_root,
source.canonical_hostname,
project['name']),
os.path.join(self.jobdir.src_root,
source.canonical_hostname,
project['name']))
repo.remotes.origin.config_writer.set('url', url)
repos.append(repo)
repo = merger.getRepo(project['connection'],
project['name'])
repos[project['canonical_name']] = repo
merge_items = [i for i in args['items'] if i.get('refspec')]
if merge_items:
if not self.doMergeChanges(merge_items, args['repo_state']):
if not self.doMergeChanges(merger, merge_items,
args['repo_state']):
# There was a merge conflict and we have already sent
# a work complete result, don't run any jobs
return
# Delete the origin remote from each repo we set up since
# it will not be valid within the jobs.
for repo in repos:
repo.delete_remote(repo.remotes.origin)
for repo in repos.values():
repo.deleteRemote('origin')
# is the playbook in a repo that we have already prepared?
trusted, untrusted = self.preparePlaybookRepos(args)
@ -633,9 +628,7 @@ class AnsibleJob(object):
result = dict(result=result)
self.job.sendWorkComplete(json.dumps(result))
def doMergeChanges(self, items, repo_state):
# Get a merger in order to update the repos involved in this job.
merger = self.executor_server._getMerger(self.jobdir.src_root)
def doMergeChanges(self, merger, items, repo_state):
ret = merger.mergeChanges(items, repo_state=repo_state)
if not ret: # merge conflict
result = dict(result='MERGER_FAILURE')

51
zuul/merger/merger.py
View File

@ -44,11 +44,12 @@ class ZuulReference(git.Reference):
class Repo(object):
log = logging.getLogger("zuul.Repo")
def __init__(self, remote, local, email, username):
def __init__(self, remote, local, email, username, cache_path=None):
self.remote_url = remote
self.local_path = local
self.email = email
self.username = username
self.cache_path = cache_path
self._initialized = False
try:
self._ensure_cloned()
@ -60,17 +61,32 @@ class Repo(object):
if self._initialized and repo_is_cloned:
return
# If the repo does not exist, clone the repo.
rewrite_url = False
if not repo_is_cloned:
self.log.debug("Cloning from %s to %s" % (self.remote_url,
self.local_path))
git.Repo.clone_from(self.remote_url, self.local_path)
if self.cache_path:
git.Repo.clone_from(self.cache_path, self.local_path)
rewrite_url = True
else:
git.Repo.clone_from(self.remote_url, self.local_path)
repo = git.Repo(self.local_path)
# Create local branches corresponding to all the remote branches
if not repo_is_cloned:
origin = repo.remotes.origin
for ref in origin.refs:
if ref.remote_head == 'HEAD':
continue
repo.create_head(ref.remote_head, ref, force=True)
with repo.config_writer() as config_writer:
if self.email:
config_writer.set_value('user', 'email', self.email)
if self.username:
config_writer.set_value('user', 'name', self.username)
config_writer.write()
if rewrite_url:
with repo.remotes.origin.config_writer as config_writer:
config_writer.set('url', self.remote_url)
self._initialized = True
def isInitialized(self):
@ -157,6 +173,11 @@ class Repo(object):
reset_repo_to_head(repo)
return repo.head.commit
def checkoutLocalBranch(self, branch):
repo = self.createRepoObject()
ref = repo.heads[branch].commit
self.checkout(ref)
def cherryPick(self, ref):
repo = self.createRepoObject()
self.log.debug("Cherry-picking %s" % ref)
@ -230,11 +251,16 @@ class Repo(object):
ret[fn] = None
return ret
def deleteRemote(self, remote):
repo = self.createRepoObject()
repo.delete_remote(repo.remotes[remote])
class Merger(object):
log = logging.getLogger("zuul.Merger")
def __init__(self, working_root, connections, email, username):
def __init__(self, working_root, connections, email, username,
cache_root=None):
self.repos = {}
self.working_root = working_root
if not os.path.exists(working_root):
@ -242,6 +268,7 @@ class Merger(object):
self.connections = connections
self.email = email
self.username = username
self.cache_root = cache_root
def _get_ssh_cmd(self, connection_name):
sshkey = self.connections.connections.get(connection_name).\
@ -264,7 +291,12 @@ class Merger(object):
key = '/'.join([hostname, project_name])
try:
path = os.path.join(self.working_root, hostname, project_name)
repo = Repo(url, path, self.email, self.username)
if self.cache_root:
cache_path = os.path.join(self.cache_root, hostname,
project_name)
else:
cache_path = None
repo = Repo(url, path, self.email, self.username, cache_path)
self.repos[key] = repo
except Exception:
@ -301,15 +333,10 @@ class Merger(object):
connection_name, project_name)
def checkoutBranch(self, connection_name, project_name, branch):
self.log.info("Checking out %s/%s branch %s",
connection_name, project_name, branch)
repo = self.getRepo(connection_name, project_name)
if repo.hasBranch(branch):
self.log.info("Checking out branch %s of %s/%s" %
(branch, connection_name, project_name))
head = repo.getBranchHead(branch)
repo.checkout(head)
else:
raise Exception("Project %s/%s does not have branch %s" %
(connection_name, project_name, branch))
repo.checkoutLocalBranch(branch)
def _saveRepoState(self, connection_name, project_name, repo,
repo_state):