Merge "Merger: retry network operations"

This commit is contained in:
Zuul 2018-02-01 20:33:33 +00:00 committed by Gerrit Code Review
commit cd46f694a5
3 changed files with 69 additions and 9 deletions

17
tests/fixtures/git_fetch_error.sh vendored Executable file
View File

@ -0,0 +1,17 @@
#!/bin/sh
echo $*
case "$1" in
fetch)
if [ -f ./stamp1 ]; then
touch ./stamp2
exit 0
fi
touch ./stamp1
exit 1
;;
version)
echo "git version 1.0.0"
exit 0
;;
esac

View File

@ -82,7 +82,7 @@ class TestMergerRepo(ZuulTestCase):
os.path.join(FIXTURE_DIR, 'fake_git.sh'))
work_repo = Repo(parent_path, self.workspace_root,
'none@example.org', 'User Name', '0', '0',
git_timeout=0.001)
git_timeout=0.001, retry_attempts=1)
# TODO: have the merger and repo classes catch fewer
# exceptions, including this one on initialization. For the
# test, we try cloning again.
@ -93,10 +93,26 @@ class TestMergerRepo(ZuulTestCase):
def test_fetch_timeout(self):
parent_path = os.path.join(self.upstream_root, 'org/project1')
work_repo = Repo(parent_path, self.workspace_root,
'none@example.org', 'User Name', '0', '0')
'none@example.org', 'User Name', '0', '0',
retry_attempts=1)
work_repo.git_timeout = 0.001
self.patch(git.Git, 'GIT_PYTHON_GIT_EXECUTABLE',
os.path.join(FIXTURE_DIR, 'fake_git.sh'))
with testtools.ExpectedException(git.exc.GitCommandError,
'.*exit code\(-9\)'):
work_repo.update()
def test_fetch_retry(self):
parent_path = os.path.join(self.upstream_root, 'org/project1')
work_repo = Repo(parent_path, self.workspace_root,
'none@example.org', 'User Name', '0', '0',
retry_interval=1)
self.patch(git.Git, 'GIT_PYTHON_GIT_EXECUTABLE',
os.path.join(FIXTURE_DIR, 'git_fetch_error.sh'))
work_repo.update()
# This is created on the first fetch
self.assertTrue(os.path.exists(os.path.join(
self.workspace_root, 'stamp1')))
# This is created on the second fetch
self.assertTrue(os.path.exists(os.path.join(
self.workspace_root, 'stamp2')))

View File

@ -17,6 +17,7 @@ from contextlib import contextmanager
import logging
import os
import shutil
import time
import git
import gitdb
@ -54,7 +55,8 @@ def timeout_handler(path):
class Repo(object):
def __init__(self, remote, local, email, username, speed_limit, speed_time,
sshkey=None, cache_path=None, logger=None, git_timeout=300):
sshkey=None, cache_path=None, logger=None, git_timeout=300,
retry_attempts=3, retry_interval=30):
if logger is None:
self.log = logging.getLogger("zuul.Repo")
else:
@ -73,6 +75,8 @@ class Repo(object):
self.username = username
self.cache_path = cache_path
self._initialized = False
self.retry_attempts = retry_attempts
self.retry_interval = retry_interval
try:
self._ensure_cloned()
except Exception:
@ -118,14 +122,37 @@ class Repo(object):
def _git_clone(self, url):
mygit = git.cmd.Git(os.getcwd())
mygit.update_environment(**self.env)
with timeout_handler(self.local_path):
mygit.clone(git.cmd.Git.polish_url(url), self.local_path,
kill_after_timeout=self.git_timeout)
for attempt in range(1, self.retry_attempts + 1):
try:
with timeout_handler(self.local_path):
mygit.clone(git.cmd.Git.polish_url(url), self.local_path,
kill_after_timeout=self.git_timeout)
break
except Exception as e:
if attempt < self.retry_attempts:
time.sleep(self.retry_interval)
self.log.warning("Retry %s: Clone %s" % (
attempt, self.local_path))
else:
raise
def _git_fetch(self, repo, remote, ref=None, **kwargs):
with timeout_handler(self.local_path):
repo.git.fetch(remote, ref, kill_after_timeout=self.git_timeout,
**kwargs)
for attempt in range(1, self.retry_attempts + 1):
try:
with timeout_handler(self.local_path):
repo.git.fetch(remote, ref,
kill_after_timeout=self.git_timeout,
**kwargs)
break
except Exception as e:
if attempt < self.retry_attempts:
time.sleep(self.retry_interval)
self.log.exception("Retry %s: Fetch %s %s %s" % (
attempt, self.local_path, remote, ref))
self._ensure_cloned()
else:
raise
def createRepoObject(self):
self._ensure_cloned()