Use tenacity to retry image pulls

Image pulls often fail due to network or registry load. This change
implements pull retries, up to 4 times with jittered exponential
backoff.

Closes-Bug #1752036

Change-Id: I09c601f357200e734aa9d887c6ce6b4e53fc2589
This commit is contained in:
Steve Baker 2018-02-27 11:17:04 +00:00
parent b533f8f5ac
commit 5c7a785053
3 changed files with 52 additions and 14 deletions

View File

@ -13,6 +13,7 @@
import json
import logging
import tenacity
LOG = logging.getLogger(__name__)
@ -215,24 +216,39 @@ class ComposeV1Builder(object):
if self.runner.inspect(image, format='exists', type='image'):
continue
cmd = [self.runner.docker_cmd, 'pull', image]
(cmd_stdout, cmd_stderr, rc) = self.runner.execute(cmd)
try:
(cmd_stdout, cmd_stderr) = self._pull(image)
except PullException as e:
returncode = e.rc
cmd_stdout = e.stdout
cmd_stderr = e.stderr
LOG.error("Error pulling %s. [%s]\n" % (image, returncode))
LOG.error("stdout: %s" % e.stdout)
LOG.error("stderr: %s" % e.stderr)
else:
LOG.debug('Pulled %s' % image)
LOG.info("stdout: %s" % cmd_stdout)
LOG.info("stderr: %s" % cmd_stderr)
if cmd_stdout:
stdout.append(cmd_stdout)
if cmd_stderr:
stderr.append(cmd_stderr)
if rc != 0:
returncode = rc
LOG.error("Error running %s. [%s]\n" % (cmd, returncode))
LOG.error("stdout: %s" % cmd_stdout)
LOG.error("stderr: %s" % cmd_stderr)
else:
LOG.debug('Completed $ %s' % ' '.join(cmd))
LOG.info("stdout: %s" % cmd_stdout)
LOG.info("stderr: %s" % cmd_stderr)
return returncode
@tenacity.retry( # Retry up to 4 times with jittered exponential backoff
reraise=True,
wait=tenacity.wait_random_exponential(multiplier=1, max=10),
stop=tenacity.stop_after_attempt(4)
)
def _pull(self, image):
cmd = [self.runner.docker_cmd, 'pull', image]
(stdout, stderr, rc) = self.runner.execute(cmd)
if rc != 0:
raise PullException(stdout, stderr, rc)
return stdout, stderr
@staticmethod
def command_argument(command):
if not command:
@ -240,3 +256,11 @@ class ComposeV1Builder(object):
if not isinstance(command, list):
return command.split()
return command
class PullException(Exception):
def __init__(self, stdout, stderr, rc):
self.stdout = stdout
self.stderr = stderr
self.rc = rc

View File

@ -23,7 +23,9 @@ from paunch.tests import base
class TestComposeV1Builder(base.TestCase):
def test_apply(self):
@mock.patch('tenacity.wait.wait_random_exponential.__call__')
def test_apply(self, mock_wait):
mock_wait.return_value = 0
config = {
'one': {
'start_order': 0,
@ -53,7 +55,8 @@ class TestComposeV1Builder(base.TestCase):
exe.side_effect = [
('exists', '', 0), # inspect for image centos:6
('', '', 1), # inspect for missing image centos:7
('Pulled centos:7', '', 0), # pull centos:6
('Pulled centos:7', 'ouch', 1), # pull centos:6 fails
('Pulled centos:7', '', 0), # pull centos:6 succeeds
('', '', 0), # ps for delete_missing_and_updated container_names
('', '', 0), # ps for after delete_missing_and_updated renames
('', '', 0), # ps to only create containers which don't exist
@ -91,6 +94,11 @@ class TestComposeV1Builder(base.TestCase):
['docker', 'inspect', '--type', 'image',
'--format', 'exists', 'centos:7']
),
# first pull attempt fails
mock.call(
['docker', 'pull', 'centos:7']
),
# second pull attempt succeeds
mock.call(
['docker', 'pull', 'centos:7']
),
@ -301,7 +309,9 @@ three-12345678 three''', '', 0),
),
])
def test_apply_failed_pull(self):
@mock.patch('tenacity.wait.wait_random_exponential.__call__')
def test_apply_failed_pull(self, mock_wait):
mock_wait.return_value = 0
config = {
'one': {
'start_order': 0,
@ -332,6 +342,9 @@ three-12345678 three''', '', 0),
('exists', '', 0), # inspect for image centos:6
('', '', 1), # inspect for missing image centos:7
('Pulling centos:7', 'ouch', 1), # pull centos:7 failure
('Pulling centos:7', 'ouch', 1), # pull centos:7 retry 2
('Pulling centos:7', 'ouch', 1), # pull centos:7 retry 3
('Pulling centos:7', 'ouch', 1), # pull centos:7 retry 4
]
r.execute = exe

View File

@ -5,3 +5,4 @@
pbr>=2.0.0,!=2.1.0 # Apache-2.0
cliff>=2.6.0 # Apache-2.0
tenacity>=3.2.1 # Apache-2.0