Merge "Use tenacity to retry image pulls"
This commit is contained in:
commit
17943fc611
@ -13,6 +13,7 @@
|
||||
|
||||
import json
|
||||
import logging
|
||||
import tenacity
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
@ -215,24 +216,39 @@ class ComposeV1Builder(object):
|
||||
if self.runner.inspect(image, format='exists', type='image'):
|
||||
continue
|
||||
|
||||
cmd = [self.runner.docker_cmd, 'pull', image]
|
||||
(cmd_stdout, cmd_stderr, rc) = self.runner.execute(cmd)
|
||||
try:
|
||||
(cmd_stdout, cmd_stderr) = self._pull(image)
|
||||
except PullException as e:
|
||||
returncode = e.rc
|
||||
cmd_stdout = e.stdout
|
||||
cmd_stderr = e.stderr
|
||||
LOG.error("Error pulling %s. [%s]\n" % (image, returncode))
|
||||
LOG.error("stdout: %s" % e.stdout)
|
||||
LOG.error("stderr: %s" % e.stderr)
|
||||
else:
|
||||
LOG.debug('Pulled %s' % image)
|
||||
LOG.info("stdout: %s" % cmd_stdout)
|
||||
LOG.info("stderr: %s" % cmd_stderr)
|
||||
|
||||
if cmd_stdout:
|
||||
stdout.append(cmd_stdout)
|
||||
if cmd_stderr:
|
||||
stderr.append(cmd_stderr)
|
||||
|
||||
if rc != 0:
|
||||
returncode = rc
|
||||
LOG.error("Error running %s. [%s]\n" % (cmd, returncode))
|
||||
LOG.error("stdout: %s" % cmd_stdout)
|
||||
LOG.error("stderr: %s" % cmd_stderr)
|
||||
else:
|
||||
LOG.debug('Completed $ %s' % ' '.join(cmd))
|
||||
LOG.info("stdout: %s" % cmd_stdout)
|
||||
LOG.info("stderr: %s" % cmd_stderr)
|
||||
return returncode
|
||||
|
||||
@tenacity.retry( # Retry up to 4 times with jittered exponential backoff
|
||||
reraise=True,
|
||||
wait=tenacity.wait_random_exponential(multiplier=1, max=10),
|
||||
stop=tenacity.stop_after_attempt(4)
|
||||
)
|
||||
def _pull(self, image):
|
||||
cmd = [self.runner.docker_cmd, 'pull', image]
|
||||
(stdout, stderr, rc) = self.runner.execute(cmd)
|
||||
if rc != 0:
|
||||
raise PullException(stdout, stderr, rc)
|
||||
return stdout, stderr
|
||||
|
||||
@staticmethod
|
||||
def command_argument(command):
|
||||
if not command:
|
||||
@ -240,3 +256,11 @@ class ComposeV1Builder(object):
|
||||
if not isinstance(command, list):
|
||||
return command.split()
|
||||
return command
|
||||
|
||||
|
||||
class PullException(Exception):
|
||||
|
||||
def __init__(self, stdout, stderr, rc):
|
||||
self.stdout = stdout
|
||||
self.stderr = stderr
|
||||
self.rc = rc
|
||||
|
@ -23,7 +23,9 @@ from paunch.tests import base
|
||||
|
||||
class TestComposeV1Builder(base.TestCase):
|
||||
|
||||
def test_apply(self):
|
||||
@mock.patch('tenacity.wait.wait_random_exponential.__call__')
|
||||
def test_apply(self, mock_wait):
|
||||
mock_wait.return_value = 0
|
||||
config = {
|
||||
'one': {
|
||||
'start_order': 0,
|
||||
@ -53,7 +55,8 @@ class TestComposeV1Builder(base.TestCase):
|
||||
exe.side_effect = [
|
||||
('exists', '', 0), # inspect for image centos:6
|
||||
('', '', 1), # inspect for missing image centos:7
|
||||
('Pulled centos:7', '', 0), # pull centos:6
|
||||
('Pulled centos:7', 'ouch', 1), # pull centos:6 fails
|
||||
('Pulled centos:7', '', 0), # pull centos:6 succeeds
|
||||
('', '', 0), # ps for delete_missing_and_updated container_names
|
||||
('', '', 0), # ps for after delete_missing_and_updated renames
|
||||
('', '', 0), # ps to only create containers which don't exist
|
||||
@ -91,6 +94,11 @@ class TestComposeV1Builder(base.TestCase):
|
||||
['docker', 'inspect', '--type', 'image',
|
||||
'--format', 'exists', 'centos:7']
|
||||
),
|
||||
# first pull attempt fails
|
||||
mock.call(
|
||||
['docker', 'pull', 'centos:7']
|
||||
),
|
||||
# second pull attempt succeeds
|
||||
mock.call(
|
||||
['docker', 'pull', 'centos:7']
|
||||
),
|
||||
@ -301,7 +309,9 @@ three-12345678 three''', '', 0),
|
||||
),
|
||||
])
|
||||
|
||||
def test_apply_failed_pull(self):
|
||||
@mock.patch('tenacity.wait.wait_random_exponential.__call__')
|
||||
def test_apply_failed_pull(self, mock_wait):
|
||||
mock_wait.return_value = 0
|
||||
config = {
|
||||
'one': {
|
||||
'start_order': 0,
|
||||
@ -332,6 +342,9 @@ three-12345678 three''', '', 0),
|
||||
('exists', '', 0), # inspect for image centos:6
|
||||
('', '', 1), # inspect for missing image centos:7
|
||||
('Pulling centos:7', 'ouch', 1), # pull centos:7 failure
|
||||
('Pulling centos:7', 'ouch', 1), # pull centos:7 retry 2
|
||||
('Pulling centos:7', 'ouch', 1), # pull centos:7 retry 3
|
||||
('Pulling centos:7', 'ouch', 1), # pull centos:7 retry 4
|
||||
]
|
||||
r.execute = exe
|
||||
|
||||
|
@ -5,3 +5,4 @@
|
||||
pbr>=2.0.0,!=2.1.0 # Apache-2.0
|
||||
|
||||
cliff>=2.6.0 # Apache-2.0
|
||||
tenacity>=3.2.1 # Apache-2.0
|
||||
|
Loading…
Reference in New Issue
Block a user