From 9a4744e1107f0c81487dc3f8250544715da7896a Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Wed, 16 Dec 2020 13:04:31 +0000 Subject: [PATCH] Changed retry logic on buildah build - Enable logging of retries as warnings - Increase number of retries to 5 and switched delay from 1s to exponential, capping at 60s Example: WARNING...:Finished call to '__main__.foo' after 9.201(s), this was the 6th time calling it. Change-Id: If13571b9e21251ced9a43fc1cc7f7d69fad06f0a Tested-By: https://review.rdoproject.org/r/#/c/29917/ (cherry picked from commit a584a5bd4c3a06240a25fccda5f80edc67a25fd6) --- tripleo_common/image/builder/buildah.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tripleo_common/image/builder/buildah.py b/tripleo_common/image/builder/buildah.py index 225f8f3ec..89c30456c 100644 --- a/tripleo_common/image/builder/buildah.py +++ b/tripleo_common/image/builder/buildah.py @@ -29,12 +29,13 @@ from tripleo_common.image.builder import base from tripleo_common.utils import process CONF = cfg.CONF +LOG = logging.getLogger(__name__ + ".BuildahBuilder") class BuildahBuilder(base.BaseBuilder): """Builder to build container images with Buildah.""" - log = logging.getLogger(__name__ + ".BuildahBuilder") + log = LOG def __init__(self, work_dir, deps, base='fedora', img_type='binary', tag='latest', namespace='master', @@ -144,10 +145,13 @@ class BuildahBuilder(base.BaseBuilder): self.log.exception(e) raise - @tenacity.retry( # Retry up to 3 times with 1 second delay + @tenacity.retry( + # Retry up to 5 times: 0, 1, 5, 21, 85 + # http://exponentialbackoffcalculator.com/ reraise=True, - wait=tenacity.wait_fixed(1), - stop=tenacity.stop_after_attempt(3) + wait=tenacity.wait_random_exponential(multiplier=4, max=60), + stop=tenacity.stop_after_attempt(5), + before_sleep=tenacity.after_log(LOG, logging.WARNING) ) def build(self, container_name, container_build_path): """Build an image from a given directory. @@ -190,7 +194,8 @@ class BuildahBuilder(base.BaseBuilder): @tenacity.retry( # Retry up to 10 times with jittered exponential backoff reraise=True, wait=tenacity.wait_random_exponential(multiplier=1, max=15), - stop=tenacity.stop_after_attempt(10) + stop=tenacity.stop_after_attempt(10), + before_sleep=tenacity.after_log(LOG, logging.WARNING) ) def push(self, destination): """Push an image to a container registry.