From 64718a796125dd1197fb4c64ca5cd8db44b750be Mon Sep 17 00:00:00 2001 From: Luke Short Date: Thu, 16 Jan 2020 11:28:27 -0500 Subject: [PATCH] container_update: Retry removing containers. Change-Id: Ifd28fd60e4bc1218acc24381d13534c227f2877e Related-Bug: #1856086 Signed-off-by: Luke Short (cherry picked from commit ef77f081a21e8483b143558dbf5ace5d909a368f) --- scripts/container-update.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/scripts/container-update.py b/scripts/container-update.py index 800ab938c..7c7428082 100755 --- a/scripts/container-update.py +++ b/scripts/container-update.py @@ -16,6 +16,7 @@ import logging import multiprocessing import subprocess import sys +import tenacity import yaml import yum @@ -50,6 +51,12 @@ def parse_opts(argv): return opts +@tenacity.retry( + reraise=True, + retry=tenacity.retry_if_exception_type(RuntimeError), + stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_fixed(1) +) def rm_container(name): log.info('Removing container: %s' % name) subproc = subprocess.Popen(['/usr/bin/docker', 'rm', name], @@ -57,12 +64,20 @@ def rm_container(name): stderr=subprocess.PIPE, universal_newlines=True) cmd_stdout, cmd_stderr = subproc.communicate() + rc = subproc.returncode + if cmd_stdout: log.debug(cmd_stdout) - if cmd_stderr and \ - cmd_stderr != 'Error response from daemon: ' \ - 'No such container: {}\n'.format(name): - log.debug(cmd_stderr) + + if rc != 0: + + if 'No such container' in cmd_stderr: + log.warn('Container that does not exist cannot be deleted: ' + '%s' % name) + else: + log.error('Error removing container: %s' % name) + log.error(cmd_stderr) + raise RuntimeError(cmd_stdout, cmd_stderr, rc) def populate_container_rpms_list(container):