From 9ab61955a2155b3ad62fa657a31a02f5123dfe5c Mon Sep 17 00:00:00 2001 From: Luke Short Date: Thu, 16 Jan 2020 11:28:27 -0500 Subject: [PATCH] container_update: Retry removing containers. Change-Id: Ifd28fd60e4bc1218acc24381d13534c227f2877e Related-Bug: #1856086 Signed-off-by: Luke Short (cherry picked from commit ef77f081a21e8483b143558dbf5ace5d909a368f) (cherry picked from commit 64718a796125dd1197fb4c64ca5cd8db44b750be) (cherry picked from commit 179f6953cf4b9623fd7a5f8ada37888b3aad34ca) (cherry picked from commit b91249bf7e60b2436f32496dd3df0f1e90c355fb) --- scripts/container-update.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/scripts/container-update.py b/scripts/container-update.py index 8c3763cd2..34f83491e 100755 --- a/scripts/container-update.py +++ b/scripts/container-update.py @@ -16,6 +16,7 @@ import logging import multiprocessing import subprocess import sys +import tenacity import yaml import yum @@ -47,18 +48,32 @@ def parse_opts(argv): return opts +@tenacity.retry( + reraise=True, + retry=tenacity.retry_if_exception_type(RuntimeError), + stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_fixed(1) +) def rm_container(name): log.info('Removing container: %s' % name) subproc = subprocess.Popen(['/usr/bin/docker', 'rm', name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) cmd_stdout, cmd_stderr = subproc.communicate() + rc = subproc.returncode + if cmd_stdout: log.debug(cmd_stdout) - if cmd_stderr and \ - cmd_stderr != 'Error response from daemon: ' \ - 'No such container: {}\n'.format(name): - log.debug(cmd_stderr) + + if rc != 0: + + if 'No such container' in cmd_stderr: + log.warn('Container that does not exist cannot be deleted: ' + '%s' % name) + else: + log.error('Error removing container: %s' % name) + log.error(cmd_stderr) + raise RuntimeError(cmd_stdout, cmd_stderr, rc) def populate_container_rpms_list(container):