From 506bd278900deaf34ed36ff1fee71ce9615c2434 Mon Sep 17 00:00:00 2001 From: Arx Cruz Date: Thu, 11 Feb 2021 16:49:39 +0100 Subject: [PATCH] Add alternative image to be downloaded Sometimes the image is being download from some other place rather than the C.DEFAULT_IMAGE, and then it fails due the fact the url is down. Add the --alt-image option allows tempestconf to instead of fail, try again download a different image Change-Id: Ia1af9e684dc4488841e2b54c5817902e19e51b7f --- config_tempest/constants.py | 7 +++++- config_tempest/main.py | 10 ++++++-- config_tempest/services/image.py | 43 ++++++++++++++++++++++++++------ requirements.txt | 1 + 4 files changed, 50 insertions(+), 11 deletions(-) diff --git a/config_tempest/constants.py b/config_tempest/constants.py index 38a8203e..b00dcfb3 100644 --- a/config_tempest/constants.py +++ b/config_tempest/constants.py @@ -23,9 +23,14 @@ TEMPEST_WORKSPACE = os.getcwd() DEPLOYER_INPUT = os.path.join(os.path.expanduser("~"), "tempest-deployer-input.conf") +DEFAULT_IMAGE_DIR = 'etc' DEFAULT_IMAGE = ("https://download.cirros-cloud.net/0.4.0/" "cirros-0.4.0-x86_64-disk.img") -DEFAULT_IMAGE_DIR = 'etc' + +DEFAULT_IMAGES = [("https://download.cirros-cloud.net/0.4.0/" + "cirros-0.4.0-x86_64-disk.img"), + ("http://images.rdoproject.org/cirros/" + "cirros-0.4.0-x86_64-disk.img")] DEFAULT_IMAGE_FORMAT = 'qcow2' DEFAULT_FLAVOR_RAM = 128 diff --git a/config_tempest/main.py b/config_tempest/main.py index eb636bc6..2aa3a645 100755 --- a/config_tempest/main.py +++ b/config_tempest/main.py @@ -295,6 +295,10 @@ def get_arg_parser(): glance if it's not already there. The name of the image is the leaf name of the path. Default is '%s'""" % C.DEFAULT_IMAGE) + parser.add_argument('--retry-image', default=False, action='store_true', + help="""Allow tempestconf to retry download an image, + in case of failure, from these urls: '%s' + """ % C.DEFAULT_IMAGES) parser.add_argument('--flavor-min-mem', default=C.DEFAULT_FLAVOR_RAM, type=int, help="""Specify minimum memory for new flavours, default is '%s'.""" % C.DEFAULT_FLAVOR_RAM) @@ -539,7 +543,8 @@ def config_tempest(**kwargs): no_rng=kwargs.get('no_rng', False), convert=kwargs.get('convert_to_raw', False)) - image.create_tempest_images(conf) + retry_alt = kwargs.get('retry_image', False) + image.create_tempest_images(conf, retry_alt=retry_alt) if services.is_service(**{"type": "network"}): network = services.get_service("network") @@ -599,7 +604,8 @@ def main(): overrides=args.overrides, remove=args.remove, test_accounts=args.test_accounts, - verbose=args.verbose + verbose=args.verbose, + retry_image=args.retry_image ) diff --git a/config_tempest/services/image.py b/config_tempest/services/image.py index 72a1e09c..68c15893 100644 --- a/config_tempest/services/image.py +++ b/config_tempest/services/image.py @@ -13,14 +13,18 @@ # License for the specific language governing permissions and limitations # under the License. -from functools import wraps import os import shutil import subprocess import time +from functools import wraps + from six.moves import urllib from tempest.lib import exceptions +from tenacity import RetryError +from tenacity import Retrying +from tenacity import stop_after_attempt from config_tempest import constants as C from config_tempest.services.base import VersionedService @@ -78,7 +82,7 @@ class ImageService(VersionedService): def set_versions(self): super(ImageService, self).set_versions(top_level=False) - def create_tempest_images(self, conf): + def create_tempest_images(self, conf, retry_alt=False): """Uploads an image to the glance. The method creates images specified in conf, if they're not created @@ -113,20 +117,22 @@ class ImageService(VersionedService): image_id = conf.get('compute', 'image_ref') image_id = self.find_or_upload_image(image_id, name, image_source=image_path, - image_dest=img_path) + image_dest=img_path, + retry_alt=retry_alt) alt_image_id = None if conf.has_option('compute', 'image_ref_alt'): alt_image_id = conf.get('compute', 'image_ref_alt') alt_image_id = self.find_or_upload_image(alt_image_id, alt_name, image_source=image_path, - image_dest=img_path) + image_dest=img_path, + retry_alt=retry_alt) # get name of the image_id conf.set('scenario', 'img_file', img_path) conf.set('compute', 'image_ref', image_id) conf.set('compute', 'image_ref_alt', alt_image_id) def find_or_upload_image(self, image_id, image_name, image_source='', - image_dest=''): + image_dest='', retry_alt=False): """If the image is not found, uploads it. :type image_id: string @@ -145,11 +151,17 @@ class ImageService(VersionedService): C.LOG.info("Creating image '%s'", image_name) if image_source.startswith("http:") or \ image_source.startswith("https:"): - self._download_file(image_source, image_dest) + try: + self._download_file(image_source, image_dest) + # We only download alternative image if the default image + # fails + except Exception: + if retry_alt: + self._download_with_retry(image_dest) else: try: shutil.copyfile(image_source, image_dest) - except IOError: + except Exception: # let's try if this is the case when a user uses already # existing image in glance which is not uploaded as *_alt if image_name[-4:] == "_alt": @@ -159,10 +171,25 @@ class ImageService(VersionedService): if not os.path.isfile(path): self._download_image(image['id'], path) else: - raise IOError + if retry_alt: + self._download_with_retry(image_dest) + else: + raise IOError image = self._upload_image(image_name, image_dest) return image['id'] + def _download_with_retry(self, destination): + retry_attempt = -1 + attempts = len(C.DEFAULT_IMAGES) + try: + for attempt in Retrying(stop=stop_after_attempt(attempts)): + retry_attempt += 1 + with attempt: + self._download_file(C.DEFAULT_IMAGES[retry_attempt], + destination) + except RetryError: + pass + def _find_image(self, image_id, image_name): """Find image by ID or name (the image client doesn't have this). diff --git a/requirements.txt b/requirements.txt index 7ca7002f..b88e111c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,5 @@ tempest>=14.0.0 # Apache-2.0 requests>=2.10.0,!=2.12.2 # Apache-2.0 openstacksdk>=0.11.3 # Apache-2.0 oslo.config>=3.23.0 # Apache-2.0 +tenacity PyYAML>=3.12 # MIT