# Copyright 2013 Red Hat, Inc. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import os import shutil import subprocess import time from functools import wraps from six.moves import urllib from tempest.lib import exceptions from tenacity import retry from tenacity import stop_after_attempt from config_tempest import constants as C from config_tempest.services.base import VersionedService stop = stop_after_attempt(len(C.DEFAULT_IMAGES)) class ImageService(VersionedService): def __init__(self, name, s_type, service_url, token, disable_ssl_validation, client=None, **kwargs): super(ImageService, self).__init__( name, s_type, service_url, token, disable_ssl_validation, client, **kwargs) self.retry_attempt = -1 def set_image_preferences(self, disk_format, non_admin, no_rng=False, convert=False): """Sets image prefferences. :type disk_format: string :type non_admin: bool :type no_rng: bool :type convert: bool """ self.disk_format = disk_format self.non_admin = non_admin self.no_rng = no_rng self.convert = convert def set_default_tempest_options(self, conf): # set 'image-feature-enabled' only if multiple stores available num_stores = self._get_number_of_stores() if num_stores > 1: conf.set('image-feature-enabled', 'import_image', 'True') # When cirros is the image, set validation.image_ssh_user to cirros. # The option is heavily used in CI and it's also usefull for refstack, # because we don't have to specify overrides. if 'cirros' in conf.get_defaulted('image', 'image_path').rsplit('/')[-1]: conf.set('validation', 'image_ssh_user', 'cirros') # image.http_image is a tempest option which defines 'http accessible # image', it can be in a compressed format so it can't be mistaken # for an image which will be uploaded to the glance. # image.http_image and image.image_path can be 2 different images. # If image.http_image wasn't set as an override, it will be set to # image.image_path or to DEFAULT_IMAGE image_path = conf.get_defaulted('image', 'image_path') if self._find_image_by_name(image_path) is None: conf.set('image', 'http_image', image_path) else: # image.image_path is name of the image already present in glance, # this value can't be set to image.http_image, therefor set the # default value conf.set('image', 'http_image', C.DEFAULT_IMAGE) def _get_number_of_stores(self): try: stores = self.client.info_stores()['stores'] return len(stores) except exceptions.Forbidden: C.LOG.info('Can not retrieve stores, user are not allowed') return 1 def get_supported_versions(self): return ['v1', 'v2'] @staticmethod def get_service_type(): return ['image'] @staticmethod def get_codename(): return 'glance' def set_versions(self): super(ImageService, self).set_versions(top_level=False) def create_tempest_images(self, conf, retry_alt=False): """Uploads an image to the glance. The method creates images specified in conf, if they're not created already. Then it sets their IDs to the conf. :type conf: TempestConf object """ # the absolute path is necessary for supporting older tempest versions, # which had CONF.scenario.img_dir option, see this line of code: # https://github.com/openstack/tempest/blob/a0ee8b4ccfc512a09 # e1ddb135950b767110aae9b/tempest/scenario/manager.py#L534 # If the path is not an absolute one, the concatenation of strings ^^ # will result in an invalid path # Moreover the absolute path is needed so that users can move the # generated tempest.conf outside of python-tempestconf destination, # otherwise tempest would fail accessing the CONF.scenario.img_file img_dir = os.path.abspath(os.path.join(C.DEFAULT_IMAGE_DIR)) image_path = conf.get_defaulted('image', 'image_path') img_path = os.path.join(img_dir, os.path.basename(image_path)) name = image_path[image_path.rfind('/') + 1:] if self.convert and name[-4:] == ".img": name = name[:-4] + ".raw" if not os.path.exists(img_dir): try: os.makedirs(img_dir) except OSError: raise alt_name = name + "_alt" image_id = None if conf.has_option('compute', 'image_ref'): image_id = conf.get('compute', 'image_ref') image_id = self.find_or_upload_image(image_id, name, image_source=image_path, image_dest=img_path, retry_alt=retry_alt) alt_image_id = None if conf.has_option('compute', 'image_ref_alt'): alt_image_id = conf.get('compute', 'image_ref_alt') alt_image_id = self.find_or_upload_image(alt_image_id, alt_name, image_source=image_path, image_dest=img_path, retry_alt=retry_alt) # get name of the image_id conf.set('scenario', 'img_file', img_path) conf.set('compute', 'image_ref', image_id) conf.set('compute', 'image_ref_alt', alt_image_id) def find_or_upload_image(self, image_id, image_name, image_source='', image_dest='', retry_alt=False): """If the image is not found, uploads it. :type image_id: string :type image_name: string :type image_source: string :type image_dest: string """ image = self._find_image(image_id, image_name) if image: C.LOG.info("(no change) Found image '%s'", image['name']) path = os.path.abspath(image_dest) if not os.path.isfile(path): self._download_image(image['id'], path) else: C.LOG.info("Creating image '%s'", image_name) if image_source.startswith("http:") or \ image_source.startswith("https:"): try: self._download_file(image_source, image_dest) # We only download alternative image if the default image # fails except Exception: if retry_alt: self._download_with_retry(image_dest) else: try: shutil.copyfile(image_source, image_dest) except Exception: # let's try if this is the case when a user uses already # existing image in glance which is not uploaded as *_alt if image_name[-4:] == "_alt": image = self._find_image(None, image_name[:-4]) if image: path = os.path.abspath(image_dest) if not os.path.isfile(path): self._download_image(image['id'], path) else: if retry_alt: self._download_with_retry(image_dest) else: raise IOError image = self._upload_image(image_name, image_dest) return image['id'] @retry(stop=stop) def _download_with_retry(self, destination): self.retry_attempt += 1 self._download_file(C.DEFAULT_IMAGES[self.retry_attempt], destination) def _find_image(self, image_id, image_name): """Find image by ID or name (the image client doesn't have this). :type image_id: string :type image_name: string """ if image_id: try: return self.client.show_image(image_id) except exceptions.NotFound: pass return self._find_image_by_name(image_name) def _find_image_by_name(self, image_name): """Find image by name. :type image_name: string :return: Information in a dict about the found image :rtype: dict or None if image was not found """ for x in self.client.list_images()['images']: if x['name'] == image_name: return x return None def _upload_image(self, name, path): """Upload image file from `path` into Glance with `name`. :type name: string :type path: string """ if self.convert: path = self.convert_image_to_raw(path) C.LOG.info("Uploading image '%s' from '%s'", name, os.path.abspath(path)) if self.non_admin: visibility = 'community' else: visibility = 'public' with open(path, 'rb') as data: args = {'name': name, 'disk_format': self.disk_format, 'container_format': 'bare', 'visibility': visibility, 'hw_rng_model': 'virtio'} if self.no_rng: args.pop('hw_rng_model') image = self.client.create_image(**args) self.client.store_image_file(image['id'], data) return image def _download_image(self, id, path): """Download image from glance. :type id: string :type path: string """ C.LOG.info("Downloading image %s to %s", id, path) body = self.client.show_image_file(id) C.LOG.debug(type(body.data)) with open(path, 'wb') as out: out.write(body.data) def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None): """Retry calling the decorated function using exponential backoff http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/ original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry Licensed under the BSD 3-Clause "New" or "Revised" License (https://github.com/saltycrane/retry-decorator/blob/master/LICENSE) :param ExceptionToCheck: the exception to check :type ExceptionToCheck: Exception or tuple :param tries: number of times before giving up :type type: int :param delay: initial delay between retries in seconds :type type: int :param backoff: backoff multiplier e.g. value of 2 will double the delay each retry :type backoff: int :param logger: logger to use. If None, print :type logger: logging. Logger instance """ def deco_retry(f): @wraps(f) def f_retry(*args, **kwargs): mtries, mdelay = tries, delay while mtries > 1: try: return f(*args, **kwargs) except ExceptionToCheck as e: msg = "%s, Retrying in %d seconds." % (str(e), mdelay) if logger: logger.warning(msg) else: print(msg) time.sleep(mdelay) mtries -= 1 mdelay *= backoff return f(*args, **kwargs) return f_retry return deco_retry @retry(urllib.error.URLError, logger=C.LOG) def retry_urlopen(self, url): """Opens url using urlopen. If it fails, it will try again. :type url: string """ return urllib.request.urlopen(url) def _download_file(self, url, destination): """Downloads a file specified by `url` to `destination`. :type url: string :type destination: string """ if os.path.exists(destination): C.LOG.info("Image '%s' already fetched to '%s'.", url, destination) return C.LOG.info("Downloading '%s' and saving as '%s'", url, destination) f = self.retry_urlopen(url) data = f.read() with open(destination, "wb") as dest: dest.write(data) def convert_image_to_raw(self, path): """Converts given image to raw format. :type path: string :return: path of the converted image :rtype: string """ head, tail = os.path.split(path) name = tail.rsplit('.', 1)[0] + '.raw' raw_path = os.path.join(head, name) # check if converted already if os.path.exists(raw_path): C.LOG.info("Image already converted in '%s'.", raw_path) else: C.LOG.info("Converting image '%s' to '%s'", os.path.abspath(path), os.path.abspath(raw_path)) rc = subprocess.call(['qemu-img', 'convert', path, raw_path]) if rc != 0: raise Exception("Converting of the image has finished with " "non-zero return code. The return code was " "'%d'", rc) self.disk_format = 'raw' return raw_path