fb978cf770
This is an attempt to populate tempestconf with multistore feature enabled if and only if multiple stores are available. In Rocky and Stein, Glance has added the ability to configure multiple stores as an EXPERIMENTAL feature. This feature is fully supported from the Train cycle. Please refer [1] for more information [1] https://specs.openstack.org/openstack/cinder-specs/specs/ussuri/support-glance-multiple-backend.html Here are two test-projects exercising the results of this change 1. With multi-store enabled https://review.rdoproject.org/r/c/testproject/+/37159 2. With multi-store not enabled https://review.rdoproject.org/r/c/testproject/+/37160 Signed-off-by: Soniya Vyas <svyas@redhat.com> Change-Id: I45f9dce14b60e9385600c57ee56b50aba3a4476f
358 lines
14 KiB
Python
358 lines
14 KiB
Python
# Copyright 2013 Red Hat, Inc.
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
|
|
from functools import wraps
|
|
|
|
from six.moves import urllib
|
|
from tempest.lib import exceptions
|
|
from tenacity import retry
|
|
from tenacity import stop_after_attempt
|
|
|
|
from config_tempest import constants as C
|
|
from config_tempest.services.base import VersionedService
|
|
|
|
|
|
stop = stop_after_attempt(len(C.DEFAULT_IMAGES))
|
|
|
|
|
|
class ImageService(VersionedService):
|
|
|
|
def __init__(self, name, s_type, service_url, token,
|
|
disable_ssl_validation, client=None, **kwargs):
|
|
super(ImageService, self).__init__(
|
|
name, s_type, service_url, token, disable_ssl_validation,
|
|
client, **kwargs)
|
|
self.retry_attempt = -1
|
|
|
|
def set_image_preferences(self, disk_format, non_admin, no_rng=False,
|
|
convert=False):
|
|
"""Sets image prefferences.
|
|
|
|
:type disk_format: string
|
|
:type non_admin: bool
|
|
:type no_rng: bool
|
|
:type convert: bool
|
|
"""
|
|
self.disk_format = disk_format
|
|
self.non_admin = non_admin
|
|
self.no_rng = no_rng
|
|
self.convert = convert
|
|
|
|
def set_default_tempest_options(self, conf):
|
|
# set 'image-feature-enabled' only if multiple stores available
|
|
if self._is_multistore_enabled():
|
|
conf.set('image-feature-enabled', 'import_image', 'True')
|
|
# When cirros is the image, set validation.image_ssh_user to cirros.
|
|
# The option is heavily used in CI and it's also usefull for refstack,
|
|
# because we don't have to specify overrides.
|
|
if 'cirros' in conf.get_defaulted('image',
|
|
'image_path').rsplit('/')[-1]:
|
|
conf.set('validation', 'image_ssh_user', 'cirros')
|
|
# image.http_image is a tempest option which defines 'http accessible
|
|
# image', it can be in a compressed format so it can't be mistaken
|
|
# for an image which will be uploaded to the glance.
|
|
# image.http_image and image.image_path can be 2 different images.
|
|
# If image.http_image wasn't set as an override, it will be set to
|
|
# image.image_path or to DEFAULT_IMAGE
|
|
image_path = conf.get_defaulted('image', 'image_path')
|
|
if self._find_image_by_name(image_path) is None:
|
|
conf.set('image', 'http_image', image_path)
|
|
else:
|
|
# image.image_path is name of the image already present in glance,
|
|
# this value can't be set to image.http_image, therefor set the
|
|
# default value
|
|
conf.set('image', 'http_image', C.DEFAULT_IMAGE)
|
|
|
|
def _is_multistore_enabled(self):
|
|
try:
|
|
self.client.info_stores()['stores']
|
|
except Exception:
|
|
C.LOG.info('Can not retrieve stores, either multiple stores are '
|
|
'not configured or user are not allowed access '
|
|
'to the stores information')
|
|
return False
|
|
return True
|
|
|
|
def get_supported_versions(self):
|
|
return ['v1', 'v2']
|
|
|
|
@staticmethod
|
|
def get_service_type():
|
|
return ['image']
|
|
|
|
@staticmethod
|
|
def get_codename():
|
|
return 'glance'
|
|
|
|
def set_versions(self):
|
|
super(ImageService, self).set_versions(top_level=False)
|
|
|
|
def create_tempest_images(self, conf, retry_alt=False):
|
|
"""Uploads an image to the glance.
|
|
|
|
The method creates images specified in conf, if they're not created
|
|
already. Then it sets their IDs to the conf.
|
|
|
|
:type conf: TempestConf object
|
|
"""
|
|
# the absolute path is necessary for supporting older tempest versions,
|
|
# which had CONF.scenario.img_dir option, see this line of code:
|
|
# https://github.com/openstack/tempest/blob/a0ee8b4ccfc512a09
|
|
# e1ddb135950b767110aae9b/tempest/scenario/manager.py#L534
|
|
# If the path is not an absolute one, the concatenation of strings ^^
|
|
# will result in an invalid path
|
|
# Moreover the absolute path is needed so that users can move the
|
|
# generated tempest.conf outside of python-tempestconf destination,
|
|
# otherwise tempest would fail accessing the CONF.scenario.img_file
|
|
img_dir = os.path.abspath(os.path.join(C.DEFAULT_IMAGE_DIR))
|
|
image_path = conf.get_defaulted('image', 'image_path')
|
|
img_path = os.path.join(img_dir,
|
|
os.path.basename(image_path))
|
|
name = image_path[image_path.rfind('/') + 1:]
|
|
if self.convert and name[-4:] == ".img":
|
|
name = name[:-4] + ".raw"
|
|
if not os.path.exists(img_dir):
|
|
try:
|
|
os.makedirs(img_dir)
|
|
except OSError:
|
|
raise
|
|
alt_name = name + "_alt"
|
|
image_id = None
|
|
if conf.has_option('compute', 'image_ref'):
|
|
image_id = conf.get('compute', 'image_ref')
|
|
image_id = self.find_or_upload_image(image_id, name,
|
|
image_source=image_path,
|
|
image_dest=img_path,
|
|
retry_alt=retry_alt)
|
|
alt_image_id = None
|
|
if conf.has_option('compute', 'image_ref_alt'):
|
|
alt_image_id = conf.get('compute', 'image_ref_alt')
|
|
alt_image_id = self.find_or_upload_image(alt_image_id, alt_name,
|
|
image_source=image_path,
|
|
image_dest=img_path,
|
|
retry_alt=retry_alt)
|
|
# get name of the image_id
|
|
conf.set('scenario', 'img_file', img_path)
|
|
conf.set('compute', 'image_ref', image_id)
|
|
conf.set('compute', 'image_ref_alt', alt_image_id)
|
|
|
|
def find_or_upload_image(self, image_id, image_name, image_source='',
|
|
image_dest='', retry_alt=False):
|
|
"""If the image is not found, uploads it.
|
|
|
|
:type image_id: string
|
|
:type image_name: string
|
|
:type image_source: string
|
|
:type image_dest: string
|
|
"""
|
|
image = self._find_image(image_id, image_name)
|
|
|
|
if image:
|
|
C.LOG.info("(no change) Found image '%s'", image['name'])
|
|
path = os.path.abspath(image_dest)
|
|
if not os.path.isfile(path):
|
|
self._download_image(image['id'], path)
|
|
else:
|
|
C.LOG.info("Creating image '%s'", image_name)
|
|
if image_source.startswith("http:") or \
|
|
image_source.startswith("https:"):
|
|
try:
|
|
self._download_file(image_source, image_dest)
|
|
# We only download alternative image if the default image
|
|
# fails
|
|
except Exception:
|
|
if retry_alt:
|
|
self._download_with_retry(image_dest)
|
|
else:
|
|
try:
|
|
shutil.copyfile(image_source, image_dest)
|
|
except Exception:
|
|
# let's try if this is the case when a user uses already
|
|
# existing image in glance which is not uploaded as *_alt
|
|
if image_name[-4:] == "_alt":
|
|
image = self._find_image(None, image_name[:-4])
|
|
if image:
|
|
path = os.path.abspath(image_dest)
|
|
if not os.path.isfile(path):
|
|
self._download_image(image['id'], path)
|
|
else:
|
|
if retry_alt:
|
|
self._download_with_retry(image_dest)
|
|
else:
|
|
raise IOError
|
|
image = self._upload_image(image_name, image_dest)
|
|
return image['id']
|
|
|
|
@retry(stop=stop)
|
|
def _download_with_retry(self, destination):
|
|
self.retry_attempt += 1
|
|
self._download_file(C.DEFAULT_IMAGES[self.retry_attempt], destination)
|
|
|
|
def _find_image(self, image_id, image_name):
|
|
"""Find image by ID or name (the image client doesn't have this).
|
|
|
|
:type image_id: string
|
|
:type image_name: string
|
|
"""
|
|
if image_id:
|
|
try:
|
|
return self.client.show_image(image_id)
|
|
except exceptions.NotFound:
|
|
pass
|
|
return self._find_image_by_name(image_name)
|
|
|
|
def _find_image_by_name(self, image_name):
|
|
"""Find image by name.
|
|
|
|
:type image_name: string
|
|
:return: Information in a dict about the found image
|
|
:rtype: dict or None if image was not found
|
|
"""
|
|
for x in self.client.list_images()['images']:
|
|
if x['name'] == image_name:
|
|
return x
|
|
return None
|
|
|
|
def _upload_image(self, name, path):
|
|
"""Upload image file from `path` into Glance with `name`.
|
|
|
|
:type name: string
|
|
:type path: string
|
|
"""
|
|
if self.convert:
|
|
path = self.convert_image_to_raw(path)
|
|
|
|
C.LOG.info("Uploading image '%s' from '%s'",
|
|
name, os.path.abspath(path))
|
|
if self.non_admin:
|
|
visibility = 'community'
|
|
else:
|
|
visibility = 'public'
|
|
|
|
with open(path, 'rb') as data:
|
|
args = {'name': name, 'disk_format': self.disk_format,
|
|
'container_format': 'bare', 'visibility': visibility,
|
|
'hw_rng_model': 'virtio'}
|
|
if self.no_rng:
|
|
args.pop('hw_rng_model')
|
|
image = self.client.create_image(**args)
|
|
self.client.store_image_file(image['id'], data)
|
|
return image
|
|
|
|
def _download_image(self, id, path):
|
|
"""Download image from glance.
|
|
|
|
:type id: string
|
|
:type path: string
|
|
"""
|
|
C.LOG.info("Downloading image %s to %s", id, path)
|
|
body = self.client.show_image_file(id)
|
|
C.LOG.debug(type(body.data))
|
|
with open(path, 'wb') as out:
|
|
out.write(body.data)
|
|
|
|
def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None):
|
|
"""Retry calling the decorated function using exponential backoff
|
|
|
|
http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
|
|
original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
|
|
|
|
Licensed under the BSD 3-Clause "New" or "Revised" License
|
|
(https://github.com/saltycrane/retry-decorator/blob/master/LICENSE)
|
|
|
|
:param ExceptionToCheck: the exception to check
|
|
:type ExceptionToCheck: Exception or tuple
|
|
:param tries: number of times before giving up
|
|
:type type: int
|
|
:param delay: initial delay between retries in seconds
|
|
:type type: int
|
|
:param backoff: backoff multiplier e.g. value of 2 will double the
|
|
delay each retry
|
|
:type backoff: int
|
|
:param logger: logger to use. If None, print
|
|
:type logger: logging. Logger instance
|
|
"""
|
|
def deco_retry(f):
|
|
@wraps(f)
|
|
def f_retry(*args, **kwargs):
|
|
mtries, mdelay = tries, delay
|
|
while mtries > 1:
|
|
try:
|
|
return f(*args, **kwargs)
|
|
except ExceptionToCheck as e:
|
|
msg = "%s, Retrying in %d seconds." % (str(e), mdelay)
|
|
if logger:
|
|
logger.warning(msg)
|
|
else:
|
|
print(msg)
|
|
time.sleep(mdelay)
|
|
mtries -= 1
|
|
mdelay *= backoff
|
|
return f(*args, **kwargs)
|
|
return f_retry
|
|
return deco_retry
|
|
|
|
@retry(urllib.error.URLError, logger=C.LOG)
|
|
def retry_urlopen(self, url):
|
|
"""Opens url using urlopen. If it fails, it will try again.
|
|
|
|
:type url: string
|
|
"""
|
|
return urllib.request.urlopen(url)
|
|
|
|
def _download_file(self, url, destination):
|
|
"""Downloads a file specified by `url` to `destination`.
|
|
|
|
:type url: string
|
|
:type destination: string
|
|
"""
|
|
if os.path.exists(destination):
|
|
C.LOG.info("Image '%s' already fetched to '%s'.", url, destination)
|
|
return
|
|
C.LOG.info("Downloading '%s' and saving as '%s'", url, destination)
|
|
f = self.retry_urlopen(url)
|
|
data = f.read()
|
|
with open(destination, "wb") as dest:
|
|
dest.write(data)
|
|
|
|
def convert_image_to_raw(self, path):
|
|
"""Converts given image to raw format.
|
|
|
|
:type path: string
|
|
:return: path of the converted image
|
|
:rtype: string
|
|
"""
|
|
head, tail = os.path.split(path)
|
|
name = tail.rsplit('.', 1)[0] + '.raw'
|
|
raw_path = os.path.join(head, name)
|
|
# check if converted already
|
|
if os.path.exists(raw_path):
|
|
C.LOG.info("Image already converted in '%s'.", raw_path)
|
|
else:
|
|
C.LOG.info("Converting image '%s' to '%s'",
|
|
os.path.abspath(path), os.path.abspath(raw_path))
|
|
rc = subprocess.call(['qemu-img', 'convert', path, raw_path])
|
|
if rc != 0:
|
|
raise Exception("Converting of the image has finished with "
|
|
"non-zero return code. The return code was "
|
|
"'%d'", rc)
|
|
self.disk_format = 'raw'
|
|
return raw_path
|