Automatic Tempest Configuration Generator
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

356 lines
14 KiB

# Copyright 2013 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import shutil
import subprocess
import time
from functools import wraps
from six.moves import urllib
from tempest.lib import exceptions
from tenacity import retry
from tenacity import stop_after_attempt
from config_tempest import constants as C
from config_tempest.services.base import VersionedService
stop = stop_after_attempt(len(C.DEFAULT_IMAGES))
class ImageService(VersionedService):
def __init__(self, name, s_type, service_url, token,
disable_ssl_validation, client=None, **kwargs):
super(ImageService, self).__init__(
name, s_type, service_url, token, disable_ssl_validation,
client, **kwargs)
self.retry_attempt = -1
def set_image_preferences(self, disk_format, non_admin, no_rng=False,
convert=False):
"""Sets image prefferences.
:type disk_format: string
:type non_admin: bool
:type no_rng: bool
:type convert: bool
"""
self.disk_format = disk_format
self.non_admin = non_admin
self.no_rng = no_rng
self.convert = convert
def set_default_tempest_options(self, conf):
# set 'image-feature-enabled' only if multiple stores available
num_stores = self._get_number_of_stores()
if num_stores > 1:
conf.set('image-feature-enabled', 'import_image', 'True')
# When cirros is the image, set validation.image_ssh_user to cirros.
# The option is heavily used in CI and it's also usefull for refstack,
# because we don't have to specify overrides.
if 'cirros' in conf.get_defaulted('image',
'image_path').rsplit('/')[-1]:
conf.set('validation', 'image_ssh_user', 'cirros')
# image.http_image is a tempest option which defines 'http accessible
# image', it can be in a compressed format so it can't be mistaken
# for an image which will be uploaded to the glance.
# image.http_image and image.image_path can be 2 different images.
# If image.http_image wasn't set as an override, it will be set to
# image.image_path or to DEFAULT_IMAGE
image_path = conf.get_defaulted('image', 'image_path')
if self._find_image_by_name(image_path) is None:
conf.set('image', 'http_image', image_path)
else:
# image.image_path is name of the image already present in glance,
# this value can't be set to image.http_image, therefor set the
# default value
conf.set('image', 'http_image', C.DEFAULT_IMAGE)
def _get_number_of_stores(self):
try:
stores = self.client.info_stores()['stores']
return len(stores)
except exceptions.Forbidden:
C.LOG.info('Can not retrieve stores, user are not allowed')
return 1
def get_supported_versions(self):
return ['v1', 'v2']
@staticmethod
def get_service_type():
return ['image']
@staticmethod
def get_codename():
return 'glance'
def set_versions(self):
super(ImageService, self).set_versions(top_level=False)
def create_tempest_images(self, conf, retry_alt=False):
"""Uploads an image to the glance.
The method creates images specified in conf, if they're not created
already. Then it sets their IDs to the conf.
:type conf: TempestConf object
"""
# the absolute path is necessary for supporting older tempest versions,
# which had CONF.scenario.img_dir option, see this line of code:
# https://github.com/openstack/tempest/blob/a0ee8b4ccfc512a09
# e1ddb135950b767110aae9b/tempest/scenario/manager.py#L534
# If the path is not an absolute one, the concatenation of strings ^^
# will result in an invalid path
# Moreover the absolute path is needed so that users can move the
# generated tempest.conf outside of python-tempestconf destination,
# otherwise tempest would fail accessing the CONF.scenario.img_file
img_dir = os.path.abspath(os.path.join(C.DEFAULT_IMAGE_DIR))
image_path = conf.get_defaulted('image', 'image_path')
img_path = os.path.join(img_dir,
os.path.basename(image_path))
name = image_path[image_path.rfind('/') + 1:]
if self.convert and name[-4:] == ".img":
name = name[:-4] + ".raw"
if not os.path.exists(img_dir):
try:
os.makedirs(img_dir)
except OSError:
raise
alt_name = name + "_alt"
image_id = None
if conf.has_option('compute', 'image_ref'):
image_id = conf.get('compute', 'image_ref')
image_id = self.find_or_upload_image(image_id, name,
image_source=image_path,
image_dest=img_path,
retry_alt=retry_alt)
alt_image_id = None
if conf.has_option('compute', 'image_ref_alt'):
alt_image_id = conf.get('compute', 'image_ref_alt')
alt_image_id = self.find_or_upload_image(alt_image_id, alt_name,
image_source=image_path,
image_dest=img_path,
retry_alt=retry_alt)
# get name of the image_id
conf.set('scenario', 'img_file', img_path)
conf.set('compute', 'image_ref', image_id)
conf.set('compute', 'image_ref_alt', alt_image_id)
def find_or_upload_image(self, image_id, image_name, image_source='',
image_dest='', retry_alt=False):
"""If the image is not found, uploads it.
:type image_id: string
:type image_name: string
:type image_source: string
:type image_dest: string
"""
image = self._find_image(image_id, image_name)
if image:
C.LOG.info("(no change) Found image '%s'", image['name'])
path = os.path.abspath(image_dest)
if not os.path.isfile(path):
self._download_image(image['id'], path)
else:
C.LOG.info("Creating image '%s'", image_name)
if image_source.startswith("http:") or \
image_source.startswith("https:"):
try:
self._download_file(image_source, image_dest)
# We only download alternative image if the default image
# fails
except Exception:
if retry_alt:
self._download_with_retry(image_dest)
else:
try:
shutil.copyfile(image_source, image_dest)
except Exception:
# let's try if this is the case when a user uses already
# existing image in glance which is not uploaded as *_alt
if image_name[-4:] == "_alt":
image = self._find_image(None, image_name[:-4])
if image:
path = os.path.abspath(image_dest)
if not os.path.isfile(path):
self._download_image(image['id'], path)
else:
if retry_alt:
self._download_with_retry(image_dest)
else:
raise IOError
image = self._upload_image(image_name, image_dest)
return image['id']
@retry(stop=stop)
def _download_with_retry(self, destination):
self.retry_attempt += 1
self._download_file(C.DEFAULT_IMAGES[self.retry_attempt], destination)
def _find_image(self, image_id, image_name):
"""Find image by ID or name (the image client doesn't have this).
:type image_id: string
:type image_name: string
"""
if image_id:
try:
return self.client.show_image(image_id)
except exceptions.NotFound:
pass
return self._find_image_by_name(image_name)
def _find_image_by_name(self, image_name):
"""Find image by name.
:type image_name: string
:return: Information in a dict about the found image
:rtype: dict or None if image was not found
"""
for x in self.client.list_images()['images']:
if x['name'] == image_name:
return x
return None
def _upload_image(self, name, path):
"""Upload image file from `path` into Glance with `name`.
:type name: string
:type path: string
"""
if self.convert:
path = self.convert_image_to_raw(path)
C.LOG.info("Uploading image '%s' from '%s'",
name, os.path.abspath(path))
if self.non_admin:
visibility = 'community'
else:
visibility = 'public'
with open(path, 'rb') as data:
args = {'name': name, 'disk_format': self.disk_format,
'container_format': 'bare', 'visibility': visibility,
'hw_rng_model': 'virtio'}
if self.no_rng:
args.pop('hw_rng_model')
image = self.client.create_image(**args)
self.client.store_image_file(image['id'], data)
return image
def _download_image(self, id, path):
"""Download image from glance.
:type id: string
:type path: string
"""
C.LOG.info("Downloading image %s to %s", id, path)
body = self.client.show_image_file(id)
C.LOG.debug(type(body.data))
with open(path, 'wb') as out:
out.write(body.data)
def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None):
"""Retry calling the decorated function using exponential backoff
http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
Licensed under the BSD 3-Clause "New" or "Revised" License
(https://github.com/saltycrane/retry-decorator/blob/master/LICENSE)
:param ExceptionToCheck: the exception to check
:type ExceptionToCheck: Exception or tuple
:param tries: number of times before giving up
:type type: int
:param delay: initial delay between retries in seconds
:type type: int
:param backoff: backoff multiplier e.g. value of 2 will double the
delay each retry
:type backoff: int
:param logger: logger to use. If None, print
:type logger: logging. Logger instance
"""
def deco_retry(f):
@wraps(f)
def f_retry(*args, **kwargs):
mtries, mdelay = tries, delay
while mtries > 1:
try:
return f(*args, **kwargs)
except ExceptionToCheck as e:
msg = "%s, Retrying in %d seconds." % (str(e), mdelay)
if logger:
logger.warning(msg)
else:
print(msg)
time.sleep(mdelay)
mtries -= 1
mdelay *= backoff
return f(*args, **kwargs)
return f_retry
return deco_retry
@retry(urllib.error.URLError, logger=C.LOG)
def retry_urlopen(self, url):
"""Opens url using urlopen. If it fails, it will try again.
:type url: string
"""
return urllib.request.urlopen(url)
def _download_file(self, url, destination):
"""Downloads a file specified by `url` to `destination`.
:type url: string
:type destination: string
"""
if os.path.exists(destination):
C.LOG.info("Image '%s' already fetched to '%s'.", url, destination)
return
C.LOG.info("Downloading '%s' and saving as '%s'", url, destination)
f = self.retry_urlopen(url)
data = f.read()
with open(destination, "wb") as dest:
dest.write(data)
def convert_image_to_raw(self, path):
"""Converts given image to raw format.
:type path: string
:return: path of the converted image
:rtype: string
"""
head, tail = os.path.split(path)
name = tail.rsplit('.', 1)[0] + '.raw'
raw_path = os.path.join(head, name)
# check if converted already
if os.path.exists(raw_path):
C.LOG.info("Image already converted in '%s'.", raw_path)
else:
C.LOG.info("Converting image '%s' to '%s'",
os.path.abspath(path), os.path.abspath(raw_path))
rc = subprocess.call(['qemu-img', 'convert', path, raw_path])
if rc != 0:
raise Exception("Converting of the image has finished with "
"non-zero return code. The return code was "
"'%d'", rc)
self.disk_format = 'raw'
return raw_path