x
/
tobiko
Code Issues Proposed changes

Implement Glance image management (setup and cleanup) 

Change-Id: Ibba2b2ea4ca2e919aca3bbdde225db8f0b79450c
This commit is contained in:
Federico Ressi 2019-06-24 09:36:28 +02:00
parent ae4accef4e
commit 07ea422696
13 changed files with 414 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
devstack/plugin.sh
View File

@ -18,6 +18,8 @@ function configure_tobiko {
fi
configure_tobiko_default "${tobiko_config}"
configure_tobiko_cirros "${tobiko_config}"
configure_tobiko_glance "${tobiko_config}"
configure_tobiko_keystone "${tobiko_config}"
configure_tobiko_nova "${tobiko_config}"
configure_tobiko_neutron "${tobiko_config}"
@ -34,6 +36,18 @@ function configure_tobiko {
}
function configure_tobiko_cirros {
echo_summary "Write [cirros] section to ${TOBIKO_CONFIG}"
local tobiko_config=$1
iniset_nonempty "${tobiko_config}" cirros name "${TOBIKO_CIRROS_IMAGE_NAME}"
iniset_nonempty "${tobiko_config}" cirros url "${TOBIKO_CIRROS_IMAGE_URL}"
iniset_nonempty "${tobiko_config}" cirros file "${TOBIKO_CIRROS_IMAGE_FILE}"
iniset_nonempty "${tobiko_config}" cirros username "${TOBIKO_CIRROS_USERNAME}"
iniset_nonempty "${tobiko_config}" cirros password "${TOBIKO_CIRROS_PASSWORD}"
}
function configure_tobiko_default {
echo_summary "Write [DEFAULT] section to ${TOBIKO_CONFIG}"
local tobiko_config=$1
@ -44,6 +58,15 @@ function configure_tobiko_default {
iniset ${tobiko_config} DEFAULT debug "${TOBIKO_DEBUG}"
}
function configure_tobiko_glance {
echo_summary "Write [glance] section to ${TOBIKO_CONFIG}"
local tobiko_config=$1
iniset_nonempty "${tobiko_config}" glance image_dir "${TOBIKO_GLANCE_IMAGE_DIR}"
}
function configure_tobiko_keystone {
echo_summary "Write [keystone] section to ${TOBIKO_CONFIG}"
local tobiko_config=$1
@ -95,15 +118,6 @@ function configure_tobiko_nova {
echo_summary "Write [nova] section to ${TOBIKO_CONFIG}"
local tobiko_config=$1
# Write image ID
local image_name=${TOBIKO_NOVA_IMAGE:-}
if [ "${image_name}" != "" ]; then
local image_id=$(openstack image show -f value -c id "${image_name}")
else
local image_id=$(openstack image list --limit 1 -f value -c ID --public --status active)
fi
iniset "${tobiko_config}" nova image "${image_id}"
# Write flavor ID
local flavor_name=${TOBIKO_NOVA_FLAVOR:-}
if [ "${flavor_name}" != "" ]; then
@ -138,6 +152,14 @@ function configure_tobiko_neutron {
}
function iniset_nonempty {
# Calls iniset only when option value is not an empty string
if [ -n "$4" ]; then
iniset "$@"
fi
}
if [[ "$1" == "stack" ]]; then
case "$2" in
install)

11
devstack/settings
View File

@ -13,6 +13,16 @@ TOBIKO_DEBUG=${TOBIKO_DEBUG:-True}
TOBIKO_LOG_DIR=${TOBIKO_LOG_DIR:-${LOGDIR:-}}
TOBIKO_LOG_FILE=${TOBIKO_LOG_FILE:-tobiko.log}
# --- Glance settings ---
TOBIKO_GLANCE_IMAGE_DIR=${TOBIKO_GLANCE_IMAGE_DIR:-}
# --- Cirros image settings ---
TOBIKO_CIRROS_IMAGE_NAME=${TOBIKO_CIRROS_IMAGE_NAME:-${DEFAULT_IMAGE_NAME}}
TOBIKO_CIRROS_IMAGE_URL=${TOBIKO_CIRROS_IMAGE_URL:-}
TOBIKO_CIRROS_IMAGE_FILE=${TOBIKO_CIRROS_IMAGE_FILE:-}
TOBIKO_CIRROS_USERNAME=${TOBIKO_CIRROS_USERNAME:-}
TOBIKO_CIRROS_PASSWORD=${TOBIKO_CIRROS_PASSWORD:-}
# --- Keystone settings ---
# See ``lib/keystone`` where these users and tenants are set up
TOBIKO_KEYSTONE_USERNAME=${TOBIKO_KEYSTONE_USERNAME:-${ADMIN_USERNAME:-admin}}
@ -24,7 +34,6 @@ TOBIKO_KEYSTONE_TRUST_ID=${TOBIKO_KEYSTONE_TRUST_ID:-}
TOBIKO_KEYSTONE_USER_ROLE=${TOBIKO_KEYSTONE_USER_ROLE:-admin}
# --- Nova settings ---
TOBIKO_NOVA_IMAGE=${TOBIKO_NOVA_IMAGE:-${DEFAULT_IMAGE_NAME}}
TOBIKO_NOVA_FLAVOR=${TOBIKO_NOVA_FLAVOR:-${DEFAULT_INSTANCE_TYPE}}
TOBIKO_NOVA_KEY_FILE=${TOBIKO_NOVA_KEY_FILE:-~/.ssh/id_rsa}

8
doc/source/user/config.rst
View File

@ -238,12 +238,17 @@ set::
for Nova instances created by Tobiko::
wget http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img
openstack image create cirros \
openstack image create cirros-0.4.0 \
--file cirros-0.4.0-x86_64-disk.img \
--disk-format qcow2 \
--container-format bare \
--public
Add reference to above image into your :ref:`tobiko-conf` file::
[glance]
cirros_image = cirros-0.4.0
Create a flavor to be used with above image::
openstack flavor create --vcpus 1 --ram 64 --disk 1 m1.tiny
@ -255,7 +260,6 @@ Create an SSH key file to be used to ssh to Nova server instances::
Add reference to above resources into your :ref:`tobiko-conf` file::
[nova]
image = cirros
flavor = m1.tiny
key_file=~/.ssh/id_rsa

8
infrared/tasks/templates/tobiko.conf.j2
View File

@ -2,9 +2,13 @@
debug = true
log_file = tobiko.log
log_dir = .
[cirros]
image_name = cirros
[nova]
image = cirros
flavor = m1.tiny
key_file = ~/.ssh/id_rsa
[neutron]
floating_network = "{{ test.floating_network }}"
floating_network = "{{ test.floating_network }}"

9
tobiko/openstack/glance/__init__.py
View File

@ -13,13 +13,22 @@
# under the License.
from __future__ import absolute_import
from tobiko.openstack import _find
from tobiko.openstack.glance import _client
from tobiko.openstack.glance import _image
glance_client = _client.glance_client
get_glance_client = _client.get_glance_client
GlanceClientFixture = _client.GlanceClientFixture
GlanceImageNotFound = _client.GlanceImageNotFound
create_image = _client.create_image
get_image = _client.get_image
find_image = _client.find_image
list_images = _client.list_images
delete_image = _client.delete_image
ResourceNotFound = _find.ResourceNotFound
GlanceImageFixture = _image.GlanceImageFixture
FileGlanceImageFixture = _image.FileGlanceImageFixture
URLGlanceImageFixture = _image.URLGlanceImageFixture

65
tobiko/openstack/glance/_client.py
View File

@ -14,6 +14,7 @@
from __future__ import absolute_import
from glanceclient.v2 import client as glanceclient
from glanceclient import exc
import tobiko
from tobiko.openstack import _client
@ -35,36 +36,64 @@ class GlanceClientManager(_client.OpenstackClientManager):
CLIENTS = GlanceClientManager()
def glance_client(obj):
if not obj:
return get_glance_client()
def glance_client(obj=None):
obj = obj or default_glance_client()
if tobiko.is_fixture(obj):
obj = tobiko.setup_fixture(obj).client
return tobiko.check_valid_type(obj, glanceclient.Client)
if isinstance(obj, glanceclient.Client):
return obj
fixture = tobiko.setup_fixture(obj)
if isinstance(fixture, GlanceClientFixture):
return fixture.client
message = "Object {!r} is not a NovaClientFixture".format(obj)
raise TypeError(message)
def default_glance_client():
return get_glance_client()
def get_glance_client(session=None, shared=True, init_client=None,
manager=None):
manager = manager or CLIENTS
client = manager.get_client(session=session, shared=shared,
init_client=init_client)
tobiko.setup_fixture(client)
return client.client
fixture = manager.get_client(session=session, shared=shared,
init_client=init_client)
return glance_client(fixture)
def create_image(client=None, **params):
"""Look for the unique network matching some property values"""
return glance_client(client).images.create(**params)
def delete_image(image_id, client=None, **params):
try:
glance_client(client).images.delete(image_id, **params)
except exc.HTTPNotFound:
pass
def get_image(image_id, client=None):
try:
return glance_client(client).images.get(image_id=image_id)
except exc.HTTPNotFound as ex:
raise GlanceImageNotFound(cause=ex)
def find_image(obj=None, properties=None, client=None, **params):
"""Look for the unique network matching some property values"""
return _find.find_resource(
obj=obj, resource_type='image', properties=properties,
resources=list_images(client=client, **params), **params)
images = list_images(client=client, limit=1, **params)
for image in _find.find_resources(obj, images, properties=properties):
return image
raise GlanceImageNotFound(obj=obj, properties=properties, params=params)
def list_images(client=None, **params):
return list(glance_client(client).images.list(**params))
def upload_image(image_id, image_data, client=None, **params):
"""Look for the unique network matching some property values"""
return glance_client(client).images.upload(
image_id=image_id, image_data=image_data, **params)
class GlanceImageNotFound(_find.ResourceNotFound):
message = ("No such image found for obj={obj!r}, "
"properties={properties!r} and params={params!r}")

284
tobiko/openstack/glance/_image.py
View File

@ -13,47 +13,295 @@
# under the License.
from __future__ import absolute_import
import io
import os
import tempfile
import time
from oslo_log import log
import requests
import tobiko
from tobiko.openstack.glance import _client
from tobiko.openstack import _find
LOG = log.getLogger(__name__)
class GlanceImageStatus(object):
#: The Image service reserved an image ID for the image in the catalog but
# did not yet upload any image data.
QUEUED = 'queued'
#: The Image service is in the process of saving the raw data for the
# image into the backing store.
SAVING = 'saving'
#: The image is active and ready for consumption in the Image service.
ACTIVE = 'active'
#: An image data upload error occurred.
KILLED = 'killed'
#: The Image service retains information about the image but the image is
# no longer available for use.
DELETED = 'deleted'
#: Similar to the deleted status. An image in this state is not
# recoverable.
PENDING_DELETE = 'pending_delete'
#: The image data is not available for use.
DEACTIVATE = 'deactivated'
#: Data has been staged as part of the interoperable image import process.
# It is not yet available for use. (Since Image API 2.6)
UPLOADING = 'uploading'
#: The image data is being processed as part of the interoperable image
# import process, but is not yet available for use. (Since Image API 2.6)
IMPORTING = 'importing'
class GlanceImageFixture(tobiko.SharedFixture):
client = None
image = None
image_details = None
image_name = None
username = None
password = None
_image = None
sleep_interval = 1.
def __init__(self, client=None, image=None):
def __init__(self, image_name=None, username=None, password=None,
client=None):
super(GlanceImageFixture, self).__init__()
if client:
self.client = client
if image:
self.image = image
elif not self.image:
self.image = self.fixture_name
if image_name:
self.image_name = image_name
elif not self.image_name:
self.image_name = self.fixture_name
tobiko.check_valid_type(self.image_name, str)
if username:
self.username = username
if password:
self.password = password
def setup_fixture(self):
self.setup_client()
self.setup_image()
def cleanup_fixture(self):
self.delete_image()
def setup_client(self):
self.client = _client.glance_client(self.client)
def setup_image(self):
try:
self.image_details = _client.find_image(self.image,
client=self.client)
except _find.ResourceNotFound:
self.image_details = self.create_image()
return self.wait_for_image_active()
def create_image(self):
raise NotImplementedError
def wait_for_image_active(self):
image = self.get_image()
while GlanceImageStatus.ACTIVE != image.status:
check_image_status(image, {GlanceImageStatus.QUEUED,
GlanceImageStatus.SAVING})
LOG.debug('Waiting for image %r to change from %r to %r...',
self.image_name, image.status, GlanceImageStatus.ACTIVE)
time.sleep(self.sleep_interval)
image = self.get_image()
@property
def image(self):
return self._image or self.get_image()
def get_image(self, **kwargs):
self._image = image = _client.find_image(
self.image_name, client=self.client, **kwargs)
LOG.debug('Got image %r: %r', self.image_name, image)
return image
def delete_image(self, image_id=None):
try:
if not image_id:
image_id = self.image_id
self._image = None
_client.delete_image(image_id=image_id, client=self.client)
except _client.GlanceImageNotFound:
LOG.debug('Image %r not deleted because not found',
image_id or self.image_name)
return None
else:
LOG.debug("Deleted image %r: %r", self.image_name, image_id)
@property
def image_id(self):
return self.image_details['id']
return self.image.id
@property
def image_name(self):
return self.image_details['name']
def image_status(self):
return self.image.status
class UploadGranceImageFixture(GlanceImageFixture):
disk_format = "raw"
container_format = "bare"
def __init__(self, disk_format=None, container_format=None, **kwargs):
super(UploadGranceImageFixture, self).__init__(**kwargs)
if container_format:
self.container_format = disk_format
tobiko.check_valid_type(self.container_format, str)
if disk_format:
self.disk_format = disk_format
tobiko.check_valid_type(self.disk_format, str)
def setup_image(self):
try:
return self.wait_for_image_active()
except _client.GlanceImageNotFound:
pass
except InvalidGlanceImageStatus as ex:
self.delete_image(image_id=ex.image_id)
new_image = self.create_image()
image = self.get_image()
if image['id'] != new_image['id']:
self.delete_image(image_id=new_image['id'])
else:
check_image_status(image, {GlanceImageStatus.QUEUED})
self.upload_image()
return self.wait_for_image_active()
def create_image(self):
image = _client.create_image(client=self.client,
name=self.image_name,
disk_format=self.disk_format,
container_format=self.container_format)
LOG.debug("Created image %r: %r", self.image_name, image)
return image
def upload_image(self):
image_data, image_size = self.get_image_data()
with image_data:
_client.upload_image(image_id=self.image_id,
image_data=image_data,
image_size=image_size)
LOG.debug("Image uploaded %r", self.image_name)
def get_image_data(self):
raise NotImplementedError
class FileGlanceImageFixture(UploadGranceImageFixture):
image_file = None
image_dir = None
def __init__(self, image_file=None, image_dir=None, **kwargs):
super(FileGlanceImageFixture, self).__init__(**kwargs)
if image_file:
self.image_file = image_file
elif not self.image_file:
self.image_file = self.fixture_name
tobiko.check_valid_type(self.image_file, str)
if image_dir:
self.image_dir = image_dir
elif not self.image_dir:
from tobiko import config
CONF = config.CONF
self.image_dir = CONF.tobiko.glance.image_dir or "."
tobiko.check_valid_type(self.image_dir, str)
@property
def real_image_dir(self):
return os.path.realpath(os.path.expanduser(self.image_dir))
@property
def real_image_file(self):
return os.path.join(self.real_image_dir, self.image_file)
def get_image_data(self):
image_file = self.real_image_file
image_size = os.path.getsize(image_file)
image_data = io.open(image_file, 'rb')
LOG.debug('Reading image %r data from file %r (%d bytes)',
self.image_name, image_file, image_size)
return image_data, image_size
class URLGlanceImageFixture(FileGlanceImageFixture):
image_url = None
def __init__(self, image_url=None, **kwargs):
super(URLGlanceImageFixture, self).__init__(**kwargs)
if image_url:
self.image_url = image_url
else:
image_url = self.image_url
tobiko.check_valid_type(image_url, str)
def get_image_data(self):
http_request = requests.get(self.image_url, stream=True)
expected_size = int(http_request.headers.get('content-length', 0))
image_file = self.real_image_file
chunks = http_request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE)
try:
if expected_size:
actual_size = os.path.getsize(image_file)
if actual_size == expected_size:
LOG.debug("Cached image %r file %r found (%d bytes)",
self.image_name, image_file, actual_size)
return super(URLGlanceImageFixture, self).get_image_data()
except Exception as ex:
LOG.debug("Unable to get image %r file %r size: %s",
self.image_name, image_file, ex)
LOG.debug('Downloading image %r from URL %r to file %r (%d bytes)',
self.image_name, self.image_url, image_file,
expected_size)
image_dir = os.path.dirname(image_file)
if not os.path.isdir(image_dir):
LOG.debug('Creating image directory: %r', image_dir)
os.makedirs(image_dir)
fd, temp_file = tempfile.mkstemp(dir=image_dir)
with io.open(fd, 'wb', io.DEFAULT_BUFFER_SIZE) as image_data:
for chunk in chunks:
image_data.write(chunk)
actual_size = os.path.getsize(temp_file)
LOG.debug('Downloaded image %r from URL %r to file %r (%d bytes)',
self.image_name, self.image_url, image_file,
actual_size)
if expected_size and actual_size != expected_size:
message = "Download file size mismatch: {!s} != {!r}".format(
expected_size, actual_size)
raise RuntimeError(message)
os.rename(temp_file, image_file)
return super(URLGlanceImageFixture, self).get_image_data()
def check_image_status(image, expected_status):
if image.status not in expected_status:
raise InvalidGlanceImageStatus(image_name=image.name,
image_id=image.id,
actual_status=image.status,
expected_status=expected_status)
class InvalidGlanceImageStatus(tobiko.TobikoException):
message = ("Invalid image {image_name!r} (id {image_id!r}) status: "
"{actual_status!r} not in {expected_status!r}")

22
tobiko/openstack/glance/config.py
View File

@ -16,7 +16,27 @@ from __future__ import absolute_import
from oslo_config import cfg
CIRROS_IMAGE_URL = \
'http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img'
def register_tobiko_options(conf):
conf.register_opts(
group=cfg.OptGroup('glance'),
opts=[])
opts=[cfg.StrOpt('image_dir',
default='~/.tobiko/cache/glance/images',
help=("Default directory where to look for image "
"files")), ])
conf.register_opts(
group=cfg.OptGroup('cirros'),
opts=[cfg.StrOpt('image_name',
help="Default CirrOS image name"),
cfg.StrOpt('image_url',
help="Default CirrOS image URL"),
cfg.StrOpt('image_file',
help="Default CirrOS image filename"),
cfg.StrOpt('username',
help="Default CirrOS username"),
cfg.StrOpt('password',
help="Default CirrOS password"), ])

2
tobiko/openstack/images/__init__.py
View File

@ -16,4 +16,4 @@ from __future__ import absolute_import
from tobiko.openstack.images import _cirros
CirrosImageFixture = _cirros.CirrosImageFixture
CirrosGlanceImageFixture = _cirros.CirrosGlanceImageFixture

25
tobiko/openstack/images/_cirros.py
View File

@ -13,29 +13,20 @@
# under the License.
from __future__ import absolute_import
from tobiko import config
from tobiko.openstack import glance
CONF = config.CONF
class CirrosImageFixture(glance.GlanceImageFixture):
CIRROS_IMAGE_URL = \
'http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img'
@property
def image(self):
"""glance image used to create a Nova server instance"""
return CONF.tobiko.nova.image
@property
def username(self):
"""username used to login to a Nova server instance"""
return CONF.tobiko.nova.username
class CirrosGlanceImageFixture(glance.URLGlanceImageFixture):
@property
def password(self):
"""password used to login to a Nova server instance"""
return CONF.tobiko.nova.password
def create_image(self):
raise NotImplementedError
image_url = CIRROS_IMAGE_URL
image_name = CONF.tobiko.cirros.image_name
image_file = CONF.tobiko.cirros.image_file
username = CONF.tobiko.cirros.username or 'cirros'
password = CONF.tobiko.cirros.password or 'gocubsgo'

8
tobiko/openstack/nova/config.py
View File

@ -19,14 +19,8 @@ from oslo_config import cfg
def register_tobiko_options(conf):
conf.register_opts(
group=cfg.OptGroup('nova'),
opts=[cfg.StrOpt('image',
help="Default image for new server instances"),
cfg.StrOpt('flavor',
opts=[cfg.StrOpt('flavor',
help="Default flavor for new server instances"),
cfg.StrOpt('key_file', default='~/.ssh/id_rsa',
help="Default SSH key to login to server instances"),
cfg.StrOpt('username', default='cirros',
help="Default username to login to server instances"),
cfg.StrOpt('password', default='gocubsgo',
help="Default password to login to server instances"),
])

3
tobiko/openstack/stacks/_neutron.py
View File

@ -169,7 +169,8 @@ class FloatingIpServerStackFixture(heat.HeatStackFixture):
network_stack = tobiko.required_setup_fixture(NetworkStackFixture)
#: Glance image used to create a Nova server instance
image_fixture = tobiko.required_setup_fixture(images.CirrosImageFixture)
image_fixture = tobiko.required_setup_fixture(
images.CirrosGlanceImageFixture)
@property
def image(self):

8
tobiko/tests/functional/openstack/test_glance.py
View File

@ -23,10 +23,14 @@ from tobiko.openstack import images
class GlanceApiTestCase(testtools.TestCase):
"""Tests network creation"""
"""Tests glance images API"""
#: Stack of resources with a network with a gateway router
fixture = tobiko.required_setup_fixture(images.CirrosImageFixture)
fixture = tobiko.required_setup_fixture(images.CirrosGlanceImageFixture)
def test_get_image(self):
image = glance.get_image(self.fixture.image_id)
self.assertEqual(self.fixture.image_id, image['id'])
def test_find_image_with_id(self):
image = glance.find_image(self.fixture.image_id)