x/tobiko
Code Issues Proposed changes

Allow to disable image customizations

Browse Source 
Tobiko currently customizes Ubuntu images during test execution, using
the `virt-customize` command for this.
With this patch, this behavior in changed:
- classes FileGlanceImageFixture and URLGlanceImageFixture are joint; if
  a local file path is provided and the file exists, the image is not
  downloaded from the URL
- when the new config option ubuntu.customized_image_provided is set to
  True, the Ubuntu image is not customized (it is expected to provide an
  already customized image from a local path or URL)

Related: #TOBIKO-126

Change-Id: Iac3b42fd4965361e22e23c2f24ee510f63b5b30c
This commit is contained in:
Eduardo Olivares 2024-11-28 16:41:32 +01:00
parent fc4a3943ba
commit 24b9dc019b
7 changed files with 66 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
roles/tobiko-configure/tasks/main.yaml
View File

@ -23,8 +23,8 @@
{% endfor %}
{% for file_name, dict_value in download_images.items() %}
- section: "{{ dict_value.type }}"
option: image_file
value: "{{ download_images_dir }}/{{ file_name }}"
option: image_url
value: "file://{{ download_images_dir }}/{{ file_name }}"
{% endfor %}
vars:
sections: "{{ test_default_conf | combine(test_conf, recursive=True) }}"

3
tobiko/openstack/glance/__init__.py
View File

@ -28,10 +28,9 @@ find_image = _client.find_image
list_images = _client.list_images
delete_image = _client.delete_image
FileGlanceImageFixture = _image.FileGlanceImageFixture
GlanceImageFixture = _image.GlanceImageFixture
HasImageMixin = _image.HasImageMixin
URLGlanceImageFixture = _image.URLGlanceImageFixture
UrlGlanceImageFixture = _image.UrlGlanceImageFixture
CustomizedGlanceImageFixture = _image.CustomizedGlanceImageFixture
open_image_file = _io.open_image_file

85
tobiko/openstack/glance/_image.py
View File

@ -19,6 +19,8 @@ import os
import tempfile
import time
import typing # noqa
from abc import ABC, abstractmethod
from urllib.parse import urlparse
from oslo_log import log
import requests
@ -82,7 +84,8 @@ class GlanceImageStatus(object):
@keystone.skip_unless_has_keystone_credentials()
class GlanceImageFixture(_client.HasGlanceClientMixin, tobiko.SharedFixture):
class GlanceImageFixture(
_client.HasGlanceClientMixin, tobiko.SharedFixture, ABC):
image_name: typing.Optional[str] = None
username: typing.Optional[str] = None
@ -185,7 +188,7 @@ class GlanceImageFixture(_client.HasGlanceClientMixin, tobiko.SharedFixture):
expected_status=expected_status)
class UploadGlanceImageFixture(GlanceImageFixture):
class UploadGlanceImageFixture(GlanceImageFixture, ABC):
disk_format = "raw"
container_format = "bare"
@ -296,20 +299,27 @@ class UploadGlanceImageFixture(GlanceImageFixture):
raise NotImplementedError
class FileGlanceImageFixture(UploadGlanceImageFixture):
class UrlGlanceImageFixture(UploadGlanceImageFixture, ABC):
image_file = None
image_dir = None
compression_type = None
image_url: str = ''
image_dir: str = ''
image_file: str = ''
compression_type: typing.Optional[str] = None
def __init__(self, image_file=None, image_dir=None, **kwargs):
super(FileGlanceImageFixture, self).__init__(**kwargs)
if image_file:
self.image_file = image_file
elif not self.image_file:
self.image_file = self.fixture_name
tobiko.check_valid_type(self.image_file, str)
def __init__(self,
image_url: str = None,
image_dir: str = None,
**kwargs):
super().__init__(**kwargs)
if image_url:
self.image_url = image_url
tobiko.check_valid_type(self.image_url, str)
# self.image_url has to be a URL - if it refers to a local file, it
# should start with file://
url = urlparse(self.image_url)
if not self.image_file:
self.image_file = (url.path if url.scheme == 'file'
else os.path.basename(url.path))
if image_dir:
self.image_dir = image_dir
@ -330,9 +340,17 @@ class FileGlanceImageFixture(UploadGlanceImageFixture):
return os.path.join(self.real_image_dir, self.image_file)
def get_image_data(self):
return self.get_image_file(image_file=self.real_image_file)
real_image_file = self.real_image_file
# if the file exists, then skip the download part
if os.path.exists(real_image_file):
return self.get_image_from_file(real_image_file)
# else, download the image
return self.get_image_from_url(real_image_file)
def get_image_file(self, image_file: str):
def customize_image_file(self, base_file: str) -> str:
return base_file
def get_image_from_file(self, image_file: str):
image_file = self.customize_image_file(base_file=image_file)
image_size = os.path.getsize(image_file)
LOG.debug('Uploading image %r data from file %r (%d bytes)',
@ -342,25 +360,7 @@ class FileGlanceImageFixture(UploadGlanceImageFixture):
compression_type=self.compression_type)
return image_data, image_size
def customize_image_file(self, base_file: str) -> str:
return base_file
class URLGlanceImageFixture(FileGlanceImageFixture):
image_url: str
def __init__(self,
image_url: str = None,
**kwargs):
super().__init__(**kwargs)
if image_url is None:
image_url = self.image_url
else:
self.image_url = image_url
tobiko.check_valid_type(image_url, str)
def get_image_file(self, image_file: str):
def get_image_from_url(self, image_file: str):
# pylint: disable=missing-timeout
http_request = requests.get(self.image_url, stream=True)
expected_size = int(http_request.headers.get('content-length', 0))
@ -394,8 +394,7 @@ class URLGlanceImageFixture(FileGlanceImageFixture):
self._download_image_file(image_file=image_file,
chunks=chunks,
expected_size=expected_size)
return super(URLGlanceImageFixture, self).get_image_file(
image_file=image_file)
return self.get_image_from_file(image_file=image_file)
def _download_image_file(self, image_file, chunks, expected_size):
image_dir = os.path.dirname(image_file)
@ -419,7 +418,7 @@ class URLGlanceImageFixture(FileGlanceImageFixture):
os.rename(temp_file, image_file)
class CustomizedGlanceImageFixture(FileGlanceImageFixture):
class CustomizedGlanceImageFixture(UrlGlanceImageFixture, ABC):
@property
def firstboot_commands(self) -> typing.List[str]:
@ -437,6 +436,11 @@ class CustomizedGlanceImageFixture(FileGlanceImageFixture):
def write_files(self) -> typing.Dict[str, str]:
return {}
@property
@abstractmethod
def customization_required(self) -> bool:
pass
username: str = ''
password: str = ''
@ -444,7 +448,6 @@ class CustomizedGlanceImageFixture(FileGlanceImageFixture):
return 'customized'
def customize_image_file(self, base_file: str) -> str:
def workaround_passt(full_command, exc):
which_passt = sh.execute(
'which passt', expect_exit_status=None).stdout.rstrip()
@ -459,6 +462,10 @@ class CustomizedGlanceImageFixture(FileGlanceImageFixture):
sh.execute(cmd, sudo=True)
sh.execute(full_command)
# if the image does not have to be customized, then do nothing
if not self.customization_required:
return base_file
customized_file = f'{base_file}-{self._get_customized_suffix()}'
if os.path.isfile(customized_file):
if (os.stat(base_file).st_mtime_ns <

4
tobiko/openstack/glance/config.py
View File

@ -70,6 +70,10 @@ def get_images_options():
help=("Allow to disable SSH auth algorithms"
"in order to SSH to old servers like"
"CirrOS ones")),
cfg.BoolOpt('customized_image_provided',
default=False,
help=("Whether the provided image (URL or file) is"
"already customized or not"))
]
)]

3
tobiko/openstack/stacks/_cirros.py
View File

@ -40,11 +40,10 @@ CIRROS_IMAGE_URL = (
version=CIRROS_IMAGE_VERSION)
class CirrosImageFixture(glance.URLGlanceImageFixture):
class CirrosImageFixture(glance.UrlGlanceImageFixture):
image_url = CONF.tobiko.cirros.image_url or CIRROS_IMAGE_URL
image_name = CONF.tobiko.cirros.image_name
image_file = CONF.tobiko.cirros.image_file
container_format = CONF.tobiko.cirros.container_format or "bare"
disk_format = CONF.tobiko.cirros.disk_format or "qcow2"
username = CONF.tobiko.cirros.username or 'cirros'

13
tobiko/openstack/stacks/_nova.py
View File

@ -15,7 +15,6 @@
# under the License.
from __future__ import absolute_import
import abc
import typing
from abc import ABC
@ -77,7 +76,7 @@ class FlavorStackFixture(heat.HeatStackFixture):
@neutron.skip_if_missing_networking_extensions('port-security')
class ServerStackFixture(heat.HeatStackFixture, abc.ABC):
class ServerStackFixture(heat.HeatStackFixture, ABC):
#: Heat template file
template = _hot.heat_template_file('nova/server.yaml')
@ -481,7 +480,7 @@ class CloudInitServerStackFixture(ServerStackFixture, ABC):
**params)
class ExternalServerStackFixture(ServerStackFixture, abc.ABC):
class ExternalServerStackFixture(ServerStackFixture, ABC):
# pylint: disable=abstract-method
#: stack with the network where the server port is created
@ -505,7 +504,7 @@ class ExternalServerStackFixture(ServerStackFixture, abc.ABC):
return self.network_stack.network_id
class PeerServerStackFixture(ServerStackFixture, abc.ABC):
class PeerServerStackFixture(ServerStackFixture, ABC):
"""Server witch networking access requires passing by another Nova server
"""
@ -537,7 +536,7 @@ class PeerServerStackFixture(ServerStackFixture, abc.ABC):
@nova.skip_if_missing_hypervisors(count=2, state='up', status='enabled')
class DifferentHostServerStackFixture(PeerServerStackFixture, abc.ABC):
class DifferentHostServerStackFixture(PeerServerStackFixture, ABC):
# pylint: disable=abstract-method
@property
@ -545,7 +544,7 @@ class DifferentHostServerStackFixture(PeerServerStackFixture, abc.ABC):
return [self.peer_stack.server_id]
class SameHostServerStackFixture(PeerServerStackFixture, abc.ABC):
class SameHostServerStackFixture(PeerServerStackFixture, ABC):
@property
def same_host(self):
@ -559,7 +558,7 @@ def as_str(text):
return text.decode()
class HttpServerStackFixture(PeerServerStackFixture, abc.ABC):
class HttpServerStackFixture(PeerServerStackFixture, ABC):
http_server_port = 80

7
tobiko/openstack/stacks/_ubuntu.py
View File

@ -58,8 +58,9 @@ class UbuntuImageFixture(glance.CustomizedGlanceImageFixture):
- nginx HTTP server listening on TCP port 80
- iperf3 server listening on TCP port 5201
"""
image_name = CONF.tobiko.ubuntu.image_name
image_url = CONF.tobiko.ubuntu.image_url
image_file = CONF.tobiko.ubuntu.image_file
image_name = CONF.tobiko.ubuntu.image_name
disk_format = CONF.tobiko.ubuntu.disk_format or "qcow2"
container_format = CONF.tobiko.ubuntu.container_format or "bare"
username = CONF.tobiko.ubuntu.username or 'ubuntu'
@ -74,6 +75,10 @@ class UbuntuImageFixture(glance.CustomizedGlanceImageFixture):
super().__init__(**kwargs)
self._ethernet_device = ethernet_devide
@property
def customization_required(self) -> bool:
return not CONF.tobiko.ubuntu.customized_image_provided
@property
def firstboot_commands(self) -> typing.List[str]:
return super().firstboot_commands + [