x/tobiko
Code Issues Proposed changes

Add support for XZ image decompression

Browse Source 
- Delete created images if it is not capable of making it active
  before leaving create_image method.

Change-Id: I30453a7e9074b1ac880f2aa2c41c7462d10c836c
This commit is contained in:
Federico Ressi 2019-07-15 12:48:12 +02:00
parent b62a2a1122
commit 5acbc53823
4 changed files with 115 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
tobiko/openstack/glance/__init__.py
View File

@ -13,9 +13,11 @@
# under the License.
from __future__ import absolute_import
from tobiko.openstack import _find
from tobiko.openstack.glance import _client
from tobiko.openstack.glance import _image
from tobiko.openstack.glance import _io
from tobiko.openstack.glance import _lzma
glance_client = _client.glance_client
get_glance_client = _client.get_glance_client
@ -26,8 +28,10 @@ find_image = _client.find_image
list_images = _client.list_images
delete_image = _client.delete_image
ResourceNotFound = _find.ResourceNotFound
GlanceImageFixture = _image.GlanceImageFixture
FileGlanceImageFixture = _image.FileGlanceImageFixture
URLGlanceImageFixture = _image.URLGlanceImageFixture
open_image_file = _io.open_image_file
has_lzma = _lzma.has_lzma

50
tobiko/openstack/glance/_image.py
View File

@ -235,12 +235,17 @@ class UploadGranceImageFixture(GlanceImageFixture):
LOG.debug('Image created: %r (id=%r)',
self.image_name, image.id)
self.upload_image()
cleanup_image_ids.remove(image.id)
else:
LOG.debug('Existing image found: %r (id=%r)',
self.image_name, image.id)
self.wait_for_image_active()
image = self.wait_for_image_active()
if image and image.id in cleanup_image_ids:
# Having an active image we can remove image created from this
# process from cleanup image list so that it is not going to
# be deleted
cleanup_image_ids.remove(image.id)
return image
@contextlib.contextmanager
@ -252,7 +257,11 @@ class UploadGranceImageFixture(GlanceImageFixture):
for image_id in created_image_ids:
LOG.warning("Delete duplicate image %r (id=%r).",
self.image_name, image_id)
self.delete_image(image_id)
try:
self.delete_image(image_id)
except Exception:
LOG.exception('Error deleting image %r (%r)',
self.image_name, image_id)
def upload_image(self):
self.check_image_status(self.image, {GlanceImageStatus.QUEUED})
@ -303,9 +312,9 @@ class FileGlanceImageFixture(UploadGranceImageFixture):
image_size = os.path.getsize(image_file)
LOG.debug('Uploading image %r data from file %r (%d bytes)',
self.image_name, image_file, image_size)
image_data = _io.open_file(filename=image_file,
mode='rb',
compression_type=self.compression_type)
image_data = _io.open_image_file(
filename=image_file, mode='rb',
compression_type=self.compression_type)
return image_data, image_size
@ -326,22 +335,29 @@ class URLGlanceImageFixture(FileGlanceImageFixture):
expected_size = int(http_request.headers.get('content-length', 0))
image_file = self.real_image_file
chunks = http_request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE)
try:
if expected_size:
download_image = True
if expected_size:
try:
actual_size = os.path.getsize(image_file)
except Exception as ex:
LOG.debug("Unable to get image %r data from file %r: %s",
self.image_name, image_file, ex)
else:
if actual_size == expected_size:
LOG.debug("Cached image %r file %r found (%d bytes)",
self.image_name, image_file, actual_size)
return super(URLGlanceImageFixture, self).get_image_data()
download_image = False
except Exception as ex:
LOG.debug("Unable to get image %r file %r size: %s",
self.image_name, image_file, ex)
LOG.debug('Downloading image %r from URL %r to file %r (%d bytes)',
self.image_name, self.image_url, image_file,
expected_size)
if download_image:
LOG.debug("Downloading image %r from URL %r to file %r "
"(%d bytes)", self.image_name, self.image_url,
image_file, expected_size)
self._download_image_file(image_file=image_file,
chunks=chunks,
expected_size=expected_size)
return super(URLGlanceImageFixture, self).get_image_data()
def _download_image_file(self, image_file, chunks, expected_size):
image_dir = os.path.dirname(image_file)
if not os.path.isdir(image_dir):
LOG.debug('Creating image directory: %r', image_dir)
@ -363,8 +379,6 @@ class URLGlanceImageFixture(FileGlanceImageFixture):
raise RuntimeError(message)
os.rename(temp_file, image_file)
return super(URLGlanceImageFixture, self).get_image_data()
class InvalidGlanceImageStatus(tobiko.TobikoException):
message = ("Invalid image {image_name!r} (id {image_id!r}) status: "

41
tobiko/openstack/glance/_io.py
View File

@ -20,40 +20,54 @@ import zipfile
from oslo_log import log
from tobiko.openstack.glance import _lzma
LOG = log.getLogger(__name__)
COMPRESSED_FILE_TYPES = {}
def comressed_file_type(cls):
COMPRESSED_FILE_TYPES[cls.compression_type] = cls
return cls
@comressed_file_type
class BZ2FileType(object):
file_magic = b'\x42\x5a\x68'
compression_type = 'bz2'
open = bz2.BZ2File
open_file = bz2.BZ2File
@comressed_file_type
class GzipFileType(gzip.GzipFile):
file_magic = b'\x1f\x8b\x08'
compression_type = 'gz'
open = gzip.GzipFile
open_file = gzip.GzipFile
@comressed_file_type
class XzFileType(object):
file_magic = b'\xfd7zXZ\x00'
compression_type = 'xz'
open_file = staticmethod(_lzma.open_file)
@comressed_file_type
class ZipFileType(object):
file_magic = b'\x50\x4b\x03\x04'
compression_type = 'zip'
open = zipfile.ZipFile
open_file = zipfile.ZipFile
COMPRESSION_FILE_TYPES = {'bz2': BZ2FileType,
'gz': GzipFileType,
'zip': ZipFileType}
def open_file(filename, mode, compression_type=None):
def open_image_file(filename, mode, compression_type=None):
if compression_type is None:
max_magic_len = max(len(cls.file_magic)
for cls in COMPRESSION_FILE_TYPES.values())
for cls in COMPRESSED_FILE_TYPES.values())
with io.open(filename, 'rb') as f:
magic = f.read(max_magic_len)
for cls in COMPRESSION_FILE_TYPES.values():
for cls in COMPRESSED_FILE_TYPES.values():
if magic.startswith(cls.file_magic):
compression_type = cls.compression_type
LOG.debug("Compression type %r of file %r got from file magic",
@ -63,10 +77,9 @@ def open_file(filename, mode, compression_type=None):
if compression_type:
LOG.debug("Open compressed file %r (mode=%r, compression_type=%r)",
filename, mode, compression_type)
open_func = COMPRESSION_FILE_TYPES[compression_type].open
open_func = COMPRESSED_FILE_TYPES[compression_type].open_file
else:
LOG.debug("Open flat file %r (mode=%r)",
filename, mode)
LOG.debug("Open flat file %r (mode=%r)", filename, mode)
open_func = io.open
return open_func(filename, mode)

49
tobiko/openstack/glance/_lzma.py Normal file
View File

@ -0,0 +1,49 @@
# Copyright 2019 Red Hat
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
import sys
import tobiko
def import_lzma():
try:
import lzma
except ImportError:
from backports import lzma
return lzma
def has_lzma():
try:
return import_lzma()
except ImportError:
return None
def open_file(filename, mode):
try:
lzma = import_lzma()
except ImportError:
tobiko.skip("Package lzma or backports.lzma is required to decompress "
"{filename!r} (mode={mode!r}) XZ image file "
"({python_version!r}).",
filename=filename,
mode=mode,
python_version=sys.version)
return lzma.LZMAFile(filename=filename, mode=mode)