Add support for XZ image decompression
- Delete created images if it is not capable of making it active before leaving create_image method. Change-Id: I30453a7e9074b1ac880f2aa2c41c7462d10c836c
This commit is contained in:
parent
b62a2a1122
commit
5acbc53823
@ -13,9 +13,11 @@
|
|||||||
# under the License.
|
# under the License.
|
||||||
from __future__ import absolute_import
|
from __future__ import absolute_import
|
||||||
|
|
||||||
from tobiko.openstack import _find
|
|
||||||
from tobiko.openstack.glance import _client
|
from tobiko.openstack.glance import _client
|
||||||
from tobiko.openstack.glance import _image
|
from tobiko.openstack.glance import _image
|
||||||
|
from tobiko.openstack.glance import _io
|
||||||
|
from tobiko.openstack.glance import _lzma
|
||||||
|
|
||||||
|
|
||||||
glance_client = _client.glance_client
|
glance_client = _client.glance_client
|
||||||
get_glance_client = _client.get_glance_client
|
get_glance_client = _client.get_glance_client
|
||||||
@ -26,8 +28,10 @@ find_image = _client.find_image
|
|||||||
list_images = _client.list_images
|
list_images = _client.list_images
|
||||||
delete_image = _client.delete_image
|
delete_image = _client.delete_image
|
||||||
|
|
||||||
ResourceNotFound = _find.ResourceNotFound
|
|
||||||
|
|
||||||
GlanceImageFixture = _image.GlanceImageFixture
|
GlanceImageFixture = _image.GlanceImageFixture
|
||||||
FileGlanceImageFixture = _image.FileGlanceImageFixture
|
FileGlanceImageFixture = _image.FileGlanceImageFixture
|
||||||
URLGlanceImageFixture = _image.URLGlanceImageFixture
|
URLGlanceImageFixture = _image.URLGlanceImageFixture
|
||||||
|
|
||||||
|
open_image_file = _io.open_image_file
|
||||||
|
|
||||||
|
has_lzma = _lzma.has_lzma
|
||||||
|
@ -235,12 +235,17 @@ class UploadGranceImageFixture(GlanceImageFixture):
|
|||||||
LOG.debug('Image created: %r (id=%r)',
|
LOG.debug('Image created: %r (id=%r)',
|
||||||
self.image_name, image.id)
|
self.image_name, image.id)
|
||||||
self.upload_image()
|
self.upload_image()
|
||||||
cleanup_image_ids.remove(image.id)
|
|
||||||
else:
|
else:
|
||||||
LOG.debug('Existing image found: %r (id=%r)',
|
LOG.debug('Existing image found: %r (id=%r)',
|
||||||
self.image_name, image.id)
|
self.image_name, image.id)
|
||||||
|
|
||||||
self.wait_for_image_active()
|
image = self.wait_for_image_active()
|
||||||
|
if image and image.id in cleanup_image_ids:
|
||||||
|
# Having an active image we can remove image created from this
|
||||||
|
# process from cleanup image list so that it is not going to
|
||||||
|
# be deleted
|
||||||
|
cleanup_image_ids.remove(image.id)
|
||||||
|
|
||||||
return image
|
return image
|
||||||
|
|
||||||
@contextlib.contextmanager
|
@contextlib.contextmanager
|
||||||
@ -252,7 +257,11 @@ class UploadGranceImageFixture(GlanceImageFixture):
|
|||||||
for image_id in created_image_ids:
|
for image_id in created_image_ids:
|
||||||
LOG.warning("Delete duplicate image %r (id=%r).",
|
LOG.warning("Delete duplicate image %r (id=%r).",
|
||||||
self.image_name, image_id)
|
self.image_name, image_id)
|
||||||
self.delete_image(image_id)
|
try:
|
||||||
|
self.delete_image(image_id)
|
||||||
|
except Exception:
|
||||||
|
LOG.exception('Error deleting image %r (%r)',
|
||||||
|
self.image_name, image_id)
|
||||||
|
|
||||||
def upload_image(self):
|
def upload_image(self):
|
||||||
self.check_image_status(self.image, {GlanceImageStatus.QUEUED})
|
self.check_image_status(self.image, {GlanceImageStatus.QUEUED})
|
||||||
@ -303,9 +312,9 @@ class FileGlanceImageFixture(UploadGranceImageFixture):
|
|||||||
image_size = os.path.getsize(image_file)
|
image_size = os.path.getsize(image_file)
|
||||||
LOG.debug('Uploading image %r data from file %r (%d bytes)',
|
LOG.debug('Uploading image %r data from file %r (%d bytes)',
|
||||||
self.image_name, image_file, image_size)
|
self.image_name, image_file, image_size)
|
||||||
image_data = _io.open_file(filename=image_file,
|
image_data = _io.open_image_file(
|
||||||
mode='rb',
|
filename=image_file, mode='rb',
|
||||||
compression_type=self.compression_type)
|
compression_type=self.compression_type)
|
||||||
return image_data, image_size
|
return image_data, image_size
|
||||||
|
|
||||||
|
|
||||||
@ -326,22 +335,29 @@ class URLGlanceImageFixture(FileGlanceImageFixture):
|
|||||||
expected_size = int(http_request.headers.get('content-length', 0))
|
expected_size = int(http_request.headers.get('content-length', 0))
|
||||||
image_file = self.real_image_file
|
image_file = self.real_image_file
|
||||||
chunks = http_request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE)
|
chunks = http_request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE)
|
||||||
try:
|
download_image = True
|
||||||
if expected_size:
|
if expected_size:
|
||||||
|
try:
|
||||||
actual_size = os.path.getsize(image_file)
|
actual_size = os.path.getsize(image_file)
|
||||||
|
except Exception as ex:
|
||||||
|
LOG.debug("Unable to get image %r data from file %r: %s",
|
||||||
|
self.image_name, image_file, ex)
|
||||||
|
else:
|
||||||
if actual_size == expected_size:
|
if actual_size == expected_size:
|
||||||
LOG.debug("Cached image %r file %r found (%d bytes)",
|
LOG.debug("Cached image %r file %r found (%d bytes)",
|
||||||
self.image_name, image_file, actual_size)
|
self.image_name, image_file, actual_size)
|
||||||
return super(URLGlanceImageFixture, self).get_image_data()
|
download_image = False
|
||||||
|
|
||||||
except Exception as ex:
|
if download_image:
|
||||||
LOG.debug("Unable to get image %r file %r size: %s",
|
LOG.debug("Downloading image %r from URL %r to file %r "
|
||||||
self.image_name, image_file, ex)
|
"(%d bytes)", self.image_name, self.image_url,
|
||||||
|
image_file, expected_size)
|
||||||
LOG.debug('Downloading image %r from URL %r to file %r (%d bytes)',
|
self._download_image_file(image_file=image_file,
|
||||||
self.image_name, self.image_url, image_file,
|
chunks=chunks,
|
||||||
expected_size)
|
expected_size=expected_size)
|
||||||
|
return super(URLGlanceImageFixture, self).get_image_data()
|
||||||
|
|
||||||
|
def _download_image_file(self, image_file, chunks, expected_size):
|
||||||
image_dir = os.path.dirname(image_file)
|
image_dir = os.path.dirname(image_file)
|
||||||
if not os.path.isdir(image_dir):
|
if not os.path.isdir(image_dir):
|
||||||
LOG.debug('Creating image directory: %r', image_dir)
|
LOG.debug('Creating image directory: %r', image_dir)
|
||||||
@ -363,8 +379,6 @@ class URLGlanceImageFixture(FileGlanceImageFixture):
|
|||||||
raise RuntimeError(message)
|
raise RuntimeError(message)
|
||||||
os.rename(temp_file, image_file)
|
os.rename(temp_file, image_file)
|
||||||
|
|
||||||
return super(URLGlanceImageFixture, self).get_image_data()
|
|
||||||
|
|
||||||
|
|
||||||
class InvalidGlanceImageStatus(tobiko.TobikoException):
|
class InvalidGlanceImageStatus(tobiko.TobikoException):
|
||||||
message = ("Invalid image {image_name!r} (id {image_id!r}) status: "
|
message = ("Invalid image {image_name!r} (id {image_id!r}) status: "
|
||||||
|
@ -20,40 +20,54 @@ import zipfile
|
|||||||
|
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
|
|
||||||
|
from tobiko.openstack.glance import _lzma
|
||||||
|
|
||||||
|
|
||||||
LOG = log.getLogger(__name__)
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
|
COMPRESSED_FILE_TYPES = {}
|
||||||
|
|
||||||
|
|
||||||
|
def comressed_file_type(cls):
|
||||||
|
COMPRESSED_FILE_TYPES[cls.compression_type] = cls
|
||||||
|
return cls
|
||||||
|
|
||||||
|
|
||||||
|
@comressed_file_type
|
||||||
class BZ2FileType(object):
|
class BZ2FileType(object):
|
||||||
file_magic = b'\x42\x5a\x68'
|
file_magic = b'\x42\x5a\x68'
|
||||||
compression_type = 'bz2'
|
compression_type = 'bz2'
|
||||||
open = bz2.BZ2File
|
open_file = bz2.BZ2File
|
||||||
|
|
||||||
|
|
||||||
|
@comressed_file_type
|
||||||
class GzipFileType(gzip.GzipFile):
|
class GzipFileType(gzip.GzipFile):
|
||||||
file_magic = b'\x1f\x8b\x08'
|
file_magic = b'\x1f\x8b\x08'
|
||||||
compression_type = 'gz'
|
compression_type = 'gz'
|
||||||
open = gzip.GzipFile
|
open_file = gzip.GzipFile
|
||||||
|
|
||||||
|
|
||||||
|
@comressed_file_type
|
||||||
|
class XzFileType(object):
|
||||||
|
file_magic = b'\xfd7zXZ\x00'
|
||||||
|
compression_type = 'xz'
|
||||||
|
open_file = staticmethod(_lzma.open_file)
|
||||||
|
|
||||||
|
|
||||||
|
@comressed_file_type
|
||||||
class ZipFileType(object):
|
class ZipFileType(object):
|
||||||
file_magic = b'\x50\x4b\x03\x04'
|
file_magic = b'\x50\x4b\x03\x04'
|
||||||
compression_type = 'zip'
|
compression_type = 'zip'
|
||||||
open = zipfile.ZipFile
|
open_file = zipfile.ZipFile
|
||||||
|
|
||||||
|
|
||||||
COMPRESSION_FILE_TYPES = {'bz2': BZ2FileType,
|
def open_image_file(filename, mode, compression_type=None):
|
||||||
'gz': GzipFileType,
|
|
||||||
'zip': ZipFileType}
|
|
||||||
|
|
||||||
|
|
||||||
def open_file(filename, mode, compression_type=None):
|
|
||||||
if compression_type is None:
|
if compression_type is None:
|
||||||
max_magic_len = max(len(cls.file_magic)
|
max_magic_len = max(len(cls.file_magic)
|
||||||
for cls in COMPRESSION_FILE_TYPES.values())
|
for cls in COMPRESSED_FILE_TYPES.values())
|
||||||
with io.open(filename, 'rb') as f:
|
with io.open(filename, 'rb') as f:
|
||||||
magic = f.read(max_magic_len)
|
magic = f.read(max_magic_len)
|
||||||
for cls in COMPRESSION_FILE_TYPES.values():
|
for cls in COMPRESSED_FILE_TYPES.values():
|
||||||
if magic.startswith(cls.file_magic):
|
if magic.startswith(cls.file_magic):
|
||||||
compression_type = cls.compression_type
|
compression_type = cls.compression_type
|
||||||
LOG.debug("Compression type %r of file %r got from file magic",
|
LOG.debug("Compression type %r of file %r got from file magic",
|
||||||
@ -63,10 +77,9 @@ def open_file(filename, mode, compression_type=None):
|
|||||||
if compression_type:
|
if compression_type:
|
||||||
LOG.debug("Open compressed file %r (mode=%r, compression_type=%r)",
|
LOG.debug("Open compressed file %r (mode=%r, compression_type=%r)",
|
||||||
filename, mode, compression_type)
|
filename, mode, compression_type)
|
||||||
open_func = COMPRESSION_FILE_TYPES[compression_type].open
|
open_func = COMPRESSED_FILE_TYPES[compression_type].open_file
|
||||||
else:
|
else:
|
||||||
LOG.debug("Open flat file %r (mode=%r)",
|
LOG.debug("Open flat file %r (mode=%r)", filename, mode)
|
||||||
filename, mode)
|
|
||||||
open_func = io.open
|
open_func = io.open
|
||||||
|
|
||||||
return open_func(filename, mode)
|
return open_func(filename, mode)
|
||||||
|
49
tobiko/openstack/glance/_lzma.py
Normal file
49
tobiko/openstack/glance/_lzma.py
Normal file
@ -0,0 +1,49 @@
|
|||||||
|
# Copyright 2019 Red Hat
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
from __future__ import absolute_import
|
||||||
|
|
||||||
|
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
import tobiko
|
||||||
|
|
||||||
|
|
||||||
|
def import_lzma():
|
||||||
|
try:
|
||||||
|
import lzma
|
||||||
|
except ImportError:
|
||||||
|
from backports import lzma
|
||||||
|
return lzma
|
||||||
|
|
||||||
|
|
||||||
|
def has_lzma():
|
||||||
|
try:
|
||||||
|
return import_lzma()
|
||||||
|
except ImportError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def open_file(filename, mode):
|
||||||
|
try:
|
||||||
|
lzma = import_lzma()
|
||||||
|
except ImportError:
|
||||||
|
tobiko.skip("Package lzma or backports.lzma is required to decompress "
|
||||||
|
"{filename!r} (mode={mode!r}) XZ image file "
|
||||||
|
"({python_version!r}).",
|
||||||
|
filename=filename,
|
||||||
|
mode=mode,
|
||||||
|
python_version=sys.version)
|
||||||
|
|
||||||
|
return lzma.LZMAFile(filename=filename, mode=mode)
|
Loading…
x
Reference in New Issue
Block a user