Add support for XZ image decompression

- Delete created images if it is not capable of making it active
  before leaving create_image method.

Change-Id: I30453a7e9074b1ac880f2aa2c41c7462d10c836c

tobiko/openstack/glance/__init__.py

@@ -13,9 +13,11 @@
 #    under the License.
 from __future__ import absolute_import
 
-from tobiko.openstack import _find
 from tobiko.openstack.glance import _client
 from tobiko.openstack.glance import _image
+from tobiko.openstack.glance import _io
+from tobiko.openstack.glance import _lzma
+
 
 glance_client = _client.glance_client
 get_glance_client = _client.get_glance_client
@@ -26,8 +28,10 @@ find_image = _client.find_image
 list_images = _client.list_images
 delete_image = _client.delete_image
 
-ResourceNotFound = _find.ResourceNotFound
-
 GlanceImageFixture = _image.GlanceImageFixture
 FileGlanceImageFixture = _image.FileGlanceImageFixture
 URLGlanceImageFixture = _image.URLGlanceImageFixture
+
+open_image_file = _io.open_image_file
+
+has_lzma = _lzma.has_lzma

tobiko/openstack/glance/_image.py

@@ -235,12 +235,17 @@ class UploadGranceImageFixture(GlanceImageFixture):
                     LOG.debug('Image created: %r (id=%r)',
                               self.image_name, image.id)
                     self.upload_image()
-                    cleanup_image_ids.remove(image.id)
                 else:
                     LOG.debug('Existing image found: %r (id=%r)',
                               self.image_name, image.id)
 
-        self.wait_for_image_active()
+            image = self.wait_for_image_active()
+            if image and image.id in cleanup_image_ids:
+                # Having an active image we can remove image created from this
+                # process from cleanup image list so that it is not going to
+                # be deleted
+                cleanup_image_ids.remove(image.id)
+
         return image
 
     @contextlib.contextmanager
@@ -252,7 +257,11 @@ class UploadGranceImageFixture(GlanceImageFixture):
             for image_id in created_image_ids:
                 LOG.warning("Delete duplicate image %r (id=%r).",
                             self.image_name, image_id)
-                self.delete_image(image_id)
+                try:
+                    self.delete_image(image_id)
+                except Exception:
+                    LOG.exception('Error deleting image %r (%r)',
+                                  self.image_name, image_id)
 
     def upload_image(self):
         self.check_image_status(self.image, {GlanceImageStatus.QUEUED})
@@ -303,9 +312,9 @@ class FileGlanceImageFixture(UploadGranceImageFixture):
         image_size = os.path.getsize(image_file)
         LOG.debug('Uploading image %r data from file %r (%d bytes)',
                   self.image_name, image_file, image_size)
-        image_data = _io.open_file(filename=image_file,
+        image_data = _io.open_image_file(
-                                   mode='rb',
+            filename=image_file, mode='rb',
-                                   compression_type=self.compression_type)
+            compression_type=self.compression_type)
         return image_data, image_size
 
 
@@ -326,22 +335,29 @@ class URLGlanceImageFixture(FileGlanceImageFixture):
         expected_size = int(http_request.headers.get('content-length', 0))
         image_file = self.real_image_file
         chunks = http_request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE)
-        try:
+        download_image = True
-            if expected_size:
+        if expected_size:
+            try:
                 actual_size = os.path.getsize(image_file)
+            except Exception as ex:
+                LOG.debug("Unable to get image %r data from file %r: %s",
+                          self.image_name, image_file, ex)
+            else:
                 if actual_size == expected_size:
                     LOG.debug("Cached image %r file %r found (%d bytes)",
                               self.image_name, image_file, actual_size)
-                    return super(URLGlanceImageFixture, self).get_image_data()
+                    download_image = False
 
-        except Exception as ex:
+        if download_image:
-            LOG.debug("Unable to get image %r file %r size: %s",
+            LOG.debug("Downloading image %r from URL %r to file %r "
-                      self.image_name, image_file, ex)
+                      "(%d bytes)", self.image_name, self.image_url,
-
+                      image_file, expected_size)
-        LOG.debug('Downloading image %r from URL %r to file %r (%d bytes)',
+            self._download_image_file(image_file=image_file,
-                  self.image_name, self.image_url, image_file,
+                                      chunks=chunks,
-                  expected_size)
+                                      expected_size=expected_size)
+        return super(URLGlanceImageFixture, self).get_image_data()
 
+    def _download_image_file(self, image_file, chunks, expected_size):
         image_dir = os.path.dirname(image_file)
         if not os.path.isdir(image_dir):
             LOG.debug('Creating image directory: %r', image_dir)
@@ -363,8 +379,6 @@ class URLGlanceImageFixture(FileGlanceImageFixture):
             raise RuntimeError(message)
         os.rename(temp_file, image_file)
 
-        return super(URLGlanceImageFixture, self).get_image_data()
-
 
 class InvalidGlanceImageStatus(tobiko.TobikoException):
     message = ("Invalid image {image_name!r} (id {image_id!r}) status: "

tobiko/openstack/glance/_io.py

@@ -20,40 +20,54 @@ import zipfile
 
 from oslo_log import log
 
+from tobiko.openstack.glance import _lzma
+
 
 LOG = log.getLogger(__name__)
 
+COMPRESSED_FILE_TYPES = {}
 
+
+def comressed_file_type(cls):
+    COMPRESSED_FILE_TYPES[cls.compression_type] = cls
+    return cls
+
+
+@comressed_file_type
 class BZ2FileType(object):
     file_magic = b'\x42\x5a\x68'
     compression_type = 'bz2'
-    open = bz2.BZ2File
+    open_file = bz2.BZ2File
 
 
+@comressed_file_type
 class GzipFileType(gzip.GzipFile):
     file_magic = b'\x1f\x8b\x08'
     compression_type = 'gz'
-    open = gzip.GzipFile
+    open_file = gzip.GzipFile
 
 
+@comressed_file_type
+class XzFileType(object):
+    file_magic = b'\xfd7zXZ\x00'
+    compression_type = 'xz'
+    open_file = staticmethod(_lzma.open_file)
+
+
+@comressed_file_type
 class ZipFileType(object):
     file_magic = b'\x50\x4b\x03\x04'
     compression_type = 'zip'
-    open = zipfile.ZipFile
+    open_file = zipfile.ZipFile
 
 
-COMPRESSION_FILE_TYPES = {'bz2': BZ2FileType,
+def open_image_file(filename, mode, compression_type=None):
-                          'gz': GzipFileType,
-                          'zip': ZipFileType}
-
-
-def open_file(filename, mode, compression_type=None):
     if compression_type is None:
         max_magic_len = max(len(cls.file_magic)
-                            for cls in COMPRESSION_FILE_TYPES.values())
+                            for cls in COMPRESSED_FILE_TYPES.values())
         with io.open(filename, 'rb') as f:
             magic = f.read(max_magic_len)
-        for cls in COMPRESSION_FILE_TYPES.values():
+        for cls in COMPRESSED_FILE_TYPES.values():
             if magic.startswith(cls.file_magic):
                 compression_type = cls.compression_type
                 LOG.debug("Compression type %r of file %r got from file magic",
@@ -63,10 +77,9 @@ def open_file(filename, mode, compression_type=None):
     if compression_type:
         LOG.debug("Open compressed file %r (mode=%r, compression_type=%r)",
                   filename, mode, compression_type)
-        open_func = COMPRESSION_FILE_TYPES[compression_type].open
+        open_func = COMPRESSED_FILE_TYPES[compression_type].open_file
     else:
-        LOG.debug("Open flat file %r (mode=%r)",
+        LOG.debug("Open flat file %r (mode=%r)", filename, mode)
-                  filename, mode)
         open_func = io.open
 
     return open_func(filename, mode)

tobiko/openstack/glance/_lzma.py

@@ -0,0 +1,49 @@
+# Copyright 2019 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import absolute_import
+
+
+import sys
+
+
+import tobiko
+
+
+def import_lzma():
+    try:
+        import lzma
+    except ImportError:
+        from backports import lzma
+    return lzma
+
+
+def has_lzma():
+    try:
+        return import_lzma()
+    except ImportError:
+        return None
+
+
+def open_file(filename, mode):
+    try:
+        lzma = import_lzma()
+    except ImportError:
+        tobiko.skip("Package lzma or backports.lzma is required to decompress "
+                    "{filename!r} (mode={mode!r}) XZ image file "
+                    "({python_version!r}).",
+                    filename=filename,
+                    mode=mode,
+                    python_version=sys.version)
+
+    return lzma.LZMAFile(filename=filename, mode=mode)