Merge "qemu: Make disk image conversion dramatically faster"

nova/privsep/qemu.py

@@ -35,23 +35,30 @@ def convert_image(source, dest, in_format, out_format, instances_path,
 # NOTE(mikal): this method is deliberately not wrapped in a privsep entrypoint
 def unprivileged_convert_image(source, dest, in_format, out_format,
                                instances_path, compress):
-    # NOTE(mdbooth): qemu-img convert defaults to cache=unsafe, which means
+    # NOTE(mdbooth, kchamart): `qemu-img convert` defaults to
-    # that data is not synced to disk at completion. We explicitly use
+    # 'cache=writeback' for the source image, and 'cache=unsafe' for the
-    # cache=none here to (1) ensure that we don't interfere with other
+    # target, which means that data is not synced to disk at completion.
-    # applications using the host's io cache, and (2) ensure that the data is
+    # We explicitly use 'cache=none' here, for the target image, to (1)
-    # on persistent storage when the command exits. Without (2), a host crash
+    # ensure that we don't interfere with other applications using the
-    # may leave a corrupt image in the image cache, which Nova cannot recover
+    # host's I/O cache, and (2) ensure that the data is on persistent
-    # automatically.
+    # storage when the command exits. Without (2), a host crash may
-    # NOTE(zigo): we cannot use -t none if the instances dir is mounted on a
+    # leave a corrupt image in the image cache, which Nova cannot
-    # filesystem that doesn't have support for O_DIRECT, which is the case
+    # recover automatically.
-    # for example with tmpfs. This simply crashes "openstack server create"
+
-    # in environments like live distributions. In such case, the best choice
+    # NOTE(zigo, kchamart): We cannot use `qemu-img convert -t none` if
-    # is writethrough, which is power-failure safe, but still faster than
+    # the 'instance_dir' is mounted on a filesystem that doesn't support
-    # writeback.
+    # O_DIRECT, which is the case, for example, with 'tmpfs'. This
+    # simply crashes `openstack server create` in environments like live
+    # distributions. In such cases, the best choice is 'writeback',
+    # which (a) makes the conversion multiple times faster; and (b) is
+    # as safe as it can be, because at the end of the conversion it,
+    # just like 'writethrough', calls fsync(2)|fdatasync(2), which
+    # ensures to safely write the data to the physical disk.
+
     if nova.privsep.utils.supports_direct_io(instances_path):
         cache_mode = 'none'
     else:
-        cache_mode = 'writethrough'
+        cache_mode = 'writeback'
     cmd = ('qemu-img', 'convert', '-t', cache_mode, '-O', out_format)
 
     if in_format is not None:

nova/tests/unit/virt/libvirt/test_driver.py

@@ -19099,7 +19099,7 @@ class LibvirtDriverTestCase(test.NoDBTestCase, TraitsComparisonMixin):
         mock_disk_op_sema.__enter__.assert_called_once()
         mock_direct_io.assert_called_once_with(CONF.instances_path)
         mock_execute.assert_has_calls([
-            mock.call('qemu-img', 'convert', '-t', 'writethrough',
+            mock.call('qemu-img', 'convert', '-t', 'writeback',
                       '-O', 'qcow2', '-f', 'raw', path, _path_qcow)])
         mock_rename.assert_has_calls([
             mock.call(_path_qcow, path)])

nova/tests/unit/virt/libvirt/test_utils.py

@@ -532,7 +532,7 @@ disk size: 4.4M
 
         libvirt_utils.extract_snapshot('/path/to/disk/image', src_format,
                                        '/extracted/snap', dest_format)
-        qemu_img_cmd = ('qemu-img', 'convert', '-t', 'writethrough',
+        qemu_img_cmd = ('qemu-img', 'convert', '-t', 'writeback',
                         '-O', out_format, '-f', src_format, )
         if CONF.libvirt.snapshot_compression and dest_format == "qcow2":
             qemu_img_cmd += ('-c',)

nova/tests/unit/virt/test_images.py

@@ -125,7 +125,7 @@ class QemuTestCase(test.NoDBTestCase):
                                                      mock_disk_op_sema):
         images._convert_image('source', 'dest', 'in_format', 'out_format',
                               run_as_root=False)
-        expected = ('qemu-img', 'convert', '-t', 'writethrough',
+        expected = ('qemu-img', 'convert', '-t', 'writeback',
                     '-O', 'out_format', '-f', 'in_format', 'source', 'dest')
         mock_disk_op_sema.__enter__.assert_called_once()
         self.assertTupleEqual(expected, mock_execute.call_args[0])

releasenotes/notes/make-disk-image-conversion-faster-c4abe83ae702888b.yaml

@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    By using ``writeback`` QEMU cache mode, make Nova's disk image
+    conversion (e.g. from raw to QCOW2 or vice versa) dramatically
+    faster, without compromising data integrity.  `Bug 1818847`_.
+
+    .. _Bug 1818847: https://bugs.launchpad.net/nova/+bug/1818847