Libvirt: Fsfreeze during live-snapshot of qemu/kvm instances

Currently it is required to execute fsfreeze command manually before
taking snapshots of active instances, otherwise the snapshot may be
inconsistent. When qemu-guest-agent is installed in a kvm instance,
we can request the instance to fsfreeze via libvirt's fsFreeze API
which is introduced in libvirt 1.2.5. To utilize this feature,
the image metadata must have 'os_require_quiesce=yes' property.
'hw_qemu_guest_agent=yes' is also needed to enable qemu-guest-agent.

Implements: blueprint quiesced-image-snapshots-with-qemu-guest-agent
Change-Id: I137efe00839ae0cb35e142b0c8f8d75ae61cfb64
Signed-off-by: Tomoki Sekiyama <tomoki.sekiyama@hds.com>

nova/exception.py

@@ -1801,3 +1801,8 @@ class InvalidToken(Invalid):
 
 class InvalidConnectionInfo(Invalid):
     msg_fmt = _("Invalid Connection Info")
+
+
+class InstanceQuiesceNotSupported(Invalid):
+    msg_fmt = _('Quiescing is not supported in instance %(instance_id)s: '
+                '%(reason)s')

nova/tests/unit/virt/libvirt/test_driver.py

@@ -243,6 +243,12 @@ class FakeVirtDomain(object):
     def destroy(self):
         pass
 
+    def fsFreeze(self, disks=None, flags=0):
+        pass
+
+    def fsThaw(self, disks=None, flags=0):
+        pass
+
 
 class CacheConcurrencyTestCase(test.NoDBTestCase):
     def setUp(self):
@@ -4138,6 +4144,36 @@ class LibvirtConnTestCase(test.NoDBTestCase):
         self.assertEqual(devices, ['/path/to/dev/1', '/path/to/dev/3'])
         mock_list.assert_called_with()
 
+    @mock.patch.object(host.Host, "has_min_version", return_value=True)
+    def test_quiesce(self, mock_has_min_version):
+        self.create_fake_libvirt_mock(lookupByName=self.fake_lookup)
+        with mock.patch.object(FakeVirtDomain, "fsFreeze") as mock_fsfreeze:
+            conn = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI())
+            instance = {'name': 'test', 'uuid': 'uuid'}
+            img_meta = {"properties": {"hw_qemu_guest_agent": "yes",
+                                       "os_require_quiesce": "yes"}}
+            self.assertIsNone(conn.quiesce(self.context, instance, img_meta))
+            mock_fsfreeze.assert_called_once_with()
+
+    def test_quiesce_not_supported(self):
+        self.create_fake_libvirt_mock()
+        conn = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI())
+        instance = {'name': 'test', 'uuid': 'uuid'}
+        self.assertRaises(exception.InstanceQuiesceNotSupported,
+                      conn.quiesce, self.context, instance, None)
+
+    @mock.patch.object(host.Host, "has_min_version", return_value=True)
+    def test_unquiesce(self, mock_has_min_version):
+        self.create_fake_libvirt_mock(getLibVersion=lambda: 1002005,
+                                      lookupByName=self.fake_lookup)
+        with mock.patch.object(FakeVirtDomain, "fsThaw") as mock_fsthaw:
+            conn = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI())
+            instance = {'name': 'test', 'uuid': 'uuid'}
+            img_meta = {"properties": {"hw_qemu_guest_agent": "yes",
+                                       "os_require_quiesce": "yes"}}
+            self.assertIsNone(conn.unquiesce(self.context, instance, img_meta))
+            mock_fsthaw.assert_called_once_with()
+
     def test_snapshot_in_ami_format(self):
         expected_calls = [
             {'args': (),
@@ -10386,7 +10422,8 @@ Active:          8381604 kB
             mock_size.return_value = 1004009
             mock_backing.return_value = bckfile
 
-            drvr._live_snapshot(mock_dom, srcfile, dstfile, "qcow2")
+            drvr._live_snapshot(self.context, self.test_instance, mock_dom,
+                                srcfile, dstfile, "qcow2", {})
 
             mock_dom.XMLDesc.assert_called_once_with(
                 fakelibvirt.VIR_DOMAIN_XML_INACTIVE |

nova/virt/driver.py

@@ -1352,6 +1352,36 @@ class ComputeDriver(object):
         #                 virt layer.
         return False
 
+    def quiesce(self, context, instance, image_meta):
+        """Quiesce the specified instance to prepare for snapshots.
+
+        If the specified instance doesn't support quiescing,
+        InstanceQuiesceNotSupported is raised. When it fails to quiesce by
+        other errors (e.g. agent timeout), NovaException is raised.
+
+        :param context:  request context
+        :param instance: nova.objects.instance.Instance to be quiesced
+        :param image_meta: image object returned by nova.image.glance that
+                           defines the image from which this instance
+                           was created
+        """
+        raise NotImplementedError()
+
+    def unquiesce(self, context, instance, image_meta):
+        """Unquiesce the specified instance after snapshots.
+
+        If the specified instance doesn't support quiescing,
+        InstanceQuiesceNotSupported is raised. When it fails to quiesce by
+        other errors (e.g. agent timeout), NovaException is raised.
+
+        :param context:  request context
+        :param instance: nova.objects.instance.Instance to be unquiesced
+        :param image_meta: image object returned by nova.image.glance that
+                           defines the image from which this instance
+                           was created
+        """
+        raise NotImplementedError()
+
 
 def load_compute_driver(virtapi, compute_driver=None):
     """Load a compute driver module.

nova/virt/fake.py

@@ -479,6 +479,12 @@ class FakeDriver(driver.ComputeDriver):
     def instance_on_disk(self, instance):
         return False
 
+    def quiesce(self, context, instance, image_meta):
+        pass
+
+    def unquiesce(self, context, instance, image_meta):
+        pass
+
 
 class FakeVirtAPI(virtapi.VirtAPI):
     def provider_fw_rule_get_all(self, context):

nova/virt/libvirt/driver.py

@@ -356,6 +356,8 @@ MIN_QEMU_DISCARD_VERSION = (1, 6, 0)
 REQ_HYPERVISOR_DISCARD = "QEMU"
 # libvirt numa topology support
 MIN_LIBVIRT_NUMA_TOPOLOGY_VERSION = (1, 0, 4)
+# fsFreeze/fsThaw requirement
+MIN_LIBVIRT_FSFREEZE_VERSION = (1, 2, 5)
 
 
 class LibvirtDriver(driver.ComputeDriver):
@@ -1411,8 +1413,8 @@ class LibvirtDriver(driver.ComputeDriver):
                 if live_snapshot:
                     # NOTE(xqueralt): libvirt needs o+x in the temp directory
                     os.chmod(tmpdir, 0o701)
-                    self._live_snapshot(virt_dom, disk_path, out_path,
-                                        image_format)
+                    self._live_snapshot(context, instance, virt_dom, disk_path,
+                                        out_path, image_format, base)
                 else:
                     snapshot_backend.snapshot_extract(out_path, image_format)
             finally:
@@ -1474,7 +1476,55 @@ class LibvirtDriver(driver.ComputeDriver):
 
         return not job_ended
 
-    def _live_snapshot(self, domain, disk_path, out_path, image_format):
+    def _can_quiesce(self, image_meta):
+        if CONF.libvirt.virt_type not in ('kvm', 'qemu'):
+            return (False, _('Only KVM and QEMU are supported'))
+
+        if not self._host.has_min_version(MIN_LIBVIRT_FSFREEZE_VERSION):
+            ver = ".".join([str(x) for x in MIN_LIBVIRT_FSFREEZE_VERSION])
+            return (False, _('Quiescing requires libvirt version %(version)s '
+                             'or greater') % {'version': ver})
+
+        img_meta_prop = image_meta.get('properties', {}) if image_meta else {}
+        hw_qga = img_meta_prop.get('hw_qemu_guest_agent', 'no')
+        if hw_qga.lower() == 'no':
+            return (False, _('QEMU guest agent is not enabled'))
+
+        return (True, None)
+
+    def _set_quiesced(self, context, instance, image_meta, quiesced):
+        supported, reason = self._can_quiesce(image_meta)
+        if not supported:
+            raise exception.InstanceQuiesceNotSupported(
+                instance_id=instance['uuid'], reason=reason)
+
+        try:
+            domain = self._lookup_by_name(instance['name'])
+            if quiesced:
+                domain.fsFreeze()
+            else:
+                domain.fsThaw()
+        except libvirt.libvirtError as ex:
+            error_code = ex.get_error_code()
+            msg = (_('Error from libvirt while quiescing %(instance_name)s: '
+                     '[Error Code %(error_code)s] %(ex)s')
+                   % {'instance_name': instance['name'],
+                      'error_code': error_code, 'ex': ex})
+            raise exception.NovaException(msg)
+
+    def quiesce(self, context, instance, image_meta):
+        """Freeze the guest filesystems to prepare for snapshot.
+
+        The qemu-guest-agent must be setup to execute fsfreeze.
+        """
+        self._set_quiesced(context, instance, image_meta, True)
+
+    def unquiesce(self, context, instance, image_meta):
+        """Thaw the guest filesystems after snapshot."""
+        self._set_quiesced(context, instance, image_meta, False)
+
+    def _live_snapshot(self, context, instance, domain, disk_path, out_path,
+                       image_format, image_meta):
         """Snapshot an instance without downtime."""
         # Save a copy of the domain's persistent XML file
         xml = domain.XMLDesc(
@@ -1499,6 +1549,11 @@ class LibvirtDriver(driver.ComputeDriver):
         libvirt_utils.create_cow_image(src_back_path, disk_delta,
                                        src_disk_size)
 
+        img_meta_prop = image_meta.get('properties', {}) if image_meta else {}
+        require_quiesce = img_meta_prop.get('os_require_quiesce', 'no')
+        if require_quiesce.lower() == 'yes':
+            self.quiesce(context, instance, image_meta)
+
         try:
             # NOTE (rmk): blockRebase cannot be executed on persistent
             #             domains, so we need to temporarily undefine it.
@@ -1521,6 +1576,8 @@ class LibvirtDriver(driver.ComputeDriver):
             libvirt_utils.chown(disk_delta, os.getuid())
         finally:
             self._conn.defineXML(xml)
+            if require_quiesce.lower() == 'yes':
+                self.unquiesce(context, instance, image_meta)
 
         # Convert the delta (CoW) image with a backing file to a flat
         # image with no backing file.