From 9fea934c71d3c2fa7fdd80c67d94e18466c5cf9a Mon Sep 17 00:00:00 2001 From: Brett Milford Date: Thu, 4 Aug 2022 16:52:33 +1000 Subject: [PATCH] Handle "no RAM info was set" migration case This handles the case where the live migration monitoring thread may race and call jobStats() after the migration has completed resulting in the following error: libvirt.libvirtError: internal error: migration was active, but no RAM info was set Closes-Bug: #1982284 Change-Id: I77fdfa9cffbd44b2889f49f266b2582bcc6a4267 --- nova/tests/unit/virt/libvirt/test_guest.py | 22 +++++++++++++++++++ nova/virt/libvirt/guest.py | 7 ++++++ ...-no-ram-info-was-set-99784934ed80fd72.yaml | 11 ++++++++++ 3 files changed, 40 insertions(+) create mode 100644 releasenotes/notes/bug-1982284-libvirt-handle-no-ram-info-was-set-99784934ed80fd72.yaml diff --git a/nova/tests/unit/virt/libvirt/test_guest.py b/nova/tests/unit/virt/libvirt/test_guest.py index f662c108a92b..3dae7e75eb07 100644 --- a/nova/tests/unit/virt/libvirt/test_guest.py +++ b/nova/tests/unit/virt/libvirt/test_guest.py @@ -1041,3 +1041,25 @@ class JobInfoTestCase(test.NoDBTestCase): mock_stats.assert_called_once_with() mock_info.assert_called_once_with() + + @mock.patch.object(fakelibvirt.virDomain, "jobInfo") + @mock.patch.object(fakelibvirt.virDomain, "jobStats") + def test_job_stats_no_ram(self, mock_stats, mock_info): + mock_stats.side_effect = fakelibvirt.make_libvirtError( + fakelibvirt.libvirtError, + "internal error: migration was active, but no RAM info was set", + error_code=fakelibvirt.VIR_ERR_INTERNAL_ERROR, + error_message="migration was active, but no RAM info was set") + + info = self.guest.get_job_info() + + self.assertIsInstance(info, libvirt_guest.JobInfo) + self.assertEqual(fakelibvirt.VIR_DOMAIN_JOB_NONE, info.type) + self.assertEqual(0, info.time_elapsed) + self.assertEqual(0, info.time_remaining) + self.assertEqual(0, info.memory_total) + self.assertEqual(0, info.memory_processed) + self.assertEqual(0, info.memory_remaining) + + mock_stats.assert_called_once_with() + self.assertFalse(mock_info.called) diff --git a/nova/virt/libvirt/guest.py b/nova/virt/libvirt/guest.py index 7dd84973b50f..4898f3fccb97 100644 --- a/nova/virt/libvirt/guest.py +++ b/nova/virt/libvirt/guest.py @@ -655,6 +655,7 @@ class Guest(object): stats = self._domain.jobStats() return JobInfo(**stats) except libvirt.libvirtError as ex: + errmsg = ex.get_error_message() if ex.get_error_code() == libvirt.VIR_ERR_NO_SUPPORT: # Remote libvirt doesn't support new API LOG.debug("Missing remote virDomainGetJobStats: %s", ex) @@ -667,6 +668,12 @@ class Guest(object): # away completclsely LOG.debug("Domain has shutdown/gone away: %s", ex) return JobInfo(type=libvirt.VIR_DOMAIN_JOB_COMPLETED) + elif (ex.get_error_code() == libvirt.VIR_ERR_INTERNAL_ERROR and + errmsg and "migration was active, " + "but no RAM info was set" in errmsg): + LOG.debug("Migration is active or completed but " + "virDomainGetJobStats is missing ram: %s", ex) + return JobInfo(type=libvirt.VIR_DOMAIN_JOB_NONE) else: LOG.debug("Failed to get job stats: %s", ex) raise diff --git a/releasenotes/notes/bug-1982284-libvirt-handle-no-ram-info-was-set-99784934ed80fd72.yaml b/releasenotes/notes/bug-1982284-libvirt-handle-no-ram-info-was-set-99784934ed80fd72.yaml new file mode 100644 index 000000000000..943aa99a4364 --- /dev/null +++ b/releasenotes/notes/bug-1982284-libvirt-handle-no-ram-info-was-set-99784934ed80fd72.yaml @@ -0,0 +1,11 @@ +--- +other: + - | + A workaround has been added to the libvirt driver to catch and pass + migrations that were previously failing with the error: + + ``libvirt.libvirtError: internal error: migration was active, but no RAM info was set`` + + See `bug 1982284`_ for more details. + + .. _bug 1982284: https://bugs.launchpad.net/nova/+bug/1982284