libvirt: set caps on maximum live migration time
Currently Nova launches live migration and then just leaves it to run, assuming it'll eventually finish. If the guest is dirtying memory quicker than the network can transfer it, it is entirely possible that a migration will never complete. In such a case it is highly undesirable to leave it running forever since it wastes valuable CPU and network resources Rather than trying to come up with a new policy for aborting migration in OpenStack, copy the existing logic that is successfully battle tested by the oVirt project in VDSM. This introduces two new host level configuration parameters that cloud administrators can use to ensure migrations which don't appear likely to complete will be aborted. First is an overall cap on the total running time of migration. The configured value is scaled by the number of GB of guest RAM, since larger guests will obviously take proportionally longer to migrate. The second is a timeout that is applied when Nova detects that memory is being dirtied faster than it can be transferred. It tracks a low watermark of data remaining and if that low watermark doesn't decrease in the given time, it assumes the VM is stuck and aborts migration NB with the default values for the config parameters, the code for detecting stuck migrations is only going to kick in for guests which have more than 2 GB of RAM allocated, as for smaller guests the overall time limit will abort migration before this happens. This is reasonable as small guests are less likely to get stuck during migration, as the network will generally be able to keep up with dirtying data well enough to get to a point where the final switchover can be performed. Related-bug: #1429220 DocImpact: two new libvirt configuration parameters in nova.conf allow the administrator to control length of migration before aborting it Change-Id: I461affe6c85aaf2a6bf6e8749586bfbfe0ebc146
This commit is contained in:
parent
07c7e5caf2
commit
da33ab4f7b
|
@ -768,6 +768,9 @@ class Domain(object):
|
|||
def injectNMI(self, flags=0):
|
||||
return 0
|
||||
|
||||
def abortJob(self):
|
||||
pass
|
||||
|
||||
|
||||
class DomainSnapshot(object):
|
||||
def __init__(self, name, domain):
|
||||
|
|
|
@ -6225,16 +6225,22 @@ class LibvirtConnTestCase(test.NoDBTestCase):
|
|||
self.assertFalse(mock_exist.called)
|
||||
self.assertFalse(mock_shutil.called)
|
||||
|
||||
EXPECT_SUCCESS = 1
|
||||
EXPECT_FAILURE = 2
|
||||
EXPECT_ABORT = 3
|
||||
|
||||
@mock.patch.object(time, "time")
|
||||
@mock.patch.object(time, "sleep",
|
||||
side_effect=lambda x: eventlet.sleep(0))
|
||||
@mock.patch.object(host.DomainJobInfo, "for_domain")
|
||||
@mock.patch.object(objects.Instance, "save")
|
||||
@mock.patch.object(fakelibvirt.Connection, "_mark_running")
|
||||
@mock.patch.object(fakelibvirt.virDomain, "abortJob")
|
||||
def _test_live_migration_monitoring(self,
|
||||
job_info_records,
|
||||
time_records,
|
||||
expect_success,
|
||||
expect_result,
|
||||
mock_abort,
|
||||
mock_running,
|
||||
mock_save,
|
||||
mock_job_info,
|
||||
|
@ -6286,12 +6292,20 @@ class LibvirtConnTestCase(test.NoDBTestCase):
|
|||
dom,
|
||||
finish_event)
|
||||
|
||||
if expect_success:
|
||||
if expect_result == self.EXPECT_SUCCESS:
|
||||
self.assertFalse(fake_recover_method.called,
|
||||
'Recover method called when success expected')
|
||||
self.assertFalse(mock_abort.called,
|
||||
'abortJob not called when success expected')
|
||||
fake_post_method.assert_called_once_with(
|
||||
self.context, instance, dest, False, migrate_data)
|
||||
else:
|
||||
if expect_result == self.EXPECT_ABORT:
|
||||
self.assertTrue(mock_abort.called,
|
||||
'abortJob called when abort expected')
|
||||
else:
|
||||
self.assertFalse(mock_abort.called,
|
||||
'abortJob not called when failure expected')
|
||||
self.assertFalse(fake_post_method.called,
|
||||
'Post method called when success not expected')
|
||||
fake_recover_method.assert_called_once_with(
|
||||
|
@ -6314,7 +6328,8 @@ class LibvirtConnTestCase(test.NoDBTestCase):
|
|||
type=fakelibvirt.VIR_DOMAIN_JOB_COMPLETED),
|
||||
]
|
||||
|
||||
self._test_live_migration_monitoring(domain_info_records, [], True)
|
||||
self._test_live_migration_monitoring(domain_info_records, [],
|
||||
self.EXPECT_SUCCESS)
|
||||
|
||||
def test_live_migration_monitor_success_race(self):
|
||||
# A normalish sequence but we're too slow to see the
|
||||
|
@ -6334,7 +6349,8 @@ class LibvirtConnTestCase(test.NoDBTestCase):
|
|||
type=fakelibvirt.VIR_DOMAIN_JOB_NONE),
|
||||
]
|
||||
|
||||
self._test_live_migration_monitoring(domain_info_records, [], True)
|
||||
self._test_live_migration_monitoring(domain_info_records, [],
|
||||
self.EXPECT_SUCCESS)
|
||||
|
||||
def test_live_migration_monitor_failed(self):
|
||||
# A failed sequence where we see all the expected events
|
||||
|
@ -6352,7 +6368,8 @@ class LibvirtConnTestCase(test.NoDBTestCase):
|
|||
type=fakelibvirt.VIR_DOMAIN_JOB_FAILED),
|
||||
]
|
||||
|
||||
self._test_live_migration_monitoring(domain_info_records, [], False)
|
||||
self._test_live_migration_monitoring(domain_info_records, [],
|
||||
self.EXPECT_FAILURE)
|
||||
|
||||
def test_live_migration_monitor_failed_race(self):
|
||||
# A failed sequence where we are too slow to see the
|
||||
|
@ -6371,7 +6388,8 @@ class LibvirtConnTestCase(test.NoDBTestCase):
|
|||
type=fakelibvirt.VIR_DOMAIN_JOB_NONE),
|
||||
]
|
||||
|
||||
self._test_live_migration_monitoring(domain_info_records, [], False)
|
||||
self._test_live_migration_monitoring(domain_info_records, [],
|
||||
self.EXPECT_FAILURE)
|
||||
|
||||
def test_live_migration_monitor_cancelled(self):
|
||||
# A cancelled sequence where we see all the events
|
||||
|
@ -6390,13 +6408,17 @@ class LibvirtConnTestCase(test.NoDBTestCase):
|
|||
type=fakelibvirt.VIR_DOMAIN_JOB_CANCELLED),
|
||||
]
|
||||
|
||||
self._test_live_migration_monitoring(domain_info_records, [], False)
|
||||
self._test_live_migration_monitoring(domain_info_records, [],
|
||||
self.EXPECT_FAILURE)
|
||||
|
||||
@mock.patch.object(fakelibvirt.virDomain, "migrateSetMaxDowntime")
|
||||
@mock.patch.object(libvirt_driver.LibvirtDriver,
|
||||
"_migration_downtime_steps")
|
||||
def test_live_migration_monitor_downtime(self, mock_downtime_steps,
|
||||
mock_set_downtime):
|
||||
self.flags(live_migration_completion_timeout=1000000,
|
||||
live_migration_progress_timeout=1000000,
|
||||
group='libvirt')
|
||||
# We've setup 4 fake downtime steps - first value is the
|
||||
# time delay, second is the downtime value
|
||||
downtime_steps = [
|
||||
|
@ -6434,12 +6456,74 @@ class LibvirtConnTestCase(test.NoDBTestCase):
|
|||
]
|
||||
|
||||
self._test_live_migration_monitoring(domain_info_records,
|
||||
fake_times, True)
|
||||
fake_times, self.EXPECT_SUCCESS)
|
||||
|
||||
mock_set_downtime.assert_has_calls([mock.call(10),
|
||||
mock.call(50),
|
||||
mock.call(200)])
|
||||
|
||||
def test_live_migration_monitor_completion(self):
|
||||
self.flags(live_migration_completion_timeout=100,
|
||||
live_migration_progress_timeout=1000000,
|
||||
group='libvirt')
|
||||
# Each one of these fake times is used for time.time()
|
||||
# when a new domain_info_records entry is consumed.
|
||||
fake_times = [0, 40, 80, 120, 160, 200, 240, 280, 320]
|
||||
|
||||
# A normal sequence where see all the normal job states
|
||||
domain_info_records = [
|
||||
host.DomainJobInfo(
|
||||
type=fakelibvirt.VIR_DOMAIN_JOB_NONE),
|
||||
host.DomainJobInfo(
|
||||
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
|
||||
host.DomainJobInfo(
|
||||
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
|
||||
host.DomainJobInfo(
|
||||
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
|
||||
host.DomainJobInfo(
|
||||
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
|
||||
host.DomainJobInfo(
|
||||
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
|
||||
"thread-finish",
|
||||
"domain-stop",
|
||||
host.DomainJobInfo(
|
||||
type=fakelibvirt.VIR_DOMAIN_JOB_CANCELLED),
|
||||
]
|
||||
|
||||
self._test_live_migration_monitoring(domain_info_records,
|
||||
fake_times, self.EXPECT_ABORT)
|
||||
|
||||
def test_live_migration_monitor_progress(self):
|
||||
self.flags(live_migration_completion_timeout=1000000,
|
||||
live_migration_progress_timeout=150,
|
||||
group='libvirt')
|
||||
# Each one of these fake times is used for time.time()
|
||||
# when a new domain_info_records entry is consumed.
|
||||
fake_times = [0, 40, 80, 120, 160, 200, 240, 280, 320]
|
||||
|
||||
# A normal sequence where see all the normal job states
|
||||
domain_info_records = [
|
||||
host.DomainJobInfo(
|
||||
type=fakelibvirt.VIR_DOMAIN_JOB_NONE),
|
||||
host.DomainJobInfo(
|
||||
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
|
||||
host.DomainJobInfo(
|
||||
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
|
||||
host.DomainJobInfo(
|
||||
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
|
||||
host.DomainJobInfo(
|
||||
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
|
||||
host.DomainJobInfo(
|
||||
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
|
||||
"thread-finish",
|
||||
"domain-stop",
|
||||
host.DomainJobInfo(
|
||||
type=fakelibvirt.VIR_DOMAIN_JOB_CANCELLED),
|
||||
]
|
||||
|
||||
self._test_live_migration_monitoring(domain_info_records,
|
||||
fake_times, self.EXPECT_ABORT)
|
||||
|
||||
def test_live_migration_downtime_steps(self):
|
||||
self.flags(live_migration_downtime=400, group='libvirt')
|
||||
self.flags(live_migration_downtime_steps=10, group='libvirt')
|
||||
|
|
|
@ -184,6 +184,19 @@ libvirt_opts = [
|
|||
'of the migration downtime. Minimum delay is %d seconds. '
|
||||
'Value is per GiB of guest RAM, with lower bound of a '
|
||||
'minimum of 2 GiB' % LIVE_MIGRATION_DOWNTIME_DELAY_MIN),
|
||||
cfg.IntOpt('live_migration_completion_timeout',
|
||||
default=800,
|
||||
help='Time to wait, in seconds, for migration to successfully '
|
||||
'complete transferring data before aborting the '
|
||||
'operation. Value is per GiB of guest RAM, with lower '
|
||||
'bound of a minimum of 2 GiB. Should usually be larger '
|
||||
'than downtime delay * downtime steps. Set to 0 to '
|
||||
'disable timeouts.'),
|
||||
cfg.IntOpt('live_migration_progress_timeout',
|
||||
default=150,
|
||||
help='Time to wait, in seconds, for migration to make forward '
|
||||
'progress in transferring data before aborting the '
|
||||
'operation. Set to 0 to disable timeouts.'),
|
||||
cfg.StrOpt('snapshot_image_format',
|
||||
choices=('raw', 'qcow2', 'vmdk', 'vdi'),
|
||||
help='Snapshot image format. Defaults to same as source image'),
|
||||
|
@ -5683,9 +5696,14 @@ class LibvirtDriver(driver.ComputeDriver):
|
|||
migrate_data, dom, finish_event):
|
||||
data_gb = self._live_migration_data_gb(instance)
|
||||
downtime_steps = list(self._migration_downtime_steps(data_gb))
|
||||
completion_timeout = int(
|
||||
CONF.libvirt.live_migration_completion_timeout * data_gb)
|
||||
progress_timeout = CONF.libvirt.live_migration_progress_timeout
|
||||
|
||||
n = 0
|
||||
start = time.time()
|
||||
progress_time = start
|
||||
progress_watermark = None
|
||||
while True:
|
||||
info = host.DomainJobInfo.for_domain(dom)
|
||||
|
||||
|
@ -5743,6 +5761,32 @@ class LibvirtDriver(driver.ComputeDriver):
|
|||
# the operation, change max bandwidth
|
||||
now = time.time()
|
||||
elapsed = now - start
|
||||
abort = False
|
||||
|
||||
if ((progress_watermark is None) or
|
||||
(progress_watermark > info.data_remaining)):
|
||||
progress_watermark = info.data_remaining
|
||||
progress_time = now
|
||||
|
||||
if (progress_timeout != 0 and
|
||||
(now - progress_time) > progress_timeout):
|
||||
LOG.warn(_LW("Live migration stuck for %d sec"),
|
||||
(now - progress_time), instance=instance)
|
||||
abort = True
|
||||
|
||||
if (completion_timeout != 0 and
|
||||
elapsed > completion_timeout):
|
||||
LOG.warn(_LW("Live migration not completed after %d sec"),
|
||||
completion_timeout, instance=instance)
|
||||
abort = True
|
||||
|
||||
if abort:
|
||||
try:
|
||||
dom.abortJob()
|
||||
except libvirt.libvirtError as e:
|
||||
LOG.warn(_LW("Failed to abort migration %s"),
|
||||
e, instance=instance)
|
||||
raise
|
||||
|
||||
# See if we need to increase the max downtime. We
|
||||
# ignore failures, since we'd rather continue trying
|
||||
|
@ -5801,6 +5845,13 @@ class LibvirtDriver(driver.ComputeDriver):
|
|||
"processed_memory": info.memory_processed,
|
||||
"remaining_memory": info.memory_remaining,
|
||||
"total_memory": info.memory_total}, instance=instance)
|
||||
if info.data_remaining > progress_watermark:
|
||||
lg(_LI("Data remaining %(remaining)d bytes, "
|
||||
"low watermark %(watermark)d bytes "
|
||||
"%(last)d seconds ago"),
|
||||
{"remaining": info.data_remaining,
|
||||
"watermark": progress_watermark,
|
||||
"last": (now - progress_time)}, instance=instance)
|
||||
|
||||
n = n + 1
|
||||
elif info.type == libvirt.VIR_DOMAIN_JOB_COMPLETED:
|
||||
|
|
Loading…
Reference in New Issue