diff --git a/nova/tests/virt/libvirt/test_libvirt.py b/nova/tests/virt/libvirt/test_libvirt.py index 857a98b20a46..85f5d099c87a 100644 --- a/nova/tests/virt/libvirt/test_libvirt.py +++ b/nova/tests/virt/libvirt/test_libvirt.py @@ -380,6 +380,8 @@ class LibvirtConnTestCase(test.TestCase): self.useFixture(fixtures.MonkeyPatch( 'nova.virt.libvirt.imagebackend.libvirt_utils', fake_libvirt_utils)) + self.stubs.Set(libvirt_driver.LibvirtDriver, + 'set_host_enabled', mock.Mock()) def fake_extend(image, size, use_cow=False): pass @@ -665,12 +667,16 @@ class LibvirtConnTestCase(test.TestCase): self.stubs.Set(self.conn, "getLibVersion", get_lib_version_stub) self.mox.StubOutWithMock(conn, '_connect') self.mox.StubOutWithMock(self.conn, 'registerCloseCallback') + self.mox.StubOutWithMock(conn, 'set_host_enabled') conn._connect(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(self.conn) self.conn.registerCloseCallback( mox.IgnoreArg(), mox.IgnoreArg()).WithSideEffects( set_close_callback) + conn.set_host_enabled('fake-mini', True) + conn.set_host_enabled('fake-mini', 'Connection to libvirt lost: 1') conn._connect(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(self.conn) + conn.set_host_enabled('fake-mini', True) self.conn.registerCloseCallback(mox.IgnoreArg(), mox.IgnoreArg()) self.mox.ReplayAll() @@ -682,6 +688,7 @@ class LibvirtConnTestCase(test.TestCase): self.close_callback(self.conn, 1, None) conn._get_connection() + self.mox.UnsetStubs() def test_cpu_features_bug_1217630(self): conn = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) @@ -3881,15 +3888,67 @@ class LibvirtConnTestCase(test.TestCase): self.mox.StubOutWithMock(libvirt, "openAuth") self.mox.StubOutWithMock(libvirt.libvirtError, "get_error_code") self.mox.StubOutWithMock(libvirt.libvirtError, "get_error_domain") + self.mox.StubOutWithMock(conn, 'set_host_enabled') libvirt.openAuth(mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg()).AndRaise( libvirt.libvirtError("fake failure")) + conn.set_host_enabled('fake-mini', 'Connection to libvirt lost: ERROR') + conn.set_host_enabled('fake-mini', False) self.mox.ReplayAll() conn._close_callback(conn._wrapped_conn, 'ERROR', '') self.assertRaises(exception.HypervisorUnavailable, conn.get_num_instances) + def test_broken_connection_disable_service(self): + disabled_reason = 'Connection to libvirt lost: ERROR!' + self.mox.UnsetStubs() + conn = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False) + self.mox.StubOutWithMock(libvirt, "openAuth") + self.mox.StubOutWithMock(libvirt.libvirtError, "get_error_code") + self.mox.StubOutWithMock(libvirt.libvirtError, "get_error_domain") + libvirt.openAuth(mox.IgnoreArg(), mox.IgnoreArg(), + mox.IgnoreArg()).AndRaise( + libvirt.libvirtError("fake failure")) + + from nova.objects import service as service_obj + service_mock = mock.MagicMock() + service_mock.__getitem__.return_value = False + service_mock_failed_conn = mock.MagicMock() + service_mock_failed_conn.__getitem__.return_value = True + self.mox.StubOutWithMock(service_obj.Service, + 'get_by_compute_host') + service_obj.Service.get_by_compute_host(mox.IgnoreArg(), + 'fake-mini').AndReturn(service_mock) + service_obj.Service.get_by_compute_host(mox.IgnoreArg(), + 'fake-mini').AndReturn(service_mock_failed_conn) + self.mox.ReplayAll() + + conn._close_callback(conn._wrapped_conn, 'ERROR!', '') + self.assertTrue(service_mock.disabled and + service_mock.disabled_reason == disabled_reason) + self.assertRaises(exception.HypervisorUnavailable, + conn.get_num_instances) + + def test_service_resume_after_broken_connection(self): + self.mox.UnsetStubs() + conn = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False) + self.mox.StubOutWithMock(libvirt, "openAuth") + libvirt.openAuth(mox.IgnoreArg(), mox.IgnoreArg(), + mox.IgnoreArg()).AndReturn(mock.MagicMock()) + + from nova.objects import service as service_obj + service_mock = mock.MagicMock() + service_mock.__getitem__.return_value = True + self.mox.StubOutWithMock(service_obj.Service, + 'get_by_compute_host') + service_obj.Service.get_by_compute_host(mox.IgnoreArg(), + 'fake-mini').AndReturn(service_mock) + self.mox.ReplayAll() + conn.get_num_instances() + self.assertTrue(not service_mock.disabled and + not service_mock.disabled_reason) + def test_immediate_delete(self): def fake_lookup_by_name(instance_name): raise exception.InstanceNotFound(instance_id=instance_name) @@ -7169,6 +7228,7 @@ class LibvirtNonblockingTestCase(test.TestCase): # Test bug 962840. import nova.virt.libvirt.driver as libvirt_driver connection = libvirt_driver.LibvirtDriver('') + connection.set_host_enabled = mock.Mock() jsonutils.to_primitive(connection._conn, convert_instances=True) diff --git a/nova/tests/virt/test_virt_drivers.py b/nova/tests/virt/test_virt_drivers.py index 31d877b4c970..dfa9af9af4a1 100644 --- a/nova/tests/virt/test_virt_drivers.py +++ b/nova/tests/virt/test_virt_drivers.py @@ -19,6 +19,7 @@ import fixtures import sys import traceback +from mock import MagicMock import netaddr from nova.compute import manager @@ -706,6 +707,14 @@ class LibvirtConnTestCase(_VirtDriverTestCase, test.TestCase): # Point _VirtDriverTestCase at the right module self.driver_module = 'nova.virt.libvirt.LibvirtDriver' super(LibvirtConnTestCase, self).setUp() + self.stubs.Set(self.connection, + 'set_host_enabled', MagicMock()) + self.useFixture(fixtures.MonkeyPatch( + 'nova.context.get_admin_context', + self._fake_admin_context)) + + def _fake_admin_context(self, *args, **kwargs): + return self.ctxt def test_force_hard_reboot(self): self.flags(libvirt_wait_soft_reboot_seconds=0) @@ -715,3 +724,19 @@ class LibvirtConnTestCase(_VirtDriverTestCase, test.TestCase): # there is lack of fake stuff to execute this method. so pass. self.skipTest("Test nothing, but this method" " needed to override superclass.") + + def test_set_host_enabled(self): + self.mox.UnsetStubs() + service_mock = MagicMock() + + # Previous status of the service: disabled: False + service_mock.__getitem__.return_value = False + from nova.objects import service as service_obj + self.mox.StubOutWithMock(service_obj.Service, + 'get_by_compute_host') + service_obj.Service.get_by_compute_host(self.ctxt, + 'fake-mini').AndReturn(service_mock) + self.mox.ReplayAll() + self.connection.set_host_enabled('my_test_host', 'ERROR!') + self.assertTrue(service_mock.disabled and + service_mock.disabled_reason == 'ERROR!') diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 2d65469e3b13..b9fac2448d2f 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -73,6 +73,7 @@ from nova import exception from nova.image import glance from nova import notifier from nova.objects import instance as instance_obj +from nova.objects import service as service_obj from nova.openstack.common import excutils from nova.openstack.common import fileutils from nova.openstack.common.gettextutils import _ @@ -588,12 +589,20 @@ class LibvirtDriver(driver.ComputeDriver): if not wrapped_conn or not self._test_connection(wrapped_conn): LOG.debug(_('Connecting to libvirt: %s'), self.uri()) - if not CONF.libvirt_nonblocking: - wrapped_conn = self._connect(self.uri(), self.read_only) - else: - wrapped_conn = tpool.proxy_call( - (libvirt.virDomain, libvirt.virConnect), - self._connect, self.uri(), self.read_only) + try: + if not CONF.libvirt_nonblocking: + wrapped_conn = self._connect(self.uri(), + self.read_only) + else: + wrapped_conn = tpool.proxy_call( + (libvirt.virDomain, libvirt.virConnect), + self._connect, self.uri(), self.read_only) + finally: + # Enabling the compute service, in case it was disabled + # since the connection was successful. + is_connected = bool(wrapped_conn) + self.set_host_enabled(CONF.host, is_connected) + self._wrapped_conn = wrapped_conn try: @@ -627,9 +636,14 @@ class LibvirtDriver(driver.ComputeDriver): def _close_callback(self, conn, reason, opaque): with self._wrapped_conn_lock: if conn == self._wrapped_conn: - LOG.info(_("Connection to libvirt lost: %s") % reason) + _error = _("Connection to libvirt lost: %s") % reason + LOG.warn(_error) self._wrapped_conn = None + # Disable compute service to avoid + # new instances of being scheduled on this host. + self.set_host_enabled(CONF.host, _error) + @staticmethod def _test_connection(conn): try: @@ -2589,6 +2603,37 @@ class LibvirtDriver(driver.ComputeDriver): % {'dev': pci_devs, 'dom': dom.ID()}) raise + def set_host_enabled(self, host, enabled): + """Sets the specified host's ability to accept new instances.""" + + status_name = {True: 'Enabled', + False: 'Disabled'} + + if isinstance(enabled, bool): + disable_service = not enabled + disable_reason = '' + else: + disable_service = bool(enabled) + disable_reason = enabled + + ctx = nova_context.get_admin_context() + try: + service = service_obj.Service.get_by_compute_host(ctx, CONF.host) + + if service.disabled != disable_service: + service.disabled = disable_service + service.disabled_reason = disable_reason + service.save() + LOG.debug(_('Updating compute service status to: %s'), + status_name[disable_service]) + except exception.ComputeHostNotFound: + LOG.warn(_('Cannot update service status on host: %s,' + 'since it is not registered.') % CONF.host) + except Exception: + LOG.warn(_('Cannot update service status on host: %s,' + 'due to an unexpected exception.') % CONF.host, + exc_info=True) + def get_host_capabilities(self): """Returns an instance of config.LibvirtConfigCaps representing the capabilities of the host.