Merge "Don't send heartbeats if driver not initializing correctly"

manila/manager.py

@@ -115,6 +115,15 @@ class Manager(base.Base, PeriodicTasks):
             config[key] = CONF.get(key, None)
         return config
 
+    def is_service_ready(self):
+        """Method indicating if service is ready.
+
+        This method should be overridden by subclasses which will return False
+        when the back end is not ready yet.
+
+        """
+        return True
+
 
 class SchedulerDependentManager(Manager):
     """Periodically send capability updates to the Scheduler services.

manila/service.py

@@ -245,6 +245,14 @@ class Service(service.Service):
 
     def report_state(self):
         """Update the state of this service in the datastore."""
+        if not self.manager.is_service_ready():
+            # NOTE(haixin): If the service is still initializing or failed to
+            #               intialize.
+            LOG.error('Manager for service %s is not ready yet, skipping state'
+                      ' update routine. Service will appear "down".',
+                      self.binary)
+            return
+
         ctxt = context.get_admin_context()
         state_catalog = {}
         try:

manila/share/manager.py

@@ -331,6 +331,16 @@ class ShareManager(manager.SchedulerDependentManager):
                  {"driver": self.driver.__class__.__name__,
                   "host": self.host})
 
+    def is_service_ready(self):
+        """Return if Manager is ready to accept requests.
+
+        This is to inform Service class that in case of manila driver
+        initialization failure the manager is actually down and not ready to
+        accept any requests.
+
+        """
+        return self.driver.initialized
+
     def ensure_driver_resources(self, ctxt):
         old_backend_info = self.db.backend_info_get(ctxt, self.host)
         old_backend_info_hash = (old_backend_info.get('info_hash')

manila/tests/share/test_manager.py

@@ -174,6 +174,13 @@ class ShareManagerTestCase(test.TestCase):
                 context=self.context,
                 periodic_hook_data=hook_data_mock.return_value)
 
+    def test_is_service_ready(self):
+        self.assertTrue(self.share_manager.is_service_ready())
+
+        # switch it to false and check again
+        self.share_manager.driver.initialized = False
+        self.assertFalse(self.share_manager.is_service_ready())
+
     def test_init_host_with_no_shares(self):
         self.mock_object(self.share_manager.db,
                          'share_instances_get_all_by_host',

manila/tests/test_service.py

@@ -199,6 +199,17 @@ class ServiceTestCase(test.TestCase):
         service.db.service_update.assert_called_once_with(
             mock.ANY, service_ref['id'], mock.ANY)
 
+    def test_report_state_service_not_ready(self):
+        with mock.patch.object(service, 'db') as mock_db:
+            mock_db.service_get.return_value = service_ref
+            serv = service.Service(host, binary, topic, CONF.fake_manager)
+            serv.manager.is_service_ready = mock.Mock(return_value=False)
+            serv.start()
+            serv.report_state()
+
+            serv.manager.is_service_ready.assert_called_once()
+            mock_db.service_update.assert_not_called()
+
 
 class TestWSGIService(test.TestCase):
 

releasenotes/notes/bug-1853940-not-send-heartbeat-if-driver-not-initial-9c3cee39e8c725d1.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    This bug fix avoids request failures while drivers are still initializing
+    or when they fail to initialize by reporting the share service as down
+    until the driver has been initialized.