Don't send heartbeats if driver not initializing correctly

This commit adds is_service_ready method to base Manager class that can be
used to indicate that service is not ready or initialzed success. This is
used to block refreshing Service heartbeats if manager will return false
from is_service_ready.

Closes-Bug:#1853940

Change-Id: Ib85468c703dfa51b03d1838bd422c9b2669bc747

manila/manager.py

@@ -115,6 +115,15 @@ class Manager(base.Base, PeriodicTasks):
             config[key] = CONF.get(key, None)
         return config
 
+    def is_service_ready(self):
+        """Method indicating if service is ready.
+
+        This method should be overridden by subclasses which will return False
+        when the back end is not ready yet.
+
+        """
+        return True
+
 
 class SchedulerDependentManager(Manager):
     """Periodically send capability updates to the Scheduler services.

manila/service.py

@@ -245,6 +245,14 @@ class Service(service.Service):
 
     def report_state(self):
         """Update the state of this service in the datastore."""
+        if not self.manager.is_service_ready():
+            # NOTE(haixin): If the service is still initializing or failed to
+            #               intialize.
+            LOG.error('Manager for service %s is not ready yet, skipping state'
+                      ' update routine. Service will appear "down".',
+                      self.binary)
+            return
+
         ctxt = context.get_admin_context()
         state_catalog = {}
         try:

manila/share/manager.py

@@ -331,6 +331,16 @@ class ShareManager(manager.SchedulerDependentManager):
                  {"driver": self.driver.__class__.__name__,
                   "host": self.host})
 
+    def is_service_ready(self):
+        """Return if Manager is ready to accept requests.
+
+        This is to inform Service class that in case of manila driver
+        initialization failure the manager is actually down and not ready to
+        accept any requests.
+
+        """
+        return self.driver.initialized
+
     def ensure_driver_resources(self, ctxt):
         old_backend_info = self.db.backend_info_get(ctxt, self.host)
         old_backend_info_hash = (old_backend_info.get('info_hash')

manila/tests/share/test_manager.py

@@ -174,6 +174,13 @@ class ShareManagerTestCase(test.TestCase):
                 context=self.context,
                 periodic_hook_data=hook_data_mock.return_value)
 
+    def test_is_service_ready(self):
+        self.assertTrue(self.share_manager.is_service_ready())
+
+        # switch it to false and check again
+        self.share_manager.driver.initialized = False
+        self.assertFalse(self.share_manager.is_service_ready())
+
     def test_init_host_with_no_shares(self):
         self.mock_object(self.share_manager.db,
                          'share_instances_get_all_by_host',

manila/tests/test_service.py

@@ -199,6 +199,17 @@ class ServiceTestCase(test.TestCase):
         service.db.service_update.assert_called_once_with(
             mock.ANY, service_ref['id'], mock.ANY)
 
+    def test_report_state_service_not_ready(self):
+        with mock.patch.object(service, 'db') as mock_db:
+            mock_db.service_get.return_value = service_ref
+            serv = service.Service(host, binary, topic, CONF.fake_manager)
+            serv.manager.is_service_ready = mock.Mock(return_value=False)
+            serv.start()
+            serv.report_state()
+
+            serv.manager.is_service_ready.assert_called_once()
+            mock_db.service_update.assert_not_called()
+
 
 class TestWSGIService(test.TestCase):
 

releasenotes/notes/bug-1853940-not-send-heartbeat-if-driver-not-initial-9c3cee39e8c725d1.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    This bug fix avoids request failures while drivers are still initializing
+    or when they fail to initialize by reporting the share service as down
+    until the driver has been initialized.