From bffd16608ad5a557345608bab688abf9f292352c Mon Sep 17 00:00:00 2001
From: LisaLi <xiaoyan.li@intel.com>
Date: Mon, 19 Nov 2018 10:14:56 -0500
Subject: [PATCH] Driver reinitialization after failure

This is a proposal to reinitialize a volume driver
when it fails during starting.

Change-Id: I01b37a15f03f0a16690745362404d741a225bad7
Co-authored-by: Ovidiu Poncea <ovidiu.poncea@windriver.com>
Implements: blueprint driver-initialization-after-fail
---
 cinder/tests/unit/volume/test_init_host.py    | 29 +++++++++++++++++++
 cinder/volume/manager.py                      | 20 ++++++++++++-
 ...ver_reinitialization-b26a8b3e665567ec.yaml |  6 ++++
 3 files changed, 54 insertions(+), 1 deletion(-)
 create mode 100644 releasenotes/notes/driver_reinitialization-b26a8b3e665567ec.yaml

diff --git a/cinder/tests/unit/volume/test_init_host.py b/cinder/tests/unit/volume/test_init_host.py
index d0b806573b3..9a97ebe47c6 100644
--- a/cinder/tests/unit/volume/test_init_host.py
+++ b/cinder/tests/unit/volume/test_init_host.py
@@ -17,8 +17,10 @@
 
 import mock
 from oslo_config import cfg
+from oslo_utils import importutils
 
 from cinder import context
+from cinder import exception
 from cinder import objects
 from cinder.tests.unit import utils as tests_utils
 from cinder.tests.unit import volume as base
@@ -281,3 +283,30 @@ class VolumeInitHostTestCase(base.BaseVolumeTestCase):
         mock_add_threadpool.assert_called_once_with(
             mock_migrate_fixed_key,
             volumes=mock_get_my_volumes())
+
+    @mock.patch('time.sleep')
+    def test_init_host_retry(self, mock_sleep):
+        kwargs = {'service_id': 2}
+        self.volume = importutils.import_object(CONF.volume_manager)
+        self.volume.driver.do_setup = mock.MagicMock()
+        self.volume.driver.do_setup.side_effect = [
+            exception.CinderException("Test driver error."),
+            exception.InvalidConfigurationValue('Test config error.'),
+            ImportError]
+
+        self.volume.init_host(added_to_cluster=False, **kwargs)
+
+        self.assertEqual(4, self.volume.driver.do_setup.call_count)
+        self.assertFalse(self.volume.is_working())
+
+    @mock.patch('time.sleep')
+    def test_init_host_retry_once(self, mock_sleep):
+        kwargs = {'service_id': 2}
+        self.volume = importutils.import_object(CONF.volume_manager)
+        self.volume.driver.do_setup = mock.MagicMock()
+        self.volume.driver.do_setup.side_effect = [ImportError, None]
+
+        self.volume.init_host(added_to_cluster=False, **kwargs)
+
+        self.assertEqual(2, self.volume.driver.do_setup.call_count)
+        self.assertTrue(self.volume.is_working())
diff --git a/cinder/volume/manager.py b/cinder/volume/manager.py
index 6df939aff7b..c404cd2b525 100644
--- a/cinder/volume/manager.py
+++ b/cinder/volume/manager.py
@@ -124,6 +124,11 @@ volume_manager_opts = [
     cfg.StrOpt('zoning_mode',
                help="FC Zoning mode configured, only 'fabric' is "
                     "supported now."),
+    cfg.IntOpt('reinit_driver_count',
+               default=3,
+               help='Maximum times to reintialize the driver '
+                    'if volume initialization fails. The interval of retry is '
+                    'exponentially backoff, and will be 1s, 2s, 4s etc.'),
 ]
 
 volume_backend_opts = [
@@ -409,7 +414,6 @@ class VolumeManager(manager.CleanableManager,
 
     def init_host(self, added_to_cluster=None, **kwargs):
         """Perform any required initialization."""
-        ctxt = context.get_admin_context()
         if not self.driver.supported:
             utils.log_unsupported_driver_warning(self.driver)
 
@@ -422,6 +426,20 @@ class VolumeManager(manager.CleanableManager,
                                     'id': self.__class__.__name__})
                 return
 
+        self._init_host(added_to_cluster, **kwargs)
+
+        if not self.driver.initialized:
+            reinit_count = 0
+            while reinit_count < CONF.reinit_driver_count:
+                time.sleep(2 ** reinit_count)
+                self._init_host(added_to_cluster, **kwargs)
+                if self.driver.initialized:
+                    return
+                reinit_count += 1
+
+    def _init_host(self, added_to_cluster=None, **kwargs):
+        ctxt = context.get_admin_context()
+
         # If we have just added this host to a cluster we have to include all
         # our resources in that cluster.
         if added_to_cluster:
diff --git a/releasenotes/notes/driver_reinitialization-b26a8b3e665567ec.yaml b/releasenotes/notes/driver_reinitialization-b26a8b3e665567ec.yaml
new file mode 100644
index 00000000000..d1a9acf37ff
--- /dev/null
+++ b/releasenotes/notes/driver_reinitialization-b26a8b3e665567ec.yaml
@@ -0,0 +1,6 @@
+---
+features:
+  - Added a new config ``reinit_driver_count`` in volume driver, which
+    indicates the maximum retry limit for driver re-initialization when
+    it fails to initialize a volume driver. Its default value is 3. The
+    interval of retry is exponentially backoff, and will be 1s, 2s, 4s etc.