Add retry to `nvme connect` in nvme connector

It is noticed that there could be some delays for a successful `nvme
connect` after "initialize_connection()" is completed in Cinder RSD
driver.

So add retries here, while keep the original logic intact.

Change-Id: I049f1d701c99612959d8262342aa63a043a97db0

os_brick/initiator/connectors/nvme.py

@@ -123,6 +123,11 @@ class NVMeConnector(base.BaseLinuxConnector):
             raise exception.VolumePathsNotFound()
         return list(path)
 
+    @utils.retry(exceptions=putils.ProcessExecutionError)
+    def _try_connect_nvme(self, cmd):
+        self._execute(*cmd, root_helper=self._root_helper,
+                      run_as_root=True)
+
     @utils.trace
     @synchronized('connect_volume')
     def connect_volume(self, connection_properties):
@@ -154,14 +159,8 @@ class NVMeConnector(base.BaseLinuxConnector):
             '-s', port]
         if host_nqn:
             cmd.extend(['-q', host_nqn])
-        try:
-            self._execute(*cmd, root_helper=self._root_helper,
-                          run_as_root=True)
-        except putils.ProcessExecutionError:
-            LOG.error(
-                "Failed to connect to NVMe nqn "
-                "%(conn_nqn)s", {'conn_nqn': conn_nqn})
-            raise
+
+        self._try_connect_nvme(cmd)
 
         path = self._get_device_path(current_nvme_devices)
         device_info['path'] = path[0]

os_brick/tests/initiator/connectors/test_nvme.py

@@ -183,6 +183,28 @@ class NVMeConnectorTestCase(test_connector.ConnectorTestCase):
         self.assertEqual('/dev/nvme0n2', device_info['path'])
         self.assertEqual('block', device_info['type'])
 
+    @mock.patch.object(nvme.NVMeConnector, '_get_nvme_devices')
+    @mock.patch.object(nvme.NVMeConnector, '_execute')
+    @mock.patch('time.sleep')
+    def test_connect_nvme_retry_success(
+            self, mock_sleep, mock_execute, mock_devices):
+        connection_properties = {'target_portal': 'portal',
+                                 'target_port': 1,
+                                 'nqn': 'nqn.volume_123',
+                                 'device_path': '',
+                                 'transport_type': 'rdma'}
+        mock_devices.side_effect = [
+            ['/dev/nvme0n1'],
+            ['/dev/nvme0n1', '/dev/nvme0n2']]
+        device_info = self.connector.connect_volume(
+            connection_properties)
+        mock_execute.side_effect = [
+            putils.ProcessExecutionError,
+            putils.ProcessExecutionError,
+            None]
+        self.assertEqual('/dev/nvme0n2', device_info['path'])
+        self.assertEqual('block', device_info['type'])
+
     @mock.patch.object(nvme.NVMeConnector, '_get_nvme_devices')
     @mock.patch.object(nvme.NVMeConnector, '_execute')
     @mock.patch('time.sleep')