NFS: Retry on intermittent mount failure

Concurrent backup operations using NFS driver may fail with
error 'busy or already mounted' when trying to mount the
share. Adding a config option 'backup_mount_attempts' to
specify the number of times to retry the mount operation
before raising the exception.

Closes-bug: 1813851
Change-Id: I35dcc5d61237fa65488eb9b2c8cbd26ee7b2adc1

cinder/backup/drivers/nfs.py

@@ -19,6 +19,7 @@
 import os
 import stat
 
+from os_brick import exception as brick_exception
 from os_brick.remotefs import remotefs as remotefs_brick
 from oslo_concurrency import processutils as putils
 from oslo_config import cfg
@@ -42,6 +43,11 @@ nfsbackup_service_opts = [
     cfg.StrOpt('backup_mount_options',
                help=('Mount options passed to the NFS client. See NFS '
                      'man page for details.')),
+    cfg.IntOpt('backup_mount_attempts',
+               min=1,
+               default=3,
+               help='The number of attempts to mount NFS shares before '
+                    'raising an error.'),
 ]
 
 CONF = cfg.CONF
@@ -83,8 +89,14 @@ class NFSBackupDriver(posix.PosixBackupDriver):
             self._root_helper,
             nfs_mount_point_base=self.backup_mount_point_base,
             nfs_mount_options=self.mount_options)
-        remotefsclient.mount(self.backup_share)
 
+        @utils.retry(
+            (brick_exception.BrickException, putils.ProcessExecutionError),
+            retries=CONF.backup_mount_attempts)
+        def mount():
+            remotefsclient.mount(self.backup_share)
+
+        mount()
         # Ensure we can write to this share
         mount_path = remotefsclient.get_mount_point(self.backup_share)
 

cinder/tests/unit/backup/drivers/test_backup_nfs.py

@@ -29,6 +29,7 @@ import zlib
 
 from eventlet import tpool
 import mock
+from os_brick import exception as brick_exception
 from os_brick.remotefs import remotefs as remotefs_brick
 from oslo_config import cfg
 import six
@@ -155,6 +156,26 @@ class BackupNFSShareTestCase(test.TestCase):
 
         self.assertEqual(0, mock_remotefsclient.call_count)
 
+    @mock.patch('time.sleep')
+    def test_init_backup_repo_path_mount_retry(self, mock_sleep):
+        self.override_config('backup_share', FAKE_BACKUP_SHARE)
+        self.override_config('backup_mount_attempts', 2)
+
+        mock_remotefsclient = mock.Mock()
+        self.mock_object(remotefs_brick, 'RemoteFsClient',
+                         return_value=mock_remotefsclient)
+
+        mock_remotefsclient.mount.side_effect = [
+            brick_exception.BrickException] * 2
+        with mock.patch.object(nfs.NFSBackupDriver, '_init_backup_repo_path'):
+            driver = nfs.NFSBackupDriver(self.ctxt)
+
+        self.assertRaises(brick_exception.BrickException,
+                          driver._init_backup_repo_path)
+        self.assertEqual([mock.call(FAKE_BACKUP_SHARE),
+                          mock.call(FAKE_BACKUP_SHARE)],
+                         mock_remotefsclient.mount.call_args_list)
+
 
 def fake_md5(arg):
     class result(object):

releasenotes/notes/bug-1813851-60a4f0ffe386d9b6.yaml

@@ -0,0 +1,5 @@
+---
+upgrade:
+  - |
+    Added config option ``backup_mount_attempts`` to specify the
+    number of attempts to mount NFS share in the NFS backup driver.