Merge "Storwize SVC multiple management IPs"

cinder/tests/unit/test_storwize_svc.py

@@ -18,6 +18,7 @@
 Tests for the IBM Storwize family and SVC volume driver.
 """
 
+import paramiko
 import random
 import re
 import time
@@ -33,6 +34,7 @@ from cinder import context
 from cinder import exception
 from cinder.i18n import _
 from cinder.objects import fields
+from cinder import ssh_utils
 from cinder import test
 from cinder.tests.unit import utils as testutils
 from cinder import utils
@@ -2525,8 +2527,11 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase):
         if self.USESIM:
             self.driver = StorwizeSVCISCSIFakeDriver(
                 configuration=conf.Configuration(None))
+            self._driver = storwize_svc_iscsi.StorwizeSVCISCSIDriver(
+                configuration=conf.Configuration(None))
 
             self._def_flags = {'san_ip': 'hostname',
+                               'storwize_san_secondary_ip': 'secondaryname',
                                'san_login': 'user',
                                'san_password': 'pass',
                                'storwize_svc_volpool_name': 'openstack',
@@ -2639,6 +2644,59 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase):
         # Finally, check with good parameters
         self.driver.do_setup(None)
 
+    @mock.patch.object(ssh_utils, 'SSHPool')
+    @mock.patch.object(processutils, 'ssh_execute')
+    def test_run_ssh_set_up_with_san_ip(self, mock_ssh_execute, mock_ssh_pool):
+        ssh_cmd = ['svcinfo']
+        self._driver._run_ssh(ssh_cmd)
+
+        mock_ssh_pool.assert_called_once_with(
+            self._driver.configuration.san_ip,
+            self._driver.configuration.san_ssh_port,
+            self._driver.configuration.ssh_conn_timeout,
+            self._driver.configuration.san_login,
+            password=self._driver.configuration.san_password,
+            privatekey=self._driver.configuration.san_private_key,
+            min_size=self._driver.configuration.ssh_min_pool_conn,
+            max_size=self._driver.configuration.ssh_max_pool_conn)
+
+    @mock.patch.object(ssh_utils, 'SSHPool')
+    @mock.patch.object(processutils, 'ssh_execute')
+    def test_run_ssh_set_up_with_secondary_ip(self, mock_ssh_execute,
+                                              mock_ssh_pool):
+        mock_ssh_pool.side_effect = [paramiko.SSHException, mock.MagicMock()]
+        ssh_cmd = ['svcinfo']
+        self._driver._run_ssh(ssh_cmd)
+
+        mock_ssh_pool.assert_called_with(
+            self._driver.configuration.storwize_san_secondary_ip,
+            self._driver.configuration.san_ssh_port,
+            self._driver.configuration.ssh_conn_timeout,
+            self._driver.configuration.san_login,
+            password=self._driver.configuration.san_password,
+            privatekey=self._driver.configuration.san_private_key,
+            min_size=self._driver.configuration.ssh_min_pool_conn,
+            max_size=self._driver.configuration.ssh_max_pool_conn)
+
+    @mock.patch.object(ssh_utils, 'SSHPool')
+    @mock.patch.object(processutils, 'ssh_execute')
+    def test_run_ssh_fail_to_secondary_ip(self, mock_ssh_execute,
+                                          mock_ssh_pool):
+        mock_ssh_execute.side_effect = [processutils.ProcessExecutionError,
+                                        mock.MagicMock()]
+        ssh_cmd = ['svcinfo']
+        self._driver._run_ssh(ssh_cmd)
+
+        mock_ssh_pool.assert_called_with(
+            self._driver.configuration.storwize_san_secondary_ip,
+            self._driver.configuration.san_ssh_port,
+            self._driver.configuration.ssh_conn_timeout,
+            self._driver.configuration.san_login,
+            password=self._driver.configuration.san_password,
+            privatekey=self._driver.configuration.san_private_key,
+            min_size=self._driver.configuration.ssh_min_pool_conn,
+            max_size=self._driver.configuration.ssh_max_pool_conn)
+
     def _generate_vol_info(self, vol_name, vol_id):
         rand_id = six.text_type(random.randint(10000, 99999))
         if vol_name:

cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py

@@ -15,6 +15,7 @@
 #
 
 import math
+import paramiko
 import random
 import re
 import string
@@ -33,6 +34,8 @@ import six
 
 from cinder import context
 from cinder import exception
+from cinder import ssh_utils
+from cinder import utils as cinder_utils
 from cinder.i18n import _, _LE, _LI, _LW
 from cinder.objects import fields
 from cinder.volume import driver
@@ -97,6 +100,10 @@ storwize_svc_opts = [
                help='If operating in stretched cluster mode, specify the '
                     'name of the pool in which mirrored copies are stored.'
                     'Example: "pool2"'),
+    cfg.StrOpt('storwize_san_secondary_ip',
+               default=None,
+               help='Specifies secondary management IP or hostname to be '
+                    'used if san_ip is invalid or becomes inaccessible.'),
     cfg.BoolOpt('storwize_svc_vol_nofmtdisk',
                 default=False,
                 help='Specifies that the volume not be formatted during '
@@ -1965,6 +1972,100 @@ class StorwizeSVCCommonDriver(san.SanDriver,
 
         LOG.debug('leave: check_for_setup_error')
 
+    def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1):
+        cinder_utils.check_ssh_injection(cmd_list)
+        command = ' '.join(cmd_list)
+        if not self.sshpool:
+            try:
+                self.sshpool = self._set_up_sshpool(self.configuration.san_ip)
+            except paramiko.SSHException:
+                LOG.warning(_LW('Unable to use san_ip to create SSHPool. Now '
+                                'attempting to use storwize_san_secondary_ip '
+                                'to create SSHPool.'))
+                if self.configuration.storwize_san_secondary_ip is not None:
+                    self.sshpool = self._set_up_sshpool(
+                        self.configuration.storwize_san_secondary_ip)
+                else:
+                    LOG.warning(_LW('Unable to create SSHPool using san_ip '
+                                    'and not able to use '
+                                    'storwize_san_secondary_ip since it is '
+                                    'not configured.'))
+                    raise
+        try:
+            self._ssh_execute(self.sshpool.command,
+                              check_exit_code, attempts)
+
+        except Exception:
+            # Need to check if creating an SSHPool storwize_san_secondary_ip
+            # before raising an error.
+
+            if self.configuration.storwize_san_secondary_ip is not None:
+
+                LOG.warning(_LW("Unable to execute SSH command. "
+                                "Attempting to switch IP to %s."),
+                            self.configuration.storwize_san_secondary_ip)
+                self.sshpool = self._set_up_sshpool(
+                    self.configuration.storwize_san_secondary_ip)
+                self._ssh_execute(self.sshpool.command,
+                                  check_exit_code, attempts)
+            else:
+                LOG.warning(_LW('Unable to execute SSH command. '
+                                'Not able to use '
+                                'storwize_san_secondary_ip since it is '
+                                'not configured.'))
+                with excutils.save_and_reraise_exception():
+                    LOG.error(_LE("Error running SSH command: %s"),
+                              command)
+
+    def _set_up_sshpool(self, ip):
+        password = self.configuration.san_password
+        privatekey = self.configuration.san_private_key
+        min_size = self.configuration.ssh_min_pool_conn
+        max_size = self.configuration.ssh_max_pool_conn
+        sshpool = ssh_utils.SSHPool(
+            ip,
+            self.configuration.san_ssh_port,
+            self.configuration.ssh_conn_timeout,
+            self.configuration.san_login,
+            password=password,
+            privatekey=privatekey,
+            min_size=min_size,
+            max_size=max_size)
+
+        return sshpool
+
+    def _ssh_execute(self, sshpool, command,
+                     check_exit_code = True, attempts=1):
+        try:
+            with sshpool.item() as ssh:
+                while attempts > 0:
+                    attempts -= 1
+                    try:
+                        return processutils.ssh_execute(
+                            ssh,
+                            command,
+                            check_exit_code=check_exit_code)
+                    except Exception as e:
+                            LOG.error(_LE('Error has occurred: %s'), e)
+                            last_exception = e
+                            greenthread.sleep(random.randint(20, 500) / 100.0)
+                    try:
+                        raise processutils.ProcessExecutionError(
+                            exit_code=last_exception.exit_code,
+                            stdout=last_exception.stdout,
+                            stderr=last_exception.stderr,
+                            cmd=last_exception.cmd)
+                    except AttributeError:
+                        raise processutils.ProcessExecutionError(
+                            exit_code=-1,
+                            stdout="",
+                            stderr="Error running SSH command",
+                            cmd=command)
+
+        except Exception:
+            with excutils.save_and_reraise_exception():
+                LOG.error(_LE("Error running SSH command: %s"), command)
+
     def ensure_export(self, ctxt, volume):
         """Check that the volume exists on the storage.
 

releasenotes/notes/storwize-multiple-management-ip-1cd364d63879d9b8.yaml

@@ -0,0 +1,3 @@
+---
+features:
+  - Add multiple management IP support to Storwize SVC driver.