diff --git a/cinder/tests/unit/test_storwize_svc.py b/cinder/tests/unit/test_storwize_svc.py index 013bdfa25..9cb4ce1b7 100644 --- a/cinder/tests/unit/test_storwize_svc.py +++ b/cinder/tests/unit/test_storwize_svc.py @@ -18,6 +18,7 @@ Tests for the IBM Storwize family and SVC volume driver. """ +import paramiko import random import re import time @@ -33,6 +34,7 @@ from cinder import context from cinder import exception from cinder.i18n import _ from cinder.objects import fields +from cinder import ssh_utils from cinder import test from cinder.tests.unit import utils as testutils from cinder import utils @@ -2525,8 +2527,11 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase): if self.USESIM: self.driver = StorwizeSVCISCSIFakeDriver( configuration=conf.Configuration(None)) + self._driver = storwize_svc_iscsi.StorwizeSVCISCSIDriver( + configuration=conf.Configuration(None)) self._def_flags = {'san_ip': 'hostname', + 'storwize_san_secondary_ip': 'secondaryname', 'san_login': 'user', 'san_password': 'pass', 'storwize_svc_volpool_name': 'openstack', @@ -2639,6 +2644,59 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase): # Finally, check with good parameters self.driver.do_setup(None) + @mock.patch.object(ssh_utils, 'SSHPool') + @mock.patch.object(processutils, 'ssh_execute') + def test_run_ssh_set_up_with_san_ip(self, mock_ssh_execute, mock_ssh_pool): + ssh_cmd = ['svcinfo'] + self._driver._run_ssh(ssh_cmd) + + mock_ssh_pool.assert_called_once_with( + self._driver.configuration.san_ip, + self._driver.configuration.san_ssh_port, + self._driver.configuration.ssh_conn_timeout, + self._driver.configuration.san_login, + password=self._driver.configuration.san_password, + privatekey=self._driver.configuration.san_private_key, + min_size=self._driver.configuration.ssh_min_pool_conn, + max_size=self._driver.configuration.ssh_max_pool_conn) + + @mock.patch.object(ssh_utils, 'SSHPool') + @mock.patch.object(processutils, 'ssh_execute') + def test_run_ssh_set_up_with_secondary_ip(self, mock_ssh_execute, + mock_ssh_pool): + mock_ssh_pool.side_effect = [paramiko.SSHException, mock.MagicMock()] + ssh_cmd = ['svcinfo'] + self._driver._run_ssh(ssh_cmd) + + mock_ssh_pool.assert_called_with( + self._driver.configuration.storwize_san_secondary_ip, + self._driver.configuration.san_ssh_port, + self._driver.configuration.ssh_conn_timeout, + self._driver.configuration.san_login, + password=self._driver.configuration.san_password, + privatekey=self._driver.configuration.san_private_key, + min_size=self._driver.configuration.ssh_min_pool_conn, + max_size=self._driver.configuration.ssh_max_pool_conn) + + @mock.patch.object(ssh_utils, 'SSHPool') + @mock.patch.object(processutils, 'ssh_execute') + def test_run_ssh_fail_to_secondary_ip(self, mock_ssh_execute, + mock_ssh_pool): + mock_ssh_execute.side_effect = [processutils.ProcessExecutionError, + mock.MagicMock()] + ssh_cmd = ['svcinfo'] + self._driver._run_ssh(ssh_cmd) + + mock_ssh_pool.assert_called_with( + self._driver.configuration.storwize_san_secondary_ip, + self._driver.configuration.san_ssh_port, + self._driver.configuration.ssh_conn_timeout, + self._driver.configuration.san_login, + password=self._driver.configuration.san_password, + privatekey=self._driver.configuration.san_private_key, + min_size=self._driver.configuration.ssh_min_pool_conn, + max_size=self._driver.configuration.ssh_max_pool_conn) + def _generate_vol_info(self, vol_name, vol_id): rand_id = six.text_type(random.randint(10000, 99999)) if vol_name: diff --git a/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py b/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py index 98fc51e82..d8a2e9316 100644 --- a/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py +++ b/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py @@ -15,6 +15,7 @@ # import math +import paramiko import random import re import string @@ -33,6 +34,8 @@ import six from cinder import context from cinder import exception +from cinder import ssh_utils +from cinder import utils as cinder_utils from cinder.i18n import _, _LE, _LI, _LW from cinder.objects import fields from cinder.volume import driver @@ -97,6 +100,10 @@ storwize_svc_opts = [ help='If operating in stretched cluster mode, specify the ' 'name of the pool in which mirrored copies are stored.' 'Example: "pool2"'), + cfg.StrOpt('storwize_san_secondary_ip', + default=None, + help='Specifies secondary management IP or hostname to be ' + 'used if san_ip is invalid or becomes inaccessible.'), cfg.BoolOpt('storwize_svc_vol_nofmtdisk', default=False, help='Specifies that the volume not be formatted during ' @@ -1965,6 +1972,100 @@ class StorwizeSVCCommonDriver(san.SanDriver, LOG.debug('leave: check_for_setup_error') + def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1): + cinder_utils.check_ssh_injection(cmd_list) + command = ' '.join(cmd_list) + if not self.sshpool: + try: + self.sshpool = self._set_up_sshpool(self.configuration.san_ip) + except paramiko.SSHException: + LOG.warning(_LW('Unable to use san_ip to create SSHPool. Now ' + 'attempting to use storwize_san_secondary_ip ' + 'to create SSHPool.')) + if self.configuration.storwize_san_secondary_ip is not None: + self.sshpool = self._set_up_sshpool( + self.configuration.storwize_san_secondary_ip) + else: + LOG.warning(_LW('Unable to create SSHPool using san_ip ' + 'and not able to use ' + 'storwize_san_secondary_ip since it is ' + 'not configured.')) + raise + try: + self._ssh_execute(self.sshpool.command, + check_exit_code, attempts) + + except Exception: + # Need to check if creating an SSHPool storwize_san_secondary_ip + # before raising an error. + + if self.configuration.storwize_san_secondary_ip is not None: + + LOG.warning(_LW("Unable to execute SSH command. " + "Attempting to switch IP to %s."), + self.configuration.storwize_san_secondary_ip) + self.sshpool = self._set_up_sshpool( + self.configuration.storwize_san_secondary_ip) + self._ssh_execute(self.sshpool.command, + check_exit_code, attempts) + else: + LOG.warning(_LW('Unable to execute SSH command. ' + 'Not able to use ' + 'storwize_san_secondary_ip since it is ' + 'not configured.')) + with excutils.save_and_reraise_exception(): + LOG.error(_LE("Error running SSH command: %s"), + command) + + def _set_up_sshpool(self, ip): + password = self.configuration.san_password + privatekey = self.configuration.san_private_key + min_size = self.configuration.ssh_min_pool_conn + max_size = self.configuration.ssh_max_pool_conn + sshpool = ssh_utils.SSHPool( + ip, + self.configuration.san_ssh_port, + self.configuration.ssh_conn_timeout, + self.configuration.san_login, + password=password, + privatekey=privatekey, + min_size=min_size, + max_size=max_size) + + return sshpool + + def _ssh_execute(self, sshpool, command, + check_exit_code = True, attempts=1): + try: + with sshpool.item() as ssh: + while attempts > 0: + attempts -= 1 + try: + return processutils.ssh_execute( + ssh, + command, + check_exit_code=check_exit_code) + except Exception as e: + LOG.error(_LE('Error has occurred: %s'), e) + last_exception = e + greenthread.sleep(random.randint(20, 500) / 100.0) + try: + raise processutils.ProcessExecutionError( + exit_code=last_exception.exit_code, + stdout=last_exception.stdout, + stderr=last_exception.stderr, + cmd=last_exception.cmd) + except AttributeError: + raise processutils.ProcessExecutionError( + exit_code=-1, + stdout="", + stderr="Error running SSH command", + cmd=command) + + except Exception: + with excutils.save_and_reraise_exception(): + LOG.error(_LE("Error running SSH command: %s"), command) + def ensure_export(self, ctxt, volume): """Check that the volume exists on the storage. diff --git a/releasenotes/notes/storwize-multiple-management-ip-1cd364d63879d9b8.yaml b/releasenotes/notes/storwize-multiple-management-ip-1cd364d63879d9b8.yaml new file mode 100644 index 000000000..6390ca70e --- /dev/null +++ b/releasenotes/notes/storwize-multiple-management-ip-1cd364d63879d9b8.yaml @@ -0,0 +1,3 @@ +--- +features: + - Add multiple management IP support to Storwize SVC driver.