From ca06025a4ab7fbe817e6d29f12bdfcf378c85cde Mon Sep 17 00:00:00 2001 From: Jacob Gregor Date: Wed, 24 Feb 2016 11:06:32 -0600 Subject: [PATCH] Storwize SVC multiple management IPs Right now Storwize SVC does not support multiple management IPs. This patch adds this feature so that if the primary IP fails, it will switch to the secondary IP that the user sets. DocImpact Adds config option 'storwize_san_secondary_ip' Implements: blueprint storwize-add-support-for-multiple-management-ips Change-Id: Ib82ba5b43e92027bfe39873a556baec796bb457e --- cinder/tests/unit/test_storwize_svc.py | 58 ++++++++++ .../ibm/storwize_svc/storwize_svc_common.py | 101 ++++++++++++++++++ ...ltiple-management-ip-1cd364d63879d9b8.yaml | 3 + 3 files changed, 162 insertions(+) create mode 100644 releasenotes/notes/storwize-multiple-management-ip-1cd364d63879d9b8.yaml diff --git a/cinder/tests/unit/test_storwize_svc.py b/cinder/tests/unit/test_storwize_svc.py index f510da8cf..16feb926c 100644 --- a/cinder/tests/unit/test_storwize_svc.py +++ b/cinder/tests/unit/test_storwize_svc.py @@ -18,6 +18,7 @@ Tests for the IBM Storwize family and SVC volume driver. """ +import paramiko import random import re import time @@ -33,6 +34,7 @@ from cinder import context from cinder import exception from cinder.i18n import _ from cinder.objects import fields +from cinder import ssh_utils from cinder import test from cinder.tests.unit import utils as testutils from cinder import utils @@ -2507,8 +2509,11 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase): if self.USESIM: self.driver = StorwizeSVCISCSIFakeDriver( configuration=conf.Configuration(None)) + self._driver = storwize_svc_iscsi.StorwizeSVCISCSIDriver( + configuration=conf.Configuration(None)) self._def_flags = {'san_ip': 'hostname', + 'storwize_san_secondary_ip': 'secondaryname', 'san_login': 'user', 'san_password': 'pass', 'storwize_svc_volpool_name': 'openstack', @@ -2621,6 +2626,59 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase): # Finally, check with good parameters self.driver.do_setup(None) + @mock.patch.object(ssh_utils, 'SSHPool') + @mock.patch.object(processutils, 'ssh_execute') + def test_run_ssh_set_up_with_san_ip(self, mock_ssh_execute, mock_ssh_pool): + ssh_cmd = ['svcinfo'] + self._driver._run_ssh(ssh_cmd) + + mock_ssh_pool.assert_called_once_with( + self._driver.configuration.san_ip, + self._driver.configuration.san_ssh_port, + self._driver.configuration.ssh_conn_timeout, + self._driver.configuration.san_login, + password=self._driver.configuration.san_password, + privatekey=self._driver.configuration.san_private_key, + min_size=self._driver.configuration.ssh_min_pool_conn, + max_size=self._driver.configuration.ssh_max_pool_conn) + + @mock.patch.object(ssh_utils, 'SSHPool') + @mock.patch.object(processutils, 'ssh_execute') + def test_run_ssh_set_up_with_secondary_ip(self, mock_ssh_execute, + mock_ssh_pool): + mock_ssh_pool.side_effect = [paramiko.SSHException, mock.MagicMock()] + ssh_cmd = ['svcinfo'] + self._driver._run_ssh(ssh_cmd) + + mock_ssh_pool.assert_called_with( + self._driver.configuration.storwize_san_secondary_ip, + self._driver.configuration.san_ssh_port, + self._driver.configuration.ssh_conn_timeout, + self._driver.configuration.san_login, + password=self._driver.configuration.san_password, + privatekey=self._driver.configuration.san_private_key, + min_size=self._driver.configuration.ssh_min_pool_conn, + max_size=self._driver.configuration.ssh_max_pool_conn) + + @mock.patch.object(ssh_utils, 'SSHPool') + @mock.patch.object(processutils, 'ssh_execute') + def test_run_ssh_fail_to_secondary_ip(self, mock_ssh_execute, + mock_ssh_pool): + mock_ssh_execute.side_effect = [processutils.ProcessExecutionError, + mock.MagicMock()] + ssh_cmd = ['svcinfo'] + self._driver._run_ssh(ssh_cmd) + + mock_ssh_pool.assert_called_with( + self._driver.configuration.storwize_san_secondary_ip, + self._driver.configuration.san_ssh_port, + self._driver.configuration.ssh_conn_timeout, + self._driver.configuration.san_login, + password=self._driver.configuration.san_password, + privatekey=self._driver.configuration.san_private_key, + min_size=self._driver.configuration.ssh_min_pool_conn, + max_size=self._driver.configuration.ssh_max_pool_conn) + def _generate_vol_info(self, vol_name, vol_id): rand_id = six.text_type(random.randint(10000, 99999)) if vol_name: diff --git a/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py b/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py index 98fc51e82..d8a2e9316 100644 --- a/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py +++ b/cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py @@ -15,6 +15,7 @@ # import math +import paramiko import random import re import string @@ -33,6 +34,8 @@ import six from cinder import context from cinder import exception +from cinder import ssh_utils +from cinder import utils as cinder_utils from cinder.i18n import _, _LE, _LI, _LW from cinder.objects import fields from cinder.volume import driver @@ -97,6 +100,10 @@ storwize_svc_opts = [ help='If operating in stretched cluster mode, specify the ' 'name of the pool in which mirrored copies are stored.' 'Example: "pool2"'), + cfg.StrOpt('storwize_san_secondary_ip', + default=None, + help='Specifies secondary management IP or hostname to be ' + 'used if san_ip is invalid or becomes inaccessible.'), cfg.BoolOpt('storwize_svc_vol_nofmtdisk', default=False, help='Specifies that the volume not be formatted during ' @@ -1965,6 +1972,100 @@ class StorwizeSVCCommonDriver(san.SanDriver, LOG.debug('leave: check_for_setup_error') + def _run_ssh(self, cmd_list, check_exit_code=True, attempts=1): + cinder_utils.check_ssh_injection(cmd_list) + command = ' '.join(cmd_list) + if not self.sshpool: + try: + self.sshpool = self._set_up_sshpool(self.configuration.san_ip) + except paramiko.SSHException: + LOG.warning(_LW('Unable to use san_ip to create SSHPool. Now ' + 'attempting to use storwize_san_secondary_ip ' + 'to create SSHPool.')) + if self.configuration.storwize_san_secondary_ip is not None: + self.sshpool = self._set_up_sshpool( + self.configuration.storwize_san_secondary_ip) + else: + LOG.warning(_LW('Unable to create SSHPool using san_ip ' + 'and not able to use ' + 'storwize_san_secondary_ip since it is ' + 'not configured.')) + raise + try: + self._ssh_execute(self.sshpool.command, + check_exit_code, attempts) + + except Exception: + # Need to check if creating an SSHPool storwize_san_secondary_ip + # before raising an error. + + if self.configuration.storwize_san_secondary_ip is not None: + + LOG.warning(_LW("Unable to execute SSH command. " + "Attempting to switch IP to %s."), + self.configuration.storwize_san_secondary_ip) + self.sshpool = self._set_up_sshpool( + self.configuration.storwize_san_secondary_ip) + self._ssh_execute(self.sshpool.command, + check_exit_code, attempts) + else: + LOG.warning(_LW('Unable to execute SSH command. ' + 'Not able to use ' + 'storwize_san_secondary_ip since it is ' + 'not configured.')) + with excutils.save_and_reraise_exception(): + LOG.error(_LE("Error running SSH command: %s"), + command) + + def _set_up_sshpool(self, ip): + password = self.configuration.san_password + privatekey = self.configuration.san_private_key + min_size = self.configuration.ssh_min_pool_conn + max_size = self.configuration.ssh_max_pool_conn + sshpool = ssh_utils.SSHPool( + ip, + self.configuration.san_ssh_port, + self.configuration.ssh_conn_timeout, + self.configuration.san_login, + password=password, + privatekey=privatekey, + min_size=min_size, + max_size=max_size) + + return sshpool + + def _ssh_execute(self, sshpool, command, + check_exit_code = True, attempts=1): + try: + with sshpool.item() as ssh: + while attempts > 0: + attempts -= 1 + try: + return processutils.ssh_execute( + ssh, + command, + check_exit_code=check_exit_code) + except Exception as e: + LOG.error(_LE('Error has occurred: %s'), e) + last_exception = e + greenthread.sleep(random.randint(20, 500) / 100.0) + try: + raise processutils.ProcessExecutionError( + exit_code=last_exception.exit_code, + stdout=last_exception.stdout, + stderr=last_exception.stderr, + cmd=last_exception.cmd) + except AttributeError: + raise processutils.ProcessExecutionError( + exit_code=-1, + stdout="", + stderr="Error running SSH command", + cmd=command) + + except Exception: + with excutils.save_and_reraise_exception(): + LOG.error(_LE("Error running SSH command: %s"), command) + def ensure_export(self, ctxt, volume): """Check that the volume exists on the storage. diff --git a/releasenotes/notes/storwize-multiple-management-ip-1cd364d63879d9b8.yaml b/releasenotes/notes/storwize-multiple-management-ip-1cd364d63879d9b8.yaml new file mode 100644 index 000000000..6390ca70e --- /dev/null +++ b/releasenotes/notes/storwize-multiple-management-ip-1cd364d63879d9b8.yaml @@ -0,0 +1,3 @@ +--- +features: + - Add multiple management IP support to Storwize SVC driver.