From 45bc8abc47456bae6b6b3dd7857eccbea102b5fd Mon Sep 17 00:00:00 2001 From: Tom Swanson Date: Fri, 26 Aug 2016 14:54:53 -0500 Subject: [PATCH] Dell SC: Error attaching after LV-AFO Fixed miscellaneous errors dealing with the state of Dell's Live Volumes after a LV AFO event. Was looking for the original volume and sc server object after the primary was down. We now look for the new primary and attach that. Ensure export was failing to properly kick off a swap back to the original primary if it was available. Fixed an error with deleting replications. Added a workaround to finding a ScLiveVolume object. Previous method could return stale information. Change-Id: I58889adbcf279d4b0107cbc51e1dace888a3fb15 Closes-Bug: #1617401 --- .../unit/volume/drivers/dell/test_dellfc.py | 79 +++++++- .../unit/volume/drivers/dell/test_dellsc.py | 121 ++++++++++--- .../volume/drivers/dell/test_dellscapi.py | 126 +++++++++---- .../drivers/dell/dell_storagecenter_api.py | 100 ++++++++--- .../drivers/dell/dell_storagecenter_common.py | 43 +++-- .../drivers/dell/dell_storagecenter_fc.py | 168 ++++++++++-------- .../drivers/dell/dell_storagecenter_iscsi.py | 145 ++++++++------- 7 files changed, 531 insertions(+), 251 deletions(-) diff --git a/cinder/tests/unit/volume/drivers/dell/test_dellfc.py b/cinder/tests/unit/volume/drivers/dell/test_dellfc.py index 0362d8797..7bcd03079 100644 --- a/cinder/tests/unit/volume/drivers/dell/test_dellfc.py +++ b/cinder/tests/unit/volume/drivers/dell/test_dellfc.py @@ -262,7 +262,8 @@ class DellSCSanFCDriverTestCase(test.TestCase): sclivevol = {'instanceId': '101.101', 'secondaryVolume': {'instanceId': '102.101', 'instanceName': fake.VOLUME_ID}, - 'secondaryScSerialNumber': 102} + 'secondaryScSerialNumber': 102, + 'secondaryRole': 'Secondary'} mock_is_live_volume.return_value = True mock_find_wwns.return_value = ( 1, [u'5000D31000FCBE3D', u'5000D31000FCBE35'], @@ -272,7 +273,7 @@ class DellSCSanFCDriverTestCase(test.TestCase): 1, [u'5000D31000FCBE3E', u'5000D31000FCBE36'], {u'21000024FF30441E': [u'5000D31000FCBE36'], u'21000024FF30441F': [u'5000D31000FCBE3E']}) - mock_get_live_volume.return_value = (sclivevol, False) + mock_get_live_volume.return_value = sclivevol res = self.driver.initialize_connection(volume, connector) expected = {'data': {'discard': True, @@ -292,6 +293,74 @@ class DellSCSanFCDriverTestCase(test.TestCase): mock_find_volume.assert_called_once_with(fake.VOLUME_ID, None, True) mock_get_volume.assert_called_once_with(self.VOLUME[u'instanceId']) + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + 'find_server', + return_value=SCSERVER) + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + 'find_volume') + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + 'get_volume') + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + 'map_volume', + return_value=MAPPING) + @mock.patch.object(dell_storagecenter_fc.DellStorageCenterFCDriver, + '_is_live_vol') + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + 'find_wwns') + @mock.patch.object(dell_storagecenter_fc.DellStorageCenterFCDriver, + 'initialize_secondary') + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + 'get_live_volume') + def test_initialize_connection_live_vol_afo(self, + mock_get_live_volume, + mock_initialize_secondary, + mock_find_wwns, + mock_is_live_volume, + mock_map_volume, + mock_get_volume, + mock_find_volume, + mock_find_server, + mock_close_connection, + mock_open_connection, + mock_init): + volume = {'id': fake.VOLUME_ID, 'provider_id': '101.101'} + scvol = {'instanceId': '102.101'} + mock_find_volume.return_value = scvol + mock_get_volume.return_value = scvol + connector = self.connector + sclivevol = {'instanceId': '101.10001', + 'primaryVolume': {'instanceId': '102.101', + 'instanceName': fake.VOLUME_ID}, + 'primaryScSerialNumber': 102, + 'secondaryVolume': {'instanceId': '101.101', + 'instanceName': fake.VOLUME_ID}, + 'secondaryScSerialNumber': 101, + 'secondaryRole': 'Activated'} + + mock_is_live_volume.return_value = True + mock_find_wwns.return_value = ( + 1, [u'5000D31000FCBE3D', u'5000D31000FCBE35'], + {u'21000024FF30441C': [u'5000D31000FCBE35'], + u'21000024FF30441D': [u'5000D31000FCBE3D']}) + mock_get_live_volume.return_value = sclivevol + res = self.driver.initialize_connection(volume, connector) + expected = {'data': + {'discard': True, + 'initiator_target_map': + {u'21000024FF30441C': [u'5000D31000FCBE35'], + u'21000024FF30441D': [u'5000D31000FCBE3D']}, + 'target_discovered': True, + 'target_lun': 1, + 'target_wwn': [u'5000D31000FCBE3D', u'5000D31000FCBE35']}, + 'driver_volume_type': 'fibre_channel'} + + self.assertEqual(expected, res, 'Unexpected return data') + # verify find_volume has been called and that is has been called twice + self.assertFalse(mock_initialize_secondary.called) + mock_find_volume.assert_called_once_with( + fake.VOLUME_ID, '101.101', True) + mock_get_volume.assert_called_once_with('102.101') + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, 'find_server', return_value=SCSERVER) @@ -581,11 +650,7 @@ class DellSCSanFCDriverTestCase(test.TestCase): volume = {'id': fake.VOLUME_ID} connector = self.connector mock_terminate_secondary.return_value = (None, [], {}) - sclivevol = {'instanceId': '101.101', - 'secondaryVolume': {'instanceId': '102.101', - 'instanceName': fake.VOLUME_ID}, - 'secondaryScSerialNumber': 102} - mock_is_live_vol.return_value = sclivevol + mock_is_live_vol.return_value = True res = self.driver.terminate_connection(volume, connector) mock_unmap_volume.assert_called_once_with(self.VOLUME, self.SCSERVER) expected = {'driver_volume_type': 'fibre_channel', diff --git a/cinder/tests/unit/volume/drivers/dell/test_dellsc.py b/cinder/tests/unit/volume/drivers/dell/test_dellsc.py index c91ec851e..9f3734fcf 100644 --- a/cinder/tests/unit/volume/drivers/dell/test_dellsc.py +++ b/cinder/tests/unit/volume/drivers/dell/test_dellsc.py @@ -518,9 +518,9 @@ class DellSCSanISCSIDriverTestCase(test.TestCase): sclivevol = {'instanceId': '101.101', 'secondaryVolume': {'instanceId': '102.101', 'instanceName': fake.VOLUME_ID}, - 'secondaryScSerialNumber': 102} - mock_api.get_live_volume = mock.MagicMock(return_value=(sclivevol, - False)) + 'secondaryScSerialNumber': 102, + 'secondaryRole': 'Secondary'} + mock_api.get_live_volume = mock.MagicMock(return_value=sclivevol) # No replication driver data. ret = self.driver._delete_live_volume(mock_api, vol) self.assertFalse(ret) @@ -538,7 +538,7 @@ class DellSCSanISCSIDriverTestCase(test.TestCase): ret = self.driver._delete_live_volume(mock_api, vol) self.assertFalse(ret) # No live volume found. - mock_api.get_live_volume.return_value = (None, False) + mock_api.get_live_volume.return_value = None ret = self.driver._delete_live_volume(mock_api, vol) self.assertFalse(ret) @@ -786,7 +786,8 @@ class DellSCSanISCSIDriverTestCase(test.TestCase): data = self.driver.initialize_connection(volume, connector) self.assertEqual('iscsi', data['driver_volume_type']) # verify find_volume has been called and that is has been called twice - mock_find_volume.assert_called_once_with(fake.VOLUME_ID, provider_id) + mock_find_volume.assert_called_once_with( + fake.VOLUME_ID, provider_id, False) mock_get_volume.assert_called_once_with(provider_id) expected = {'data': self.ISCSI_PROPERTIES, 'driver_volume_type': 'iscsi'} @@ -990,11 +991,7 @@ class DellSCSanISCSIDriverTestCase(test.TestCase): mock_init): volume = {'id': fake.VOLUME_ID} connector = self.connector - sclivevol = {'instanceId': '101.101', - 'secondaryVolume': {'instanceId': '102.101', - 'instanceName': fake.VOLUME_ID}, - 'secondaryScSerialNumber': 102} - mock_is_live_vol.return_value = sclivevol + mock_is_live_vol.return_value = True lvol_properties = {'access_mode': 'rw', 'target_discovered': False, 'target_iqn': @@ -1018,6 +1015,75 @@ class DellSCSanISCSIDriverTestCase(test.TestCase): 'driver_volume_type': 'iscsi'} self.assertEqual(expected, ret) + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + 'find_server', + return_value=None) + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + 'create_server', + return_value=SCSERVER) + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + 'find_volume') + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + 'get_volume') + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + 'map_volume', + return_value=MAPPINGS[0]) + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + 'find_iscsi_properties') + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + 'get_live_volume') + @mock.patch.object(dell_storagecenter_iscsi.DellStorageCenterISCSIDriver, + '_is_live_vol') + @mock.patch.object(dell_storagecenter_iscsi.DellStorageCenterISCSIDriver, + 'initialize_secondary') + def test_initialize_connection_live_volume_afo(self, + mock_initialize_secondary, + mock_is_live_vol, + mock_get_live_vol, + mock_find_iscsi_props, + mock_map_volume, + mock_get_volume, + mock_find_volume, + mock_create_server, + mock_find_server, + mock_close_connection, + mock_open_connection, + mock_init): + volume = {'id': fake.VOLUME_ID, 'provider_id': '101.101'} + scvol = {'instanceId': '102.101'} + mock_find_volume.return_value = scvol + mock_get_volume.return_value = scvol + connector = self.connector + sclivevol = {'instanceId': '101.10001', + 'primaryVolume': {'instanceId': '101.101', + 'instanceName': fake.VOLUME_ID}, + 'primaryScSerialNumber': 101, + 'secondaryVolume': {'instanceId': '102.101', + 'instanceName': fake.VOLUME_ID}, + 'secondaryScSerialNumber': 102, + 'secondaryRole': 'Activated'} + mock_is_live_vol.return_value = True + mock_get_live_vol.return_value = sclivevol + props = { + 'access_mode': 'rw', + 'target_discovered': False, + 'target_iqn': u'iqn:1', + 'target_iqns': [u'iqn:1', + u'iqn:2'], + 'target_lun': 1, + 'target_luns': [1, 1], + 'target_portal': u'192.168.1.21:3260', + 'target_portals': [u'192.168.1.21:3260', + u'192.168.1.22:3260'] + } + mock_find_iscsi_props.return_value = props + ret = self.driver.initialize_connection(volume, connector) + expected = {'data': props, + 'driver_volume_type': 'iscsi'} + expected['data']['discard'] = True + self.assertEqual(expected, ret) + self.assertFalse(mock_initialize_secondary.called) + @mock.patch.object(dell_storagecenter_iscsi.DellStorageCenterISCSIDriver, '_get_replication_specs', return_value={'enabled': True, 'live': True}) @@ -1230,9 +1296,10 @@ class DellSCSanISCSIDriverTestCase(test.TestCase): sclivevol = {'instanceId': '101.101', 'secondaryVolume': {'instanceId': '102.101', 'instanceName': fake.VOLUME_ID}, - 'secondaryScSerialNumber': 102} + 'secondaryScSerialNumber': 102, + 'secondaryRole': 'Secondary'} mock_is_live_vol.return_value = True - mock_get_live_vol.return_value = (sclivevol, False) + mock_get_live_vol.return_value = sclivevol connector = self.connector res = self.driver.terminate_connection(volume, connector) mock_unmap_volume.assert_called_once_with(self.VOLUME, self.SCSERVER) @@ -2661,36 +2728,36 @@ class DellSCSanISCSIDriverTestCase(test.TestCase): 'instanceName': fake.VOLUME2_ID}, 'secondaryVolume': {'instanceId': '102.101', 'instanceName': fake.VOLUME_ID}, - 'secondaryScSerialNumber': 102} + 'secondaryScSerialNumber': 102, + 'secondaryRole': 'Secondary'} postfail = {'instanceId': '101.100', 'primaryVolume': {'instanceId': '102.101', 'instanceName': fake.VOLUME_ID}, 'secondaryVolume': {'instanceId': '101.101', 'instanceName': fake.VOLUME2_ID}, - 'secondaryScSerialNumber': 102} + 'secondaryScSerialNumber': 102, + 'secondaryRole': 'Secondary'} mock_api.get_live_volume = mock.MagicMock() - mock_api.get_live_volume.side_effect = [(sclivevol, False), - (postfail, True), - (sclivevol, False), - (sclivevol, False) - ] + mock_api.get_live_volume.side_effect = [sclivevol, postfail, + sclivevol, sclivevol] # Good run. + mock_api.is_swapped = mock.MagicMock(return_value=False) mock_api.swap_roles_live_volume = mock.MagicMock(return_value=True) model_update = {'provider_id': '102.101', 'replication_status': 'failed-over'} ret = self.driver._failover_live_volume(mock_api, fake.VOLUME_ID, - '101.100') + '101.101') self.assertEqual(model_update, ret) # Swap fail mock_api.swap_roles_live_volume.return_value = False model_update = {'status': 'error'} ret = self.driver._failover_live_volume(mock_api, fake.VOLUME_ID, - '101.100') + '101.101') self.assertEqual(model_update, ret) # Can't find live volume. - mock_api.get_live_volume.return_value = (None, False) + mock_api.get_live_volume.return_value = None ret = self.driver._failover_live_volume(mock_api, fake.VOLUME_ID, - '101.100') + '101.101') self.assertEqual(model_update, ret) def test__failover_replication(self, @@ -3206,16 +3273,16 @@ class DellSCSanISCSIDriverTestCase(test.TestCase): {'id': fake.VOLUME2_ID, 'replication_driver_data': '12345', 'provider_id': '12345.2'}] - mock_get_live_volume.side_effect = [( + mock_get_live_volume.side_effect = [ {'instanceId': '11111.101', 'secondaryVolume': {'instanceId': '11111.1001', 'instanceName': fake.VOLUME_ID}, - 'secondaryScSerialNumber': 11111}, True), ( + 'secondaryScSerialNumber': 11111}, {'instanceId': '11111.102', 'secondaryVolume': {'instanceId': '11111.1002', 'instanceName': fake.VOLUME2_ID}, - 'secondaryScSerialNumber': 11111}, True - )] + 'secondaryScSerialNumber': 11111} + ] mock_get_replication_specs.return_value = {'enabled': True, 'live': True} mock_swap_roles_live_volume.side_effect = [True, True] diff --git a/cinder/tests/unit/volume/drivers/dell/test_dellscapi.py b/cinder/tests/unit/volume/drivers/dell/test_dellscapi.py index f52e91e51..ed4d82173 100644 --- a/cinder/tests/unit/volume/drivers/dell/test_dellscapi.py +++ b/cinder/tests/unit/volume/drivers/dell/test_dellscapi.py @@ -21,6 +21,7 @@ import uuid from cinder import context from cinder import exception from cinder import test +from cinder.tests.unit import fake_constants as fake from cinder.volume.drivers.dell import dell_storagecenter_api @@ -5977,7 +5978,7 @@ class DellSCSanAPITestCase(test.TestCase): expected = 'StorageCenter/ScReplication/%s' % ( self.SCREPL[0]['instanceId']) expected_payload = {'DeleteDestinationVolume': True, - 'RecycleDestinationVolume': False, + 'RecycleDestinationVolume': True, 'DeleteRestorePoint': True} ret = self.scapi.delete_replication(self.VOLUME, destssn) mock_delete.assert_any_call(expected, payload=expected_payload, @@ -6014,7 +6015,7 @@ class DellSCSanAPITestCase(test.TestCase): expected = 'StorageCenter/ScReplication/%s' % ( self.SCREPL[0]['instanceId']) expected_payload = {'DeleteDestinationVolume': True, - 'RecycleDestinationVolume': False, + 'RecycleDestinationVolume': True, 'DeleteRestorePoint': True} ret = self.scapi.delete_replication(self.VOLUME, destssn) mock_delete.assert_any_call(expected, payload=expected_payload, @@ -6399,37 +6400,60 @@ class DellSCSanAPITestCase(test.TestCase): scvol, 'a,b') - @mock.patch.object(dell_storagecenter_api.HttpClient, - 'get') @mock.patch.object(dell_storagecenter_api.StorageCenterApi, - '_get_json') + '_sc_live_volumes') + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + '_get_live_volumes') def test_get_live_volume(self, - mock_get_json, - mock_get, + mock_get_live_volumes, + mock_sc_live_volumes, mock_close_connection, mock_open_connection, mock_init): # Basic check - retlv, retswapped = self.scapi.get_live_volume(None) + retlv = self.scapi.get_live_volume(None) self.assertIsNone(retlv) - self.assertFalse(retswapped) lv1 = {'primaryVolume': {'instanceId': '12345.1'}, 'secondaryVolume': {'instanceId': '67890.1'}} lv2 = {'primaryVolume': {'instanceId': '12345.2'}} - mock_get_json.return_value = [lv1, lv2] - mock_get.return_value = self.RESPONSE_200 + mock_sc_live_volumes.return_value = [lv1, lv2] # Good Run - retlv, retswapped = self.scapi.get_live_volume('12345.2') + retlv = self.scapi.get_live_volume('12345.2') self.assertEqual(lv2, retlv) - self.assertFalse(retswapped) + mock_sc_live_volumes.assert_called_once_with('12345') + self.assertFalse(mock_get_live_volumes.called) - @mock.patch.object(dell_storagecenter_api.HttpClient, - 'get') @mock.patch.object(dell_storagecenter_api.StorageCenterApi, - '_get_json') + '_sc_live_volumes') + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + '_get_live_volumes') + def test_get_live_volume_on_secondary(self, + mock_get_live_volumes, + mock_sc_live_volumes, + mock_close_connection, + mock_open_connection, + mock_init): + # Basic check + retlv = self.scapi.get_live_volume(None) + self.assertIsNone(retlv) + lv1 = {'primaryVolume': {'instanceId': '12345.1'}, + 'secondaryVolume': {'instanceId': '67890.1'}} + lv2 = {'primaryVolume': {'instanceId': '12345.2'}} + mock_sc_live_volumes.return_value = [] + mock_get_live_volumes.return_value = [lv1, lv2] + # Good Run + retlv = self.scapi.get_live_volume('12345.2') + self.assertEqual(lv2, retlv) + mock_sc_live_volumes.assert_called_once_with('12345') + mock_get_live_volumes.assert_called_once_with() + + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + '_sc_live_volumes') + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + '_get_live_volumes') def test_get_live_volume_not_found(self, - mock_get_json, - mock_get, + mock_get_live_volumes, + mock_sc_live_volumes, mock_close_connection, mock_open_connection, mock_init): @@ -6437,19 +6461,20 @@ class DellSCSanAPITestCase(test.TestCase): 'secondaryVolume': {'instanceId': '67890.1'}} lv2 = {'primaryVolume': {'instanceId': '12345.2'}, 'secondaryVolume': {'instanceId': '67890.2'}} - mock_get_json.return_value = [lv1, lv2] - mock_get.return_value = self.RESPONSE_200 - retlv, retswapped = self.scapi.get_live_volume('12345.3') + mock_get_live_volumes.return_value = [lv1, lv2] + mock_sc_live_volumes.return_value = [] + retlv = self.scapi.get_live_volume('12345.3') self.assertIsNone(retlv) - self.assertFalse(retswapped) + mock_sc_live_volumes.assert_called_once_with('12345') + mock_get_live_volumes.assert_called_once_with() - @mock.patch.object(dell_storagecenter_api.HttpClient, - 'get') @mock.patch.object(dell_storagecenter_api.StorageCenterApi, - '_get_json') + '_sc_live_volumes') + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + '_get_live_volumes') def test_get_live_volume_swapped(self, - mock_get_json, - mock_get, + mock_get_live_volumes, + mock_sc_live_volumes, mock_close_connection, mock_open_connection, mock_init): @@ -6457,23 +6482,50 @@ class DellSCSanAPITestCase(test.TestCase): 'secondaryVolume': {'instanceId': '67890.1'}} lv2 = {'primaryVolume': {'instanceId': '67890.2'}, 'secondaryVolume': {'instanceId': '12345.2'}} - mock_get_json.return_value = [lv1, lv2] - mock_get.return_value = self.RESPONSE_200 - retlv, retswapped = self.scapi.get_live_volume('12345.2') + mock_get_live_volumes.return_value = [lv1, lv2] + mock_sc_live_volumes.return_value = [] + retlv = self.scapi.get_live_volume('12345.2') self.assertEqual(lv2, retlv) - self.assertTrue(retswapped) + mock_sc_live_volumes.assert_called_once_with('12345') + mock_get_live_volumes.assert_called_once_with() - @mock.patch.object(dell_storagecenter_api.HttpClient, - 'get') + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + '_sc_live_volumes') + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + '_get_live_volumes') def test_get_live_volume_error(self, - mock_get, + mock_get_live_volumes, + mock_sc_live_volumes, mock_close_connection, mock_open_connection, mock_init): - mock_get.return_value = self.RESPONSE_400 - retlv, retswapped = self.scapi.get_live_volume('12345.2') + mock_get_live_volumes.return_value = [] + mock_sc_live_volumes.return_value = [] + retlv = self.scapi.get_live_volume('12345.2') self.assertIsNone(retlv) - self.assertFalse(retswapped) + + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + '_sc_live_volumes') + @mock.patch.object(dell_storagecenter_api.StorageCenterApi, + '_get_live_volumes') + def test_get_live_volume_by_name(self, + mock_get_live_volumes, + mock_sc_live_volumes, + mock_close_connection, + mock_open_connection, + mock_init): + lv1 = {'primaryVolume': {'instanceId': '12345.1'}, + 'secondaryVolume': {'instanceId': '67890.1'}, + 'instanceName': 'Live volume of ' + fake.VOLUME2_ID} + lv2 = {'primaryVolume': {'instanceId': '67890.2'}, + 'secondaryVolume': {'instanceId': '12345.2'}, + 'instanceName': 'Live volume of ' + fake.VOLUME_ID} + mock_get_live_volumes.return_value = [lv1, lv2] + mock_sc_live_volumes.return_value = [] + retlv = self.scapi.get_live_volume('12345.2', fake.VOLUME_ID) + self.assertEqual(lv2, retlv) + mock_sc_live_volumes.assert_called_once_with('12345') + mock_get_live_volumes.assert_called_once_with() @mock.patch.object(dell_storagecenter_api.HttpClient, 'post') diff --git a/cinder/volume/drivers/dell/dell_storagecenter_api.py b/cinder/volume/drivers/dell/dell_storagecenter_api.py index d2903d094..6a0edd1af 100644 --- a/cinder/volume/drivers/dell/dell_storagecenter_api.py +++ b/cinder/volume/drivers/dell/dell_storagecenter_api.py @@ -1091,20 +1091,26 @@ class StorageCenterApi(object): def _autofailback(self, lv): # if we have a working replication state. ret = False - if (lv['ReplicationState'] == 'Up' and - lv['failoverState'] == 'AutoFailedOver'): + LOG.debug('Attempting autofailback of %s', lv) + if (lv and lv['status'] == 'Up' and lv['replicationState'] == 'Up' and + lv['failoverState'] == 'Protected' and lv['secondaryStatus'] == 'Up' + and lv['primarySwapRoleState'] == 'NotSwapping'): ret = self.swap_roles_live_volume(lv) return ret - def _find_volume_primary(self, provider_id): + def _find_volume_primary(self, provider_id, name): # if there is no live volume then we return our provider_id. primary_id = provider_id - lv, swapped = self.get_live_volume(provider_id) - # if we swapped see if we can autofailback. Unless the admin - # failed us over, that is. - if swapped and not self.failed_over: + lv = self.get_live_volume(provider_id, name) + LOG.info(_LI('Volume %(provider)s at primary %(primary)s.'), + {'provider': provider_id, 'primary': primary_id}) + # If we have a live volume and are swapped and are not failed over + # at least give failback a shot. + if lv and self.is_swapped(provider_id, lv) and not self.failed_over: if self._autofailback(lv): - ls, swapped = self.get_live_volume(provider_id) + lv = self.get_live_volume(provider_id) + LOG.info(_LI('After failback %s'), lv) + if lv: primary_id = lv['primaryVolume']['instanceId'] return primary_id @@ -1127,7 +1133,7 @@ class StorageCenterApi(object): scvolume = None if islivevol: # Just get the primary from the sc live vol. - primary_id = self._find_volume_primary(provider_id) + primary_id = self._find_volume_primary(provider_id, name) scvolume = self.get_volume(primary_id) elif self._use_provider_id(provider_id): # just get our volume @@ -2733,7 +2739,7 @@ class StorageCenterApi(object): if replication: payload = {} payload['DeleteDestinationVolume'] = deletedestvolume - payload['RecycleDestinationVolume'] = False + payload['RecycleDestinationVolume'] = deletedestvolume payload['DeleteRestorePoint'] = True r = self.client.delete('StorageCenter/ScReplication/%s' % self._get_id(replication), payload=payload, @@ -3053,24 +3059,73 @@ class StorageCenterApi(object): progress) return None, None - def get_live_volume(self, primaryid): + def is_swapped(self, provider_id, sclivevolume): + if (sclivevolume.get('primaryVolume') and + sclivevolume['primaryVolume']['instanceId'] != provider_id): + return True + return False + + def is_failed_over(self, provider_id, sclivevolume): + # either the secondary is active or the secondary is now our primary. + if (sclivevolume.get('secondaryRole') == 'Activated' or + self.is_swapped(provider_id, sclivevolume)): + return True + return False + + def _sc_live_volumes(self, ssn): + if ssn: + r = self.client.get('StorageCenter/StorageCenter/%s/LiveVolumeList' + % ssn) + if self._check_result(r): + return self._get_json(r) + return [] + + def _get_live_volumes(self): + # Work around for a FW bug. Instead of grabbing the entire list at + # once we have to Trundle through each SC's list. + lvs = [] + pf = self._get_payload_filter() + pf.append('connected', True) + r = self.client.post('StorageCenter/StorageCenter/GetList', + pf.payload) + if self._check_result(r): + # Should return [] if nothing there. + # Just in case do the or. + scs = self._get_json(r) or [] + for sc in scs: + lvs += self._sc_live_volumes(self._get_id(sc)) + return lvs + + def get_live_volume(self, primaryid, name=None): """Get's the live ScLiveVolume object for the vol with primaryid. :param primaryid: InstanceId of the primary volume. - :return: ScLiveVolume object or None, swapped True/False. + :parma name: Volume name associated with this live volume. + :return: ScLiveVolume object or None """ + sclivevol = None if primaryid: - r = self.client.get('StorageCenter/ScLiveVolume') - if self._check_result(r): - lvs = self._get_json(r) + # Try from our primary SSN. This would be the authoritay on the + # Live Volume in question. + lvs = self._sc_live_volumes(primaryid.split('.')[0]) + # No, grab them all and see if we are on the secondary. + if not lvs: + lvs = self._get_live_volumes() + if lvs: + # Look for our primaryid. for lv in lvs: - if (lv.get('primaryVolume') and - lv['primaryVolume']['instanceId'] == primaryid): - return lv, False - if (lv.get('secondaryVolume') and - lv['secondaryVolume']['instanceId'] == primaryid): - return lv, True - return None, False + if ((lv.get('primaryVolume') and + lv['primaryVolume']['instanceId'] == primaryid) or + (lv.get('secondaryVolume') and + lv['secondaryVolume']['instanceId'] == primaryid)): + sclivevol = lv + break + # Sometimes the lv object returns without a secondary + # volume. Make sure we find this by name if we have to. + if (name and sclivevol is None and + lv['instanceName'].endswith(name)): + sclivevol = lv + return sclivevol def _get_hbas(self, serverid): # Helper to get the hba's of a given server. @@ -3170,6 +3225,7 @@ class StorageCenterApi(object): payload['ConvertToReplication'] = False payload['DeleteSecondaryVolume'] = deletesecondaryvolume payload['RecycleSecondaryVolume'] = deletesecondaryvolume + payload['DeleteRestorePoint'] = deletesecondaryvolume r = self.client.delete('StorageCenter/ScLiveVolume/%s' % self._get_id(sclivevolume), payload, True) if self._check_result(r): diff --git a/cinder/volume/drivers/dell/dell_storagecenter_common.py b/cinder/volume/drivers/dell/dell_storagecenter_common.py index c0bdf2ebf..4dbec6787 100644 --- a/cinder/volume/drivers/dell/dell_storagecenter_common.py +++ b/cinder/volume/drivers/dell/dell_storagecenter_common.py @@ -362,8 +362,7 @@ class DellCommonDriver(driver.ConsistencyGroupVD, driver.ManageableVD, ssnstrings = self._split_driver_data(replication_driver_data) if ssnstrings: ssn = int(ssnstrings[0]) - sclivevolume, swapped = api.get_live_volume( - volume.get('provider_id')) + sclivevolume = api.get_live_volume(volume.get('provider_id')) # Have we found the live volume? if (sclivevolume and sclivevolume.get('secondaryScSerialNumber') == ssn and @@ -682,12 +681,7 @@ class DellCommonDriver(driver.ConsistencyGroupVD, driver.ManageableVD, def _update_volume_stats(self): """Retrieve stats info from volume group.""" with self._client.open_connection() as api: - storageusage = api.get_storage_usage() - if not storageusage: - msg = _('Unable to retrieve volume stats.') - raise exception.VolumeBackendAPIException(message=msg) - - # all of this is basically static for now + # Static stats. data = {} data['volume_backend_name'] = self.backend_name data['vendor_name'] = 'Dell' @@ -696,12 +690,6 @@ class DellCommonDriver(driver.ConsistencyGroupVD, driver.ManageableVD, data['reserved_percentage'] = 0 data['consistencygroup_support'] = True data['thin_provisioning_support'] = True - totalcapacity = storageusage.get('availableSpace') - totalcapacitygb = self._bytes_to_gb(totalcapacity) - data['total_capacity_gb'] = totalcapacitygb - freespace = storageusage.get('freeSpace') - freespacegb = self._bytes_to_gb(freespace) - data['free_capacity_gb'] = freespacegb data['QoS_support'] = False data['replication_enabled'] = self.replication_enabled if self.replication_enabled: @@ -715,6 +703,22 @@ class DellCommonDriver(driver.ConsistencyGroupVD, driver.ManageableVD, replication_targets.append(target_device_id) data['replication_targets'] = replication_targets + # Get our capacity. + storageusage = api.get_storage_usage() + if storageusage: + # Get actual stats. + totalcapacity = storageusage.get('availableSpace') + totalcapacitygb = self._bytes_to_gb(totalcapacity) + data['total_capacity_gb'] = totalcapacitygb + freespace = storageusage.get('freeSpace') + freespacegb = self._bytes_to_gb(freespace) + data['free_capacity_gb'] = freespacegb + else: + # Soldier on. Just return 0 for this iteration. + LOG.error(_LE('Unable to retrieve volume stats.')) + data['total_capacity_gb'] = 0 + data['free_capacity_gb'] = 0 + self._stats = data LOG.debug('Total cap %(total)s Free cap %(free)s', {'total': data['total_capacity_gb'], @@ -1390,7 +1394,10 @@ class DellCommonDriver(driver.ConsistencyGroupVD, driver.ManageableVD, :return: model_update dict """ model_update = {} - sclivevolume, swapped = api.get_live_volume(provider_id) + # We do not search by name. Only failback if we have a complete + # LV object. + sclivevolume = api.get_live_volume(provider_id) + # TODO(tswanson): Check swapped state first. if sclivevolume and api.swap_roles_live_volume(sclivevolume): LOG.info(_LI('Success swapping sclivevolume roles %s'), id) model_update = { @@ -1489,8 +1496,10 @@ class DellCommonDriver(driver.ConsistencyGroupVD, driver.ManageableVD, def _failover_live_volume(self, api, id, provider_id): model_update = {} - sclivevolume, swapped = api.get_live_volume(provider_id) + # Search for volume by id if we have to. + sclivevolume = api.get_live_volume(provider_id, id) if sclivevolume: + swapped = api.is_swapped(provider_id, sclivevolume) # If we aren't swapped try it. If fail error out. if not swapped and not api.swap_roles_live_volume(sclivevolume): LOG.info(_LI('Failure swapping roles %s'), id) @@ -1498,7 +1507,7 @@ class DellCommonDriver(driver.ConsistencyGroupVD, driver.ManageableVD, return model_update LOG.info(_LI('Success swapping sclivevolume roles %s'), id) - sclivevolume, swapped = api.get_live_volume(provider_id) + sclivevolume = api.get_live_volume(provider_id) model_update = { 'replication_status': fields.ReplicationStatus.FAILED_OVER, diff --git a/cinder/volume/drivers/dell/dell_storagecenter_fc.py b/cinder/volume/drivers/dell/dell_storagecenter_fc.py index 20d9ac4d8..f5fecbfd1 100644 --- a/cinder/volume/drivers/dell/dell_storagecenter_fc.py +++ b/cinder/volume/drivers/dell/dell_storagecenter_fc.py @@ -90,52 +90,59 @@ class DellStorageCenterFCDriver(dell_storagecenter_common.DellCommonDriver, with self._client.open_connection() as api: try: wwpns = connector.get('wwpns') - # Find our server. - scserver = self._find_server(api, wwpns) - - # No? Create it. - if scserver is None: - scserver = api.create_server( - wwpns, self.configuration.dell_server_os) - # Find the volume on the storage center. + # Find the volume on the storage center. Note that if this + # is live volume and we are swapped this will be the back + # half of the live volume. scvolume = api.find_volume(volume_name, provider_id, islivevol) - if scserver is not None and scvolume is not None: - mapping = api.map_volume(scvolume, scserver) - if mapping is not None: - # Since we just mapped our volume we had best update - # our sc volume object. - scvolume = api.get_volume(scvolume['instanceId']) - lun, targets, init_targ_map = api.find_wwns(scvolume, - scserver) + if scvolume: + # Get the SSN it is on. + ssn = scvolume['instanceId'].split('.')[0] + # Find our server. + scserver = self._find_server(api, wwpns, ssn) - # Do we have extra live volume work? - if islivevol: - # Get our volume and our swap state. - sclivevolume, swapped = api.get_live_volume( - provider_id) - # Do not map to a failed over volume. - if sclivevolume and not swapped: - # Now map our secondary. - lvlun, lvtargets, lvinit_targ_map = ( - self.initialize_secondary(api, - sclivevolume, - wwpns)) - # Unmapped. Add info to our list. - targets += lvtargets - init_targ_map.update(lvinit_targ_map) + # No? Create it. + if scserver is None: + scserver = api.create_server( + wwpns, self.configuration.dell_server_os, ssn) + # We have a volume and a server. Map them. + if scserver is not None: + mapping = api.map_volume(scvolume, scserver) + if mapping is not None: + # Since we just mapped our volume we had + # best update our sc volume object. + scvolume = api.get_volume(scvolume['instanceId']) + lun, targets, init_targ_map = api.find_wwns( + scvolume, scserver) - # Roll up our return data. - if lun is not None and len(targets) > 0: - data = {'driver_volume_type': 'fibre_channel', - 'data': {'target_lun': lun, - 'target_discovered': True, - 'target_wwn': targets, - 'initiator_target_map': - init_targ_map, - 'discard': True}} - LOG.debug('Return FC data: %s', data) - return data - LOG.error(_LE('Lun mapping returned null!')) + # Do we have extra live volume work? + if islivevol: + # Get our live volume. + sclivevolume = api.get_live_volume(provider_id) + # Do not map to a failed over volume. + if (sclivevolume and not + api.is_failed_over(provider_id, + sclivevolume)): + # Now map our secondary. + lvlun, lvtargets, lvinit_targ_map = ( + self.initialize_secondary(api, + sclivevolume, + wwpns)) + # Unmapped. Add info to our list. + targets += lvtargets + init_targ_map.update(lvinit_targ_map) + + # Roll up our return data. + if lun is not None and len(targets) > 0: + data = {'driver_volume_type': 'fibre_channel', + 'data': {'target_lun': lun, + 'target_discovered': True, + 'target_wwn': targets, + 'initiator_target_map': + init_targ_map, + 'discard': True}} + LOG.debug('Return FC data: %s', data) + return data + LOG.error(_LE('Lun mapping returned null!')) except Exception: with excutils.save_and_reraise_exception(): @@ -191,46 +198,53 @@ class DellStorageCenterFCDriver(dell_storagecenter_common.DellCommonDriver, with self._client.open_connection() as api: try: wwpns = connector.get('wwpns') - scserver = self._find_server(api, wwpns) - # Find the volume on the storage center. scvolume = api.find_volume(volume_name, provider_id, islivevol) - # Get our target map so we can return it to free up a zone. - lun, targets, init_targ_map = api.find_wwns(scvolume, scserver) + if scvolume: + # Get the SSN it is on. + ssn = scvolume['instanceId'].split('.')[0] - # Do we have extra live volume work? - if islivevol: - # Get our volume and our swap state. - sclivevolume, swapped = api.get_live_volume( - provider_id) - # Do not map to a failed over volume. - if sclivevolume and not swapped: - lvlun, lvtargets, lvinit_targ_map = ( - self.terminate_secondary(api, sclivevolume, wwpns)) - # Add to our return. - if lvlun: - targets += lvtargets - init_targ_map.update(lvinit_targ_map) + scserver = self._find_server(api, wwpns, ssn) - # If we have a server and a volume lets unmap them. - if (scserver is not None and - scvolume is not None and - api.unmap_volume(scvolume, scserver) is True): - LOG.debug('Connection terminated') - else: - raise exception.VolumeBackendAPIException( - _('Terminate connection failed')) + # Get our target map so we can return it to free up a zone. + lun, targets, init_targ_map = api.find_wwns(scvolume, + scserver) - # basic return info... - info = {'driver_volume_type': 'fibre_channel', - 'data': {}} + # Do we have extra live volume work? + if islivevol: + # Get our live volume. + sclivevolume = api.get_live_volume(provider_id) + # Do not map to a failed over volume. + if (sclivevolume and not + api.is_failed_over(provider_id, + sclivevolume)): + lvlun, lvtargets, lvinit_targ_map = ( + self.terminate_secondary( + api, sclivevolume, wwpns)) + # Add to our return. + if lvlun: + targets += lvtargets + init_targ_map.update(lvinit_targ_map) - # if not then we return the target map so that - # the zone can be freed up. - if api.get_volume_count(scserver) == 0: - info['data'] = {'target_wwn': targets, - 'initiator_target_map': init_targ_map} - return info + # If we have a server and a volume lets unmap them. + if (scserver is not None and + scvolume is not None and + api.unmap_volume(scvolume, scserver) is True): + LOG.debug('Connection terminated') + else: + raise exception.VolumeBackendAPIException( + _('Terminate connection failed')) + + # basic return info... + info = {'driver_volume_type': 'fibre_channel', + 'data': {}} + + # if not then we return the target map so that + # the zone can be freed up. + if api.get_volume_count(scserver) == 0: + info['data'] = {'target_wwn': targets, + 'initiator_target_map': init_targ_map} + return info except Exception: with excutils.save_and_reraise_exception(): diff --git a/cinder/volume/drivers/dell/dell_storagecenter_iscsi.py b/cinder/volume/drivers/dell/dell_storagecenter_iscsi.py index fd1957d70..f84efd5bc 100644 --- a/cinder/volume/drivers/dell/dell_storagecenter_iscsi.py +++ b/cinder/volume/drivers/dell/dell_storagecenter_iscsi.py @@ -96,59 +96,68 @@ class DellStorageCenterISCSIDriver(dell_storagecenter_common.DellCommonDriver, with self._client.open_connection() as api: try: - # Find our server. - scserver = api.find_server(initiator_name) - # No? Create it. - if scserver is None: - scserver = api.create_server( - [initiator_name], self.configuration.dell_server_os) - # Find the volume on the storage center. - scvolume = api.find_volume(volume_name, provider_id) + # Find the volume on the storage center. Note that if this + # is live volume and we are swapped this will be the back + # half of the live volume. + scvolume = api.find_volume(volume_name, provider_id, islivevol) + if scvolume: + # Get the SSN it is on. + ssn = scvolume['instanceId'].split('.')[0] + # Find our server. + scserver = api.find_server(initiator_name, ssn) + # No? Create it. + if scserver is None: + scserver = api.create_server( + [initiator_name], + self.configuration.dell_server_os, ssn) - # if we have a server and a volume lets bring them together. - if scserver is not None and scvolume is not None: - mapping = api.map_volume(scvolume, scserver) - if mapping is not None: - # Since we just mapped our volume we had best update - # our sc volume object. - scvolume = api.get_volume(provider_id) - # Our return. - iscsiprops = {} + # if we have a server and a volume lets bring them + # together. + if scserver is not None: + mapping = api.map_volume(scvolume, scserver) + if mapping is not None: + # Since we just mapped our volume we had best + # update our sc volume object. + scvolume = api.get_volume(scvolume['instanceId']) + # Our return. + iscsiprops = {} - # Three cases that should all be satisfied with the - # same return of Target_Portal and Target_Portals. - # 1. Nova is calling us so we need to return the - # Target_Portal stuff. It should ignore the - # Target_Portals stuff. - # 2. OS brick is calling us in multipath mode so we - # want to return Target_Portals. It will ignore - # the Target_Portal stuff. - # 3. OS brick is calling us in single path mode so - # we want to return Target_Portal and - # Target_Portals as alternates. - iscsiprops = api.find_iscsi_properties(scvolume) + # Three cases that should all be satisfied with the + # same return of Target_Portal and Target_Portals. + # 1. Nova is calling us so we need to return the + # Target_Portal stuff. It should ignore the + # Target_Portals stuff. + # 2. OS brick is calling us in multipath mode so we + # want to return Target_Portals. It will ignore + # the Target_Portal stuff. + # 3. OS brick is calling us in single path mode so + # we want to return Target_Portal and + # Target_Portals as alternates. + iscsiprops = api.find_iscsi_properties(scvolume) - # If this is a live volume we need to map up our - # secondary volume. - if islivevol: - sclivevolume, swapped = api.get_live_volume( - provider_id) - # Only map if we are not swapped. - if sclivevolume and not swapped: - secondaryprops = self.initialize_secondary( - api, sclivevolume, initiator_name) - # Combine with iscsiprops - iscsiprops['target_iqns'] += ( - secondaryprops['target_iqns']) - iscsiprops['target_portals'] += ( - secondaryprops['target_portals']) - iscsiprops['target_luns'] += ( - secondaryprops['target_luns']) + # If this is a live volume we need to map up our + # secondary volume. Note that if we have failed + # over we do not wish to do this. + if islivevol: + sclivevolume = api.get_live_volume(provider_id) + # Only map if we are not failed over. + if (sclivevolume and not + api.is_failed_over(provider_id, + sclivevolume)): + secondaryprops = self.initialize_secondary( + api, sclivevolume, initiator_name) + # Combine with iscsiprops + iscsiprops['target_iqns'] += ( + secondaryprops['target_iqns']) + iscsiprops['target_portals'] += ( + secondaryprops['target_portals']) + iscsiprops['target_luns'] += ( + secondaryprops['target_luns']) - # Return our iscsi properties. - iscsiprops['discard'] = True - return {'driver_volume_type': 'iscsi', - 'data': iscsiprops} + # Return our iscsi properties. + iscsiprops['discard'] = True + return {'driver_volume_type': 'iscsi', + 'data': iscsiprops} # Re-raise any backend exception. except exception.VolumeBackendAPIException: with excutils.save_and_reraise_exception(): @@ -214,23 +223,31 @@ class DellStorageCenterISCSIDriver(dell_storagecenter_common.DellCommonDriver, 'initiator': initiator_name}) with self._client.open_connection() as api: try: - scserver = api.find_server(initiator_name) - # Find the volume on the storage center. - scvolume = api.find_volume(volume_name, provider_id) + # Find the volume on the storage center. Note that if this + # is live volume and we are swapped this will be the back + # half of the live volume. + scvolume = api.find_volume(volume_name, provider_id, islivevol) + if scvolume: + # Get the SSN it is on. + ssn = scvolume['instanceId'].split('.')[0] + # Find our server. + scserver = api.find_server(initiator_name, ssn) - # Unmap our secondary if it isn't swapped. - if islivevol: - sclivevolume, swapped = api.get_live_volume(provider_id) - if sclivevolume and not swapped: - self.terminate_secondary(api, sclivevolume, - initiator_name) + # Unmap our secondary if not failed over.. + if islivevol: + sclivevolume = api.get_live_volume(provider_id) + if (sclivevolume and not + api.is_failed_over(provider_id, + sclivevolume)): + self.terminate_secondary(api, sclivevolume, + initiator_name) - # If we have a server and a volume lets pull them apart. - if (scserver is not None and - scvolume is not None and - api.unmap_volume(scvolume, scserver) is True): - LOG.debug('Connection terminated') - return + # If we have a server and a volume lets pull them apart. + if (scserver is not None and + scvolume is not None and + api.unmap_volume(scvolume, scserver) is True): + LOG.debug('Connection terminated') + return except Exception: with excutils.save_and_reraise_exception(): LOG.error(_LE('Failed to terminate connection '