diff --git a/cinder/tests/unit/volume/drivers/open_e/test_iscsi.py b/cinder/tests/unit/volume/drivers/open_e/test_iscsi.py index c2c8d7b1b42..d4d2e79399c 100644 --- a/cinder/tests/unit/volume/drivers/open_e/test_iscsi.py +++ b/cinder/tests/unit/volume/drivers/open_e/test_iscsi.py @@ -27,7 +27,6 @@ from cinder.volume.drivers.open_e import iscsi from cinder.volume.drivers.open_e.jovian_common import exception as jexc from cinder.volume.drivers.open_e.jovian_common import jdss_common as jcom - UUID_1 = '12345678-1234-1234-1234-000000000001' UUID_2 = '12345678-1234-1234-1234-000000000002' UUID_3 = '12345678-1234-1234-1234-000000000003' @@ -36,7 +35,7 @@ UUID_4 = '12345678-1234-1234-1234-000000000004' CONFIG_OK = { 'san_hosts': ['192.168.0.2'], 'san_api_port': 82, - 'driver_use_ssl': 'https', + 'driver_use_ssl': 'true', 'jovian_rest_send_repeats': 3, 'jovian_recovery_delay': 60, 'jovian_user': 'admin', @@ -53,7 +52,7 @@ CONFIG_OK = { CONFIG_BLOCK_SIZE = { 'san_hosts': ['192.168.0.2'], 'san_api_port': 82, - 'driver_use_ssl': 'https', + 'driver_use_ssl': 'true', 'jovian_rest_send_repeats': 3, 'jovian_recovery_delay': 60, 'jovian_user': 'admin', @@ -67,10 +66,27 @@ CONFIG_BLOCK_SIZE = { 'jovian_block_size': '64K' } +CONFIG_BAD_BLOCK_SIZE = { + 'san_hosts': ['192.168.0.2'], + 'san_api_port': 82, + 'driver_use_ssl': 'true', + 'jovian_rest_send_repeats': 3, + 'jovian_recovery_delay': 60, + 'jovian_user': 'admin', + 'jovian_password': 'password', + 'jovian_ignore_tpath': [], + 'target_port': 3260, + 'jovian_pool': 'Pool-0', + 'target_prefix': 'iqn.2020-04.com.open-e.cinder:', + 'chap_password_len': 12, + 'san_thin_provision': False, + 'jovian_block_size': '61K' +} + CONFIG_BACKEND_NAME = { 'san_hosts': ['192.168.0.2'], 'san_api_port': 82, - 'driver_use_ssl': 'https', + 'driver_use_ssl': 'true', 'jovian_rest_send_repeats': 3, 'jovian_recovery_delay': 60, 'jovian_user': 'admin', @@ -89,7 +105,7 @@ CONFIG_BACKEND_NAME = { CONFIG_MULTI_HOST = { 'san_hosts': ['192.168.0.2', '192.168.0.3'], 'san_api_port': 82, - 'driver_use_ssl': 'https', + 'driver_use_ssl': 'true', 'jovian_rest_send_repeats': 3, 'jovian_recovery_delay': 60, 'jovian_user': 'admin', @@ -168,10 +184,6 @@ def get_jdss_exceptions(): return out -def fake_safe_get(value): - return CONFIG_OK[value] - - class TestOpenEJovianDSSDriver(test.TestCase): def get_driver(self, config): @@ -179,11 +191,16 @@ class TestOpenEJovianDSSDriver(test.TestCase): cfg = mock.Mock() cfg.append_config_values.return_value = None - cfg.safe_get = lambda val: config[val] + cfg.get = lambda val, default: config.get(val, default) jdssd = iscsi.JovianISCSIDriver() + jdssd.configuration = cfg - jdssd.do_setup(ctx) + lib_to_patch = ('cinder.volume.drivers.open_e.jovian_common.rest.' + 'JovianRESTAPI') + with mock.patch(lib_to_patch) as ra: + ra.is_pool_exists.return_value = True + jdssd.do_setup(ctx) jdssd.ra = mock.Mock() return jdssd, ctx @@ -195,6 +212,39 @@ class TestOpenEJovianDSSDriver(test.TestCase): for p in patches: p.stop() + def test_check_for_setup_error(self): + + cfg = mock.Mock() + cfg.append_config_values.return_value = None + + jdssd = iscsi.JovianISCSIDriver() + jdssd.configuration = cfg + + jdssd.ra = mock.Mock() + + # No IP + jdssd.ra.is_pool_exists.return_value = True + jdssd.jovian_hosts = [] + jdssd.block_size = ['64K'] + + self.assertRaises(exception.VolumeDriverException, + jdssd.check_for_setup_error) + + # No pool detected + jdssd.ra.is_pool_exists.return_value = False + jdssd.jovian_hosts = ['192.168.0.2'] + jdssd.block_size = ['64K'] + + self.assertRaises(exception.VolumeDriverException, + jdssd.check_for_setup_error) + # Bad block size + jdssd.ra.is_pool_exists.return_value = True + jdssd.jovian_hosts = ['192.168.0.2', '192.168.0.3'] + jdssd.block_size = ['61K'] + + self.assertRaises(exception.InvalidConfigurationValue, + jdssd.check_for_setup_error) + def test_get_provider_location(self): jdssd, ctx = self.get_driver(CONFIG_OK) host = CONFIG_OK["san_hosts"][0] @@ -351,17 +401,18 @@ class TestOpenEJovianDSSDriver(test.TestCase): SNAPSHOTS_EMPTY, SNAPSHOTS_EMPTY] - fake_gc = mock.Mock() - fake_hide_object = mock.Mock() - gc = mock.patch.object(jdssd, "_gc_delete", new=fake_gc) - gc.start() - hide = mock.patch.object(jdssd, "_hide_object", new=fake_hide_object) - hide.start() + patches = [mock.patch.object(jdssd, "_gc_delete"), + mock.patch.object(jdssd, "_hide_object")] + + self.start_patches(patches) + jdssd._cascade_volume_delete(o_vname, o_snaps) + jdssd._hide_object.assert_called_once_with(o_vname) - hide.stop() jdssd._gc_delete.assert_not_called() - gc.stop() + + self.stop_patches(patches) + delete_snapshot_expected = [ mock.call(o_vname, SNAPSHOTS_CASCADE_2[0]["name"], @@ -399,17 +450,17 @@ class TestOpenEJovianDSSDriver(test.TestCase): mock.call(SNAPSHOTS_CASCADE_1[1]["name"]), mock.call(o_vname)] - fake_gc = mock.Mock() - fake_hide_object = mock.Mock() - gc = mock.patch.object(jdssd, "_gc_delete", new=fake_gc) - gc.start() - hide = mock.patch.object(jdssd, "_hide_object", new=fake_hide_object) - hide.start() + patches = [mock.patch.object(jdssd, "_gc_delete"), + mock.patch.object(jdssd, "_hide_object")] + + self.start_patches(patches) + jdssd._cascade_volume_delete(o_vname, o_snaps) jdssd._hide_object.assert_has_calls(hide_object_expected) - hide.stop() jdssd._gc_delete.assert_not_called() - gc.stop() + + self.stop_patches(patches) + jdssd.ra.get_snapshots.assert_has_calls(get_snapshots) delete_snapshot_expected = [ @@ -522,7 +573,8 @@ class TestOpenEJovianDSSDriver(test.TestCase): jdssd._gc_delete(jcom.vname(UUID_1)) jdssd._delete_back_recursively.assert_not_called() - jdssd.ra.delete_lun.assert_called_once_with(jcom.vname(UUID_1)) + jdssd.ra.delete_lun.assert_called_once_with(jcom.vname(UUID_1), + force_umount=True) self.stop_patches(patches) @@ -655,6 +707,133 @@ class TestOpenEJovianDSSDriver(test.TestCase): except Exception as err: self.assertIsInstance(err, exception.VolumeBackendAPIException) + def test_revert_to_snapshot(self): + + jdssd, ctx = self.get_driver(CONFIG_OK) + vol = fake_volume.fake_volume_obj(ctx) + vol.id = UUID_1 + snap = fake_snapshot.fake_snapshot_obj(ctx) + snap.id = UUID_2 + + vname = jcom.vname(UUID_1) + sname = jcom.sname(UUID_2) + + get_lun_resp_1 = {'vscan': None, + 'full_name': 'Pool-0/' + UUID_1, + 'userrefs': None, + 'primarycache': 'all', + 'logbias': 'latency', + 'creation': '1591543140', + 'sync': 'always', + 'is_clone': False, + 'dedup': 'off', + 'sharenfs': None, + 'receive_resume_token': None, + 'volsize': '2147483648'} + + get_lun_resp_2 = {'vscan': None, + 'full_name': 'Pool-0/' + UUID_1, + 'userrefs': None, + 'primarycache': 'all', + 'logbias': 'latency', + 'creation': '1591543140', + 'sync': 'always', + 'is_clone': False, + 'dedup': 'off', + 'sharenfs': None, + 'receive_resume_token': None, + 'volsize': '1073741824'} + + jdssd.ra.get_lun.side_effect = [get_lun_resp_1, get_lun_resp_2] + + get_lun_expected = [mock.call(vname), mock.call(vname)] + + jdssd.revert_to_snapshot(ctx, vol, snap) + + jdssd.ra.get_lun.assert_has_calls(get_lun_expected) + + jdssd.ra.rollback_volume_to_snapshot.assert_called_once_with(vname, + sname) + jdssd.ra.extend_lun(vname, '2147483648') + + def test_revert_to_snapshot_exception(self): + + jdssd, ctx = self.get_driver(CONFIG_OK) + vol = fake_volume.fake_volume_obj(ctx) + vol.id = UUID_1 + snap = fake_snapshot.fake_snapshot_obj(ctx) + snap.id = UUID_2 + + vname = jcom.vname(UUID_1) + + get_lun_resp_no_size = {'vscan': None, + 'full_name': 'Pool-0/' + vname, + 'userrefs': None, + 'primarycache': 'all', + 'logbias': 'latency', + 'creation': '1591543140', + 'sync': 'always', + 'is_clone': False, + 'dedup': 'off', + 'sharenfs': None, + 'receive_resume_token': None, + 'volsize': None} + + get_lun_resp_1 = {'vscan': None, + 'full_name': 'Pool-0/' + vname, + 'userrefs': None, + 'primarycache': 'all', + 'logbias': 'latency', + 'creation': '1591543140', + 'sync': 'always', + 'is_clone': False, + 'dedup': 'off', + 'sharenfs': None, + 'receive_resume_token': None, + 'volsize': '2147483648'} + + get_lun_resp_2 = {'vscan': None, + 'full_name': 'Pool-0/' + vname, + 'userrefs': None, + 'primarycache': 'all', + 'logbias': 'latency', + 'creation': '1591543140', + 'sync': 'always', + 'is_clone': False, + 'dedup': 'off', + 'sharenfs': None, + 'receive_resume_token': None, + 'volsize': '1073741824'} + + jdssd.ra.get_lun.side_effect = [get_lun_resp_no_size, get_lun_resp_2] + + self.assertRaises(exception.VolumeDriverException, + jdssd.revert_to_snapshot, + ctx, + vol, + snap) + + jdssd.ra.get_lun.side_effect = [get_lun_resp_1, get_lun_resp_2] + + jdssd.ra.rollback_volume_to_snapshot.side_effect = [ + jexc.JDSSResourceNotFoundException(res=vname)] + self.assertRaises(exception.VolumeBackendAPIException, + jdssd.revert_to_snapshot, + ctx, + vol, + snap) + + jdssd.ra.get_lun.side_effect = [get_lun_resp_1, + jexc.JDSSException("some_error")] + + jdssd.ra.rollback_volume_to_snapshot.side_effect = [ + jexc.JDSSResourceNotFoundException(res=vname)] + self.assertRaises(exception.VolumeBackendAPIException, + jdssd.revert_to_snapshot, + ctx, + vol, + snap) + def test_clone_object(self): jdssd, ctx = self.get_driver(CONFIG_OK) origin = jcom.vname(UUID_1) @@ -1103,7 +1282,7 @@ class TestOpenEJovianDSSDriver(test.TestCase): location_info = 'JovianISCSIDriver:192.168.0.2:Pool-0' correct_out = { 'vendor_name': 'Open-E', - 'driver_version': "1.0.0", + 'driver_version': "1.0.1", 'storage_protocol': 'iSCSI', 'total_capacity_gb': 100, 'free_capacity_gb': 50, @@ -1311,6 +1490,7 @@ class TestOpenEJovianDSSDriver(test.TestCase): jdssd._create_target_volume.assert_called_once_with(vol) jdssd.ra.is_target_lun.assert_not_called() + self.stop_patches(patches) def test_remove_target_volume(self): @@ -1454,7 +1634,6 @@ class TestOpenEJovianDSSDriver(test.TestCase): 'driver_volume_type': 'iscsi', 'data': properties, } - jdssd.ra.activate_target.return_value = None ret = jdssd.initialize_connection(vol, connector) diff --git a/cinder/tests/unit/volume/drivers/open_e/test_rest.py b/cinder/tests/unit/volume/drivers/open_e/test_rest.py index 4dba89924dd..87bbc933a58 100644 --- a/cinder/tests/unit/volume/drivers/open_e/test_rest.py +++ b/cinder/tests/unit/volume/drivers/open_e/test_rest.py @@ -26,11 +26,12 @@ from cinder.volume.drivers.open_e.jovian_common import rest UUID_1 = '12345678-1234-1234-1234-000000000001' UUID_2 = '12345678-1234-1234-1234-000000000002' +UUID_3 = '12345678-1234-1234-1234-000000000003' CONFIG_OK = { 'san_hosts': ['192.168.0.2'], 'san_api_port': 82, - 'driver_use_ssl': 'https', + 'driver_use_ssl': 'true', 'jovian_rest_send_repeats': 3, 'jovian_recovery_delay': 60, 'san_login': 'admin', @@ -45,10 +46,6 @@ CONFIG_OK = { } -def fake_safe_get(value): - return CONFIG_OK[value] - - class TestOpenEJovianRESTAPI(test.TestCase): def get_rest(self, config): @@ -57,19 +54,11 @@ class TestOpenEJovianRESTAPI(test.TestCase): cfg = mock.Mock() cfg.append_config_values.return_value = None cfg.safe_get = lambda val: config[val] - cfg.get = lambda val, default: config[val] - jdssr = rest.JovianRESTAPI(cfg) + cfg.get = lambda val, default: config.get(val, default) + jdssr = rest.JovianRESTAPI(config) jdssr.rproxy = mock.Mock() return jdssr, ctx - def start_patches(self, patches): - for p in patches: - p.start() - - def stop_patches(self, patches): - for p in patches: - p.stop() - def test_get_active_host(self): jrest, ctx = self.get_rest(CONFIG_OK) @@ -114,7 +103,7 @@ class TestOpenEJovianRESTAPI(test.TestCase): jrest, ctx = self.get_rest(CONFIG_OK) resp = {'data': [{ 'vscan': None, - 'full_name': 'pool-0/' + UUID_1, + 'full_name': 'Pool-0/' + UUID_1, 'userrefs': None, 'primarycache': 'all', 'logbias': 'latency', @@ -148,7 +137,7 @@ class TestOpenEJovianRESTAPI(test.TestCase): jrest, ctx = self.get_rest(CONFIG_OK) resp = {'data': { 'vscan': None, - 'full_name': 'pool-0/' + jcom.vname(UUID_1), + 'full_name': 'Pool-0/' + jcom.vname(UUID_1), 'userrefs': None, 'primarycache': 'all', 'logbias': 'latency', @@ -231,7 +220,7 @@ class TestOpenEJovianRESTAPI(test.TestCase): jrest, ctx = self.get_rest(CONFIG_OK) resp = {'data': { "vscan": None, - "full_name": "pool-0/" + jcom.vname(UUID_1), + "full_name": "Pool-0/" + jcom.vname(UUID_1), "userrefs": None, "primarycache": "all", "logbias": "latency", @@ -268,7 +257,7 @@ class TestOpenEJovianRESTAPI(test.TestCase): def test_get_lun(self): jrest, ctx = self.get_rest(CONFIG_OK) resp = {'data': {"vscan": None, - "full_name": "pool-0/v_" + UUID_1, + "full_name": "Pool-0/v_" + UUID_1, "userrefs": None, "primarycache": "all", "logbias": "latency", @@ -995,3 +984,498 @@ class TestOpenEJovianRESTAPI(test.TestCase): self.assertRaises(jexc.JDSSException, jrest.detach_target_vol, tname, vname) jrest.rproxy.pool_request.assert_has_calls(detach_target_vol_expected) + + def test_create_snapshot(self): + + jrest, ctx = self.get_rest(CONFIG_OK) + vname = jcom.vname(UUID_1) + sname = jcom.sname(UUID_2) + + data = {'name': jcom.sname(UUID_2)} + resp = {'data': data, + 'error': None, + 'code': 201} + + jrest.rproxy.pool_request.return_value = resp + self.assertIsNone(jrest.create_snapshot(vname, sname)) + + def test_create_snapshot_exception(self): + + jrest, ctx = self.get_rest(CONFIG_OK) + vname = jcom.vname(UUID_1) + sname = jcom.sname(UUID_2) + + addr = '/volumes/{vol}/snapshots'.format(vol=vname) + req = {'snapshot_name': sname} + + url = ('http://192.168.0.2:82/api/v3/pools/Pool-0/volumes/{vol}/' + 'snapshots').format(vol=UUID_1) + resp = {'data': None, + 'error': { + 'class': "zfslib.zfsapi.resources.ZfsResourceError", + 'errno': 1, + 'message': ('Zfs resource: Pool-0/{vol} not found in ' + 'this collection.'.format(vol=vname)), + "url": url}, + 'code': 500} + + jrest.rproxy.pool_request.return_value = resp + create_snapshot_expected = [ + mock.call('POST', addr, json_data=req)] + + self.assertRaises(jexc.JDSSVolumeNotFoundException, + jrest.create_snapshot, + vname, + sname) + + # snapshot exists + resp = {'data': None, + 'error': { + 'class': "zfslib.zfsapi.resources.ZfsResourceError", + 'errno': 5, + 'message': 'Resource Pool-0/{vol}@{snap} already exists.', + 'url': url}, + 'code': 500} + + jrest.rproxy.pool_request.return_value = resp + create_snapshot_expected += [mock.call('POST', addr, json_data=req)] + self.assertRaises(jexc.JDSSSnapshotExistsException, + jrest.create_snapshot, + vname, + sname) + + # error unknown + err = {"class": "some test error", + "message": "test error message", + "url": url, + "errno": 123} + + resp = {'data': None, + 'error': err, + 'code': 500} + + jrest.rproxy.pool_request.return_value = resp + create_snapshot_expected += [mock.call('POST', addr, json_data=req)] + self.assertRaises(jexc.JDSSException, + jrest.create_snapshot, + vname, + sname) + jrest.rproxy.pool_request.assert_has_calls(create_snapshot_expected) + + def test_create_volume_from_snapshot(self): + + jrest, ctx = self.get_rest(CONFIG_OK) + vname = jcom.vname(UUID_1) + sname = jcom.sname(UUID_2) + cname = jcom.vname(UUID_3) + + addr = '/volumes/{vol}/clone'.format(vol=vname) + jbody = { + 'name': cname, + 'snapshot': sname, + 'sparse': False + } + + data = { + "origin": "Pool-0/{vol}@{snap}".format(vol=vname, snap=sname), + "is_clone": True, + "full_name": "Pool-0/{}".format(cname), + "name": cname + } + + resp = {'data': data, + 'error': None, + 'code': 201} + + jrest.rproxy.pool_request.return_value = resp + create_volume_from_snapshot_expected = [ + mock.call('POST', addr, json_data=jbody)] + self.assertIsNone(jrest.create_volume_from_snapshot(cname, + sname, + vname)) + + jrest.rproxy.pool_request.assert_has_calls( + create_volume_from_snapshot_expected) + + def test_create_volume_from_snapshot_exception(self): + + jrest, ctx = self.get_rest(CONFIG_OK) + vname = jcom.vname(UUID_1) + sname = jcom.sname(UUID_2) + cname = jcom.vname(UUID_3) + + addr = '/volumes/{vol}/clone'.format(vol=vname) + jbody = { + 'name': cname, + 'snapshot': sname, + 'sparse': False + } + + # volume DNE + url = ('http://192.168.0.2:82/api/v3/pools/Pool-0/volumes/{vol}/' + 'clone').format(vol=UUID_1) + resp = {'data': None, + 'error': { + 'class': "zfslib.zfsapi.resources.ZfsResourceError", + 'errno': 1, + 'message': ('Zfs resource: Pool-0/{vol} not found in ' + 'this collection.'.format(vol=vname)), + "url": url}, + 'code': 500} + + jrest.rproxy.pool_request.return_value = resp + create_volume_from_snapshot_expected = [ + mock.call('POST', addr, json_data=jbody)] + + self.assertRaises(jexc.JDSSResourceNotFoundException, + jrest.create_volume_from_snapshot, + cname, + sname, + vname) + + # clone exists + resp = {'data': None, + 'error': { + "class": "zfslib.wrap.zfs.ZfsCmdError", + "errno": 100, + "message": ("cannot create 'Pool-0/{}': " + "dataset already exists").format(vname), + 'url': url}, + 'code': 500} + + jrest.rproxy.pool_request.return_value = resp + create_volume_from_snapshot_expected += [ + mock.call('POST', addr, json_data=jbody)] + self.assertRaises(jexc.JDSSResourceExistsException, + jrest.create_volume_from_snapshot, + cname, + sname, + vname) + + # error unknown + err = {"class": "some test error", + "message": "test error message", + "url": url, + "errno": 123} + + resp = {'data': None, + 'error': err, + 'code': 500} + + jrest.rproxy.pool_request.return_value = resp + create_volume_from_snapshot_expected += [ + mock.call('POST', addr, json_data=jbody)] + self.assertRaises(jexc.JDSSException, + jrest.create_volume_from_snapshot, + cname, + sname, + vname) + jrest.rproxy.pool_request.assert_has_calls( + create_volume_from_snapshot_expected) + + def test_rollback_volume_to_snapshot(self): + + jrest, ctx = self.get_rest(CONFIG_OK) + vname = jcom.vname(UUID_1) + sname = jcom.sname(UUID_2) + + req = ('/volumes/{vol}/snapshots/' + '{snap}/rollback').format(vol=vname, snap=sname) + + resp = {'data': None, + 'error': None, + 'code': 200} + + jrest.rproxy.pool_request.return_value = resp + rollback_volume_to_snapshot_expected = [ + mock.call('POST', req)] + self.assertIsNone(jrest.rollback_volume_to_snapshot(vname, sname)) + + jrest.rproxy.pool_request.assert_has_calls( + rollback_volume_to_snapshot_expected) + + def test_rollback_volume_to_snapshot_exception(self): + + jrest, ctx = self.get_rest(CONFIG_OK) + vname = jcom.vname(UUID_1) + sname = jcom.sname(UUID_2) + + req = ('/volumes/{vol}/snapshots/' + '{snap}/rollback').format(vol=vname, + snap=sname) + + # volume DNE + msg = ('Zfs resource: Pool-0/{vname}' + ' not found in this collection.').format(vname=vname) + + url = ('http://192.168.0.2:82/api/v3/pools/Pool-0/volumes/{vol}/' + 'snapshots/{snap}/rollback').format(vol=vname, snap=sname) + err = {"class": "zfslib.zfsapi.resources.ZfsResourceError", + "message": msg, + "url": url, + "errno": 123} + + resp = {'data': None, + 'error': err, + 'code': 500} + + jrest.rproxy.pool_request.return_value = resp + rollback_volume_to_snapshot_expected = [ + mock.call('POST', req)] + self.assertRaises(jexc.JDSSException, + jrest.rollback_volume_to_snapshot, + vname, + sname) + jrest.rproxy.pool_request.assert_has_calls( + rollback_volume_to_snapshot_expected) + + # error unknown + err = {"class": "some test error", + "message": "test error message", + "url": url, + "errno": 123} + + resp = {'data': None, + 'error': err, + 'code': 500} + + jrest.rproxy.pool_request.return_value = resp + rollback_volume_to_snapshot_expected += [ + mock.call('POST', req)] + self.assertRaises(jexc.JDSSException, + jrest.rollback_volume_to_snapshot, + vname, + sname) + jrest.rproxy.pool_request.assert_has_calls( + rollback_volume_to_snapshot_expected) + + def test_delete_snapshot(self): + + jrest, ctx = self.get_rest(CONFIG_OK) + vname = jcom.vname(UUID_1) + sname = jcom.sname(UUID_2) + + addr = '/volumes/{vol}/snapshots/{snap}'.format(vol=vname, snap=sname) + + jbody = { + 'recursively_children': True, + 'recursively_dependents': True, + 'force_umount': True + } + + resp = {'data': None, + 'error': None, + 'code': 204} + + jrest.rproxy.pool_request.return_value = resp + delete_snapshot_expected = [mock.call('DELETE', addr)] + self.assertIsNone(jrest.delete_snapshot(vname, sname)) + + delete_snapshot_expected += [ + mock.call('DELETE', addr, json_data=jbody)] + self.assertIsNone(jrest.delete_snapshot(vname, + sname, + recursively_children=True, + recursively_dependents=True, + force_umount=True)) + + jrest.rproxy.pool_request.assert_has_calls(delete_snapshot_expected) + + def test_delete_snapshot_exception(self): + + jrest, ctx = self.get_rest(CONFIG_OK) + vname = jcom.vname(UUID_1) + sname = jcom.sname(UUID_2) + cname = jcom.sname(UUID_3) + + addr = '/volumes/{vol}/snapshots/{snap}'.format(vol=vname, snap=sname) + + # snapshot busy + url = ('http://192.168.0.2:82/api/v3/pools/Pool-0/volumes/{vol}/' + 'snapshots/{snap}').format(vol=vname, snap=sname) + msg = ('cannot destroy "Pool-0/{vol}@{snap}": snapshot has dependent ' + 'clones use "-R" to destroy the following datasets: ' + 'Pool-0/{clone}').format(vol=vname, snap=sname, clone=cname) + err = {'class': 'zfslib.wrap.zfs.ZfsCmdError', + 'message': msg, + 'url': url, + 'errno': 1000} + + resp = {'data': None, + 'error': err, + 'code': 500} + + jrest.rproxy.pool_request.return_value = resp + delete_snapshot_expected = [ + mock.call('DELETE', addr)] + + self.assertRaises(jexc.JDSSSnapshotIsBusyException, + jrest.delete_snapshot, + vname, + sname) + + # error unknown + err = {"class": "some test error", + "message": "test error message", + "url": url, + "errno": 123} + + resp = {'data': None, + 'error': err, + 'code': 500} + + jrest.rproxy.pool_request.return_value = resp + delete_snapshot_expected += [mock.call('DELETE', addr)] + self.assertRaises(jexc.JDSSException, + jrest.delete_snapshot, vname, sname) + + jrest.rproxy.pool_request.assert_has_calls(delete_snapshot_expected) + + def test_get_snapshots(self): + + jrest, ctx = self.get_rest(CONFIG_OK) + vname = jcom.vname(UUID_1) + + addr = '/volumes/{vol}/snapshots'.format(vol=vname) + + data = {"results": 2, + "entries": {"referenced": "65536", + "name": jcom.sname(UUID_2), + "defer_destroy": "off", + "userrefs": "0", + "primarycache": "all", + "type": "snapshot", + "creation": "2015-5-27 16:8:35", + "refcompressratio": "1.00x", + "compressratio": "1.00x", + "written": "65536", + "used": "0", + "clones": "", + "mlslabel": "none", + "secondarycache": "all"}} + + resp = {'data': data, + 'error': None, + 'code': 200} + + jrest.rproxy.pool_request.return_value = resp + get_snapshots_expected = [mock.call('GET', addr)] + self.assertEqual(data['entries'], jrest.get_snapshots(vname)) + + jrest.rproxy.pool_request.assert_has_calls(get_snapshots_expected) + + def test_get_snapshots_exception(self): + + jrest, ctx = self.get_rest(CONFIG_OK) + vname = jcom.vname(UUID_1) + + addr = '/volumes/{vol}/snapshots'.format(vol=vname) + + url = ('http://192.168.0.2:82/api/v3/pools/Pool-0/volumes/{vol}/' + 'snapshots').format(vol=vname) + + err = {"class": "zfslib.zfsapi.resources.ZfsResourceError", + "message": ('Zfs resource: Pool-0/{vol} not found in ' + 'this collection.').format(vol=vname), + "url": url, + "errno": 1} + + resp = {'data': None, + 'error': err, + 'code': 500} + + jrest.rproxy.pool_request.return_value = resp + get_snapshots_expected = [mock.call('GET', addr)] + self.assertRaises(jexc.JDSSResourceNotFoundException, + jrest.get_snapshots, + vname) + + # error unknown + err = {"class": "some test error", + "message": "test error message", + "url": url, + "errno": 123} + + resp = {'data': None, + 'error': err, + 'code': 500} + + jrest.rproxy.pool_request.return_value = resp + get_snapshots_expected += [ + mock.call('GET', addr)] + self.assertRaises(jexc.JDSSException, jrest.get_snapshots, vname) + + jrest.rproxy.pool_request.assert_has_calls(get_snapshots_expected) + + def test_get_pool_stats(self): + + jrest, ctx = self.get_rest(CONFIG_OK) + + addr = '' + + data = {"available": "950040707072", + "status": 26, + "name": "Pool-0", + "scan": None, + "encryption": {"enabled": False}, + "iostats": { + "read": "0", + "write": "0", + "chksum": "0"}, + "vdevs": [{"name": "wwn-0x5000cca3a8cddb2f", + "iostats": {"read": "0", + "write": "0", + "chksum": "0"}, + "disks": [{"origin": "local", + "led": "off", + "name": "sdc", + "iostats": {"read": "0", + "write": "0", + "chksum": "0"}, + "health": "ONLINE", + "sn": "JPW9K0N20ZGXWE", + "path": None, + "model": "Hitachi HUA72201", + "id": "wwn-0x5000cca3a8cddb2f", + "size": 1000204886016}], + "health": "ONLINE", + "vdev_replacings": [], + "vdev_spares": [], + "type": ""}], + "health": "ONLINE", + "operation": "none", + "id": "12413634663904564349", + "size": "996432412672"} + + resp = {'data': data, + 'error': None, + 'code': 200} + + jrest.rproxy.pool_request.return_value = resp + get_pool_stats_expected = [mock.call('GET', addr)] + self.assertEqual(data, jrest.get_pool_stats()) + + jrest.rproxy.pool_request.assert_has_calls(get_pool_stats_expected) + + def test_get_pool_stats_exception(self): + + jrest, ctx = self.get_rest(CONFIG_OK) + + addr = '' + + url = 'http://192.168.0.2:82/api/v3/pools/Pool-0/' + + err = {'class': 'zfslib.zfsapi.zpool.ZpoolError', + 'message': "Given zpool 'Pool-0' doesn't exists.", + "url": url, + "errno": 1} + + resp = {'data': None, + 'error': err, + 'code': 500} + + jrest.rproxy.pool_request.return_value = resp + get_pool_stats_expected = [mock.call('GET', addr)] + self.assertRaises(jexc.JDSSException, jrest.get_pool_stats) + + jrest.rproxy.pool_request.assert_has_calls(get_pool_stats_expected) diff --git a/cinder/tests/unit/volume/drivers/open_e/test_rest_proxy.py b/cinder/tests/unit/volume/drivers/open_e/test_rest_proxy.py new file mode 100644 index 00000000000..49d63d31a0b --- /dev/null +++ b/cinder/tests/unit/volume/drivers/open_e/test_rest_proxy.py @@ -0,0 +1,325 @@ +# Copyright (c) 2020 Open-E, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import json +from unittest import mock + +import requests + +from cinder import exception +from cinder.tests.unit import test +from cinder.volume.drivers.open_e.jovian_common import exception as jexc +from cinder.volume.drivers.open_e.jovian_common import rest_proxy + +UUID_1 = '12345678-1234-1234-1234-000000000001' +UUID_2 = '12345678-1234-1234-1234-000000000002' +UUID_3 = '12345678-1234-1234-1234-000000000003' + +CONFIG_OK = { + 'san_hosts': ['192.168.0.2'], + 'san_api_port': 82, + 'driver_use_ssl': 'true', + 'driver_ssl_cert_verify': True, + 'driver_ssl_cert_path': '/etc/cinder/joviandss.crt', + 'jovian_rest_send_repeats': 3, + 'jovian_recovery_delay': 60, + 'san_login': 'admin', + 'san_password': 'password', + 'jovian_ignore_tpath': [], + 'target_port': 3260, + 'jovian_pool': 'Pool-0', + 'iscsi_target_prefix': 'iqn.2020-04.com.open-e.cinder:', + 'chap_password_len': 12, + 'san_thin_provision': False, + 'jovian_block_size': '128K' + +} + +CONFIG_BAD_IP = { + 'san_hosts': ['asd'], + 'san_api_port': 82, + 'driver_use_ssl': 'true', + 'driver_ssl_cert_verify': True, + 'driver_ssl_cert_path': '/etc/cinder/joviandss.crt', + 'jovian_rest_send_repeats': 3, + 'jovian_recovery_delay': 60, + 'san_login': 'admin', + 'san_password': 'password', + 'jovian_ignore_tpath': [], + 'target_port': 3260, + 'jovian_pool': 'Pool-0', + 'iscsi_target_prefix': 'iqn.2020-04.com.open-e.cinder:', + 'chap_password_len': 12, + 'san_thin_provision': False, + 'jovian_block_size': '128K' + +} + +CONFIG_MULTIHOST = { + 'san_hosts': ['192.168.0.2', '192.168.0.3', '192.168.0.4'], + 'san_api_port': 82, + 'driver_use_ssl': 'true', + 'driver_ssl_cert_verify': True, + 'driver_ssl_cert_path': '/etc/cinder/joviandss.crt', + 'jovian_rest_send_repeats': 3, + 'jovian_recovery_delay': 60, + 'san_login': 'admin', + 'san_password': 'password', + 'jovian_ignore_tpath': [], + 'target_port': 3260, + 'jovian_pool': 'Pool-0', + 'iscsi_target_prefix': 'iqn.2020-04.com.open-e.cinder:', + 'chap_password_len': 12, + 'san_thin_provision': False, + 'jovian_block_size': '128K' + +} + + +class TestOpenEJovianRESTProxy(test.TestCase): + + def start_patches(self, patches): + for p in patches: + p.start() + + def stop_patches(self, patches): + for p in patches: + p.stop() + + def test_init(self): + + self.assertRaises(exception.InvalidConfigurationValue, + rest_proxy.JovianRESTProxy, + CONFIG_BAD_IP) + + def test_get_base_url(self): + + proxy = rest_proxy.JovianRESTProxy(CONFIG_OK) + + url = proxy._get_base_url() + + exp = '{proto}://{host}:{port}/api/v3'.format( + proto='https', + host='192.168.0.2', + port='82') + self.assertEqual(exp, url) + + def test_next_host(self): + + proxy = rest_proxy.JovianRESTProxy(CONFIG_MULTIHOST) + + self.assertEqual(0, proxy.active_host) + proxy._next_host() + + self.assertEqual(1, proxy.active_host) + proxy._next_host() + + self.assertEqual(2, proxy.active_host) + proxy._next_host() + + self.assertEqual(0, proxy.active_host) + + def test_request(self): + + proxy = rest_proxy.JovianRESTProxy(CONFIG_MULTIHOST) + + patches = [ + mock.patch.object(requests, "Request", return_value="request"), + mock.patch.object(proxy.session, + "prepare_request", + return_value="out_data"), + mock.patch.object(proxy, "_send", return_value="out_data")] + + addr = 'https://192.168.0.2:82/api/v3/pools/Pool-0' + + self.start_patches(patches) + proxy.request('GET', '/pools/Pool-0') + + requests.Request.assert_called_once_with('GET', addr) + self.stop_patches(patches) + + def test_request_host_failure(self): + + proxy = rest_proxy.JovianRESTProxy(CONFIG_MULTIHOST) + + patches = [ + mock.patch.object(requests, "Request", return_value="request"), + mock.patch.object(proxy.session, + "prepare_request", + return_value="out_data"), + mock.patch.object(proxy, "_send", return_value="out_data")] + + request_expected = [ + mock.call('GET', + 'https://192.168.0.2:82/api/v3/pools/Pool-0'), + mock.call('GET', + 'https://192.168.0.3:82/api/v3/pools/Pool-0'), + mock.call('GET', + 'https://192.168.0.4:82/api/v3/pools/Pool-0')] + + self.start_patches(patches) + + proxy._send.side_effect = [ + requests.exceptions.ConnectionError(), + requests.exceptions.ConnectionError(), + "out_data"] + + proxy.request('GET', '/pools/Pool-0') + self.assertEqual(2, proxy.active_host) + requests.Request.assert_has_calls(request_expected) + + self.stop_patches(patches) + + def test_pool_request(self): + + proxy = rest_proxy.JovianRESTProxy(CONFIG_OK) + + patches = [mock.patch.object(proxy, "request")] + + req = '/pools/Pool-0/volumes' + + self.start_patches(patches) + proxy.pool_request('GET', '/volumes') + + proxy.request.assert_called_once_with('GET', req, json_data=None) + self.stop_patches(patches) + + def test_send(self): + + proxy = rest_proxy.JovianRESTProxy(CONFIG_MULTIHOST) + + json_data = {"data": [{"available": "949998694400", + "status": 26, + "name": "Pool-0", + "scan": None, + "encryption": {"enabled": False}, + "iostats": {"read": "0", + "write": "0", + "chksum": "0"}, + "vdevs": [{}], + "health": "ONLINE", + "operation": "none", + "id": "12413634663904564349", + "size": "996432412672"}], + "error": None} + session_ret = mock.Mock() + session_ret.text = json.dumps(json_data) + session_ret.status_code = 200 + patches = [mock.patch.object(proxy.session, + "send", + return_value=session_ret)] + + pr = 'prepared_request' + + self.start_patches(patches) + ret = proxy._send(pr) + + proxy.session.send.assert_called_once_with(pr) + + self.assertEqual(0, proxy.active_host) + + self.assertEqual(200, ret['code']) + self.assertEqual(json_data['data'], ret['data']) + self.assertEqual(json_data['error'], ret['error']) + self.stop_patches(patches) + + def test_send_connection_error(self): + + proxy = rest_proxy.JovianRESTProxy(CONFIG_MULTIHOST) + + json_data = {"data": None, + "error": None} + + session_ret = mock.Mock() + session_ret.text = json.dumps(json_data) + session_ret.status_code = 200 + patches = [mock.patch.object(proxy.session, "send")] + + pr = 'prepared_request' + + self.start_patches(patches) + + side_effect = [requests.exceptions.ConnectionError()] * 4 + side_effect += [session_ret] + + proxy.session.send.side_effect = side_effect + + send_expected = [mock.call(pr)] * 4 + + ret = proxy._send(pr) + + proxy.session.send.assert_has_calls(send_expected) + + self.assertEqual(0, proxy.active_host) + + self.assertEqual(200, ret['code']) + self.assertEqual(json_data['data'], ret['data']) + self.assertEqual(json_data['error'], ret['error']) + self.stop_patches(patches) + + def test_send_mixed_error(self): + + proxy = rest_proxy.JovianRESTProxy(CONFIG_MULTIHOST) + + json_data = {"data": None, + "error": None} + + session_ret = mock.Mock() + session_ret.text = json.dumps(json_data) + session_ret.status_code = 200 + patches = [mock.patch.object(proxy.session, "send")] + + pr = 'prepared_request' + + self.start_patches(patches) + + side_effect = [requests.exceptions.ConnectionError()] * 4 + side_effect += [jexc.JDSSOSException()] * 4 + side_effect += [session_ret] + + proxy.session.send.side_effect = side_effect + + send_expected = [mock.call(pr)] * 7 + + self.assertRaises(jexc.JDSSOSException, proxy._send, pr) + + proxy.session.send.assert_has_calls(send_expected) + + self.assertEqual(0, proxy.active_host) + + def test_handle_500(self): + + error = {"class": "exceptions.OSError", + "errno": 17, + "message": ""} + + json_data = {"data": None, + "error": error} + + session_ret = mock.Mock() + session_ret.text = json.dumps(json_data) + session_ret.status_code = 500 + + self.assertRaises(jexc.JDSSOSException, + rest_proxy.JovianRESTProxy._handle_500, + session_ret) + + session_ret.status_code = 200 + json_data = {"data": None, + "error": None} + + session_ret.text = json.dumps(json_data) + self.assertIsNone(rest_proxy.JovianRESTProxy._handle_500(session_ret)) diff --git a/cinder/volume/drivers/open_e/iscsi.py b/cinder/volume/drivers/open_e/iscsi.py index 8175e032214..d816153cccd 100644 --- a/cinder/volume/drivers/open_e/iscsi.py +++ b/cinder/volume/drivers/open_e/iscsi.py @@ -43,23 +43,25 @@ class JovianISCSIDriver(driver.ISCSIDriver): .. code-block:: none 1.0.0 - Open-E JovianDSS driver with basic functionality + 1.0.1 - Added certificate support + Added revert to snapshot support """ # ThirdPartySystems wiki page CI_WIKI_NAME = "Open-E_JovianDSS_CI" - VERSION = "1.0.0" + VERSION = "1.0.1" def __init__(self, *args, **kwargs): super(JovianISCSIDriver, self).__init__(*args, **kwargs) self._stats = None - self._pool = 'Pool-0' self.jovian_iscsi_target_portal_port = "3260" self.jovian_target_prefix = 'iqn.2020-04.com.open-e.cinder:' self.jovian_chap_pass_len = 12 self.jovian_sparse = False self.jovian_ignore_tpath = None self.jovian_hosts = None + self._pool = 'Pool-0' self.ra = None @property @@ -67,7 +69,8 @@ class JovianISCSIDriver(driver.ISCSIDriver): """Return backend name.""" backend_name = None if self.configuration: - backend_name = self.configuration.safe_get('volume_backend_name') + backend_name = self.configuration.get('volume_backend_name', + 'JovianDSS') if not backend_name: backend_name = self.__class__.__name__ return backend_name @@ -82,26 +85,30 @@ class JovianISCSIDriver(driver.ISCSIDriver): options.jdss_volume_opts) self.configuration.append_config_values(san.san_opts) - self._pool = self.configuration.safe_get('jovian_pool') - self.jovian_iscsi_target_portal_port = self.configuration.safe_get( - 'target_port') + self._pool = self.configuration.get('jovian_pool', 'Pool-0') + self.jovian_iscsi_target_portal_port = self.configuration.get( + 'target_port', 3260) - self.jovian_target_prefix = self.configuration.safe_get( - 'target_prefix') - self.jovian_chap_pass_len = self.configuration.safe_get( - 'chap_password_len') + self.jovian_target_prefix = self.configuration.get( + 'target_prefix', + 'iqn.2020-04.com.open-e.cinder:') + self.jovian_chap_pass_len = self.configuration.get( + 'chap_password_len', 12) self.block_size = ( - self.configuration.safe_get('jovian_block_size')) + self.configuration.get('jovian_block_size', '64K')) self.jovian_sparse = ( - self.configuration.safe_get('san_thin_provision')) + self.configuration.get('san_thin_provision', True)) self.jovian_ignore_tpath = self.configuration.get( 'jovian_ignore_tpath', None) - self.jovian_hosts = self.configuration.safe_get( - 'san_hosts') + self.jovian_hosts = self.configuration.get( + 'san_hosts', []) + self.ra = rest.JovianRESTAPI(self.configuration) + self.check_for_setup_error() + def check_for_setup_error(self): - """Verify that the pool exists.""" + """Check for setup error.""" if len(self.jovian_hosts) == 0: msg = _("No hosts provided in configuration") raise exception.VolumeDriverException(msg) @@ -110,6 +117,12 @@ class JovianISCSIDriver(driver.ISCSIDriver): msg = (_("Unable to identify pool %s") % self._pool) raise exception.VolumeDriverException(msg) + valid_bsize = ['32K', '64K', '128K', '256K', '512K', '1M'] + if self.block_size not in valid_bsize: + raise exception.InvalidConfigurationValue( + value=self.block_size, + option='jovian_block_size') + def _get_target_name(self, volume_name): """Return iSCSI target name to access volume.""" return '%s%s' % (self.jovian_target_prefix, volume_name) @@ -290,14 +303,14 @@ class JovianISCSIDriver(driver.ISCSIDriver): jcom.origin_snapshot(vol['origin'])) else: try: - self.ra.delete_lun(vname) + self.ra.delete_lun(vname, force_umount=True) except jexc.JDSSRESTException as err: LOG.debug( "Unable to delete physical volume %(volume)s " "with error %(err)s.", { "volume": vname, "err": err}) - raise exception.SnapshotIsBusy(err) + raise exception.VolumeIsBusy(err) def _delete_back_recursively(self, opvname, opsname): """Deletes snapshot by removing its oldest removable parent @@ -391,6 +404,46 @@ class JovianISCSIDriver(driver.ISCSIDriver): raise exception.VolumeBackendAPIException( (_('Failed to extend volume %s.'), volume.id)) + def revert_to_snapshot(self, context, volume, snapshot): + """Revert volume to snapshot. + + Note: the revert process should not change the volume's + current size, that means if the driver shrank + the volume during the process, it should extend the + volume internally. + """ + vname = jcom.vname(volume.id) + sname = jcom.sname(snapshot.id) + LOG.debug('reverting %(vname)s to %(sname)s', { + "vname": vname, + "sname": sname}) + + vsize = None + try: + vsize = self.ra.get_lun(vname).get('volsize') + except jexc.JDSSResourceNotFoundException: + raise exception.VolumeNotFound(volume_id=volume.id) + except jexc.JDSSException as err: + raise exception.VolumeBackendAPIException(err) + + if vsize is None: + raise exception.VolumeDriverException( + _("unable to identify volume size")) + + try: + self.ra.rollback_volume_to_snapshot(vname, sname) + except jexc.JDSSException as err: + raise exception.VolumeBackendAPIException(err.message) + + try: + rvsize = self.ra.get_lun(vname).get('volsize') + if rvsize != vsize: + self.ra.extend_lun(vname, vsize) + except jexc.JDSSResourceNotFoundException: + raise exception.VolumeNotFound(volume_id=volume.id) + except jexc.JDSSException as err: + raise exception.VolumeBackendAPIException(err) + def _clone_object(self, oname, coname): """Creates a clone of specified object @@ -430,7 +483,7 @@ class JovianISCSIDriver(driver.ISCSIDriver): coname, oname, sparse=self.jovian_sparse) - except jexc.JDSSVolumeExistsException: + except jexc.JDSSResourceExistsException: raise exception.Duplicate() except jexc.JDSSException as err: try: @@ -671,7 +724,7 @@ class JovianISCSIDriver(driver.ISCSIDriver): free_capacity = math.floor(int(pool_stats["available"]) / o_units.Gi) reserved_percentage = ( - self.configuration.safe_get('reserved_percentage')) + self.configuration.get('reserved_percentage', 0)) if total_capacity is None: total_capacity = 'unknown' @@ -784,7 +837,7 @@ class JovianISCSIDriver(driver.ISCSIDriver): auth = volume.provider_auth if not auth: - msg = _("Volume {} is missing provider_auth") % volume.id + msg = _("Volume %s is missing provider_auth") % volume.id raise exception.VolumeDriverException(msg) (__, auth_username, auth_secret) = auth.split() diff --git a/cinder/volume/drivers/open_e/jovian_common/exception.py b/cinder/volume/drivers/open_e/jovian_common/exception.py index dbc1eb7127e..f893913e2f5 100644 --- a/cinder/volume/drivers/open_e/jovian_common/exception.py +++ b/cinder/volume/drivers/open_e/jovian_common/exception.py @@ -80,3 +80,9 @@ class JDSSSnapshotIsBusyException(JDSSResourceIsBusyException): """Snapshot have dependent clones""" message = _("JDSS snapshot %(snapshot)s is busy.") + + +class JDSSOSException(JDSSException): + """Storage internal system error""" + + message = _("JDSS internal system error %(message)s.") diff --git a/cinder/volume/drivers/open_e/jovian_common/rest.py b/cinder/volume/drivers/open_e/jovian_common/rest.py index f04d277754d..5a08f7b8a4d 100644 --- a/cinder/volume/drivers/open_e/jovian_common/rest.py +++ b/cinder/volume/drivers/open_e/jovian_common/rest.py @@ -20,7 +20,6 @@ import re from oslo_log import log as logging from cinder import exception -from cinder.i18n import _ from cinder.volume.drivers.open_e.jovian_common import exception as jexc from cinder.volume.drivers.open_e.jovian_common import rest_proxy @@ -32,9 +31,7 @@ class JovianRESTAPI(object): def __init__(self, config): - self.target_p = config.get('iscsi_target_prefix', - 'iqn.2020-04.com.open-e.cinder:') - self.pool = config.safe_get('jovian_pool') + self.pool = config.get('jovian_pool', 'Pool-0') self.rproxy = rest_proxy.JovianRESTProxy(config) self.resource_dne_msg = ( @@ -48,7 +45,7 @@ class JovianRESTAPI(object): code = resp.get('code', 'Unknown') msg = resp.get('message', 'Unknown') - reason = ("Request to {url} failed with code:%{code} " + reason = ("Request to {url} failed with code: {code} " "of type:{eclass} reason:{message}") reason = reason.format(eclass=eclass, code=code, @@ -638,12 +635,12 @@ class JovianRESTAPI(object): if resp["code"] == 500: if resp["error"]: - if resp["error"]["errno"] == 1: - raise jexc.JDSSVolumeNotFoundException( - volume=volume_name) if resp["error"]["errno"] == 5: raise jexc.JDSSSnapshotExistsException( snapshot=snapshot_name) + if resp["error"]["errno"] == 1: + raise jexc.JDSSVolumeNotFoundException( + volume=volume_name) self._general_error(req, resp) @@ -682,34 +679,42 @@ class JovianRESTAPI(object): if resp["error"]["errno"] == 100: raise jexc.JDSSVolumeExistsException( volume=volume_name) - args = {"vol": volume_name, "e": resp['error']['message']} - msg = _('Failed to create volume %(vol)s, err: %(e)s') % args - raise jexc.JDSSRESTException(msg) + if resp["error"]["errno"] == 1: + raise jexc.JDSSResourceNotFoundException( + res="{vol}@{snap}".format(vol=original_vol_name, + snap=snapshot_name)) - raise jexc.JDSSRESTException('unable to create volume') + self._general_error(req, resp) - def is_snapshot(self, volume_name, snapshot_name): - """is_snapshots. + def rollback_volume_to_snapshot(self, volume_name, snapshot_name): + """Rollback volume to its snapshot - GET - /volumes//snapshots//clones - - :param volume_name: that snapshot belongs to - :return: bool + POST /volumes//snapshots//rollback + :param volume_name: volume that is going to be restored + :param snapshot_name: snapshot of a volume above + :return: """ - req = '/volumes/' + volume_name + '/snapshots/' + snapshot_name + \ - '/clones' + req = ('/volumes/{vol}/snapshots/' + '{snap}/rollback').format(vol=volume_name, + snap=snapshot_name) - LOG.debug("check if snapshot %(snap)s of volume %(vol)s exists", - {'snap': snapshot_name, - 'vol': volume_name}) + LOG.debug("rollback volume %(vol)s to snapshot %(snap)s", + {'vol': volume_name, + 'snap': snapshot_name}) - resp = self.rproxy.pool_request('GET', req) + resp = self.rproxy.pool_request('POST', req) if not resp["error"] and resp["code"] == 200: - return True + return - return False + if resp["code"] == 500: + if resp["error"]: + if resp["error"]["errno"] == 1: + raise jexc.JDSSResourceNotFoundException( + res="{vol}@{snap}".format(vol=volume_name, + snap=snapshot_name)) + + self._general_error(req, resp) def delete_snapshot(self, volume_name, @@ -733,8 +738,6 @@ class JovianRESTAPI(object): umount (defualt false). :return: """ - if not self.is_snapshot(volume_name, snapshot_name): - return req = '/volumes/' + volume_name + '/snapshots/' + snapshot_name @@ -767,11 +770,7 @@ class JovianRESTAPI(object): if resp["error"]["errno"] == 1000: raise jexc.JDSSSnapshotIsBusyException( snapshot=snapshot_name) - msg = 'Failed to delete snapshot {}, err: {}'.format( - snapshot_name, resp['error']['message']) - raise jexc.JDSSRESTException(msg) - msg = 'Failed to delete snapshot {}'.format(snapshot_name) - raise jexc.JDSSRESTException(msg) + self._general_error(req, resp) def get_snapshots(self, volume_name): """get_snapshots. @@ -818,7 +817,8 @@ class JovianRESTAPI(object): if 'message' in resp['error']: if self.resource_dne_msg.match(resp['error']['message']): raise jexc.JDSSResourceNotFoundException(volume_name) - raise jexc.JDSSRESTException('unable to get snapshots') + + self._general_error(req, resp) def get_pool_stats(self): """get_pool_stats. @@ -890,4 +890,4 @@ class JovianRESTAPI(object): if not resp["error"] and resp["code"] == 200: return resp["data"] - raise jexc.JDSSRESTException('Unable to get pool info') + self._general_error(req, resp) diff --git a/cinder/volume/drivers/open_e/jovian_common/rest_proxy.py b/cinder/volume/drivers/open_e/jovian_common/rest_proxy.py index 964b19254af..07a1faf964c 100644 --- a/cinder/volume/drivers/open_e/jovian_common/rest_proxy.py +++ b/cinder/volume/drivers/open_e/jovian_common/rest_proxy.py @@ -16,7 +16,6 @@ """Network connection handling class for JovianDSS driver.""" import json -import time from oslo_log import log as logging from oslo_utils import netutils as o_netutils @@ -25,6 +24,7 @@ import urllib3 from cinder import exception from cinder.i18n import _ +from cinder.utils import retry from cinder.volume.drivers.open_e.jovian_common import exception as jexc @@ -35,17 +35,15 @@ class JovianRESTProxy(object): """Jovian REST API proxy.""" def __init__(self, config): - """:param config: config is like dict.""" + """:param config: list of config values.""" self.proto = 'http' if config.get('driver_use_ssl', True): self.proto = 'https' - self.hosts = config.safe_get('san_hosts') + self.hosts = config.get('san_hosts', []) self.port = str(config.get('san_api_port', 82)) - self.active_host = 0 - for host in self.hosts: if o_netutils.is_valid_ip(host) is False: err_msg = ('Invalid value of jovian_host property: ' @@ -55,36 +53,50 @@ class JovianRESTProxy(object): LOG.debug(err_msg) raise exception.InvalidConfigurationValue(err_msg) - self.api_path = "/api/v3" + self.active_host = 0 + self.delay = config.get('jovian_recovery_delay', 40) - self.pool = config.safe_get('jovian_pool') + self.pool = config.get('jovian_pool', 'Pool-0') self.user = config.get('san_login', 'admin') self.password = config.get('san_password', 'admin') - self.auth = requests.auth.HTTPBasicAuth(self.user, self.password) - self.verify = False - self.retry_n = config.get('jovian_rest_send_repeats', 3) - self.header = {'connection': 'keep-alive', - 'Content-Type': 'application/json', - 'authorization': 'Basic '} + self.verify = config.get('driver_ssl_cert_verify', True) + self.cert = config.get('driver_ssl_cert_path') + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - def _get_pool_url(self, host): - url = ('%(proto)s://%(host)s:%(port)s/api/v3/pools/%(pool)s' % { - 'proto': self.proto, - 'host': host, - 'port': self.port, - 'pool': self.pool}) - return url + self.session = self._get_session() + + def _get_session(self): + """Create and init new session object""" + + session = requests.Session() + session.auth = (self.user, self.password) + session.headers.update({'Connection': 'keep-alive', + 'Content-Type': 'application/json', + 'Authorization': 'Basic'}) + session.hooks['response'] = [JovianRESTProxy._handle_500] + session.verify = self.verify + if self.verify and self.cert: + session.verify = self.cert + return session + + def _get_base_url(self): + """Get url prefix with active host""" - def _get_url(self, host): url = ('%(proto)s://%(host)s:%(port)s/api/v3' % { 'proto': self.proto, - 'host': host, + 'host': self.hosts[self.active_host], 'port': self.port}) + return url + def _next_host(self): + """Set next host as active""" + + self.active_host = (self.active_host + 1) % len(self.hosts) + def request(self, request_method, req, json_data=None): """Send request to the specific url. @@ -92,39 +104,31 @@ class JovianRESTProxy(object): :param url: where to send :param json_data: data """ - for j in range(self.retry_n): - for i in range(len(self.hosts)): - host = self.hosts[self.active_host] - url = self._get_url(host) + req + out = None + for i in range(len(self.hosts)): + try: + addr = "{base}{req}".format(base=self._get_base_url(), + req=req) + LOG.debug("Sending %(t)s to %(addr)s", + {'t': request_method, 'addr': addr}) + r = None + if json_data: + r = requests.Request(request_method, + addr, + data=json.dumps(json_data)) + else: + r = requests.Request(request_method, addr) - LOG.debug( - "sending request of type %(type)s to %(url)s " - "attempt: %(num)s.", - {'type': request_method, - 'url': url, - 'num': j}) + pr = self.session.prepare_request(r) + out = self._send(pr) + except requests.exceptions.ConnectionError: + self._next_host() + continue + break - if json_data is not None: - LOG.debug( - "sending data: %s.", json_data) - try: - - ret = self._request_routine(url, request_method, json_data) - if len(ret) == 0: - self.active_host = ((self.active_host + 1) - % len(self.hosts)) - continue - return ret - - except requests.ConnectionError as err: - LOG.debug("Connection error %s", err) - self.active_host = (self.active_host + 1) % len(self.hosts) - continue - time.sleep(self.delay) - - msg = (_('%(times)s faild in a row') % {'times': j}) - - raise jexc.JDSSRESTProxyException(host=url, reason=msg) + LOG.debug("Geting %(data)s from %(t)s to %(addr)s", + {'data': out, 't': request_method, 'addr': addr}) + return out def pool_request(self, request_method, req, json_data=None): """Send request to the specific url. @@ -133,94 +137,65 @@ class JovianRESTProxy(object): :param url: where to send :param json_data: data """ - url = "" - for j in range(self.retry_n): - for i in range(len(self.hosts)): - host = self.hosts[self.active_host] - url = self._get_pool_url(host) + req + req = "/pools/{pool}{req}".format(pool=self.pool, req=req) + addr = "{base}{req}".format(base=self._get_base_url(), req=req) + LOG.debug("Sending pool request %(t)s to %(addr)s", + {'t': request_method, 'addr': addr}) + return self.request(request_method, req, json_data=json_data) - LOG.debug( - "sending pool request of type %(type)s to %(url)s " - "attempt: %(num)s.", - {'type': request_method, - 'url': url, - 'num': j}) + @retry((requests.exceptions.ConnectionError, + jexc.JDSSOSException), + interval=2, + backoff_rate=2, + retries=7) + def _send(self, pr): + """Send prepared request - if json_data is not None: - LOG.debug( - "JovianDSS: Sending data: %s.", str(json_data)) - try: + :param pr: prepared request + """ + ret = dict() - ret = self._request_routine(url, request_method, json_data) - if len(ret) == 0: - self.active_host = ((self.active_host + 1) - % len(self.hosts)) - continue - return ret + response_obj = self.session.send(pr) - except requests.ConnectionError as err: - LOG.debug("Connection error %s", err) - self.active_host = (self.active_host + 1) % len(self.hosts) - continue - time.sleep(int(self.delay)) + ret['code'] = response_obj.status_code - msg = (_('%(times)s faild in a row') % {'times': j}) - - raise jexc.JDSSRESTProxyException(host=url, reason=msg) - - def _request_routine(self, url, request_method, json_data=None): - """Make an HTTPS request and return the results.""" - - ret = None - for i in range(3): - ret = dict() - try: - response_obj = requests.request(request_method, - auth=self.auth, - url=url, - headers=self.header, - data=json.dumps(json_data), - verify=self.verify) - - LOG.debug('response code: %s', response_obj.status_code) - LOG.debug('response data: %s', response_obj.text) - - ret['code'] = response_obj.status_code - - if '{' in response_obj.text and '}' in response_obj.text: - if "error" in response_obj.text: - ret["error"] = json.loads(response_obj.text)["error"] - else: - ret["error"] = None - if "data" in response_obj.text: - ret["data"] = json.loads(response_obj.text)["data"] - else: - ret["data"] = None - - if ret["code"] == 500: - if ret["error"] is not None: - if (("errno" in ret["error"]) and - ("class" in ret["error"])): - if (ret["error"]["class"] == - "opene.tools.scstadmin.ScstAdminError"): - LOG.debug("ScstAdminError %(code)d %(msg)s", { - "code": ret["error"]["errno"], - "msg": ret["error"]["message"]}) - continue - if (ret["error"]["class"] == - "exceptions.OSError"): - LOG.debug("OSError %(code)d %(msg)s", { - "code": ret["error"]["errno"], - "msg": ret["error"]["message"]}) - continue - break - - except requests.HTTPError as err: - LOG.debug("HTTP parsing error %s", err) - self.active_host = (self.active_host + 1) % len(self.hosts) + try: + data = json.loads(response_obj.text) + ret["error"] = data.get("error") + ret["data"] = data.get("data") + except json.JSONDecodeError: + pass return ret + @staticmethod + def _handle_500(resp, *args, **kwargs): + """Handle OS error on a storage side""" + + error = None + if resp.status_code == 500: + try: + data = json.loads(resp.text) + error = data.get("error") + except json.JSONDecodeError: + return + else: + return + + if error: + if "class" in error: + if error["class"] == "opene.tools.scstadmin.ScstAdminError": + LOG.debug("ScstAdminError %(code)d %(msg)s", + {'code': error["errno"], + 'msg': error["message"]}) + raise jexc.JDSSOSException(_(error["message"])) + + if error["class"] == "exceptions.OSError": + LOG.debug("OSError %(code)d %(msg)s", + {'code': error["errno"], + 'msg': error["message"]}) + raise jexc.JDSSOSException(_(error["message"])) + def get_active_host(self): """Return address of currently used host.""" return self.hosts[self.active_host] diff --git a/cinder/volume/drivers/open_e/options.py b/cinder/volume/drivers/open_e/options.py index 5da873451b3..c8f94780820 100644 --- a/cinder/volume/drivers/open_e/options.py +++ b/cinder/volume/drivers/open_e/options.py @@ -19,9 +19,6 @@ jdss_connection_opts = [ cfg.ListOpt('san_hosts', default='', help='IP address of Open-E JovianDSS SA'), - cfg.IntOpt('jovian_rest_send_repeats', - default=3, - help='Number of retries to send REST request.'), cfg.IntOpt('jovian_recovery_delay', default=60, help='Time before HA cluster failure.'), @@ -41,8 +38,8 @@ jdss_iscsi_opts = [ jdss_volume_opts = [ cfg.StrOpt('jovian_block_size', - default='128K', - help='Block size for volumes (512 - 128K)'), + default='64K', + help='Block size can be: 32K, 64K, 128K, 256K, 512K, 1M'), ] CONF = cfg.CONF diff --git a/doc/source/configuration/block-storage/drivers/open-e-joviandss-driver.rst b/doc/source/configuration/block-storage/drivers/open-e-joviandss-driver.rst index 2dc69e092a4..0d02be9dea5 100644 --- a/doc/source/configuration/block-storage/drivers/open-e-joviandss-driver.rst +++ b/doc/source/configuration/block-storage/drivers/open-e-joviandss-driver.rst @@ -38,10 +38,11 @@ Provide settings to JovianDSS driver by adding 'jdss-0' description: backend_name = jdss-0 chap_password_len = 14 driver_use_ssl = True + driver_ssl_cert_verify = True + driver_ssl_cert_path = /etc/cinder/jdss.crt iscsi_target_prefix = iqn.2016-04.com.open-e.cinder: jovian_pool = Pool-0 jovian_block_size = 128K - jovian_rest_send_repeats = 4 san_api_port = 82 target_port = 3260 volume_driver = cinder.volume.drivers.open_e.iscsi.JovianISCSIDriver @@ -65,6 +66,12 @@ Provide settings to JovianDSS driver by adding 'jdss-0' description: * - ``driver_use_ssl`` - True - Use SSL to send requests to JovianDSS[1] + * - ``driver_ssl_cert_verify`` + - True + - Verify authenticity of JovianDSS[1] certificate + * - ``driver_ssl_cert_path`` + - None + - Path to the JovianDSS[1] certificate for verification * - ``iscsi_target_prefix`` - iqn.2016-04.com.open-e:01:cinder- - Prefix that will be used to form target name for volume @@ -74,9 +81,6 @@ Provide settings to JovianDSS driver by adding 'jdss-0' description: * - ``jovian_block_size`` - 128K - Block size for newly created volumes - * - ``jovian_rest_send_repeats`` - - 3 - - Number of times that driver will try to send REST request * - ``san_api_port`` - 82 - Rest port according to the settings in [1] @@ -94,7 +98,7 @@ Provide settings to JovianDSS driver by adding 'jdss-0' description: - Must be set according to the settings in [1] * - ``san_password`` - admin - - Jovian password [1], **should be changed** for security purpouses + - Jovian password [1], **should be changed** for security purposes * - ``san_thin_provision`` - False - Using thin provisioning for new volumes @@ -126,10 +130,10 @@ For instance if you want to add ``Pool-1`` located on the same host as backend_name = jdss-0 chap_password_len = 14 driver_use_ssl = True + driver_ssl_cert_verify = False iscsi_target_prefix = iqn.2016-04.com.open-e.cinder: jovian_pool = Pool-0 jovian_block_size = 128K - jovian_rest_send_repeats = 4 san_api_port = 82 target_port = 3260 volume_driver = cinder.volume.drivers.open_e.iscsi.JovianISCSIDriver @@ -142,10 +146,10 @@ For instance if you want to add ``Pool-1`` located on the same host as backend_name = jdss-1 chap_password_len = 14 driver_use_ssl = True + driver_ssl_cert_verify = False iscsi_target_prefix = iqn.2016-04.com.open-e.cinder: jovian_pool = Pool-1 jovian_block_size = 128K - jovian_rest_send_repeats = 4 san_api_port = 82 target_port = 3260 volume_driver = cinder.volume.drivers.open_e.iscsi.JovianISCSIDriver @@ -175,10 +179,10 @@ and 192.168.31.100 the configuration file will look like: backend_name = jdss-2 chap_password_len = 14 driver_use_ssl = True + driver_ssl_cert_verify = False iscsi_target_prefix = iqn.2016-04.com.open-e.cinder: jovian_pool = Pool-0 jovian_block_size = 128K - jovian_rest_send_repeats = 4 san_api_port = 82 target_port = 3260 volume_driver = cinder.volume.drivers.open_e.iscsi.JovianISCSIDriver diff --git a/doc/source/reference/support-matrix.ini b/doc/source/reference/support-matrix.ini index 983c9b8de09..6d8f518e5ee 100644 --- a/doc/source/reference/support-matrix.ini +++ b/doc/source/reference/support-matrix.ini @@ -150,6 +150,9 @@ title=Generic NFS Reference Driver (NFS) [driver.nimble] title=Nimble Storage Driver (iSCSI, FC) +[driver.opene_joviandss] +title=Open-E JovianDSS Storage Driver (iSCSI) + [driver.prophetstor] title=ProphetStor Flexvisor Driver (iSCSI, NFS) @@ -259,6 +262,7 @@ driver.netapp_solidfire=complete driver.nexenta=complete driver.nfs=complete driver.nimble=complete +driver.opene_joviandss=complete driver.prophetstor=missing driver.pure=complete driver.qnap=complete @@ -328,6 +332,7 @@ driver.netapp_solidfire=complete driver.nexenta=complete driver.nfs=missing driver.nimble=complete +driver.opene_joviandss=missing driver.prophetstor=complete driver.pure=complete driver.qnap=complete @@ -397,6 +402,7 @@ driver.netapp_solidfire=missing driver.nexenta=missing driver.nfs=missing driver.nimble=missing +driver.opene_joviandss=missing driver.prophetstor=missing driver.pure=missing driver.qnap=missing @@ -469,6 +475,7 @@ driver.netapp_solidfire=complete driver.nexenta=missing driver.nfs=missing driver.nimble=missing +driver.opene_joviandss=missing driver.prophetstor=missing driver.pure=complete driver.qnap=missing @@ -540,6 +547,7 @@ driver.netapp_solidfire=complete driver.nexenta=missing driver.nfs=missing driver.nimble=missing +driver.opene_joviandss=missing driver.prophetstor=missing driver.pure=complete driver.qnap=missing @@ -612,6 +620,7 @@ driver.netapp_solidfire=complete driver.nexenta=missing driver.nfs=missing driver.nimble=complete +driver.opene_joviandss=missing driver.prophetstor=complete driver.pure=complete driver.qnap=missing @@ -683,6 +692,7 @@ driver.netapp_solidfire=complete driver.nexenta=missing driver.nfs=complete driver.nimble=complete +driver.opene_joviandss=complete driver.prophetstor=missing driver.pure=complete driver.qnap=missing @@ -755,6 +765,7 @@ driver.netapp_solidfire=complete driver.nexenta=missing driver.nfs=missing driver.nimble=missing +driver.opene_joviandss=missing driver.prophetstor=missing driver.pure=missing driver.qnap=missing @@ -827,6 +838,7 @@ driver.netapp_solidfire=complete driver.nexenta=missing driver.nfs=missing driver.nimble=complete +driver.opene_joviandss=missing driver.prophetstor=missing driver.pure=complete driver.qnap=missing @@ -896,6 +908,7 @@ driver.netapp_solidfire=complete driver.nexenta=missing driver.nfs=missing driver.nimble=complete +driver.opene_joviandss=complete driver.prophetstor=missing driver.pure=complete driver.qnap=missing @@ -969,6 +982,7 @@ driver.netapp_solidfire=complete driver.nexenta=missing driver.nfs=missing driver.nimble=missing +driver.opene_joviandss=missing driver.prophetstor=missing driver.pure=complete driver.qnap=missing diff --git a/releasenotes/notes/bp-jdss-add-cert-and-snapshot-revert-b34f352754ad07de.yaml b/releasenotes/notes/bp-jdss-add-cert-and-snapshot-revert-b34f352754ad07de.yaml new file mode 100644 index 00000000000..6973f3e9291 --- /dev/null +++ b/releasenotes/notes/bp-jdss-add-cert-and-snapshot-revert-b34f352754ad07de.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Added support of authenticity verification through self-signed certificates + for JovianDSS data storage. + Added support of revert to snapshot functionality. + Expands unit-test coverage for JovianDSS driver.