From beb2879793099a0c574c97bd6683f4128f430b3e Mon Sep 17 00:00:00 2001 From: yuval brave Date: Mon, 13 Dec 2021 17:24:15 +0200 Subject: [PATCH] Lightbits LightOS driver This commit introduces the LightOS driver for cinder. LightOS is a software-defined disaggregated clustered storage solution running on commodity servers with commodity SSDs. It is developed by Lightbits Labs (https://www.lightbitslabs.com) and is actively developed and maintained. LightOS is proprietary but the openstack drivers are licensed under Apache v2.0. The Cinder driver for LightOS currently supports the following functionality: Create volume Delete volume Attach volume Detach volume Create image from volume create volume from image Live migration Volume replication Thin provisioning Multi-attach Extend volume Create snapshot Delete snapshot Create volume from snapshot Create volume from volume (clone) This driver has been developed and has been in use for a couple of years by Lightbits and our clients. We have tested it extensively internally with multiple openstack versions, including Queens, Rocky, Stein, and Train. We have also tested it with master (19.1 xena) and we are working to extend testing to cover additional openstack releases. We are glad to join the openstack community and hope to get your feedback and comments on this driver, and if it is acceptable, to see it merged into the tree. Signed-off-by: Yuval Brave yuval@lightbitslabs.com Depends-On: I2e86fa84049053b7c75421d33ad1a1af459ef4e0 Change-Id: I8e6eab23419a4e7f092bb49aa85a4aaab5a8ba38 --- cinder/opts.py | 2 + .../unit/volume/drivers/lightos/__init__.py | 0 .../drivers/lightos/test_lightos_storage.py | 702 ++++++++ cinder/volume/drivers/lightos.py | 1481 +++++++++++++++++ .../drivers/lightbits-lightos-driver.rst | 140 ++ doc/source/reference/support-matrix.ini | 13 + ...tered-nvmetcp-driver-d1ef8f83263921f2.yaml | 8 + 7 files changed, 2346 insertions(+) create mode 100644 cinder/tests/unit/volume/drivers/lightos/__init__.py create mode 100644 cinder/tests/unit/volume/drivers/lightos/test_lightos_storage.py create mode 100644 cinder/volume/drivers/lightos.py create mode 100644 doc/source/configuration/block-storage/drivers/lightbits-lightos-driver.rst create mode 100644 releasenotes/notes/bp-lightbits-lightos-clustered-nvmetcp-driver-d1ef8f83263921f2.yaml diff --git a/cinder/opts.py b/cinder/opts.py index e0343d5cd07..efa8de8c945 100644 --- a/cinder/opts.py +++ b/cinder/opts.py @@ -133,6 +133,7 @@ from cinder.volume.drivers.kioxia import kumoscale as \ cinder_volume_drivers_kioxia_kumoscale from cinder.volume.drivers.lenovo import lenovo_common as \ cinder_volume_drivers_lenovo_lenovocommon +from cinder.volume.drivers import lightos as cinder_volume_drivers_lightos from cinder.volume.drivers import linstordrv as \ cinder_volume_drivers_linstordrv from cinder.volume.drivers import lvm as cinder_volume_drivers_lvm @@ -350,6 +351,7 @@ def list_opts(): kaminario_opts, cinder_volume_drivers_lenovo_lenovocommon.common_opts, cinder_volume_drivers_lenovo_lenovocommon.iscsi_opts, + cinder_volume_drivers_lightos.lightos_opts, cinder_volume_drivers_linstordrv.linstor_opts, cinder_volume_drivers_lvm.volume_opts, cinder_volume_drivers_macrosan_driver.config.macrosan_opts, diff --git a/cinder/tests/unit/volume/drivers/lightos/__init__.py b/cinder/tests/unit/volume/drivers/lightos/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cinder/tests/unit/volume/drivers/lightos/test_lightos_storage.py b/cinder/tests/unit/volume/drivers/lightos/test_lightos_storage.py new file mode 100644 index 00000000000..6589fa74a63 --- /dev/null +++ b/cinder/tests/unit/volume/drivers/lightos/test_lightos_storage.py @@ -0,0 +1,702 @@ +# Copyright (C) 2016-2022 Lightbits Labs Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from copy import deepcopy +import functools +import hashlib +import http.client as httpstatus +import json +from typing import Dict +from typing import List +from typing import Tuple +from unittest import mock +import uuid + +from cinder import context +from cinder import db +from cinder import exception +from cinder.tests.unit import test +from cinder.tests.unit import utils as test_utils +from cinder.volume import configuration as conf +from cinder.volume.drivers import lightos + + +FAKE_LIGHTOS_CLUSTER_NODES: Dict[str, List] = { + "nodes": [ + {"UUID": "926e6df8-73e1-11ec-a624-000000000001", + "nvmeEndpoint": "192.168.75.10:4420"}, + {"UUID": "926e6df8-73e1-11ec-a624-000000000002", + "nvmeEndpoint": "192.168.75.11:4420"}, + {"UUID": "926e6df8-73e1-11ec-a624-000000000003", + "nvmeEndpoint": "192.168.75.12:4420"} + ] +} + +FAKE_LIGHTOS_CLUSTER_INFO: Dict[str, str] = { + 'UUID': "926e6df8-73e1-11ec-a624-07ba3880f6cc", + 'subsystemNQN': "nqn.2014-08.org.nvmexpress:NVMf:uuid:" + "f4a89ce0-9fc2-4900-bfa3-00ad27995e7b" +} + +FAKE_CLIENT_HOSTNQN = "hostnqn1" +VOLUME_BACKEND_NAME = "lightos_backend" +RESERVED_PERCENTAGE = 30 +DEVICE_SCAN_ATTEMPTS_DEFAULT = 5 +LIGHTOS_API_SERVICE_TIMEOUT = 30 +VOLUME_BACKEND_NAME = "lightos_backend" +RESERVED_PERCENTAGE = 30 + + +class InitiatorConnectorFactoryMocker: + @staticmethod + def factory(protocol, root_helper, driver=None, + use_multipath=False, + device_scan_attempts=DEVICE_SCAN_ATTEMPTS_DEFAULT, + arch=None, + *args, **kwargs): + return InitialConnectorMock() + + +class InitialConnectorMock: + hostnqn = FAKE_CLIENT_HOSTNQN + found_discovery_client = True + + def get_hostnqn(self): + return self.__class__.hostnqn + + def find_dsc(self): + return self.__class__.found_discovery_client + + +def get_connector_properties(): + connector = InitialConnectorMock() + return dict(hostnqn=connector.get_hostnqn(), + found_dsc=connector.find_dsc()) + + +def get_vol_etag(volume): + v = deepcopy(volume) + v.pop("ETag", None) + dump = json.dumps(v, sort_keys=True).encode('utf-8') + return hashlib.md5(dump).hexdigest() + + +class DBMock(object): + + def __init__(self): + self.data = { + "projects": {}, + } + + def get_or_create_project(self, project_name) -> Dict: + project = self.data["projects"].setdefault(project_name, {}) + return project + + def get_project(self, project_name) -> Dict: + project = self.data["projects"].get(project_name, None) + return project if project else None + + def delete_project(self, project_name) -> Dict: + assert project_name != "default", "can't delete default project" + project = self.get_project(project_name) + if not project: + return None + self.data["projects"].remove(project) + return project + + def create_volume(self, volume) -> Tuple[int, Dict]: + assert volume["project_name"] and volume["name"], "must be provided" + project = self.get_or_create_project(volume["project_name"]) + volumes = project.setdefault("volumes", []) + existing_volume = next(iter([vol for vol in volumes + if vol["name"] == volume["name"]]), None) + if not existing_volume: + volume["UUID"] = str(uuid.uuid4()) + volumes.append(volume) + return httpstatus.OK, volume + return httpstatus.CONFLICT, None + + def get_volume_by_uuid(self, project_name, + volume_uuid) -> Tuple[int, Dict]: + assert project_name and volume_uuid, "must be provided" + project = self.get_project(project_name) + if not project: + return httpstatus.NOT_FOUND, None + proj_vols = project.get("volumes", None) + if not proj_vols: + return httpstatus.NOT_FOUND, None + vol = next(iter([vol for vol in proj_vols + if vol["UUID"] == volume_uuid]), None) + return (httpstatus.OK, vol) if vol else (httpstatus.NOT_FOUND, None) + + def update_volume_by_uuid(self, project_name, + volume_uuid, **kwargs) -> Tuple[int, Dict]: + error_code, volume = self.get_volume_by_uuid(project_name, volume_uuid) + if error_code != httpstatus.OK: + return error_code, None + etag = kwargs.get("etag", None) + if etag: + vol_etag = volume.get("ETag", None) + if etag != vol_etag: + return httpstatus.BAD_REQUEST, None + if kwargs.get("size", None): + volume["size"] = kwargs["size"] + if kwargs.get("acl", None): + volume["acl"] = {'values': kwargs.get('acl')} + volume["ETag"] = get_vol_etag(volume) + return httpstatus.OK, volume + + def get_volume_by_name(self, project_name, + volume_name) -> Tuple[int, Dict]: + assert project_name and volume_name, "must be provided" + project = self.get_project(project_name) + if not project: + return httpstatus.NOT_FOUND, None + proj_vols = project.get("volumes", None) + if not proj_vols: + return httpstatus.NOT_FOUND, None + vol = next(iter([vol for vol in proj_vols + if vol["name"] == volume_name]), None) + return (httpstatus.OK, vol) if vol else (httpstatus.NOT_FOUND, None) + + def delete_volume(self, project_name, volume_uuid) -> Tuple[int, Dict]: + assert project_name and volume_uuid, "must be provided" + project = self.get_project(project_name) + if not project: + return httpstatus.NOT_FOUND, None + proj_vols = project.get("volumes", None) + if not proj_vols: + return httpstatus.NOT_FOUND, None + for vol in proj_vols: + if vol["UUID"] == volume_uuid: + proj_vols.remove(vol) + return httpstatus.OK, vol + + def update_volume(self, **kwargs): + assert("project_name" in kwargs and kwargs["project_name"]), \ + "project_name must be provided" + + def create_snapshot(self, snapshot) -> Tuple[int, Dict]: + assert snapshot["project_name"] and snapshot["name"], \ + "must be provided" + project = self.get_or_create_project(snapshot["project_name"]) + snapshots = project.setdefault("snapshots", []) + existing_snap = next(iter([snap for snap in snapshots + if snap["name"] == snapshot["name"]]), None) + if not existing_snap: + snapshot["UUID"] = str(uuid.uuid4()) + snapshots.append(snapshot) + return httpstatus.OK, snapshot + return httpstatus.CONFLICT, None + + def delete_snapshot(self, project_name, snapshot_uuid) -> Tuple[int, Dict]: + assert project_name and snapshot_uuid, "must be provided" + project = self.get_project(project_name) + if not project: + return httpstatus.NOT_FOUND, None + proj_snaps = project.get("snapshots", None) + if not proj_snaps: + return httpstatus.NOT_FOUND, None + for snap in proj_snaps: + if snap["UUID"] == snapshot_uuid: + proj_snaps.remove(snap) + return httpstatus.OK, snap + + def get_snapshot_by_name(self, project_name, + snapshot_name) -> Tuple[int, Dict]: + assert project_name and snapshot_name, "must be provided" + project = self.get_project(project_name) + if not project: + return httpstatus.NOT_FOUND, None + proj_snaps = project.get("snapshots", None) + if not proj_snaps: + return httpstatus.NOT_FOUND, None + snap = next(iter([snap for snap in proj_snaps + if snap["name"] == snapshot_name]), None) + return (httpstatus.OK, snap) if snap else (httpstatus.NOT_FOUND, None) + + def get_snapshot_by_uuid(self, project_name, + snapshot_uuid) -> Tuple[int, Dict]: + assert project_name and snapshot_uuid, "must be provided" + project = self.get_project(project_name) + if not project: + return httpstatus.NOT_FOUND, None + proj_snaps = project.get("snapshots", None) + if not proj_snaps: + return httpstatus.NOT_FOUND, None + snap = next(iter([snap for snap in proj_snaps + if snap["UUID"] == snapshot_uuid]), None) + return (httpstatus.OK, snap) if snap else (httpstatus.NOT_FOUND, None) + + +class LightOSStorageVolumeDriverTest(test.TestCase): + + def setUp(self): + """Initialize LightOS Storage Driver.""" + super(LightOSStorageVolumeDriverTest, self).setUp() + + configuration = mock.Mock(conf.Configuration) + + configuration.lightos_api_address = \ + "10.10.10.71,10.10.10.72,10.10.10.73" + configuration.lightos_api_port = 443 + configuration.lightos_jwt = None + configuration.lightos_snapshotname_prefix = 'openstack_' + configuration.lightos_intermediate_snapshot_name_prefix = 'for_clone_' + configuration.lightos_default_compression_enabled = False + configuration.lightos_default_num_replicas = 3 + configuration.num_volume_device_scan_tries = ( + DEVICE_SCAN_ATTEMPTS_DEFAULT) + configuration.lightos_api_service_timeout = LIGHTOS_API_SERVICE_TIMEOUT + configuration.driver_ssl_cert_verify = False + # for some reason this value is not initialized by the driver parent + # configs + configuration.volume_name_template = 'volume-%s' + configuration.initiator_connector = ( + "cinder.tests.unit.volume.drivers.lightos." + "test_lightos_storage.InitiatorConnectorFactoryMocker") + configuration.volume_backend_name = VOLUME_BACKEND_NAME + configuration.reserved_percentage = RESERVED_PERCENTAGE + + def mocked_safe_get(config, variable_name): + if hasattr(config, variable_name): + return config.__getattribute__(variable_name) + else: + return None + + configuration.safe_get = functools.partial(mocked_safe_get, + configuration) + self.driver = lightos.LightOSVolumeDriver(configuration=configuration) + self.ctxt = context.get_admin_context() + self.db: DBMock = DBMock() + + # define a default send_cmd override to return default values. + def send_cmd_default_mock(cmd, timeout, **kwargs): + if cmd == "get_nodes": + return (httpstatus.OK, FAKE_LIGHTOS_CLUSTER_NODES) + if cmd == "get_node": + self.assertTrue(kwargs["UUID"]) + for node in FAKE_LIGHTOS_CLUSTER_NODES["nodes"]: + if kwargs["UUID"] == node["UUID"]: + return (httpstatus.OK, node) + return (httpstatus.NOT_FOUND, node) + elif cmd == "get_cluster_info": + return (httpstatus.OK, FAKE_LIGHTOS_CLUSTER_INFO) + elif cmd == "create_volume": + project_name = kwargs["project_name"] + volume = { + "project_name": project_name, + "name": kwargs["name"], + "size": kwargs["size"], + "n_replicas": kwargs["n_replicas"], + "compression": kwargs["compression"], + "src_snapshot_name": kwargs["src_snapshot_name"], + "acl": {'values': kwargs.get('acl')}, + "state": "Available", + } + volume["ETag"] = get_vol_etag(volume) + code, new_vol = self.db.create_volume(volume) + return (code, new_vol) + elif cmd == "delete_volume": + return self.db.delete_volume(kwargs["project_name"], + kwargs["volume_uuid"]) + elif cmd == "get_volume": + return self.db.get_volume_by_uuid(kwargs["project_name"], + kwargs["volume_uuid"]) + elif cmd == "get_volume_by_name": + return self.db.get_volume_by_name(kwargs["project_name"], + kwargs["volume_name"]) + elif cmd == "extend_volume": + size = kwargs.get("size", None) + return self.db.update_volume_by_uuid(kwargs["project_name"], + kwargs["volume_uuid"], + size=size) + elif cmd == "create_snapshot": + snapshot = { + "project_name": kwargs.get("project_name", None), + "name": kwargs.get("name", None), + "state": "Available", + } + return self.db.create_snapshot(snapshot) + elif cmd == "delete_snapshot": + return self.db.delete_snapshot(kwargs["project_name"], + kwargs["snapshot_uuid"]) + elif cmd == "get_snapshot": + return self.db.get_snapshot_by_uuid(kwargs["project_name"], + kwargs["snapshot_uuid"]) + elif cmd == "get_snapshot_by_name": + return self.db.get_snapshot_by_name(kwargs["project_name"], + kwargs["snapshot_name"]) + elif cmd == "update_volume": + return self.db.update_volume_by_uuid(**kwargs) + + else: + raise RuntimeError( + f"'{cmd}' is not implemented. kwargs: {kwargs}") + + self.driver.cluster.send_cmd = send_cmd_default_mock + + def test_setup_should_fail_if_lightos_client_cant_auth_cluster(self): + """Verify lightos_client fail with bad auth.""" + + def side_effect(cmd, timeout): + if cmd == "get_cluster_info": + return (httpstatus.UNAUTHORIZED, None) + else: + raise RuntimeError() + + self.driver.cluster.send_cmd = side_effect + self.assertRaises(exception.InvalidAuthKey, + self.driver.do_setup, None) + + def test_setup_should_succeed(self): + """Test that lightos_client succeed.""" + self.driver.do_setup(None) + + def test_create_volume_should_succeed(self): + """Test that lightos_client succeed.""" + self.driver.do_setup(None) + + vol_type = test_utils.create_volume_type(self.ctxt, self, + name='my_vol_type') + volume = test_utils.create_volume(self.ctxt, size=4, + volume_type_id=vol_type.id) + + self.driver.create_volume(volume) + self.driver.delete_volume(volume) + db.volume_destroy(self.ctxt, volume.id) + + def test_create_volume_same_volume_twice_succeed(self): + """Test succeed to create an exiting volume.""" + self.driver.do_setup(None) + + vol_type = test_utils.create_volume_type(self.ctxt, self, + name='my_vol_type') + volume = test_utils.create_volume(self.ctxt, size=4, + volume_type_id=vol_type.id) + + self.driver.create_volume(volume) + self.driver.create_volume(volume) + self.driver.delete_volume(volume) + db.volume_destroy(self.ctxt, volume.id) + + def test_create_volume_in_failed_state(self): + """Verify scenario of created volume in failed state: + + Driver is expected to issue a deletion command and raise exception + """ + def send_cmd_mock(cmd, **kwargs): + if cmd == "create_volume": + project_name = kwargs["project_name"] + volume = { + "project_name": project_name, + "name": kwargs["name"], + "size": kwargs["size"], + "n_replicas": kwargs["n_replicas"], + "compression": kwargs["compression"], + "src_snapshot_name": kwargs["src_snapshot_name"], + "acl": {'values': kwargs.get('acl')}, + "state": "Failed", + } + volume["ETag"] = get_vol_etag(volume) + code, new_vol = self.db.create_volume(volume) + return (code, new_vol) + elif cmd == "delete_volume": + return self.db.delete_volume(kwargs["project_name"], + kwargs["volume_uuid"]) + elif cmd == "get_volume": + return self.db.get_volume_by_uuid(kwargs["project_name"], + kwargs["volume_uuid"]) + elif cmd == "get_volume_by_name": + return self.db.get_volume_by_name(kwargs["project_name"], + kwargs["volume_name"]) + else: + raise RuntimeError( + f"'{cmd}' is not implemented. kwargs: {kwargs}") + + self.driver.do_setup(None) + self.driver.cluster.send_cmd = send_cmd_mock + vol_type = test_utils.create_volume_type(self.ctxt, self, + name='my_vol_type') + volume = test_utils.create_volume(self.ctxt, size=4, + volume_type_id=vol_type.id) + self.assertRaises(exception.VolumeBackendAPIException, + self.driver.create_volume, volume) + proj = self.db.data["projects"][lightos.LIGHTOS_DEFAULT_PROJECT_NAME] + actual_volumes = proj["volumes"] + self.assertEqual(0, len(actual_volumes)) + db.volume_destroy(self.ctxt, volume.id) + + def test_delete_volume_fail_if_not_created(self): + """Test that lightos_client fail creating an already exists volume.""" + self.driver.do_setup(None) + + vol_type = test_utils.create_volume_type(self.ctxt, self, + name='my_vol_type') + volume = test_utils.create_volume(self.ctxt, size=4, + volume_type_id=vol_type.id) + self.driver.delete_volume(volume) + db.volume_destroy(self.ctxt, volume.id) + + def test_extend_volume_should_succeed(self): + self.driver.do_setup(None) + + vol_type = test_utils.create_volume_type(self.ctxt, self, + name='my_vol_type') + volume = test_utils.create_volume(self.ctxt, size=4, + volume_type_id=vol_type.id) + + self.driver.create_volume(volume) + self.driver.extend_volume(volume, 6) + self.driver.delete_volume(volume) + db.volume_destroy(self.ctxt, volume.id) + + def test_extend_volume_should_fail_if_volume_does_not_exist(self): + self.driver.do_setup(None) + + vol_type = test_utils.create_volume_type(self.ctxt, self, + name='my_vol_type') + volume = test_utils.create_volume(self.ctxt, size=4, + volume_type_id=vol_type.id) + + self.assertRaises(exception.VolumeNotFound, + self.driver.extend_volume, volume, 6) + db.volume_destroy(self.ctxt, volume.id) + + def test_create_snapshot(self): + self.driver.do_setup(None) + + vol_type = test_utils.create_volume_type(self.ctxt, self, + name='my_vol_type') + volume = test_utils.create_volume(self.ctxt, size=4, + volume_type_id=vol_type.id) + snapshot = test_utils.create_snapshot(self.ctxt, volume_id=volume.id) + + self.driver.create_volume(volume) + self.driver.create_snapshot(snapshot) + self.driver.delete_volume(volume) + db.volume_destroy(self.ctxt, volume.id) + + def test_delete_snapshot(self): + self.driver.do_setup(None) + + vol_type = test_utils.create_volume_type(self.ctxt, self, + name='my_vol_type') + volume = test_utils.create_volume(self.ctxt, size=4, + volume_type_id=vol_type.id) + snapshot = test_utils.create_snapshot(self.ctxt, volume_id=volume.id) + + self.driver.create_volume(volume) + self.driver.create_snapshot(snapshot) + self.driver.delete_snapshot(snapshot) + self.driver.delete_volume(volume) + db.volume_destroy(self.ctxt, volume.id) + + def test_create_volume_from_snapshot(self): + self.driver.do_setup(None) + vol_type = test_utils.create_volume_type(self.ctxt, self, + name='my_vol_type') + volume = test_utils.create_volume(self.ctxt, size=4, + volume_type_id=vol_type.id) + snapshot = test_utils.create_snapshot(self.ctxt, volume_id=volume.id) + self.driver.create_volume_from_snapshot(volume, snapshot) + proj = self.db.data["projects"][lightos.LIGHTOS_DEFAULT_PROJECT_NAME] + actual_volumes = proj["volumes"] + self.assertEqual(1, len(actual_volumes)) + self.driver.delete_snapshot(snapshot) + self.driver.delete_volume(volume) + db.volume_destroy(self.ctxt, volume.id) + db.snapshot_destroy(self.ctxt, snapshot.id) + + def test_initialize_connection(self): + InitialConnectorMock.hostnqn = "hostnqn1" + InitialConnectorMock.found_discovery_client = True + self.driver.do_setup(None) + vol_type = test_utils.create_volume_type(self.ctxt, self, + name='my_vol_type') + volume = test_utils.create_volume(self.ctxt, size=4, + volume_type_id=vol_type.id) + self.driver.create_volume(volume) + connection_props = \ + self.driver.initialize_connection(volume, + get_connector_properties()) + self.assertIn('driver_volume_type', connection_props) + self.assertEqual('lightos', connection_props['driver_volume_type']) + self.assertEqual(FAKE_CLIENT_HOSTNQN, + connection_props['data']['hostnqn']) + self.assertEqual(FAKE_LIGHTOS_CLUSTER_INFO['subsystemNQN'], + connection_props['data']['nqn']) + self.assertEqual( + self.db.data['projects']['default']['volumes'][0]['UUID'], + connection_props['data']['uuid']) + + self.driver.delete_volume(volume) + db.volume_destroy(self.ctxt, volume.id) + + def test_initialize_connection_no_hostnqn_should_fail(self): + InitialConnectorMock.hostnqn = "" + InitialConnectorMock.found_discovery_client = True + self.driver.do_setup(None) + vol_type = test_utils.create_volume_type(self.ctxt, self, + name='my_vol_type') + volume = test_utils.create_volume(self.ctxt, size=4, + volume_type_id=vol_type.id) + self.driver.create_volume(volume) + self.assertRaises(exception.VolumeBackendAPIException, + self.driver.initialize_connection, volume, + get_connector_properties()) + self.driver.delete_volume(volume) + db.volume_destroy(self.ctxt, volume.id) + + def test_initialize_connection_no_dsc_should_fail(self): + InitialConnectorMock.hostnqn = "hostnqn1" + InitialConnectorMock.found_discovery_client = False + self.driver.do_setup(None) + vol_type = test_utils.create_volume_type(self.ctxt, self, + name='my_vol_type') + volume = test_utils.create_volume(self.ctxt, size=4, + volume_type_id=vol_type.id) + self.driver.create_volume(volume) + self.assertRaises(exception.VolumeBackendAPIException, + self.driver.initialize_connection, volume, + get_connector_properties()) + self.driver.delete_volume(volume) + db.volume_destroy(self.ctxt, volume.id) + + def test_terminate_connection_with_hostnqn(self): + InitialConnectorMock.hostnqn = "hostnqn1" + InitialConnectorMock.found_discovery_client = True + self.driver.do_setup(None) + vol_type = test_utils.create_volume_type(self.ctxt, self, + name='my_vol_type') + volume = test_utils.create_volume(self.ctxt, size=4, + volume_type_id=vol_type.id) + self.driver.create_volume(volume) + self.driver.terminate_connection(volume, get_connector_properties()) + self.driver.delete_volume(volume) + db.volume_destroy(self.ctxt, volume.id) + + def test_terminate_connection_with_empty_hostnqn_should_fail(self): + InitialConnectorMock.hostnqn = "" + InitialConnectorMock.found_discovery_client = True + self.driver.do_setup(None) + vol_type = test_utils.create_volume_type(self.ctxt, self, + name='my_vol_type') + volume = test_utils.create_volume(self.ctxt, size=4, + volume_type_id=vol_type.id) + self.driver.create_volume(volume) + self.assertRaises(exception.VolumeBackendAPIException, + self.driver.terminate_connection, volume, + get_connector_properties()) + self.driver.delete_volume(volume) + db.volume_destroy(self.ctxt, volume.id) + + def test_force_terminate_connection_with_empty_hostnqn(self): + InitialConnectorMock.hostnqn = "" + InitialConnectorMock.found_discovery_client = True + self.driver.do_setup(None) + vol_type = test_utils.create_volume_type(self.ctxt, self, + name='my_vol_type') + volume = test_utils.create_volume(self.ctxt, size=4, + volume_type_id=vol_type.id) + self.driver.create_volume(volume) + self.driver.terminate_connection(volume, get_connector_properties(), + force=True) + self.driver.delete_volume(volume) + db.volume_destroy(self.ctxt, volume.id) + + def test_check_for_setup_error(self): + InitialConnectorMock.hostnqn = "hostnqn1" + InitialConnectorMock.found_discovery_client = True + self.driver.do_setup(None) + self.driver.check_for_setup_error() + + def test_check_for_setup_error_no_subsysnqn_should_fail(self): + InitialConnectorMock.hostnqn = "hostnqn1" + InitialConnectorMock.found_discovery_client = True + self.driver.do_setup(None) + self.driver.cluster.subsystemNQN = "" + self.assertRaises(exception.VolumeBackendAPIException, + self.driver.check_for_setup_error) + + def test_check_for_setup_error_no_hostnqn_should_fail(self): + InitialConnectorMock.hostnqn = "" + InitialConnectorMock.found_discovery_client = True + self.driver.do_setup(None) + self.assertRaises(exception.VolumeBackendAPIException, + self.driver.check_for_setup_error) + + def test_check_for_setup_error_no_dsc_should_succeed(self): + InitialConnectorMock.hostnqn = "hostnqn1" + InitialConnectorMock.found_discovery_client = False + self.driver.do_setup(None) + self.driver.check_for_setup_error() + + def test_create_clone(self): + self.driver.do_setup(None) + + vol_type = test_utils.create_volume_type(self.ctxt, self, + name='my_vol_type') + volume = test_utils.create_volume(self.ctxt, size=4, + volume_type_id=vol_type.id) + clone = test_utils.create_volume(self.ctxt, size=4, + volume_type_id=vol_type.id) + + self.driver.create_volume(volume) + self.driver.create_cloned_volume(clone, volume) + self.driver.delete_volume(volume) + self.driver.delete_volume(clone) + + db.volume_destroy(self.ctxt, volume.id) + db.volume_destroy(self.ctxt, clone.id) + + def test_get_volume_stats(self): + """Test that lightos_client succeed.""" + self.driver.do_setup(None) + volumes_data = self.driver.get_volume_stats(refresh=False) + assert len(volumes_data) == 0, "Expected empty config" + + volumes_data = self.driver.get_volume_stats(refresh=True) + assert volumes_data['vendor_name'] == 'LightOS Storage', \ + "Expected 'LightOS Storage', received %s" % \ + volumes_data['vendor_name'] + assert volumes_data['volume_backend_name'] == VOLUME_BACKEND_NAME, \ + "Expected %s, received %s" % \ + (VOLUME_BACKEND_NAME, volumes_data['volume_backend_name']) + assert volumes_data['driver_version'] == self.driver.VERSION, \ + "Expected %s, received %s" % \ + (self.driver.VERSION, volumes_data['driver_version']) + assert volumes_data['storage_protocol'] == "lightos", \ + "Expected 'lightos', received %s" % \ + volumes_data['storage_protocol'] + assert volumes_data['reserved_percentage'] == RESERVED_PERCENTAGE, \ + "Expected %d, received %s" % \ + (RESERVED_PERCENTAGE, volumes_data['reserved_percentage']) + assert volumes_data['QoS_support'] is False, \ + "Expected False, received %s" % volumes_data['QoS_support'] + assert volumes_data['online_extend_support'] is True, \ + "Expected True, received %s" % \ + volumes_data['online_extend_support'] + assert volumes_data['thin_provisioning_support'] is True, \ + "Expected True, received %s" % \ + volumes_data['thin_provisioning_support'] + assert volumes_data['compression'] is False, \ + "Expected False, received %s" % volumes_data['compression'] + assert volumes_data['multiattach'] is True, \ + "Expected True, received %s" % volumes_data['multiattach'] + assert volumes_data['free_capacity_gb'] == 'infinite', \ + "Expected 'infinite', received %s" % \ + volumes_data['free_capacity_gb'] diff --git a/cinder/volume/drivers/lightos.py b/cinder/volume/drivers/lightos.py new file mode 100644 index 00000000000..8324bfda0d4 --- /dev/null +++ b/cinder/volume/drivers/lightos.py @@ -0,0 +1,1481 @@ +# Copyright (C) 2016-2022 Lightbits Labs Ltd. +# Copyright (C) 2020 Intel Corporation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import http.client as httpstatus +import json +import random +import time +from typing import Dict + + +from oslo_config import cfg +from oslo_log import log as logging +from oslo_utils import importutils +from oslo_utils import units +import requests +import urllib3 + +from cinder import coordination +from cinder import exception +from cinder.i18n import _ +from cinder import interface +from cinder import utils +from cinder.volume import configuration as config +from cinder.volume import driver + + +LOG = logging.getLogger(__name__) +ENABLE_TRACE = True +LIGHTOS_DEFAULT_PROJECT_NAME = "default" + +urllib3.disable_warnings() + +lightos_opts = [ + cfg.ListOpt('lightos_api_address', + default=None, + item_type=cfg.types.IPAddress(), + help='The IP addresses of the LightOS API servers separated' + ' by commas.'), + cfg.PortOpt('lightos_api_port', + default='443', + help='The TCP/IP port at which the LightOS API' + ' endpoints listen.' + ' Port 443 is used for HTTPS and other values' + ' are used for HTTP.'), + cfg.StrOpt('lightos_jwt', + default=None, + help='JWT to be used for volume and snapshot operations with' + ' the LightOS cluster.' + ' Do not set this parameter if the cluster is installed' + ' with multi-tenancy disabled.'), + cfg.IntOpt('lightos_default_num_replicas', + min=1, + max=3, + default=3, + help='The default number of replicas to create for each' + ' volume.'), + cfg.BoolOpt('lightos_default_compression_enabled', + default=False, + help='The default compression enabled setting' + ' for new volumes.'), + cfg.IntOpt('lightos_api_service_timeout', + default=30, + help='The default amount of time (in seconds) to wait for' + ' an API endpoint response.') +] + +CONF = cfg.CONF +CONF.register_opts(lightos_opts, group=config.SHARED_CONF_GROUP) +BLOCK_SIZE = 8 +LIGHTOS = "LIGHTOS" +INTERM_SNAPSHOT_PREFIX = "for_clone_" + + +class LightOSConnection(object): + def __init__(self, conf): + self.conf = conf + self.access_key = None + self.apiservers = self._init_api_servers() + self._cur_api_server_idx = random.randint(0, len(self.apiservers) - 1) + self.targets = dict() + self.lightos_cluster_uuid = None + self.subsystemNQN = None + self._stats = {'total_capacity_gb': 0, 'free_capacity_gb': 0} + # a single API call must have been answered in this time if the API + # service/network were up + self.api_timeout = self.conf.lightos_api_service_timeout + + def _init_api_servers(self) -> Dict[int, Dict]: + # And verify that port is in range + apiservers: Dict[int, Dict] = {} + hosts = self.conf.lightos_api_address + port = str(self.conf.lightos_api_port) + apiservers = [dict(api_address=addr, api_port=port) for addr in hosts] + return apiservers + + def _generate_lightos_cmd(self, cmd, **kwargs): + """Generate command to be sent to LightOS API service""" + + def _joined_params(params): + param_str = [] + for k, v in params.items(): + param_str.append("%s=%s" % (k, v)) + return '&'.join(param_str) + + # Dictionary of applicable LightOS commands in the following format: + # 'command': (method, API_URL, {optional parameters}) + # This is constructed on the fly to include the caller-supplied kwargs + # Can be optimized by only constructing the specific + # command the user provided in cmd + + # API V2 common commands + lightos_commands = { + # cluster operations, + 'get_cluster_info': ('GET', + '/api/v2/clusterinfo', {}), + + 'get_cluster': ('GET', + '/api/v2/cluster', {}), + + # node operations + 'get_node': ('GET', + '/api/v2/nodes/%s' % kwargs.get('UUID'), {}), + + 'get_nodes': ('GET', + '/api/v2/nodes', {}), + + # volume operations + 'create_volume': ('POST', + '/api/v2/projects/%s/volumes' % kwargs.get( + "project_name"), + { + 'name': kwargs.get('name'), + 'size': kwargs.get('size'), + 'replicaCount': kwargs.get('n_replicas'), + 'compression': kwargs.get('compression'), + 'acl': { + 'values': kwargs.get('acl'), + }, + 'sourceSnapshotUUID': kwargs.get( + 'src_snapshot_uuid'), + 'sourceSnapshotName': kwargs.get( + 'src_snapshot_name'), + }), + + 'delete_volume': ('DELETE', + '/api/v2/projects/%s/volumes/%s' % (kwargs.get( + "project_name"), kwargs.get("volume_uuid")), + {}), + + 'update_volume': ('PUT', + '/api/v2/projects/%s/volumes/%s' % (kwargs.get( + "project_name"), kwargs.get("volume_uuid")), + { + 'acl': { + 'values': kwargs.get('acl'), + }, + }), + + 'extend_volume': ('PUT', + '/api/v2/projects/%s/volumes/%s' % ( + kwargs.get("project_name"), + kwargs.get("volume_uuid")), + { + 'UUID': kwargs.get('volume_uuid'), + 'size': kwargs.get('size'), + }), + + # snapshots operations + 'create_snapshot': ('POST', + '/api/v2/projects/%s/snapshots' % kwargs.get( + "project_name"), + { + 'name': kwargs.get('name'), + 'sourceVolumeUUID': kwargs.get( + 'src_volume_uuid'), + 'sourceVolumeName': kwargs.get( + 'src_volume_name'), + }), + + 'delete_snapshot': ('DELETE', + '/api/v2/projects/%s/snapshots/%s' % ( + kwargs.get("project_name"), + kwargs.get("snapshot_uuid")), + {}), + + # get operations + 'get_volume': ('GET', + '/api/v2/projects/%s/volumes/%s' % ( + kwargs.get("project_name"), + kwargs.get("volume_uuid")), + {}), + + 'get_volume_by_name': ('GET', + '/api/v2/projects/%s/volumes/?name=%s' % ( + kwargs.get("project_name"), + kwargs.get("volume_name")), + {}), + + 'list_volumes': ('GET', + '/api/v2/projects/%s/volumes' % kwargs.get( + "project_name"), + {}), + + 'get_snapshot': ('GET', + '/api/v2/projects/%s/snapshots/%s' % ( + kwargs.get("project_name"), + kwargs.get("snapshot_uuid")), + {}), + + 'get_snapshot_by_name': ('GET', + '/api/v2/projects/%s/snapshots' + '/?Name=%s' % ( + kwargs.get("project_name"), + kwargs.get("snapshot_name")), + {}) + } + if cmd not in lightos_commands: + raise exception.UnknownCmd(cmd=cmd) + else: + (method, url, params) = lightos_commands[cmd] + + if method == 'GET': + body = params + + elif method == 'DELETE': + LOG.debug("DELETE params: %s", params) + # For DELETE commands add parameters to the URL + url += '?' + _joined_params(params) + body = '' + + elif method == 'PUT': + # For PUT commands add parameters to the URL + body = params + + elif method == 'POST': + body = params + + else: + msg = (_('Method %(method)s is not defined') % + {'method': method}) + LOG.error(msg) + raise AssertionError(msg) + + return (method, url, body) + + def pretty_print_req(self, req, timeout): + request = req.method + ' ' + req.url + header = ', '.join('"{}: {}"'.format(k, v) + for k, v in req.headers.items()) + LOG.debug('Req: %s Headers: %s Body: %s Timeout: %s', + request, + header, + req.body, + timeout) + + def send_cmd(self, cmd, timeout, **kwargs): + """Send command to any LightOS REST API server.""" + start_idx = self._cur_api_server_idx + stop = time.time() + timeout + while time.time() <= stop: + server = self.apiservers[self._cur_api_server_idx] + host = server['api_address'] + port = server['api_port'] + + (success, status_code, data) = self.__send_cmd( + cmd, host, port, self.api_timeout, **kwargs) + if success: + return (status_code, data) + # go on to the next API server wrapping around as needed + self._cur_api_server_idx = ( + self._cur_api_server_idx + 1) % len(self.apiservers) + # if we only have a single API server, keep trying it + # if we have more than one and we tried all of them, give up + if (self._cur_api_server_idx == + start_idx and len(self.apiservers) > 1): + break + + raise exception.VolumeDriverException( + message="Could not get a response from any API server") + + def __send_cmd(self, cmd, host, port, timeout, **kwargs): + """Send command to LightOS REST API server. + + Returns: (success = True/False, data) + """ + ssl_verify = self.conf.driver_ssl_cert_verify + (method, url, body) = self._generate_lightos_cmd(cmd, **kwargs) + LOG.info( + 'Invoking %(cmd)s using %(method)s url: %(url)s \ + request.body: %(body)s ssl_verify: %(ssl_verify)s', + {'cmd': cmd, 'method': method, 'url': url, 'body': body, + 'ssl_verify': ssl_verify}) + + api_url = "https://%s:%s%s" % (host, port, url) + + try: + with requests.Session() as session: + req = requests.Request( + method, api_url, data=json.dumps(body) if body else None) + req.headers.update({'Accept': 'application/json'}) + # -H 'Expect:' will prevent us from getting + # the 100 Continue response from curl + req.headers.update({'Expect': ''}) + if method in ('POST', 'PUT'): + req.headers.update({'Content-Type': 'application/json'}) + if kwargs.get("etag"): + req.headers.update({'If-Match': kwargs['etag']}) + if self.conf.lightos_jwt: + req.headers.update( + {'Authorization': + 'Bearer %s' % self.conf.lightos_jwt}) + prepped = req.prepare() + self.pretty_print_req(prepped, timeout) + response = session.send( + prepped, timeout=timeout, verify=ssl_verify) + except Exception: + LOG.exception("REST server not responding at '%s'", api_url) + return (False, None, None) + + try: + resp = response.json() + except ValueError: + resp = response.text + data = resp + + LOG.debug( + 'Resp(%s): code %s data %s', + api_url, + response.status_code, + data) + return (True, response.status_code, data) + + +@interface.volumedriver +class LightOSVolumeDriver(driver.VolumeDriver): + """OpenStack NVMe/TCP cinder drivers for Lightbits LightOS. + + .. code-block:: default + + Version history: + 2.3.12 - Initial upstream driver version. + """ + + VERSION = '2.3.12' + # ThirdPartySystems wiki page + CI_WIKI_NAME = "LightbitsLabs_CI" + + def __init__(self, *args, **kwargs): + super(LightOSVolumeDriver, self).__init__(*args, **kwargs) + self.configuration.append_config_values(lightos_opts) + # connector implements NVMe/TCP initiator functionality. + if not self.configuration.__dict__.get("initiator_connector", None): + self.configuration.initiator_connector = ( + "os_brick.initiator.connector.InitiatorConnector") + if not self.configuration.__dict__.get("lightos_client", None): + self.configuration.lightos_client = ( + "cinder.volume.drivers.lightos.LightOSConnection") + + initiator_connector = importutils.import_class( + self.configuration.initiator_connector) + self.connector = initiator_connector.factory( + LIGHTOS, + root_helper=utils.get_root_helper(), + message_queue=None, + device_scan_attempts= + self.configuration.num_volume_device_scan_tries) + + lightos_client_ctor = importutils.import_class( + self.configuration.lightos_client) + self.cluster = lightos_client_ctor(self.configuration) + + self.logical_op_timeout = \ + self.configuration.lightos_api_service_timeout * 3 + 10 + + @classmethod + def get_driver_options(cls): + additional_opts = cls._get_oslo_driver_opts( + 'driver_ssl_cert_verify', 'reserved_percentage', + 'volume_backend_name') + return lightos_opts + additional_opts + + def create_cloned_volume(self, volume, src_vref): + """Creates a clone of the specified volume. + + If volume_type extra specs includes 'replication: True' the + driver needs to create a volume replica (secondary) + and setup replication between the newly created volume + and the secondary volume. + """ + project_name = self._get_lightos_project_name(volume) + # Create an intermediate snapshot + snapshot_name = self._interm_snapshotname(volume) + src_volume_name = self._lightos_volname(src_vref) + try: + self._create_snapshot(project_name, + snapshot_name, src_volume_name) + except Exception as e: + LOG.warning( + "Failed to create intermediate snapshot \ + %s from source volume %s.", + snapshot_name, + src_volume_name) + raise e + + # Create a volume from the intermediate snapshot + try: + self._create_volume(volume, + src_snapshot_lightos_name=snapshot_name) + except Exception as e: + LOG.error("Failed to create volume %s from intermediate " + " snapshot %s. Trying to clean up.", + src_volume_name, snapshot_name) + raise e + + # Delete the intermediate snapshot + finally: + try: + self._delete_lightos_snapshot(project_name, snapshot_name) + except Exception as e: + LOG.warning("Failed to delete the intermediate snapshot %s for" + " volume %s. Trying to clean up.", + snapshot_name, src_volume_name) + raise e + + def create_export(self, context, volume, vg=None): + """Irrelevant for lightos volumes. + + Export created during attachment. + """ + pass + + def ensure_export(self, context, volume): + """Irrelevant for lightos volumes. + + Export created during attachment. + """ + pass + + def remove_export(self, context, volume): + """Irrelevant for lightos volumes. + + Export removed during detach. + """ + pass + + def _get_lightos_volume( + self, + project_name, + timeout, + vol_uuid=None, + vol_name=None): + assert vol_uuid or vol_name, 'LightOS volume name or UUID \ + must be specified' + if vol_uuid: + return self.cluster.send_cmd( + cmd='get_volume', + project_name=project_name, + timeout=timeout, + volume_uuid=vol_uuid) + + return self.cluster.send_cmd( + cmd='get_volume_by_name', + project_name=project_name, + timeout=timeout, + volume_name=vol_name) + + def _lightos_volname(self, volume): + volid = volume.name_id + lightos_volname = CONF.volume_name_template % volid + return lightos_volname + + def _get_lightos_project_name(self, volume): + try: + extra_specs = volume.volume_type.extra_specs + project_name = extra_specs.get( + 'lightos:project_name', + LIGHTOS_DEFAULT_PROJECT_NAME) + except Exception: + LOG.debug( + "LIGHTOS volume %s has no lightos:project_name", + volume) + project_name = LIGHTOS_DEFAULT_PROJECT_NAME + + return project_name + + def _lightos_snapshotname(self, snapshot_id): + return CONF.snapshot_name_template % snapshot_id + + def _interm_snapshotname(self, snapshot): + id = snapshot['id'] + return '%s%s' % (INTERM_SNAPSHOT_PREFIX, id) + + def _get_lightos_snapshot( + self, + project_name, + timeout, + snapshot_uuid=None, + snapshot_name=None): + assert snapshot_uuid or snapshot_name, 'LightOS snapshot name or \ + UUID must be specified' + if snapshot_uuid: + return self.cluster.send_cmd( + cmd='get_snapshot', + project_name=project_name, + timeout=timeout, + snapshot_uuid=snapshot_uuid) + + return self.cluster.send_cmd( + cmd='get_snapshot_by_name', + project_name=project_name, + timeout=timeout, + snapshot_name=snapshot_name) + + def _wait_for_volume_available( + self, + project_name, + timeout, + vol_uuid=None, + vol_name=None): + """Wait until the volume is available.""" + assert vol_uuid or vol_name, 'LightOS volume UUID or name \ + must be supplied' + # while creating lightos volume we can stop on any terminal status + # possible states: Unknown, Creating, Available, Deleting, Deleted, + # Failed, Updating + states = ('Available', 'Deleting', 'Deleted', 'Failed', 'UNKNOWN') + + stop = time.time() + timeout + while time.time() <= stop: + (status_code, + resp) = self._get_lightos_volume(project_name, + timeout=self.logical_op_timeout, + vol_uuid=vol_uuid, + vol_name=vol_name) + state = resp.get('state', 'UNKNOWN') if \ + status_code == httpstatus.OK and resp else 'UNKNOWN' + if state in states and status_code != httpstatus.NOT_FOUND: + break + time.sleep(1) + + return state + + def _get_volume_specs(self, volume): + compression = 'True' if self.configuration. \ + lightos_default_compression_enabled else 'False' + num_replicas = str(self.configuration.lightos_default_num_replicas) + + if not volume.volume_type: + return (compression, num_replicas, LIGHTOS_DEFAULT_PROJECT_NAME) + + specs = getattr(volume.volume_type, 'extra_specs', {}) + compression = 'True' if specs.get('compression', None) \ + else compression + num_replicas = str(specs.get('lightos:num_replicas', num_replicas)) + project_name = specs.get( + 'lightos:project_name', + LIGHTOS_DEFAULT_PROJECT_NAME) + return (compression, num_replicas, project_name) + + def _create_new_lightos_volume(self, + os_volume, + project_name, + lightos_name, + src_snapshot_lightos_name=None): + """Create a new LightOS volume for this openstack volume.""" + (compression, num_replicas, _) = self._get_volume_specs(os_volume) + return self.cluster.send_cmd( + cmd='create_volume', + project_name=project_name, + timeout=self.logical_op_timeout, + name=lightos_name, + size=str(os_volume['size']) + ' gib', + n_replicas=num_replicas, + compression=compression, + src_snapshot_name=src_snapshot_lightos_name, + acl=['ALLOW_NONE'] + ) + + def _get_lightos_uuid(self, project_name, volume): + lightos_name = self._lightos_volname(volume) + timeout = self.logical_op_timeout + + (status, data) = self._get_lightos_volume(project_name=project_name, + timeout=timeout, + vol_name=lightos_name) + if status != httpstatus.OK or not data: + LOG.error( + 'Failed to get LightOS volume %s project %s status: \ + %s data: %s', + lightos_name, + project_name, + status, + str(data)) + raise exception.VolumeNotFound(volume_id=volume) + + lightos_uuid = data.get('UUID') + if not lightos_uuid: + LOG.error('Failed to get LightOS volume UUID status: %s, data: %s', + status, str(data)) + raise exception.VolumeNotFound(volume_id=volume) + + return lightos_uuid + + def create_volume(self, volume): + return self._create_volume(volume, src_snapshot_lightos_name=None) + + def create_volume_from_snapshot(self, volume, snapshot): + snapshotname = self._lightos_snapshotname(snapshot["id"]) + return self._create_volume(volume, + src_snapshot_lightos_name=snapshotname) + + def _create_volume(self, volume, src_snapshot_lightos_name): + lightos_name = self._lightos_volname(volume) + project_name = self._get_lightos_project_name(volume) + lightos_uuid = '' + vol_state = 'UNKNOWN' + + # first, check if such a volume exists + # if it exists, we must have created it earlier in a previous + # invocation of create volume since it takes a while for + # openstack to retry the call, it's highly unlikely that we created + # it but it does not show up yet, so assume that if it does not show + # up, it was never created + status_code, resp = self._get_lightos_volume(project_name, + timeout=self. + logical_op_timeout, + vol_name=lightos_name) + if status_code == httpstatus.NOT_FOUND: + status_code, resp = self._create_new_lightos_volume( + os_volume=volume, + project_name=project_name, + lightos_name=lightos_name, + src_snapshot_lightos_name=src_snapshot_lightos_name) + + if status_code in (httpstatus.OK, httpstatus.CREATED): + lightos_uuid = resp['UUID'] + vol_state = self._wait_for_volume_available( + project_name, + timeout=self.logical_op_timeout, + vol_uuid=lightos_uuid) + if vol_state == 'Available': + LOG.debug( + "LIGHTOS created volume name %s lightos_uuid \ + %s project %s", + lightos_name, + lightos_uuid, + project_name) + return + + # if volume was created in failed state we should clean it up + LOG.warning( + 'LightOS volume with UUID %s project %s last_state is %s', + lightos_uuid, + project_name, + vol_state) + if vol_state != 'UNKNOWN': + LOG.debug( + 'Cleaning up LightOS volume with UUID %s project %s', + lightos_uuid, + project_name) + self._delete_lightos_volume(project_name, lightos_uuid) + # wait for openstack to call us again to create it + + msg = ( + "Did not succeed creating LightOS volume with UUID %(uuid)s" + " status_code %(code)s last state %(state)s" % + dict(uuid=lightos_uuid, code=status_code, state=vol_state)) + msg = _(msg) + raise exception.VolumeBackendAPIException(message=msg) + + def _wait_for_snapshot_available(self, project_name, + timeout, + snapshot_uuid=None, + snapshot_name=None): + """Wait until the snapshot is available.""" + assert snapshot_uuid or snapshot_name, \ + 'LightOS snapshot UUID or name must be supplied' + # we can stop on any terminal status + # possible states: Unknown, Creating, Available, Deleting, Deleted, + # Failed, Updating + states = ('Available', 'Deleting', 'Deleted', 'Failed', 'UNKNOWN') + + stop = time.time() + timeout + while time.time() <= stop: + (status_code, + resp) = self._get_lightos_snapshot(project_name, + timeout= + self.logical_op_timeout, + snapshot_uuid=snapshot_uuid, + snapshot_name=snapshot_name) + state = resp.get('state', 'UNKNOWN') if \ + status_code == httpstatus.OK and resp else 'UNKNOWN' + if state in states and status_code != httpstatus.NOT_FOUND: + break + time.sleep(1) + + return state + + def _wait_for_snapshot_deleted(self, + project_name, + timeout, + snapshot_uuid): + """Wait until the snapshot has been deleted.""" + assert snapshot_uuid, 'LightOS snapshot UUID must be specified' + states = ('Deleted', 'Deleting', 'UNKNOWN') + + stop = time.time() + timeout + while time.time() <= stop: + status_code, resp = ( + self._get_lightos_snapshot(project_name, + timeout=self.logical_op_timeout, + snapshot_uuid=snapshot_uuid)) + if status_code == httpstatus.NOT_FOUND: + return 'Deleted' + state = resp.get('state', 'UNKNOWN') if \ + status_code == httpstatus.OK and resp else 'UNKNOWN' + if state in states: + break + time.sleep(1) + + return state + + def _wait_for_volume_deleted(self, project_name, timeout, vol_uuid): + """Wait until the volume has been deleted.""" + assert vol_uuid, 'LightOS volume UUID must be specified' + states = ('Deleted', 'Deleting', 'UNKNOWN') + + stop = time.time() + timeout + while time.time() <= stop: + (status_code, + resp) = self._get_lightos_volume(project_name, + timeout=self.logical_op_timeout, + vol_uuid=vol_uuid) + if status_code == httpstatus.NOT_FOUND: + return 'Deleted' + state = resp.get('state', 'UNKNOWN') if \ + status_code == httpstatus.OK and resp else 'UNKNOWN' + if state in states: + break + time.sleep(1) + + return state + + def _delete_lightos_volume(self, project_name, lightos_uuid): + end = time.time() + self.logical_op_timeout + while (time.time() < end): + status_code, resp = ( + self.cluster.send_cmd( + cmd='delete_volume', + project_name=project_name, + timeout=self. logical_op_timeout, + volume_uuid=lightos_uuid)) + if status_code == httpstatus.OK: + break + + LOG.warning( + "delete_volume for volume with LightOS UUID %s failed \ + with status code %s response %s", + lightos_uuid, + status_code, + resp) + time.sleep(1) + else: # no break + LOG.error( + "Failed to delete volume with LightOS UUID %s. Final status \ + code %s response %s", + lightos_uuid, + status_code, + resp) + return False + + deleted_state = self._wait_for_volume_deleted( + project_name, timeout=self.logical_op_timeout, + vol_uuid=lightos_uuid) + return deleted_state in ('Deleted', 'Deleting', 'UNKNOWN') + + def delete_volume(self, volume): + """Delete volume.""" + project_name = self._get_lightos_project_name(volume) + try: + lightos_uuid = self._get_lightos_uuid(project_name, volume) + except exception.VolumeNotFound: + return True + + if not self._delete_lightos_volume(project_name, lightos_uuid): + msg = ('Failed to delete LightOS volume with UUID' + ' %(uuid)s project %(project_name)s' % ( + dict(uuid=lightos_uuid, project_name=project_name))) + raise exception.VolumeBackendAPIException(message=msg) + + def get_vol_by_id(self, volume): + LOG.warning('UNIMPLEMENTED: get vol by id') + + def get_vols(self): + LOG.warning('UNIMPLEMENTED: get vols') + + def check_for_setup_error(self): + subsysnqn = self.cluster.subsystemNQN + if not subsysnqn: + msg = ('LIGHTOS: Cinder driver requires the' + ' LightOS cluster subsysnqn') + raise exception.VolumeBackendAPIException(message=msg) + + hostnqn = self.connector.get_hostnqn() + if not hostnqn: + msg = ("LIGHTOS: Cinder driver requires a local hostnqn for" + " image_to/from_volume operations") + raise exception.VolumeBackendAPIException(message=msg) + + def get_cluster_info(self): + status_code, cluster_info = self.cluster.send_cmd( + cmd='get_cluster_info', timeout=self.logical_op_timeout) + if status_code == httpstatus.UNAUTHORIZED: + msg = f'LIGHTOS: failed to connect to cluster. code: {status_code}' + raise exception.InvalidAuthKey(message=_(msg)) + if status_code != httpstatus.OK: + msg = 'LIGHTOS: Could not connect to LightOS cluster' + raise exception.VolumeBackendAPIException(message=_(msg)) + + LOG.info("Connected to LightOS cluster %s subsysnqn %s", + cluster_info['UUID'], cluster_info['subsystemNQN']) + self.cluster.lightos_cluster_uuid = cluster_info['UUID'] + self.cluster.subsystemNQN = cluster_info['subsystemNQN'] + + def get_cluster_stats(self): + status_code, cluster_info = self.cluster.send_cmd( + cmd='get_cluster', timeout=self.logical_op_timeout) + if status_code != httpstatus.OK: + msg = 'LIGHTOS: Could not connect to LightOS cluster' + raise exception.VolumeBackendAPIException(message=_(msg)) + + return cluster_info['statistics'] + + def valid_nodes_info(self, nodes_info): + if not nodes_info or 'nodes' not in nodes_info: + return False + + return True + + def wait_for_lightos_cluster(self): + cmd = 'get_nodes' + end = time.time() + self.logical_op_timeout + while (time.time() < end): + status_code, nodes_info = self.cluster.send_cmd( + cmd=cmd, timeout=self.logical_op_timeout) + if status_code != httpstatus.OK or not self.valid_nodes_info( + nodes_info): + time.sleep(1) + continue + + return nodes_info + + # bail out if we got here, timeout elapsed + msg = 'Failed to get nodes, last status was {} nodes_info {}'.format( + status_code, nodes_info) + raise exception.VolumeBackendAPIException(message=_(msg)) + + def do_setup(self, context): + + self.get_cluster_info() + nodes_info = self.wait_for_lightos_cluster() + + self.cluster.targets = dict() + node_list = nodes_info['nodes'] + for node in node_list: + self.cluster.targets[node['UUID']] = node + + # reduce the logical op timeout if single server LightOS cluster + if len(node_list) == 1: + self.logical_op_timeout = self.configuration. \ + lightos_api_service_timeout + 10 + + def extend_volume(self, volume, size): + # loop because lightos api is async + end = time.time() + self.logical_op_timeout + while (time.time() < end): + try: + finished = self._extend_volume(volume, size) + if finished: + break + except exception.VolumeNotFound as e: + raise e + except Exception as e: + # bail out if the time out elapsed... + if time.time() >= end: + LOG.warning('Timed out extend volume operation') + raise e + # if we still have more time, just print the exception + LOG.warning( + 'caught this in extend_volume() ... will retry: %s', + str(e)) + time.sleep(1) + + def _extend_volume(self, volume, size): + lightos_volname = self._lightos_volname(volume) + project_name = self._get_lightos_project_name(volume) + + try: + (status, data) = self._get_lightos_volume( + project_name, + timeout=self. + logical_op_timeout, + vol_name=lightos_volname) + if status != httpstatus.OK or not data: + LOG.error( + 'Failed to get LightOS volume status: %s data: %s', + status, + str(data)) + raise exception.VolumeNotFound(volume_id=volume.id) + lightos_uuid = data['UUID'] + etag = data.get('ETag', '') + except Exception as e: + raise e + + try: + code, message = self.cluster.send_cmd( + cmd='extend_volume', + project_name=project_name, + timeout=self.logical_op_timeout, + volume_uuid=lightos_uuid, + size=str(size) + ' gib', + etag=etag + ) + if code == httpstatus.OK: + LOG.info( + "Successfully extended volume %s project %s size:%s", + volume, + project_name, + size) + else: + raise exception.ExtendVolumeError(reason=message) + + except exception.ExtendVolumeError as e: + raise e + except Exception as e: + raise exception.ExtendVolumeError(raised_exception=e) + return True + + @staticmethod + def byte_to_gb(bbytes): + return int(int(bbytes) / units.Gi) + + def get_volume_stats(self, refresh=False): + """Retrieve stats info for the volume *service*, + + not a specific volume. + """ + + LOG.debug("getting volume stats (refresh=%s)", refresh) + + if not refresh: + return self._stats + + backend_name = self.configuration.safe_get('volume_backend_name') + res_percentage = self.configuration.safe_get('reserved_percentage') + compression = self.configuration.safe_get( + 'lightos_default_compression_enabled') + storage_protocol = 'lightos' + # as a tenant we dont have access to cluster stats + # in the future we might expose this per project via get_project API + # currently we remove this stats call. + # cluster_stats = self.get_cluster_stats() + + data = {'vendor_name': 'LightOS Storage', + 'volume_backend_name': backend_name or self.__class__.__name__, + 'driver_version': self.VERSION, + 'storage_protocol': storage_protocol, + 'reserved_percentage': res_percentage, + 'QoS_support': False, + 'online_extend_support': True, + 'thin_provisioning_support': True, + 'compression': compression, + 'multiattach': True} + # data['total_capacity_gb'] = + # self.byte_to_gb(cluster_stats['effectivePhysicalStorage']) + # It would be preferable to return + # self.byte_to_gb(cluster_stats['freePhysicalStorage']) + # here but we return 'infinite' due to the Cinder bug described in + # https://bugs.launchpad.net/cinder/+bug/1871371 + data['free_capacity_gb'] = 'infinite' + self._stats = data + + return self._stats + + def _get_connection_properties(self, project_name, volume): + lightos_targets = {} + for target in self.cluster.targets.values(): + properties = dict() + data_address, _ = target['nvmeEndpoint'].split(':') + properties['target_portal'] = data_address + properties['target_port'] = 8009 # spec specified discovery port + properties['transport_type'] = 'tcp' + lightos_targets[data_address] = properties + + server_properties = {} + server_properties['lightos_nodes'] = lightos_targets + server_properties['uuid'] = ( + self._get_lightos_uuid(project_name, volume)) + server_properties['nqn'] = self.cluster.subsystemNQN + + return server_properties + + def set_volume_acl(self, project_name, lightos_uuid, acl, etag): + return self.cluster.send_cmd( + cmd='update_volume', + project_name=project_name, + timeout=self.logical_op_timeout, + volume_uuid=lightos_uuid, + acl=acl, + etag=etag + ) + + def __add_volume_acl(self, project_name, lightos_volname, acl_to_add): + (status, data) = self._get_lightos_volume(project_name, + self.logical_op_timeout, + vol_name=lightos_volname) + if status != httpstatus.OK or not data: + LOG.error('Failed to get LightOS volume %s status %s data %s', + lightos_volname, status, data) + return False + + lightos_uuid = data.get('UUID') + if not lightos_uuid: + LOG.warning('Got LightOS volume without UUID?! data: %s', data) + return False + + acl = data.get('acl') + if not acl: + LOG.warning('Got LightOS volume without ACL?! data: %s', data) + return False + + acl = acl.get('values', []) + + # remove ALLOW_NONE and add our acl_to_add if not already there + if 'ALLOW_NONE' in acl: + acl.remove('ALLOW_NONE') + if acl_to_add not in acl: + acl.append(acl_to_add) + + return self.set_volume_acl( + project_name, + lightos_uuid, + acl, + etag=data.get( + 'ETag', + '')) + + def add_volume_acl(self, project_name, volume, acl_to_add): + LOG.debug( + 'add_volume_acl got volume %s project %s acl %s', + volume, + project_name, + acl_to_add) + lightos_volname = self._lightos_volname(volume) + return self.update_volume_acl( + self.__add_volume_acl, + project_name, + lightos_volname, + acl_to_add) + + def __remove_volume_acl( + self, + project_name, + lightos_volname, + acl_to_remove): + (status, data) = self._get_lightos_volume(project_name, + self.logical_op_timeout, + vol_name=lightos_volname) + if not data: + LOG.error( + 'Could not get data for LightOS volume %s project %s', + lightos_volname, + project_name) + return False + + lightos_uuid = data.get('UUID') + if not lightos_uuid: + LOG.warning('Got LightOS volume without UUID?! data: %s', data) + return False + + acl = data.get('acl') + if not acl: + LOG.warning('Got LightOS volume without ACL?! data: %s', data) + return False + + acl = acl.get('values') + if not acl: + LOG.warning( + 'Got LightOS volume without ACL values?! data: %s', data) + return False + + try: + acl.remove(acl_to_remove) + except ValueError: + LOG.warning( + 'Could not remove acl %s from LightOS volume %s project \ + %s with acl %s', + acl_to_remove, + lightos_volname, + project_name, + acl) + + # if the ACL is empty here, put in ALLOW_NONE + if not acl: + acl.append('ALLOW_NONE') + + return self.set_volume_acl( + project_name, + lightos_uuid, + acl, + etag=data.get( + 'ETag', + '')) + + def __overwrite_volume_acl( + self, + project_name, + lightos_volname, + acl): + status, data = self._get_lightos_volume(project_name, + self.logical_op_timeout, + vol_name=lightos_volname) + if not data: + LOG.error( + 'Could not get data for LightOS volume %s project %s', + lightos_volname, + project_name) + return False + + lightos_uuid = data.get('UUID') + if not lightos_uuid: + LOG.warning('Got LightOS volume without UUID?! data: %s', data) + return False + + return self.set_volume_acl( + project_name, + lightos_uuid, + acl, + etag=data.get( + 'ETag', + '')) + + def remove_volume_acl(self, project_name, volume, acl_to_remove): + lightos_volname = self._lightos_volname(volume) + LOG.debug('remove_volume_acl volume %s project %s acl %s', + volume, project_name, acl_to_remove) + return self.update_volume_acl( + self.__remove_volume_acl, + project_name, + lightos_volname, + acl_to_remove) + + def remove_all_volume_acls(self, project_name, volume): + lightos_volname = self._lightos_volname(volume) + LOG.debug('remove_all_volume_acls volume %s project %s', + volume, project_name) + return self.update_volume_acl( + self.__overwrite_volume_acl, + project_name, + lightos_volname, + ['ALLOW_NONE']) + + def update_volume_acl(self, func, project_name, lightos_volname, acl): + # loop because lightos api is async + end = time.time() + self.logical_op_timeout + first_iteration = True + while (time.time() < end): + if not first_iteration: + time.sleep(1) + first_iteration = False + res = func(project_name, lightos_volname, acl) + if not isinstance(res, tuple): + LOG.debug('Update_volume: func %s(%s project %s) failed', + func, lightos_volname, project_name) + continue + if len(res) != 2: + LOG.debug("Unexpected number of values to unpack") + continue + (status, resp) = res + if status != httpstatus.OK: + LOG.debug( + 'update_volume: func %s(%s project %s) got \ + http status %s', + func, + lightos_volname, + project_name, + status) + else: + break + + # bail out if the time out elapsed... + if time.time() >= end: + LOG.warning( + 'Timed out %s(%s project %s)', + func, + lightos_volname, + project_name) + return False + + # or the call succeeded and we need to wait + # for the volume to stabilize + vol_state = self._wait_for_volume_available( + project_name, timeout=end - time.time(), vol_name=lightos_volname) + if vol_state != 'Available': + LOG.warning( + 'Timed out waiting for volume %s project %s to stabilize, \ + last state %s', + lightos_volname, + project_name, + vol_state) + return False + + return True + + def _wait_for_volume_acl( + self, + project_name, + lightos_volname, + acl, + requested_membership): + end = time.time() + self.logical_op_timeout + while (time.time() < end): + (status, resp) = self._get_lightos_volume( + project_name, + self.logical_op_timeout, + vol_name=lightos_volname) + if status == httpstatus.OK: + if not resp or not resp.get('acl'): + LOG.warning( + 'Got LightOS volume %s without ACL?! data: %s', + lightos_volname, + resp) + return False + + volume_acls = resp.get('acl').get('values', []) + membership = acl in volume_acls + if membership == requested_membership: + return True + + LOG.debug( + 'ACL did not settle for volume %s project %s, status \ + %s resp %s', + lightos_volname, + project_name, + status, + resp) + time.sleep(1) + LOG.warning( + 'ACL did not settle for volume %s, giving up', + lightos_volname) + return False + + def create_snapshot(self, snapshot): + snapshot_name = self._lightos_snapshotname(snapshot["id"]) + src_volume_name = self._lightos_volname(snapshot["volume"]) + project_name = self._get_lightos_project_name(snapshot.volume) + self._create_snapshot(project_name, snapshot_name, src_volume_name) + + @coordination.synchronized('lightos-create_snapshot-{src_volume_name}') + def _create_snapshot(self, project_name, snapshot_name, src_volume_name): + (status_code_get, response) = self._get_lightos_snapshot( + project_name, self.logical_op_timeout, + snapshot_name=snapshot_name) + if status_code_get != httpstatus.OK: + end = time.time() + self.logical_op_timeout + while (time.time() < end): + (status_code_create, response) = self.cluster.send_cmd( + cmd='create_snapshot', + project_name=project_name, + timeout=self.logical_op_timeout, + name=snapshot_name, + src_volume_name=src_volume_name, + ) + + if status_code_create == httpstatus.INTERNAL_SERVER_ERROR: + pass + else: + break + + time.sleep(1) + + if status_code_create != httpstatus.OK: + msg = ('Did not succeed creating LightOS snapshot %s' + ' project %s' + ' status code %s response %s' % + (snapshot_name, project_name, status_code_create, + response)) + raise exception.VolumeBackendAPIException(message=_(msg)) + + state = self._wait_for_snapshot_available(project_name, + timeout= + self.logical_op_timeout, + snapshot_name=snapshot_name) + + if state == 'Available': + LOG.debug( + 'Successfully created LightOS snapshot %s', snapshot_name) + return + + LOG.error( + 'Failed to create snapshot %s project %s for volume %s. \ + state = %s.', + snapshot_name, + project_name, + src_volume_name, + state) + try: + self._delete_lightos_snapshot(project_name, snapshot_name) + except exception.CinderException as ex: + LOG.warning("Error deleting snapshot during cleanup: %s", ex) + + msg = ('Did not succeed creating LightOS snapshot %s project' + '%s last state %s' % (snapshot_name, project_name, state)) + raise exception.VolumeBackendAPIException(message=_(msg)) + + def delete_snapshot(self, snapshot): + lightos_snapshot_name = self._lightos_snapshotname(snapshot["id"]) + project_name = self._get_lightos_project_name(snapshot.volume) + self._delete_lightos_snapshot(project_name=project_name, + snapshot_name=lightos_snapshot_name) + + def _get_lightos_snapshot_uuid(self, project_name, lightos_snapshot_name): + (status_code, data) = self._get_lightos_snapshot( + project_name=project_name, + timeout=self.logical_op_timeout, + snapshot_name=lightos_snapshot_name) + + if status_code == httpstatus.OK: + uuid = data.get("UUID") + if uuid: + return uuid + + if status_code == httpstatus.NOT_FOUND: + return None + + msg = ('Unable to fetch UUID of snapshot named %s. status code' + ' %s data %s' % (lightos_snapshot_name, status_code, data)) + raise exception.VolumeBackendAPIException(message=_(msg)) + + def _delete_lightos_snapshot(self, project_name, snapshot_name): + snapshot_uuid = self._get_lightos_snapshot_uuid( + project_name, snapshot_name) + if snapshot_uuid is None: + LOG.warning( + "Unable to find lightos snapshot %s project %s for deletion", + snapshot_name, + project_name) + return False + + (status_code, _) = self.cluster.send_cmd(cmd='delete_snapshot', + project_name=project_name, + timeout=self. + logical_op_timeout, + snapshot_uuid=snapshot_uuid) + if status_code == httpstatus.OK: + state = self._wait_for_snapshot_deleted( + project_name, + timeout=self.logical_op_timeout, + snapshot_uuid=snapshot_uuid) + if state in ('Deleted', 'Deleting', 'UNKNOWN'): + LOG.debug( + "Successfully detected that snapshot %s was deleted.", + snapshot_name) + return True + LOG.warning("Snapshot %s was not deleted. It is in state %s.", + snapshot_name, state) + return False + LOG.warning( + "Request to delete snapshot %s" + " was rejected with status code %s.", + snapshot_name, + status_code) + return False + + def initialize_connection(self, volume, connector): + hostnqn = connector.get('hostnqn') + found_dsc = connector.get('found_dsc') + LOG.debug( + 'initialize_connection: connector hostnqn is %s found_dsc %s', + hostnqn, + found_dsc) + if not hostnqn: + msg = 'Connector (%s) did not contain a hostnqn, aborting' % ( + connector) + raise exception.VolumeBackendAPIException(message=_(msg)) + + if not found_dsc: + msg = ('Connector (%s) did not indicate a discovery' + 'client, aborting' % (connector)) + raise exception.VolumeBackendAPIException(message=_(msg)) + + lightos_volname = self._lightos_volname(volume) + project_name = self._get_lightos_project_name(volume) + success = self.add_volume_acl(project_name, volume, hostnqn) + if not success or not self._wait_for_volume_acl( + project_name, lightos_volname, hostnqn, True): + msg = ('Could not add ACL for hostnqn %s LightOS volume' + ' %s, aborting' % (hostnqn, lightos_volname)) + raise exception.VolumeBackendAPIException(message=_(msg)) + + props = self._get_connection_properties(project_name, volume) + props['hostnqn'] = hostnqn + return {'driver_volume_type': ('lightos'), 'data': props} + + def terminate_connection(self, volume, connector, **kwargs): + force = 'force' in kwargs + hostnqn = connector.get('hostnqn') if connector else None + LOG.debug( + 'terminate_connection: force %s kwargs %s hostnqn %s', + force, + kwargs, + hostnqn) + + project_name = self._get_lightos_project_name(volume) + + if not hostnqn: + if force: + LOG.debug( + 'Terminating connection with extreme prejudice for \ + volume %s', + volume) + self.remove_all_volume_acls(project_name, volume) + return + + msg = 'Connector (%s) did not return a hostnqn, aborting' % ( + connector) + raise exception.VolumeBackendAPIException(message=_(msg)) + + lightos_volname = self._lightos_volname(volume) + project_name = self._get_lightos_project_name(volume) + success = self.remove_volume_acl(project_name, volume, hostnqn) + if not success or not self._wait_for_volume_acl( + project_name, lightos_volname, hostnqn, False): + LOG.warning( + 'Could not remove ACL for hostnqn %s LightOS \ + volume %s, limping along', + hostnqn, + lightos_volname) + + def _init_vendor_properties(self): + # compression is one of the standard properties, + # no need to add it here + # see the definition of this function in cinder/volume/driver.py + properties = {} + self._set_property( + properties, + "lightos:num_replicas", + "Number of replicas for LightOS volume", + _( + "Specifies the number of replicas to create for the \ + LightOS volume."), + "integer", + minimum=1, + maximun=3, + default=3) + + return properties, 'lightos' + + def backup_use_temp_snapshot(self): + return False + + def snapshot_revert_use_temp_snapshot(self): + """Disable the use of a temporary snapshot on revert.""" + return False + + def snapshot_remote_attachable(self): + """LightOS does not support 'mount a snapshot'""" + return False diff --git a/doc/source/configuration/block-storage/drivers/lightbits-lightos-driver.rst b/doc/source/configuration/block-storage/drivers/lightbits-lightos-driver.rst new file mode 100644 index 00000000000..a495f52dbb3 --- /dev/null +++ b/doc/source/configuration/block-storage/drivers/lightbits-lightos-driver.rst @@ -0,0 +1,140 @@ +=============================== +Lightbits LightOS Cinder Driver +=============================== + +The Lightbits(TM) LightOS(R) OpenStack driver enables OpenStack +clusters to use LightOS clustered storage servers. This documentation +explains how to configure Cinder for use with the Lightbits LightOS +storage backend system. + +Supported operations +~~~~~~~~~~~~~~~~~~~~ + +- Create volume +- Delete volume +- Attach volume +- Detach volume +- Create image from volume +- Live migration +- Volume replication +- Thin provisioning +- Multi-attach +- Supported vendor driver +- Extend volume +- Create snapshot +- Delete snapshot +- Create volume from snapshot +- Create volume from volume (clone) + +LightOS OpenStack Driver Components +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The LightOS OpenStack driver has three components: +- Cinder driver +- Nova libvirt volume driver +- os_brick initiator connector + +In addition, it requires the LightOS ``discovery-client``, provided +with LightOS. The os_brick connector uses the LightOS +``discovery-client`` to communicate with LightOS NVMe/TCP discovery +services. + +The Cinder Driver +~~~~~~~~~~~~~~~~~ + +The Cinder driver integrates with Cinder and performs REST operations +against the LightOS cluster. To enable the driver, add the following +to Cinder's configuration file + +.. code-block:: ini + + enabled_backends = lightos, + +and + +.. code-block:: ini + + [lightos] + volume_driver = cinder.volume.drivers.lightos.LightOSVolumeDriver + volume_backend_name = lightos + lightos_api_address = + lightos_api_port = 443 + lightos_jwt= + lightos_default_num_replicas = 3 + lightos_default_compression_enabled = False + lightos_api_service_timeout=30 + +- ``TARGET_ACCESS_IPS`` are the LightOS cluster nodes access + IPs. Multiple nodes should be separated by commas. For example: + ``lightos_api_address = + 192.168.67.78,192.168.34.56,192.168.12.17``. These IPs are where the + driver looks for the LightOS clusters REST API servers. +- ``LIGHTOS_JWT`` is the JWT (JSON Web Token) that is located at the + LightOS installation controller. You can find the jwt at + ``~/lightos-default-admin-jwt``. +- The default number of replicas for volumes is 3, and valid values + for ``lightos_default_num_replicas`` are 1, 2, or 3. +- The default compression setting is False (i.e., data is + uncompressed). The default compression setting can also be + True. This can also be changed on a per-volume basis. +- The default time to wait for API service response is 30 seconds per + API endpoint. + +Creating volumes with non-default compression and number of replicas +settings can be done through the volume types mechanism. To create a +new volume type with compression enabled: + +.. code-block:: console + + $ openstack volume type create --property compression=' True' volume-with-compression + +To create a new volume type with one replica: + +.. code-block:: console + + $ openstack volume type create --property lightos:num_replicas=1 volume-with-one-replica + +To create a new type for a compressed volume with three replicas: + +.. code-block:: console + + $ openstack volume type create --property compression=' True' --property lightos:num_replicas=3 volume-with-three-replicas-and-compression + +Then create a new volume with one of these volume types: + +.. code-block:: console + + $ openstack volume create --size --type + +NVNe/TCP and Asymmetric Namespace Access (ANA) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The LightOS clusters expose their volumes using NVMe/TCP Asynchronous +Namespace Access (ANA). ANA is a relatively new feature in the +NVMe/TCP stack in Linux but it is fully supported in Ubuntu +20.04. Each compute host in the OpenStack cluster needs to be +ANA-capable to provide OpenStack VMs with LightOS volumes over +NVMe/TCP. For more information on how to set up the compute nodes to +use ANA, see the CentOS Linux Cluster Client Software Installation +section of the Lightbits(TM) LightOS(R) Cluster Installation and +Initial Configuration Guide. + +Note +~~~~ + +In the current version, if any of the cluster nodes changes its access +IPs, the Cinder driver's configuration file should be updated with the +cluster nodes access IPs and restarted. As long as the Cinder driver +can access at least one cluster access IP it will work, but will be +susceptible to cluster node failures. + +Driver options +~~~~~~~~~~~~~~ + +The following table contains the configuration options supported by the +Lightbits LightOS Cinder driver. + +.. config-table:: + :config-target: Lightbits LightOS + + cinder.volume.drivers.lightos diff --git a/doc/source/reference/support-matrix.ini b/doc/source/reference/support-matrix.ini index cb7c5d08218..40526cf51bb 100644 --- a/doc/source/reference/support-matrix.ini +++ b/doc/source/reference/support-matrix.ini @@ -123,6 +123,9 @@ title=Kioxia Kumoscale Driver (NVMeOF) [driver.lenovo] title=Lenovo Storage Driver (FC, iSCSI) +[driver.lightbits_lightos] +title=Lightbits LightOS Storage Driver (NVMeTCP) + [driver.linbit_linstor] title=LINBIT DRBD/LINSTOR Driver (DRBD) @@ -253,6 +256,7 @@ driver.inspur_as13000=complete driver.kaminario=complete driver.kioxia_kumoscale=complete driver.lenovo=complete +driver.lightbits_lightos=complete driver.linbit_linstor=complete driver.lvm=complete driver.macrosan=complete @@ -323,6 +327,7 @@ driver.inspur_as13000=complete driver.kaminario=complete driver.kioxia_kumoscale=complete driver.lenovo=complete +driver.lightbits_lightos=complete driver.linbit_linstor=complete driver.lvm=complete driver.macrosan=complete @@ -396,6 +401,7 @@ driver.inspur_as13000=missing driver.kaminario=missing driver.kioxia_kumoscale=missing driver.lenovo=missing +driver.lightbits_lightos=missing driver.linbit_linstor=missing driver.lvm=missing driver.macrosan=complete @@ -468,6 +474,7 @@ driver.inspur_as13000=missing driver.kaminario=complete driver.kioxia_kumoscale=missing driver.lenovo=missing +driver.lightbits_lightos=missing driver.linbit_linstor=missing driver.lvm=missing driver.macrosan=complete @@ -541,6 +548,7 @@ driver.inspur_as13000=missing driver.kaminario=missing driver.kioxia_kumoscale=missing driver.lenovo=missing +driver.lightbits_lightos=missing driver.linbit_linstor=missing driver.lvm=missing driver.macrosan=missing @@ -613,6 +621,7 @@ driver.inspur_as13000=complete driver.kaminario=complete driver.kioxia_kumoscale=complete driver.lenovo=missing +driver.lightbits_lightos=complete driver.linbit_linstor=missing driver.lvm=complete driver.macrosan=complete @@ -686,6 +695,7 @@ driver.inspur_as13000=missing driver.kaminario=missing driver.kioxia_kumoscale=missing driver.lenovo=missing +driver.lightbits_lightos=missing driver.linbit_linstor=missing driver.lvm=missing driver.macrosan=complete @@ -759,6 +769,7 @@ driver.inspur_as13000=complete driver.kaminario=missing driver.kioxia_kumoscale=missing driver.lenovo=complete +driver.lightbits_lightos=complete driver.linbit_linstor=missing driver.lvm=complete driver.macrosan=missing @@ -829,6 +840,7 @@ driver.inspur_as13000=missing driver.kaminario=missing driver.kioxia_kumoscale=missing driver.lenovo=missing +driver.lightbits_lightos=missing driver.linbit_linstor=missing driver.lvm=complete driver.macrosan=missing @@ -903,6 +915,7 @@ driver.inspur_as13000=missing driver.kaminario=missing driver.kioxia_kumoscale=missing driver.lenovo=missing +driver.lightbits_lightos=missing driver.linbit_linstor=missing driver.lvm=missing driver.macrosan=complete diff --git a/releasenotes/notes/bp-lightbits-lightos-clustered-nvmetcp-driver-d1ef8f83263921f2.yaml b/releasenotes/notes/bp-lightbits-lightos-clustered-nvmetcp-driver-d1ef8f83263921f2.yaml new file mode 100644 index 00000000000..2497f6e7597 --- /dev/null +++ b/releasenotes/notes/bp-lightbits-lightos-clustered-nvmetcp-driver-d1ef8f83263921f2.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Lightbits LightOS driver: new Cinder driver for Lightbits(TM) + LightOS(R). Lightbits Labs (http://www.lightbitslabs.com) LightOS + is software-defined, cloud native, high-performance, clustered + scale-out and redundant NVMe/TCP storage that performs like local + NVMe flash.