# Copyright (c) 2014 LINBIT HA Solutions GmbH # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. """ This driver connects Cinder to an installed DRBDmanage instance, see http://drbd.linbit.com/users-guide-9.0/ch-openstack.html for more details. """ import eventlet import json import six import socket import time import uuid from oslo_config import cfg from oslo_log import log as logging from oslo_utils import importutils from oslo_utils import units from cinder import exception from cinder.i18n import _, _LW, _LI, _LE from cinder import interface from cinder.volume import driver try: import dbus import drbdmanage.consts as dm_const import drbdmanage.exceptions as dm_exc import drbdmanage.utils as dm_utils except ImportError: # Used for the tests, when no DRBDmanage is installed dbus = None dm_const = None dm_exc = None dm_utils = None LOG = logging.getLogger(__name__) drbd_opts = [ cfg.IntOpt('drbdmanage_redundancy', default=1, help='Number of nodes that should replicate the data.'), cfg.StrOpt('drbdmanage_resource_policy', default='{"ratio": "0.51", "timeout": "60"}', help='Resource deployment completion wait policy.'), cfg.StrOpt('drbdmanage_disk_options', default='{"c-min-rate": "4M"}', help='Disk options to set on new resources. ' 'See http://www.drbd.org/en/doc/users-guide-90/re-drbdconf' ' for all the details.'), cfg.StrOpt('drbdmanage_net_options', default='{"connect-int": "4", "allow-two-primaries": "yes", ' '"ko-count": "30", "max-buffers": "20000", ' '"ping-timeout": "100"}', help='Net options to set on new resources. ' 'See http://www.drbd.org/en/doc/users-guide-90/re-drbdconf' ' for all the details.'), cfg.StrOpt('drbdmanage_resource_options', default='{"auto-promote-timeout": "300"}', help='Resource options to set on new resources. ' 'See http://www.drbd.org/en/doc/users-guide-90/re-drbdconf' ' for all the details.'), cfg.StrOpt('drbdmanage_snapshot_policy', default='{"count": "1", "timeout": "60"}', help='Snapshot completion wait policy.'), cfg.StrOpt('drbdmanage_resize_policy', default='{"timeout": "60"}', help='Volume resize completion wait policy.'), cfg.StrOpt('drbdmanage_resource_plugin', default="drbdmanage.plugins.plugins.wait_for.WaitForResource", help='Resource deployment completion wait plugin.'), cfg.StrOpt('drbdmanage_snapshot_plugin', default="drbdmanage.plugins.plugins.wait_for.WaitForSnapshot", help='Snapshot completion wait plugin.'), cfg.StrOpt('drbdmanage_resize_plugin', default="drbdmanage.plugins.plugins.wait_for.WaitForVolumeSize", help='Volume resize completion wait plugin.'), cfg.BoolOpt('drbdmanage_devs_on_controller', default=True, help='''If set, the c-vol node will receive a useable /dev/drbdX device, even if the actual data is stored on other nodes only. This is useful for debugging, maintenance, and to be able to do the iSCSI export from the c-vol node.''') # TODO(PM): offsite_redundancy? # TODO(PM): choose DRBDmanage storage pool? ] CONF = cfg.CONF CONF.register_opts(drbd_opts) AUX_PROP_CINDER_VOL_ID = "cinder-id" AUX_PROP_TEMP_CLIENT = "cinder-is-temp-client" DM_VN_PREFIX = 'CV_' # sadly 2CV isn't allowed by DRBDmanage DM_SN_PREFIX = 'SN_' # Need to be set later, so that the tests can fake CS_DEPLOYED = None CS_DISKLESS = None CS_UPD_CON = None class DrbdManageBaseDriver(driver.VolumeDriver): """Cinder driver that uses DRBDmanage for storage.""" VERSION = '1.1.0' drbdmanage_dbus_name = 'org.drbd.drbdmanaged' drbdmanage_dbus_interface = '/interface' # ThirdPartySystems wiki page CI_WIKI_NAME = "Cinder_Jenkins" def __init__(self, *args, **kwargs): self.empty_list = dbus.Array([], signature="a(s)") self.empty_dict = dbus.Array([], signature="a(ss)") super(DrbdManageBaseDriver, self).__init__(*args, **kwargs) self.configuration.append_config_values(drbd_opts) if not self.drbdmanage_dbus_name: self.drbdmanage_dbus_name = 'org.drbd.drbdmanaged' if not self.drbdmanage_dbus_interface: self.drbdmanage_dbus_interface = '/interface' self.drbdmanage_redundancy = int(getattr(self.configuration, 'drbdmanage_redundancy', 1)) self.drbdmanage_devs_on_controller = bool( getattr(self.configuration, 'drbdmanage_devs_on_controller', True)) self.dm_control_vol = ".drbdctrl" self.backend_name = self.configuration.safe_get( 'volume_backend_name') or 'drbdmanage' js_decoder = json.JSONDecoder() self.policy_resource = js_decoder.decode( self.configuration.safe_get('drbdmanage_resource_policy')) self.policy_snapshot = js_decoder.decode( self.configuration.safe_get('drbdmanage_snapshot_policy')) self.policy_resize = js_decoder.decode( self.configuration.safe_get('drbdmanage_resize_policy')) self.resource_options = js_decoder.decode( self.configuration.safe_get('drbdmanage_resource_options')) self.net_options = js_decoder.decode( self.configuration.safe_get('drbdmanage_net_options')) self.disk_options = js_decoder.decode( self.configuration.safe_get('drbdmanage_disk_options')) self.plugin_resource = self.configuration.safe_get( 'drbdmanage_resource_plugin') self.plugin_snapshot = self.configuration.safe_get( 'drbdmanage_snapshot_plugin') self.plugin_resize = self.configuration.safe_get( 'drbdmanage_resize_plugin') # needed as per pep8: # F841 local variable 'CS_DEPLOYED' is assigned to but never used global CS_DEPLOYED, CS_DISKLESS, CS_UPD_CON CS_DEPLOYED = dm_const.CSTATE_PREFIX + dm_const.FLAG_DEPLOY CS_DISKLESS = dm_const.CSTATE_PREFIX + dm_const.FLAG_DISKLESS CS_UPD_CON = dm_const.CSTATE_PREFIX + dm_const.FLAG_UPD_CON def dbus_connect(self): self.odm = dbus.SystemBus().get_object(self.drbdmanage_dbus_name, self.drbdmanage_dbus_interface) self.odm.ping() def call_or_reconnect(self, fn, *args): """Call DBUS function; on a disconnect try once to reconnect.""" try: return fn(*args) except dbus.DBusException as e: LOG.warning(_LW("Got disconnected; trying to reconnect. (%s)"), e) self.dbus_connect() # Old function object is invalid, get new one. return getattr(self.odm, fn._method_name)(*args) def _fetch_answer_data(self, res, key, level=None, req=True): for code, fmt, data in res: if code == dm_exc.DM_INFO: if level and level != fmt: continue value = [v for k, v in data if k == key] if value: if len(value) == 1: return value[0] else: return value if req: if level: l = level + ":" + key else: l = key msg = _('DRBDmanage driver error: expected key "%s" ' 'not in answer, wrong DRBDmanage version?') % l LOG.error(msg) raise exception.VolumeDriverException(message=msg) return None def do_setup(self, context): """Any initialization the volume driver does while starting.""" super(DrbdManageBaseDriver, self).do_setup(context) self.dbus_connect() def check_for_setup_error(self): """Verify that requirements are in place to use DRBDmanage driver.""" if not all((dbus, dm_exc, dm_const, dm_utils)): msg = _('DRBDmanage driver setup error: some required ' 'libraries (dbus, drbdmanage.*) not found.') LOG.error(msg) raise exception.VolumeDriverException(message=msg) if self.odm.ping() != 0: message = _('Cannot ping DRBDmanage backend') raise exception.VolumeBackendAPIException(data=message) def _clean_uuid(self): """Returns a UUID string, WITHOUT braces.""" # Some uuid library versions put braces around the result!? # We don't want them, just a plain [0-9a-f-]+ string. id = str(uuid.uuid4()) id = id.replace("{", "") id = id.replace("}", "") return id def _check_result(self, res, ignore=None, ret=0): seen_success = False seen_error = False result = ret for (code, fmt, arg_l) in res: # convert from DBUS to Python arg = dict(arg_l) if ignore and code in ignore: if not result: result = code continue if code == dm_exc.DM_SUCCESS: seen_success = True continue if code == dm_exc.DM_INFO: continue seen_error = _("Received error string: %s") % (fmt % arg) if seen_error: raise exception.VolumeBackendAPIException(data=seen_error) if seen_success: return ret # by default okay - or the ignored error code. return ret # DRBDmanage works in kiB units; Cinder uses GiB. def _vol_size_to_dm(self, size): return int(size * units.Gi / units.Ki) def _vol_size_to_cinder(self, size): return int(size * units.Ki / units.Gi) def is_clean_volume_name(self, name, prefix): try: if (name.startswith(CONF.volume_name_template % "") and uuid.UUID(name[7:]) is not None): return prefix + name[7:] except ValueError: return None try: if uuid.UUID(name) is not None: return prefix + name except ValueError: return None def _call_policy_plugin(self, plugin, pol_base, pol_this): """Returns True for done, False for timeout.""" pol_inp_data = dict(pol_base) pol_inp_data.update(pol_this, starttime=str(time.time())) retry = 0 while True: res, pol_result = self.call_or_reconnect( self.odm.run_external_plugin, plugin, pol_inp_data) self._check_result(res) if pol_result['result'] == dm_const.BOOL_TRUE: return True if pol_result['timeout'] == dm_const.BOOL_TRUE: return False eventlet.sleep(min(0.5 + retry / 5, 2)) retry += 1 def _wait_for_node_assignment(self, res_name, vol_nr, nodenames, filter_props=None, timeout=90, check_vol_deployed=True): """Return True as soon as one assignment matches the filter.""" # TODO(LINBIT): unify with policy plugins if not filter_props: filter_props = self.empty_dict end_time = time.time() + timeout retry = 0 while time.time() < end_time: res, assgs = self.call_or_reconnect(self.odm.list_assignments, nodenames, [res_name], 0, filter_props, self.empty_list) self._check_result(res) if len(assgs) > 0: for assg in assgs: vols = assg[3] for v_nr, v_prop in vols: if (v_nr == vol_nr): if not check_vol_deployed: # no need to check return True if v_prop[CS_DEPLOYED] == dm_const.BOOL_TRUE: return True retry += 1 # Not yet LOG.warning(_LW('Try #%(try)d: Volume "%(res)s"/%(vol)d ' 'not yet deployed on "%(host)s", waiting.'), {'try': retry, 'host': nodenames, 'res': res_name, 'vol': vol_nr}) eventlet.sleep(min(0.5 + retry / 5, 2)) # Timeout return False def _priv_hash_from_volume(self, volume): return dm_utils.dict_to_aux_props({ AUX_PROP_CINDER_VOL_ID: volume['id'], }) def snapshot_name_from_cinder_snapshot(self, snapshot): sn_name = self.is_clean_volume_name(snapshot['id'], DM_SN_PREFIX) return sn_name def _res_and_vl_data_for_volume(self, volume, empty_ok=False): """Find DRBD resource and volume ID. A DRBD resource might consist of several "volumes" (think consistency groups). So we have to find the number of the volume within one resource. Returns resource name, volume number, and resource and volume properties. """ # If we get a string, use it as-is. # Else it's a dictionary; then get the ID. if isinstance(volume, six.string_types): v_uuid = volume else: v_uuid = volume['id'] res, rl = self.call_or_reconnect(self.odm.list_volumes, self.empty_dict, 0, dm_utils.dict_to_aux_props( {AUX_PROP_CINDER_VOL_ID: v_uuid}), self.empty_dict) self._check_result(res) if (not rl) or (len(rl) == 0): if empty_ok: LOG.debug("No volume %s found.", v_uuid) return None, None, None, None raise exception.VolumeBackendAPIException( data=_("volume %s not found in drbdmanage") % v_uuid) if len(rl) > 1: raise exception.VolumeBackendAPIException( data=_("multiple resources with name %s found by drbdmanage") % v_uuid) (r_name, r_props, vols) = rl[0] if len(vols) != 1: raise exception.VolumeBackendAPIException( data=_("not exactly one volume with id %s") % v_uuid) (v_nr, v_props) = vols[0] LOG.debug("volume %(uuid)s is %(res)s/%(nr)d; %(rprop)s, %(vprop)s", {'uuid': v_uuid, 'res': r_name, 'nr': v_nr, 'rprop': dict(r_props), 'vprop': dict(v_props)}) return r_name, v_nr, r_props, v_props def _resource_and_snap_data_from_snapshot(self, snapshot, empty_ok=False): """Find DRBD resource and snapshot name from the snapshot ID.""" s_uuid = snapshot['id'] res, rs = self.call_or_reconnect(self.odm.list_snapshots, self.empty_dict, self.empty_dict, 0, dm_utils.dict_to_aux_props( {AUX_PROP_CINDER_VOL_ID: s_uuid}), self.empty_dict) self._check_result(res) if (not rs) or (len(rs) == 0): if empty_ok: return None else: raise exception.VolumeBackendAPIException( data=_("no snapshot with id %s found in drbdmanage") % s_uuid) if len(rs) > 1: raise exception.VolumeBackendAPIException( data=_("multiple resources with snapshot ID %s found") % s_uuid) (r_name, snaps) = rs[0] if len(snaps) != 1: raise exception.VolumeBackendAPIException( data=_("not exactly one snapshot with id %s") % s_uuid) (s_name, s_props) = snaps[0] LOG.debug("snapshot %(uuid)s is %(res)s/%(snap)s", {'uuid': s_uuid, 'res': r_name, 'snap': s_name}) return r_name, s_name, s_props def _resource_name_volnr_for_volume(self, volume, empty_ok=False): res, vol, __, __ = self._res_and_vl_data_for_volume(volume, empty_ok) return res, vol def local_path(self, volume): d_res_name, d_vol_nr = self._resource_name_volnr_for_volume(volume) res, data = self.call_or_reconnect(self.odm.text_query, [dm_const.TQ_GET_PATH, d_res_name, str(d_vol_nr)]) self._check_result(res) if len(data) == 1: return data[0] message = _('Got bad path information from DRBDmanage! (%s)') % data raise exception.VolumeBackendAPIException(data=message) def _push_drbd_options(self, d_res_name): res_opt = {'resource': d_res_name, 'target': 'resource', 'type': 'reso'} res_opt.update(self.resource_options) res = self.call_or_reconnect(self.odm.set_drbdsetup_props, res_opt) self._check_result(res) res_opt = {'resource': d_res_name, 'target': 'resource', 'type': 'neto'} res_opt.update(self.net_options) res = self.call_or_reconnect(self.odm.set_drbdsetup_props, res_opt) self._check_result(res) res_opt = {'resource': d_res_name, 'target': 'resource', 'type': 'disko'} res_opt.update(self.disk_options) res = self.call_or_reconnect(self.odm.set_drbdsetup_props, res_opt) self._check_result(res) def create_volume(self, volume): """Creates a DRBD resource. We address it later on via the ID that gets stored as a private property. """ # TODO(PM): consistency groups d_res_name = self.is_clean_volume_name(volume['id'], DM_VN_PREFIX) res = self.call_or_reconnect(self.odm.create_resource, d_res_name, self.empty_dict) self._check_result(res, ignore=[dm_exc.DM_EEXIST], ret=None) self._push_drbd_options(d_res_name) # If we get DM_EEXIST, then the volume already exists, eg. because # deploy gave an error on a previous try (like ENOSPC). # Still, there might or might not be the volume in the resource - # we have to check that explicitly. (__, drbd_vol) = self._resource_name_volnr_for_volume(volume, empty_ok=True) if not drbd_vol: props = self._priv_hash_from_volume(volume) # TODO(PM): properties - redundancy, etc res = self.call_or_reconnect(self.odm.create_volume, d_res_name, self._vol_size_to_dm(volume['size']), props) self._check_result(res) drbd_vol = self._fetch_answer_data(res, dm_const.VOL_ID) # If we crashed between create_volume and the deploy call, # the volume might be defined but not exist on any server. Oh my. res = self.call_or_reconnect(self.odm.auto_deploy, d_res_name, self.drbdmanage_redundancy, 0, True) self._check_result(res) okay = self._call_policy_plugin(self.plugin_resource, self.policy_resource, dict(resource=d_res_name, volnr=str(drbd_vol))) if not okay: message = (_('DRBDmanage timeout waiting for volume creation; ' 'resource "%(res)s", volume "%(vol)s"') % {'res': d_res_name, 'vol': volume['id']}) raise exception.VolumeBackendAPIException(data=message) if self.drbdmanage_devs_on_controller: # TODO(pm): CG res = self.call_or_reconnect(self.odm.assign, socket.gethostname(), d_res_name, [(dm_const.FLAG_DISKLESS, dm_const.BOOL_TRUE)]) self._check_result(res, ignore=[dm_exc.DM_EEXIST]) return {} def delete_volume(self, volume): """Deletes a resource.""" d_res_name, d_vol_nr = self._resource_name_volnr_for_volume( volume, empty_ok=True) if not d_res_name: # OK, already gone. return True # TODO(PM): check if in use? Ask whether Primary, or just check result? res = self.call_or_reconnect(self.odm.remove_volume, d_res_name, d_vol_nr, False) self._check_result(res, ignore=[dm_exc.DM_ENOENT]) # Ask for volumes in that resource that are not scheduled for deletion. res, rl = self.call_or_reconnect(self.odm.list_volumes, [d_res_name], 0, [(dm_const.TSTATE_PREFIX + dm_const.FLAG_REMOVE, dm_const.BOOL_FALSE)], self.empty_list) self._check_result(res) # We expect the _resource_ to be here still (we just got a volnr from # it!), so just query the volumes. # If the resource has no volumes anymore, the current DRBDmanage # version (errorneously, IMO) returns no *resource*, too. if len(rl) > 1: message = _('DRBDmanage expected one resource ("%(res)s"), ' 'got %(n)d') % {'res': d_res_name, 'n': len(rl)} raise exception.VolumeBackendAPIException(data=message) # Delete resource, if empty if (not rl) or (not rl[0]) or (len(rl[0][2]) == 0): res = self.call_or_reconnect(self.odm.remove_resource, d_res_name, False) self._check_result(res, ignore=[dm_exc.DM_ENOENT]) def create_volume_from_snapshot(self, volume, snapshot): """Creates a volume from a snapshot.""" LOG.debug("create vol from snap: from %(snap)s make %(vol)s", {'snap': snapshot['id'], 'vol': volume['id']}) # TODO(PM): Consistency groups. d_res_name, sname, sprop = self._resource_and_snap_data_from_snapshot( snapshot) new_res = self.is_clean_volume_name(volume['id'], DM_VN_PREFIX) r_props = self.empty_dict # TODO(PM): consistency groups => different volume number possible new_vol_nr = 0 v_props = [(new_vol_nr, self._priv_hash_from_volume(volume))] res = self.call_or_reconnect(self.odm.restore_snapshot, new_res, d_res_name, sname, r_props, v_props) self._check_result(res, ignore=[dm_exc.DM_ENOENT]) self._push_drbd_options(d_res_name) # TODO(PM): CG okay = self._call_policy_plugin(self.plugin_resource, self.policy_resource, dict(resource=new_res, volnr=str(new_vol_nr))) if not okay: message = (_('DRBDmanage timeout waiting for new volume ' 'after snapshot restore; ' 'resource "%(res)s", volume "%(vol)s"') % {'res': new_res, 'vol': volume['id']}) raise exception.VolumeBackendAPIException(data=message) if (('size' in volume) and (volume['size'] > snapshot['volume_size'])): LOG.debug("resize volume '%(dst_vol)s' from %(src_size)d to " "%(dst_size)d", {'dst_vol': volume['id'], 'src_size': snapshot['volume_size'], 'dst_size': volume['size']}) self.extend_volume(volume, volume['size']) def create_cloned_volume(self, volume, src_vref): temp_id = self._clean_uuid() snapshot = {'id': temp_id} self.create_snapshot({'id': temp_id, 'volume_id': src_vref['id']}) snapshot['volume_size'] = src_vref['size'] self.create_volume_from_snapshot(volume, snapshot) self.delete_snapshot(snapshot) def _update_volume_stats(self): data = {} data["vendor_name"] = 'Open Source' data["driver_version"] = self.VERSION # This has to match the name set in the cinder volume driver spec, # so keep it lowercase data["volume_backend_name"] = self.backend_name data["pools"] = [] res, free, total = self.call_or_reconnect(self.odm.cluster_free_query, self.drbdmanage_redundancy) self._check_result(res) location_info = ('DrbdManageBaseDriver:%(cvol)s:%(dbus)s' % {'cvol': self.dm_control_vol, 'dbus': self.drbdmanage_dbus_name}) # add volumes res, rl = self.call_or_reconnect(self.odm.list_volumes, self.empty_list, 0, self.empty_dict, self.empty_list) self._check_result(res) total_volumes = 0 for res in rl: total_volumes += len(res[2]) # TODO(PM): multiple DRBDmanage instances and/or multiple pools single_pool = {} single_pool.update(dict( pool_name=data["volume_backend_name"], free_capacity_gb=self._vol_size_to_cinder(free), total_capacity_gb=self._vol_size_to_cinder(total), reserved_percentage=self.configuration.reserved_percentage, location_info=location_info, total_volumes=total_volumes, filter_function=self.get_filter_function(), goodness_function=self.get_goodness_function(), QoS_support=False)) data["pools"].append(single_pool) self._stats = data return self._stats def extend_volume(self, volume, new_size): d_res_name, d_vol_nr = self._resource_name_volnr_for_volume(volume) res = self.call_or_reconnect(self.odm.resize_volume, d_res_name, d_vol_nr, -1, self._vol_size_to_dm(new_size), 0) self._check_result(res) okay = self._call_policy_plugin(self.plugin_resize, self.policy_resize, dict(resource=d_res_name, volnr=str(d_vol_nr), req_size=str(new_size))) if not okay: message = (_('DRBDmanage timeout waiting for volume size; ' 'volume ID "%(id)s" (res "%(res)s", vnr %(vnr)d)') % {'id': volume['id'], 'res': d_res_name, 'vnr': d_vol_nr}) raise exception.VolumeBackendAPIException(data=message) def create_snapshot(self, snapshot): """Creates a snapshot.""" sn_name = self.snapshot_name_from_cinder_snapshot(snapshot) d_res_name, d_vol_nr = self._resource_name_volnr_for_volume( snapshot["volume_id"]) res, data = self.call_or_reconnect(self.odm.list_assignments, self.empty_dict, [d_res_name], 0, {CS_DISKLESS: dm_const.BOOL_FALSE}, self.empty_list) self._check_result(res) nodes = [d[0] for d in data] if len(nodes) < 1: raise exception.VolumeBackendAPIException( _('Snapshot res "%s" that is not deployed anywhere?') % (d_res_name)) props = self._priv_hash_from_volume(snapshot) res = self.call_or_reconnect(self.odm.create_snapshot, d_res_name, sn_name, nodes, props) self._check_result(res) okay = self._call_policy_plugin(self.plugin_snapshot, self.policy_snapshot, dict(resource=d_res_name, snapshot=sn_name)) if not okay: message = (_('DRBDmanage timeout waiting for snapshot creation; ' 'resource "%(res)s", snapshot "%(sn)s"') % {'res': d_res_name, 'sn': sn_name}) raise exception.VolumeBackendAPIException(data=message) def delete_snapshot(self, snapshot): """Deletes a snapshot.""" d_res_name, sname, _ = self._resource_and_snap_data_from_snapshot( snapshot, empty_ok=True) if not d_res_name: # resource already gone? LOG.warning(_LW("snapshot: %s not found, " "skipping delete operation"), snapshot['id']) LOG.info(_LI('Successfully deleted snapshot: %s'), snapshot['id']) return True res = self.call_or_reconnect(self.odm.remove_snapshot, d_res_name, sname, True) return self._check_result(res, ignore=[dm_exc.DM_ENOENT]) # Class with iSCSI interface methods @interface.volumedriver class DrbdManageIscsiDriver(DrbdManageBaseDriver): """Cinder driver that uses the iSCSI protocol. """ def __init__(self, *args, **kwargs): super(DrbdManageIscsiDriver, self).__init__(*args, **kwargs) target_driver = self.target_mapping[ self.configuration.safe_get('iscsi_helper')] LOG.debug('Attempting to initialize DRBD driver with the ' 'following target_driver: %s', target_driver) self.target_driver = importutils.import_object( target_driver, configuration=self.configuration, db=self.db, executor=self._execute) def get_volume_stats(self, refresh=False): """Get volume status.""" self._update_volume_stats() self._stats["storage_protocol"] = "iSCSI" return self._stats def ensure_export(self, context, volume): volume_path = self.local_path(volume) return self.target_driver.ensure_export( context, volume, volume_path) def create_export(self, context, volume, connector): volume_path = self.local_path(volume) export_info = self.target_driver.create_export( context, volume, volume_path) return {'provider_location': export_info['location'], 'provider_auth': export_info['auth'], } def remove_export(self, context, volume): return self.target_driver.remove_export(context, volume) def initialize_connection(self, volume, connector): return self.target_driver.initialize_connection(volume, connector) def validate_connector(self, connector): return self.target_driver.validate_connector(connector) def terminate_connection(self, volume, connector, **kwargs): return self.target_driver.terminate_connection(volume, connector, **kwargs) # for backwards compatibility keep the old class name, too DrbdManageDriver = DrbdManageIscsiDriver # Class with DRBD transport mode @interface.volumedriver class DrbdManageDrbdDriver(DrbdManageBaseDriver): """Cinder driver that uses the DRBD protocol. """ def __init__(self, *args, **kwargs): super(DrbdManageDrbdDriver, self).__init__(*args, **kwargs) def get_volume_stats(self, refresh=False): """Get volume status.""" self._update_volume_stats() self._stats["storage_protocol"] = "DRBD" return self._stats def _return_local_access(self, nodename, volume, d_res_name=None, volume_path=None): if not volume_path: volume_path = self.local_path(volume) return { 'driver_volume_type': 'local', 'data': { "device_path": volume_path } } def _return_drbdadm_config(self, volume, nodename, d_res_name=None, volume_path=None): if not d_res_name: d_res_name, d_vol_nr = self._resource_name_volnr_for_volume(volume) res, data = self.call_or_reconnect( self.odm.text_query, ['export_conf_split_up', nodename, d_res_name]) self._check_result(res) config = six.text_type(data.pop(0)) subst_data = {} while len(data): k = data.pop(0) subst_data[k] = data.pop(0) if not volume_path: volume_path = self.local_path(volume) return { 'driver_volume_type': 'drbd', 'data': { 'provider_location': ' '.join('drbd', nodename), 'device': volume_path, # TODO(pm): consistency groups 'devices': [volume_path], 'provider_auth': subst_data['shared-secret'], 'config': config, 'name': d_res_name, } } def _is_external_node(self, nodename): """Return whether the given node is an "external" node.""" # If the node accessing the data (the "initiator" in iSCSI speak, # "client" or "target" otherwise) is marked as an FLAG_EXTERNAL # node, it does not have DRBDmanage active - and that means # we have to send the necessary DRBD configuration. # # If DRBDmanage is running there, just pushing the (client) # assignment is enough to make the local path available. res, nodes = self.call_or_reconnect(self.odm.list_nodes, [nodename], 0, self.empty_dict, [dm_const.FLAG_EXTERNAL]) self._check_result(res) if len(nodes) != 1: msg = _('Expected exactly one node called "%s"') % nodename LOG.error(msg) raise exception.VolumeDriverException(message=msg) __, nodeattr = nodes[0] return getattr(nodeattr, dm_const.FLAG_EXTERNAL, dm_const.BOOL_FALSE) == dm_const.BOOL_TRUE def _return_connection_data(self, nodename, volume, d_res_name=None): if nodename and self._is_external_node(nodename): return self._return_drbdadm_config(nodename, volume, d_res_name=d_res_name) else: return self._return_local_access(nodename, volume) def create_export(self, context, volume, connector): d_res_name, d_vol_nr = self._resource_name_volnr_for_volume(volume) nodename = connector["host"] # Ensure the node is known to DRBDmanage. # Note that this does *not* mean that DRBDmanage has to # be installed on it! # This is just so that DRBD allows the IP to connect. node_prop = { dm_const.NODE_ADDR: connector["ip"], dm_const.FLAG_DRBDCTRL: dm_const.BOOL_FALSE, dm_const.FLAG_STORAGE: dm_const.BOOL_FALSE, dm_const.FLAG_EXTERNAL: dm_const.BOOL_TRUE, } res = self.call_or_reconnect( self.odm.create_node, nodename, node_prop) self._check_result(res, ignore=[dm_exc.DM_EEXIST]) # Ensure the data is accessible, by creating an assignment. assg_prop = { dm_const.FLAG_DISKLESS: dm_const.BOOL_TRUE, } # If we create the assignment here, it's temporary - # and has to be removed later on again. assg_prop.update(dm_utils.aux_props_to_dict({ AUX_PROP_TEMP_CLIENT: dm_const.BOOL_TRUE, })) res = self.call_or_reconnect( self.odm.assign, nodename, d_res_name, assg_prop) self._check_result(res, ignore=[dm_exc.DM_EEXIST]) # Wait for DRBDmanage to have completed that action. # A DRBDmanage controlled node will set the cstate:deploy flag; # an external node will not be available to change it, so we have # to wait for the storage nodes to remove the upd_con flag # (ie. they're now ready to receive the connection). if self._is_external_node(nodename): self._wait_for_node_assignment( d_res_name, d_vol_nr, [], check_vol_deployed=False, filter_props={ # must be deployed CS_DEPLOYED: dm_const.BOOL_TRUE, # must be a storage node (not diskless), CS_DISKLESS: dm_const.BOOL_FALSE, # connection must be available, no need for updating CS_UPD_CON: dm_const.BOOL_FALSE, }) else: self._wait_for_node_assignment( d_res_name, d_vol_nr, [nodename], check_vol_deployed=True, filter_props={ CS_DEPLOYED: dm_const.BOOL_TRUE, }) return self._return_connection_data(nodename, volume) def ensure_export(self, context, volume): p_location = volume['provider_location'] if p_location: fields = p_location.split(" ") nodename = fields[1] else: nodename = None return self._return_connection_data(nodename, volume) def initialize_connection(self, volume, connector): nodename = connector["host"] return self._return_connection_data(nodename, volume) def terminate_connection(self, volume, connector, force=False, **kwargs): d_res_name, d_vol_nr = self._resource_name_volnr_for_volume( volume, empty_ok=True) if not d_res_name: return nodename = connector["host"] # If the DRBD volume is diskless on that node, we remove it; # if it has local storage, we keep it. res, data = self.call_or_reconnect( self.odm.list_assignments, [nodename], [d_res_name], 0, self.empty_list, self.empty_list) self._check_result(res, ignore=[dm_exc.DM_ENOENT]) if len(data) < 1: # already removed?! LOG.info(_LI('DRBD connection for %s already removed'), volume['id']) elif len(data) == 1: __, __, props, __ = data[0] my_props = dm_utils.dict_to_aux_props(props) diskless = getattr(props, dm_const.FLAG_DISKLESS, dm_const.BOOL_FALSE) temp_cli = getattr(my_props, AUX_PROP_TEMP_CLIENT, dm_const.BOOL_FALSE) # If diskless assigned, if ((diskless == dm_const.BOOL_TRUE) and (temp_cli == dm_const.BOOL_TRUE)): # remove the assignment # TODO(pm): does it make sense to relay "force" here? # What are the semantics? # TODO(pm): consistency groups shouldn't really # remove until *all* volumes are detached res = self.call_or_reconnect(self.odm.unassign, nodename, d_res_name, force) self._check_result(res, ignore=[dm_exc.DM_ENOENT]) else: # more than one assignment? LOG.error(_LE("DRBDmanage: too many assignments returned.")) return def remove_export(self, context, volume): pass