933 lines
36 KiB
Python
933 lines
36 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import errno
|
|
import glob
|
|
import json
|
|
import os.path
|
|
import re
|
|
import time
|
|
|
|
from oslo_concurrency import lockutils
|
|
from oslo_concurrency import processutils as putils
|
|
from oslo_log import log as logging
|
|
|
|
from os_brick import exception
|
|
from os_brick.i18n import _
|
|
from os_brick.initiator.connectors import base
|
|
try:
|
|
from os_brick.initiator.connectors import nvmeof_agent
|
|
except ImportError:
|
|
nvmeof_agent = None
|
|
from os_brick.privileged import rootwrap as priv_rootwrap
|
|
from os_brick import utils
|
|
|
|
DEV_SEARCH_PATH = '/dev/'
|
|
|
|
DEVICE_SCAN_ATTEMPTS_DEFAULT = 5
|
|
|
|
synchronized = lockutils.synchronized_with_prefix('os-brick-')
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
class NVMeOFConnector(base.BaseLinuxConnector):
|
|
"""Connector class to attach/detach NVMe-oF volumes."""
|
|
|
|
native_multipath_supported = None
|
|
|
|
def __init__(self, root_helper, driver=None, use_multipath=False,
|
|
device_scan_attempts=DEVICE_SCAN_ATTEMPTS_DEFAULT,
|
|
*args, **kwargs):
|
|
super(NVMeOFConnector, self).__init__(
|
|
root_helper,
|
|
driver=driver,
|
|
device_scan_attempts=device_scan_attempts,
|
|
*args, **kwargs)
|
|
self.use_multipath = use_multipath
|
|
self._set_native_multipath_supported()
|
|
if self.use_multipath and not \
|
|
NVMeOFConnector.native_multipath_supported:
|
|
LOG.warning('native multipath is not enabled')
|
|
|
|
@staticmethod
|
|
def get_search_path():
|
|
return DEV_SEARCH_PATH
|
|
|
|
def get_volume_paths(self, connection_properties):
|
|
device_path = connection_properties.get('device_path')
|
|
if device_path:
|
|
return [device_path]
|
|
volume_replicas = connection_properties.get('volume_replicas')
|
|
if not volume_replicas: # compatibility
|
|
return []
|
|
replica_count = connection_properties.get('replica_count')
|
|
|
|
try:
|
|
if volume_replicas and replica_count > 1:
|
|
return ['/dev/md/' + connection_properties.get('alias')]
|
|
if volume_replicas and replica_count == 1:
|
|
return [NVMeOFConnector.get_nvme_device_path(
|
|
self, volume_replicas[0]['target_nqn'],
|
|
volume_replicas[0]['vol_uuid'])]
|
|
else:
|
|
return [NVMeOFConnector.get_nvme_device_path(
|
|
self, connection_properties.get('target_nqn'),
|
|
connection_properties.get('vol_uuid'))]
|
|
except exception.VolumeDeviceNotFound:
|
|
return []
|
|
|
|
@classmethod
|
|
def nvme_present(cls):
|
|
try:
|
|
priv_rootwrap.custom_execute('nvme', 'version')
|
|
return True
|
|
except Exception as exc:
|
|
if isinstance(exc, OSError) and exc.errno == errno.ENOENT:
|
|
LOG.debug('nvme not present on system')
|
|
else:
|
|
LOG.warning('Unknown error when checking presence of nvme: %s',
|
|
exc)
|
|
return False
|
|
|
|
@classmethod
|
|
def get_connector_properties(cls, root_helper, *args, **kwargs):
|
|
"""The NVMe-oF connector properties (initiator uuid and nqn.)"""
|
|
execute = kwargs.get('execute') or priv_rootwrap.execute
|
|
nvmf = NVMeOFConnector(root_helper=root_helper, execute=execute)
|
|
ret = {}
|
|
nqn = None
|
|
uuid = nvmf._get_host_uuid()
|
|
suuid = nvmf._get_system_uuid()
|
|
if cls.nvme_present():
|
|
nqn = utils.get_host_nqn()
|
|
if uuid:
|
|
ret['uuid'] = uuid
|
|
if suuid:
|
|
ret['system uuid'] = suuid # compatibility
|
|
if nqn:
|
|
ret['nqn'] = nqn
|
|
ret['nvme_native_multipath'] = cls._set_native_multipath_supported()
|
|
return ret
|
|
|
|
def _get_host_uuid(self):
|
|
cmd = ('findmnt', '/', '-n', '-o', 'SOURCE')
|
|
try:
|
|
lines, err = self._execute(
|
|
*cmd, run_as_root=True, root_helper=self._root_helper)
|
|
blkid_cmd = (
|
|
'blkid', lines.split('\n')[0], '-s', 'UUID', '-o', 'value')
|
|
lines, _err = self._execute(
|
|
*blkid_cmd, run_as_root=True, root_helper=self._root_helper)
|
|
return lines.split('\n')[0]
|
|
except putils.ProcessExecutionError as e:
|
|
LOG.warning(
|
|
"Process execution error in _get_host_uuid: %s" % str(e))
|
|
return None
|
|
|
|
def _get_system_uuid(self):
|
|
# RSD requires system_uuid to let Cinder RSD Driver identify
|
|
# Nova node for later RSD volume attachment.
|
|
try:
|
|
out, err = self._execute('cat', '/sys/class/dmi/id/product_uuid',
|
|
root_helper=self._root_helper,
|
|
run_as_root=True)
|
|
except putils.ProcessExecutionError:
|
|
try:
|
|
out, err = self._execute('dmidecode', '-ssystem-uuid',
|
|
root_helper=self._root_helper,
|
|
run_as_root=True)
|
|
if not out:
|
|
LOG.warning('dmidecode returned empty system-uuid')
|
|
except putils.ProcessExecutionError as e:
|
|
LOG.debug("Unable to locate dmidecode. For Cinder RSD Backend,"
|
|
" please make sure it is installed: %s", e)
|
|
out = ""
|
|
return out.strip()
|
|
|
|
@classmethod
|
|
def _set_native_multipath_supported(cls):
|
|
if cls.native_multipath_supported is None:
|
|
cls.native_multipath_supported = \
|
|
cls._is_native_multipath_supported()
|
|
return cls.native_multipath_supported
|
|
|
|
@staticmethod
|
|
def _is_native_multipath_supported():
|
|
try:
|
|
with open('/sys/module/nvme_core/parameters/multipath', 'rt') as f:
|
|
return f.read().strip() == 'Y'
|
|
except Exception:
|
|
LOG.warning("Could not find nvme_core/parameters/multipath")
|
|
return False
|
|
|
|
def _get_nvme_devices(self):
|
|
nvme_devices = []
|
|
# match nvme devices like /dev/nvme10n10
|
|
pattern = r'/dev/nvme[0-9]+n[0-9]+'
|
|
cmd = ['nvme', 'list']
|
|
for retry in range(1, self.device_scan_attempts + 1):
|
|
try:
|
|
(out, err) = self._execute(*cmd,
|
|
root_helper=self._root_helper,
|
|
run_as_root=True)
|
|
for line in out.split('\n'):
|
|
result = re.match(pattern, line)
|
|
if result:
|
|
nvme_devices.append(result.group(0))
|
|
LOG.debug("_get_nvme_devices returned %(nvme_devices)s",
|
|
{'nvme_devices': nvme_devices})
|
|
return nvme_devices
|
|
|
|
except putils.ProcessExecutionError:
|
|
LOG.warning(
|
|
"Failed to list available NVMe connected controllers, "
|
|
"retrying.")
|
|
time.sleep(retry ** 2)
|
|
else:
|
|
msg = _("Failed to retrieve available connected NVMe controllers "
|
|
"when running nvme list.")
|
|
raise exception.CommandExecutionFailed(message=msg)
|
|
|
|
@utils.retry(exception.VolumePathsNotFound)
|
|
def _get_device_path(self, current_nvme_devices):
|
|
all_nvme_devices = self._get_nvme_devices()
|
|
LOG.debug("all_nvme_devices are %(all_nvme_devices)s",
|
|
{'all_nvme_devices': all_nvme_devices})
|
|
path = set(all_nvme_devices) - set(current_nvme_devices)
|
|
if not path:
|
|
raise exception.VolumePathsNotFound()
|
|
return list(path)[0]
|
|
|
|
@utils.retry(exception.VolumeDeviceNotFound)
|
|
def _get_device_path_by_nguid(self, nguid):
|
|
device_path = os.path.join(DEV_SEARCH_PATH,
|
|
'disk',
|
|
'by-id',
|
|
'nvme-eui.%s' % nguid)
|
|
LOG.debug("Try to retrieve symlink to %(device_path)s.",
|
|
{"device_path": device_path})
|
|
try:
|
|
path, _err = self._execute('readlink',
|
|
'-e',
|
|
device_path,
|
|
run_as_root=True,
|
|
root_helper=self._root_helper)
|
|
if not path:
|
|
raise exception.VolumePathsNotFound()
|
|
return path.rstrip()
|
|
except putils.ProcessExecutionError as e:
|
|
LOG.exception(e)
|
|
raise exception.VolumeDeviceNotFound(device=device_path)
|
|
|
|
@utils.retry(putils.ProcessExecutionError)
|
|
def _try_connect_nvme(self, cmd):
|
|
try:
|
|
self._execute(*cmd, root_helper=self._root_helper,
|
|
run_as_root=True)
|
|
except putils.ProcessExecutionError as e:
|
|
# Idempotent connection to target.
|
|
# Exit code 70 means that target is already connected.
|
|
if e.exit_code == 70:
|
|
return
|
|
raise
|
|
|
|
def _get_nvme_subsys(self):
|
|
# Example output:
|
|
# {
|
|
# 'Subsystems' : [
|
|
# {
|
|
# 'Name' : 'nvme-subsys0',
|
|
# 'NQN' : 'nqn.2016-06.io.spdk:cnode1'
|
|
# },
|
|
# {
|
|
# 'Paths' : [
|
|
# {
|
|
# 'Name' : 'nvme0',
|
|
# 'Transport' : 'rdma',
|
|
# 'Address' : 'traddr=10.0.2.15 trsvcid=4420'
|
|
# }
|
|
# ]
|
|
# }
|
|
# ]
|
|
# }
|
|
#
|
|
|
|
cmd = ['nvme', 'list-subsys', '-o', 'json']
|
|
ret_val = self._execute(*cmd, root_helper=self._root_helper,
|
|
run_as_root=True)
|
|
|
|
return ret_val
|
|
|
|
@staticmethod
|
|
def _filter_nvme_devices(current_nvme_devices, nvme_controller):
|
|
LOG.debug("Filter NVMe devices belonging to controller "
|
|
"%(nvme_controller)s.",
|
|
{"nvme_controller": nvme_controller})
|
|
nvme_name_pattern = "/dev/%sn[0-9]+" % nvme_controller
|
|
nvme_devices_filtered = list(
|
|
filter(
|
|
lambda device: re.match(nvme_name_pattern, device),
|
|
current_nvme_devices
|
|
)
|
|
)
|
|
return nvme_devices_filtered
|
|
|
|
@utils.retry(exception.NotFound, retries=5)
|
|
def _is_nvme_available(self, nvme_name):
|
|
current_nvme_devices = self._get_nvme_devices()
|
|
if self._filter_nvme_devices(current_nvme_devices, nvme_name):
|
|
return True
|
|
else:
|
|
LOG.error("Failed to find nvme device")
|
|
raise exception.NotFound()
|
|
|
|
def _wait_for_blk(self, nvme_transport_type, conn_nqn,
|
|
target_portal, port):
|
|
# Find nvme name in subsystem list and wait max 15 seconds
|
|
# until new volume will be available in kernel
|
|
nvme_name = ""
|
|
nvme_address = "traddr=%s trsvcid=%s" % (target_portal, port)
|
|
|
|
# Get nvme subsystems in order to find
|
|
# nvme name for connected nvme
|
|
try:
|
|
(out, err) = self._get_nvme_subsys()
|
|
except putils.ProcessExecutionError:
|
|
LOG.error("Failed to get nvme subsystems")
|
|
raise
|
|
|
|
# Get subsystem list. Throw exception if out is currupt or empty
|
|
try:
|
|
subsystems = json.loads(out)['Subsystems']
|
|
except Exception:
|
|
return False
|
|
|
|
# Find nvme name among subsystems
|
|
for i in range(0, int(len(subsystems) / 2)):
|
|
subsystem = subsystems[i * 2]
|
|
if 'NQN' in subsystem and subsystem['NQN'] == conn_nqn:
|
|
for path in subsystems[i * 2 + 1]['Paths']:
|
|
if (path['Transport'] == nvme_transport_type
|
|
and path['Address'] == nvme_address):
|
|
nvme_name = path['Name']
|
|
break
|
|
|
|
if not nvme_name:
|
|
return False
|
|
|
|
# Wait until nvme will be available in kernel
|
|
return self._is_nvme_available(nvme_name)
|
|
|
|
@utils.trace
|
|
@synchronized('connect_volume', external=True)
|
|
def connect_volume(self, connection_properties):
|
|
"""Discover and attach the volume.
|
|
|
|
:param connection_properties: The dictionary that describes all
|
|
of the target volume attributes.
|
|
connection_properties must include:
|
|
nqn - NVMe subsystem name to the volume to be connected
|
|
target_port - NVMe target port that hosts the nqn sybsystem
|
|
target_portal - NVMe target ip that hosts the nqn sybsystem
|
|
:type connection_properties: dict
|
|
:returns: dict
|
|
"""
|
|
if connection_properties.get('vol_uuid'): # compatibility
|
|
return self._connect_volume_by_uuid(connection_properties)
|
|
|
|
current_nvme_devices = self._get_nvme_devices()
|
|
device_info = {'type': 'block'}
|
|
conn_nqn = connection_properties['nqn']
|
|
target_portal = connection_properties['target_portal']
|
|
port = connection_properties['target_port']
|
|
nvme_transport_type = connection_properties['transport_type']
|
|
host_nqn = connection_properties.get('host_nqn')
|
|
device_nguid = connection_properties.get('volume_nguid')
|
|
cmd = [
|
|
'nvme', 'connect',
|
|
'-t', nvme_transport_type,
|
|
'-n', conn_nqn,
|
|
'-a', target_portal,
|
|
'-s', port]
|
|
if host_nqn:
|
|
cmd.extend(['-q', host_nqn])
|
|
|
|
self._try_connect_nvme(cmd)
|
|
try:
|
|
self._wait_for_blk(nvme_transport_type, conn_nqn,
|
|
target_portal, port)
|
|
except exception.NotFound:
|
|
LOG.error("Waiting for nvme failed")
|
|
raise exception.NotFound(message="nvme connect: NVMe device "
|
|
"not found")
|
|
if device_nguid:
|
|
path = self._get_device_path_by_nguid(device_nguid)
|
|
else:
|
|
path = self._get_device_path(current_nvme_devices)
|
|
device_info['path'] = path
|
|
LOG.debug("NVMe device to be connected to is %(path)s",
|
|
{'path': path})
|
|
return device_info
|
|
|
|
@utils.trace
|
|
@synchronized('connect_volume', external=True)
|
|
def disconnect_volume(self, connection_properties, device_info,
|
|
force=False, ignore_errors=False):
|
|
"""Flush the volume.
|
|
|
|
Disconnect of volumes happens on storage system side. Here we could
|
|
remove connections to subsystems if no volumes are left. But new
|
|
volumes can pop up asynchronously in the meantime. So the only thing
|
|
left is flushing or disassembly of a correspondng RAID device.
|
|
|
|
:param connection_properties: The dictionary that describes all
|
|
of the target volume attributes.
|
|
connection_properties must include:
|
|
device_path - path to the volume to be connected
|
|
:type connection_properties: dict
|
|
|
|
:param device_info: historical difference, but same as connection_props
|
|
:type device_info: dict
|
|
|
|
"""
|
|
if connection_properties.get('vol_uuid'): # compatibility
|
|
return self._disconnect_volume_replicated(
|
|
connection_properties, device_info,
|
|
force=force, ignore_errors=ignore_errors)
|
|
|
|
conn_nqn = connection_properties['nqn']
|
|
if device_info and device_info.get('path'):
|
|
device_path = device_info.get('path')
|
|
else:
|
|
device_path = connection_properties['device_path'] or ''
|
|
current_nvme_devices = self._get_nvme_devices()
|
|
if device_path not in current_nvme_devices:
|
|
LOG.warning("Trying to disconnect device %(device_path)s with "
|
|
"subnqn %(conn_nqn)s that is not connected.",
|
|
{'device_path': device_path, 'conn_nqn': conn_nqn})
|
|
return
|
|
|
|
try:
|
|
self._linuxscsi.flush_device_io(device_path)
|
|
except putils.ProcessExecutionError:
|
|
if not ignore_errors:
|
|
raise
|
|
|
|
@utils.trace
|
|
@synchronized('extend_volume', external=True)
|
|
def extend_volume(self, connection_properties):
|
|
"""Update the local kernel's size information.
|
|
|
|
Try and update the local kernel's size information
|
|
for an LVM volume.
|
|
"""
|
|
if connection_properties.get('vol_uuid'): # compatibility
|
|
return self._extend_volume_replicated(connection_properties)
|
|
volume_paths = self.get_volume_paths(connection_properties)
|
|
if volume_paths:
|
|
if connection_properties.get('volume_nguid'):
|
|
for path in volume_paths:
|
|
return self._linuxscsi.get_device_size(path)
|
|
return self._linuxscsi.extend_volume(
|
|
volume_paths, use_multipath=self.use_multipath)
|
|
else:
|
|
LOG.warning("Couldn't find any volume paths on the host to "
|
|
"extend volume for %(props)s",
|
|
{'props': connection_properties})
|
|
raise exception.VolumePathsNotFound()
|
|
|
|
@utils.trace
|
|
def _connect_volume_by_uuid(self, connection_properties):
|
|
"""connect to volume on host
|
|
|
|
connection_properties for NVMe-oF must include:
|
|
target_portals - list of ip,port,transport for each portal
|
|
target_nqn - NVMe-oF Qualified Name
|
|
vol_uuid - UUID for volume/replica
|
|
"""
|
|
volume_replicas = connection_properties.get('volume_replicas')
|
|
replica_count = connection_properties.get('replica_count')
|
|
volume_alias = connection_properties.get('alias')
|
|
|
|
if volume_replicas:
|
|
host_device_paths = []
|
|
|
|
for replica in volume_replicas:
|
|
try:
|
|
rep_host_device_path = self._connect_target_volume(
|
|
replica['target_nqn'], replica['vol_uuid'],
|
|
replica['portals'])
|
|
if rep_host_device_path:
|
|
host_device_paths.append(rep_host_device_path)
|
|
except Exception as ex:
|
|
LOG.error("_connect_target_volume: %s", ex)
|
|
if not host_device_paths:
|
|
raise exception.VolumeDeviceNotFound(
|
|
device=volume_replicas)
|
|
|
|
if replica_count > 1:
|
|
device_path = self._handle_replicated_volume(
|
|
host_device_paths, volume_alias, replica_count)
|
|
else:
|
|
device_path = self._handle_single_replica(
|
|
host_device_paths, volume_alias)
|
|
else:
|
|
device_path = self._connect_target_volume(
|
|
connection_properties['target_nqn'],
|
|
connection_properties['vol_uuid'],
|
|
connection_properties['portals'])
|
|
|
|
if nvmeof_agent:
|
|
nvmeof_agent.NVMeOFAgent.ensure_running(self)
|
|
|
|
return {'type': 'block', 'path': device_path}
|
|
|
|
@utils.trace
|
|
def _disconnect_volume_replicated(self, connection_properties, device_info,
|
|
force=False, ignore_errors=False):
|
|
device_path = None
|
|
volume_replicas = connection_properties.get('volume_replicas')
|
|
replica_count = connection_properties.get('replica_count')
|
|
if device_info and device_info.get('path'):
|
|
device_path = device_info['path']
|
|
elif connection_properties.get('device_path'):
|
|
device_path = connection_properties['device_path']
|
|
elif volume_replicas and replica_count > 1:
|
|
device_path = '/dev/md/' + connection_properties['alias']
|
|
|
|
if volume_replicas and replica_count > 1:
|
|
NVMeOFConnector.end_raid(self, device_path)
|
|
else:
|
|
if self._get_fs_type(device_path) == 'linux_raid_member':
|
|
NVMeOFConnector.end_raid(self, device_path)
|
|
|
|
def _extend_volume_replicated(self, connection_properties):
|
|
volume_replicas = connection_properties.get('volume_replicas')
|
|
replica_count = connection_properties.get('replica_count')
|
|
|
|
if volume_replicas and replica_count > 1:
|
|
device_path = '/dev/md/' + connection_properties['alias']
|
|
NVMeOFConnector.run_mdadm(
|
|
self, ['mdadm', '--grow', '--size', 'max', device_path])
|
|
else:
|
|
target_nqn = None
|
|
vol_uuid = None
|
|
if not volume_replicas:
|
|
target_nqn = connection_properties['target_nqn']
|
|
vol_uuid = connection_properties['vol_uuid']
|
|
elif len(volume_replicas) == 1:
|
|
target_nqn = volume_replicas[0]['target_nqn']
|
|
vol_uuid = volume_replicas[0]['vol_uuid']
|
|
device_path = NVMeOFConnector.get_nvme_device_path(
|
|
self, target_nqn, vol_uuid)
|
|
|
|
return self._linuxscsi.get_device_size(device_path)
|
|
|
|
def _connect_target_volume(self, target_nqn, vol_uuid, portals):
|
|
nvme_ctrls = NVMeOFConnector.rescan(self, target_nqn)
|
|
any_new_connect = NVMeOFConnector.connect_to_portals(self, target_nqn,
|
|
portals,
|
|
nvme_ctrls)
|
|
if not any_new_connect and len(nvme_ctrls) == 0:
|
|
# no new connections and any pre-exists controllers
|
|
LOG.error("No successful connections to: %s", target_nqn)
|
|
raise exception.VolumeDeviceNotFound(device=target_nqn)
|
|
if any_new_connect:
|
|
# new connections - refresh controllers map
|
|
nvme_ctrls = \
|
|
NVMeOFConnector.get_live_nvme_controllers_map(self, target_nqn)
|
|
nvme_ctrls_values = list(nvme_ctrls.values())
|
|
dev_path = NVMeOFConnector.get_nvme_device_path(self, target_nqn,
|
|
vol_uuid,
|
|
nvme_ctrls_values)
|
|
if not dev_path:
|
|
LOG.error("Target %s volume %s not found", target_nqn, vol_uuid)
|
|
raise exception.VolumeDeviceNotFound(device=vol_uuid)
|
|
return dev_path
|
|
|
|
@staticmethod
|
|
def connect_to_portals(executor, target_nqn, target_portals, nvme_ctrls):
|
|
# connect to any of NVMe-oF target portals -
|
|
# check if the controller exist before trying to connect
|
|
# in multipath connect all given target portals
|
|
any_new_connect = False
|
|
no_multipath = not executor.use_multipath or not \
|
|
NVMeOFConnector.native_multipath_supported
|
|
for portal in target_portals:
|
|
portal_address = portal[0]
|
|
portal_port = portal[1]
|
|
if NVMeOFConnector.is_portal_connected(portal_address, portal_port,
|
|
nvme_ctrls):
|
|
if no_multipath:
|
|
break
|
|
continue
|
|
if portal[2] == 'RoCEv2':
|
|
portal_transport = 'rdma'
|
|
else:
|
|
portal_transport = 'tcp'
|
|
nvme_command = (
|
|
'connect', '-a', portal_address, '-s', portal_port, '-t',
|
|
portal_transport, '-n', target_nqn, '-Q', '128', '-l', '-1')
|
|
try:
|
|
NVMeOFConnector.run_nvme_cli(executor, nvme_command)
|
|
any_new_connect = True
|
|
if no_multipath:
|
|
break
|
|
except Exception:
|
|
LOG.exception("Could not connect to portal %s", portal)
|
|
return any_new_connect
|
|
|
|
@staticmethod
|
|
def is_portal_connected(portal_address, portal_port, nvme_ctrls):
|
|
address = f"traddr={portal_address},trsvcid={portal_port}"
|
|
return address in nvme_ctrls
|
|
|
|
@staticmethod
|
|
def get_nvme_controllers(executor, target_nqn):
|
|
nvme_controllers = \
|
|
NVMeOFConnector.get_live_nvme_controllers_map(executor, target_nqn)
|
|
if len(nvme_controllers) > 0:
|
|
return nvme_controllers.values()
|
|
raise exception.VolumeDeviceNotFound(device=target_nqn)
|
|
|
|
@staticmethod
|
|
def get_live_nvme_controllers_map(executor, target_nqn):
|
|
"""returns map of all live controllers and their addresses """
|
|
nvme_controllers = dict()
|
|
ctrls = glob.glob('/sys/class/nvme-fabrics/ctl/nvme*')
|
|
for ctrl in ctrls:
|
|
try:
|
|
lines, _err = executor._execute(
|
|
'cat', ctrl + '/subsysnqn', run_as_root=True,
|
|
root_helper=executor._root_helper)
|
|
for line in lines.split('\n'):
|
|
if line == target_nqn:
|
|
state, _err = executor._execute(
|
|
'cat', ctrl + '/state', run_as_root=True,
|
|
root_helper=executor._root_helper)
|
|
if 'live' in state:
|
|
address_file = ctrl + '/address'
|
|
try:
|
|
with open(address_file, 'rt') as f:
|
|
address = f.read().strip()
|
|
except Exception:
|
|
LOG.warning("Failed to read file %s",
|
|
address_file)
|
|
continue
|
|
ctrl_name = os.path.basename(ctrl)
|
|
LOG.debug("[!] address: %s|%s", address, ctrl_name)
|
|
nvme_controllers[address] = ctrl_name
|
|
else:
|
|
LOG.debug("nvmeof ctrl device not live: %s", ctrl)
|
|
except putils.ProcessExecutionError as e:
|
|
LOG.exception(e)
|
|
return nvme_controllers
|
|
|
|
@staticmethod
|
|
@utils.retry(exception.VolumeDeviceNotFound, retries=5)
|
|
def get_nvme_device_path(executor, target_nqn, vol_uuid, nvme_ctrls=None):
|
|
if not nvme_ctrls:
|
|
nvme_ctrls = NVMeOFConnector.get_nvme_controllers(executor,
|
|
target_nqn)
|
|
LOG.debug("[!] nvme_ctrls: %s", nvme_ctrls)
|
|
for nvme_ctrl in nvme_ctrls:
|
|
uuid_paths = glob.glob('/sys/class/block/' + nvme_ctrl + 'n*/uuid')
|
|
for uuid_path in uuid_paths:
|
|
try:
|
|
uuid_lines, _err = executor._execute(
|
|
'cat', uuid_path, run_as_root=True,
|
|
root_helper=executor._root_helper)
|
|
if uuid_lines.split('\n')[0] == vol_uuid:
|
|
ignore = len('/uuid')
|
|
ns_ind = uuid_path.rfind('/', 0, -ignore)
|
|
nvme_device = uuid_path[ns_ind + 1: -ignore]
|
|
return '/dev/' + nvme_device
|
|
except putils.ProcessExecutionError as e:
|
|
LOG.exception(e)
|
|
raise exception.VolumeDeviceNotFound(device=vol_uuid)
|
|
|
|
def _handle_replicated_volume(self, host_device_paths,
|
|
volume_alias, num_of_replicas):
|
|
path_in_raid = False
|
|
for dev_path in host_device_paths:
|
|
path_in_raid = NVMeOFConnector._is_device_in_raid(self, dev_path)
|
|
if path_in_raid:
|
|
break
|
|
device_path = '/dev/md/' + volume_alias
|
|
if path_in_raid:
|
|
NVMeOFConnector.stop_and_assemble_raid(
|
|
self, host_device_paths, device_path, False)
|
|
else:
|
|
paths_found = len(host_device_paths)
|
|
if num_of_replicas > paths_found:
|
|
LOG.error(
|
|
'Cannot create MD as %s out of %s legs were found.',
|
|
paths_found, num_of_replicas)
|
|
raise exception.VolumeDeviceNotFound(device=volume_alias)
|
|
NVMeOFConnector.create_raid(self, host_device_paths, '1',
|
|
volume_alias, volume_alias, False)
|
|
|
|
return device_path
|
|
|
|
def _handle_single_replica(self, host_device_paths, volume_alias):
|
|
if self._get_fs_type(host_device_paths[0]) == 'linux_raid_member':
|
|
md_path = '/dev/md/' + volume_alias
|
|
NVMeOFConnector.stop_and_assemble_raid(
|
|
self, host_device_paths, md_path, False)
|
|
return md_path
|
|
return host_device_paths[0]
|
|
|
|
@staticmethod
|
|
def run_mdadm(executor, cmd, raise_exception=False):
|
|
cmd_output = None
|
|
try:
|
|
lines, err = executor._execute(
|
|
*cmd, run_as_root=True, root_helper=executor._root_helper)
|
|
for line in lines.split('\n'):
|
|
cmd_output = line
|
|
break
|
|
except putils.ProcessExecutionError as ex:
|
|
LOG.warning("[!] Could not run mdadm: %s", str(ex))
|
|
if raise_exception:
|
|
raise ex
|
|
return cmd_output
|
|
|
|
@staticmethod
|
|
def _is_device_in_raid(self, device_path):
|
|
cmd = ['mdadm', '--examine', device_path]
|
|
raid_expected = device_path + ':'
|
|
try:
|
|
lines, err = self._execute(
|
|
*cmd, run_as_root=True, root_helper=self._root_helper)
|
|
for line in lines.split('\n'):
|
|
if line == raid_expected:
|
|
return True
|
|
else:
|
|
return False
|
|
except putils.ProcessExecutionError:
|
|
return False
|
|
|
|
@staticmethod
|
|
def ks_readlink(dest):
|
|
try:
|
|
return os.readlink(dest)
|
|
except Exception:
|
|
return ''
|
|
|
|
@staticmethod
|
|
def get_md_name(executor, device_name):
|
|
get_md_cmd = (
|
|
'cat /proc/mdstat | grep ' + device_name +
|
|
' | awk \'{print $1;}\'')
|
|
cmd = ['bash', '-c', get_md_cmd]
|
|
LOG.debug("[!] cmd = " + str(cmd))
|
|
cmd_output = None
|
|
|
|
try:
|
|
lines, err = executor._execute(
|
|
*cmd, run_as_root=True, root_helper=executor._root_helper)
|
|
|
|
for line in lines.split('\n'):
|
|
cmd_output = line
|
|
break
|
|
|
|
LOG.debug("[!] cmd_output = " + cmd_output)
|
|
if err:
|
|
return None
|
|
|
|
return cmd_output
|
|
except putils.ProcessExecutionError as ex:
|
|
LOG.warning("[!] Could not run cmd: %s", str(ex))
|
|
return None
|
|
|
|
@staticmethod
|
|
def stop_and_assemble_raid(executor, drives, md_path, read_only):
|
|
md_name = None
|
|
i = 0
|
|
assembled = False
|
|
link = ''
|
|
while i < 5 and not assembled:
|
|
for drive in drives:
|
|
device_name = drive[5:]
|
|
md_name = NVMeOFConnector.get_md_name(executor, device_name)
|
|
link = NVMeOFConnector.ks_readlink(md_path)
|
|
if link != '':
|
|
link = os.path.basename(link)
|
|
if md_name and md_name == link:
|
|
return
|
|
LOG.debug(
|
|
"sleeping 1 sec -allow auto assemble link = " +
|
|
link + " md path = " + md_path)
|
|
time.sleep(1)
|
|
|
|
if md_name and md_name != link:
|
|
NVMeOFConnector.stop_raid(executor, md_name)
|
|
|
|
try:
|
|
assembled = NVMeOFConnector.assemble_raid(
|
|
executor, drives, md_path, read_only)
|
|
except Exception:
|
|
i += 1
|
|
|
|
@staticmethod
|
|
def assemble_raid(executor, drives, md_path, read_only):
|
|
cmd = ['mdadm', '--assemble', '--run', md_path]
|
|
|
|
if read_only:
|
|
cmd.append('-o')
|
|
|
|
for i in range(len(drives)):
|
|
cmd.append(drives[i])
|
|
|
|
try:
|
|
NVMeOFConnector.run_mdadm(executor, cmd, True)
|
|
except putils.ProcessExecutionError as ex:
|
|
LOG.warning("[!] Could not _assemble_raid: %s", str(ex))
|
|
raise ex
|
|
|
|
return True
|
|
|
|
@staticmethod
|
|
def create_raid(executor, drives, raid_type, device_name, name, read_only):
|
|
cmd = ['mdadm']
|
|
num_drives = len(drives)
|
|
cmd.append('-C')
|
|
|
|
if read_only:
|
|
cmd.append('-o')
|
|
|
|
cmd.append(device_name)
|
|
cmd.append('-R')
|
|
|
|
if name:
|
|
cmd.append('-N')
|
|
cmd.append(name)
|
|
|
|
cmd.append('--level')
|
|
cmd.append(raid_type)
|
|
cmd.append('--raid-devices=' + str(num_drives))
|
|
cmd.append('--bitmap=internal')
|
|
cmd.append('--homehost=any')
|
|
cmd.append('--failfast')
|
|
cmd.append('--assume-clean')
|
|
|
|
for i in range(len(drives)):
|
|
cmd.append(drives[i])
|
|
|
|
LOG.debug('[!] cmd = ' + str(cmd))
|
|
NVMeOFConnector.run_mdadm(executor, cmd)
|
|
# sometimes under load, md is not created right away, so we wait
|
|
for i in range(60):
|
|
try:
|
|
is_exist = os.path.exists("/dev/md/" + name)
|
|
LOG.debug("[!] md is_exist = %s", is_exist)
|
|
if is_exist:
|
|
return
|
|
time.sleep(1)
|
|
except Exception:
|
|
LOG.debug('[!] Exception_wait_raid!')
|
|
msg = _("md: /dev/md/%s not found.") % name
|
|
LOG.error(msg)
|
|
raise exception.NotFound(message=msg)
|
|
|
|
@staticmethod
|
|
def end_raid(executor, device_path):
|
|
raid_exists = NVMeOFConnector.is_raid_exists(executor, device_path)
|
|
if raid_exists:
|
|
for i in range(10):
|
|
try:
|
|
cmd_out = NVMeOFConnector.stop_raid(
|
|
executor, device_path, True)
|
|
if not cmd_out:
|
|
break
|
|
except Exception:
|
|
time.sleep(1)
|
|
try:
|
|
is_exist = os.path.exists(device_path)
|
|
LOG.debug("[!] is_exist = %s", is_exist)
|
|
if is_exist:
|
|
NVMeOFConnector.remove_raid(executor, device_path)
|
|
os.remove(device_path)
|
|
except Exception:
|
|
LOG.debug('[!] Exception_stop_raid!')
|
|
|
|
@staticmethod
|
|
def stop_raid(executor, md_path, raise_exception=False):
|
|
cmd = ['mdadm', '--stop', md_path]
|
|
LOG.debug("[!] cmd = " + str(cmd))
|
|
cmd_out = NVMeOFConnector.run_mdadm(executor, cmd, raise_exception)
|
|
return cmd_out
|
|
|
|
@staticmethod
|
|
def is_raid_exists(executor, device_path):
|
|
cmd = ['mdadm', '--detail', device_path]
|
|
LOG.debug("[!] cmd = " + str(cmd))
|
|
raid_expected = device_path + ':'
|
|
try:
|
|
lines, err = executor._execute(
|
|
*cmd, run_as_root=True, root_helper=executor._root_helper)
|
|
|
|
for line in lines.split('\n'):
|
|
LOG.debug("[!] line = " + line)
|
|
if line == raid_expected:
|
|
return True
|
|
else:
|
|
return False
|
|
except putils.ProcessExecutionError:
|
|
return False
|
|
|
|
@staticmethod
|
|
def remove_raid(executor, device_path):
|
|
cmd = ['mdadm', '--remove', device_path]
|
|
LOG.debug("[!] cmd = " + str(cmd))
|
|
NVMeOFConnector.run_mdadm(executor, cmd)
|
|
|
|
@staticmethod
|
|
def run_nvme_cli(executor, nvme_command, **kwargs):
|
|
(out, err) = executor._execute('nvme', *nvme_command, run_as_root=True,
|
|
root_helper=executor._root_helper,
|
|
check_exit_code=True)
|
|
msg = ("nvme %(nvme_command)s: stdout=%(out)s stderr=%(err)s" %
|
|
{'nvme_command': nvme_command, 'out': out, 'err': err})
|
|
LOG.debug("[!] " + msg)
|
|
|
|
return out, err
|
|
|
|
@staticmethod
|
|
def rescan(executor, target_nqn):
|
|
nvme_ctrls = NVMeOFConnector.get_live_nvme_controllers_map(executor,
|
|
target_nqn)
|
|
for nvme_ctrl in nvme_ctrls.values():
|
|
ctr_device = (NVMeOFConnector.get_search_path() + nvme_ctrl)
|
|
nvme_command = ('ns-rescan', ctr_device)
|
|
try:
|
|
NVMeOFConnector.run_nvme_cli(executor, nvme_command)
|
|
except Exception as e:
|
|
LOG.exception(e)
|
|
return nvme_ctrls
|
|
|
|
def _get_fs_type(self, device_path):
|
|
cmd = ['blkid', device_path, '-s', 'TYPE', '-o', 'value']
|
|
LOG.debug("[!] cmd = " + str(cmd))
|
|
fs_type = None
|
|
|
|
try:
|
|
lines, err = self._execute(
|
|
*cmd, run_as_root=True, root_helper=self._root_helper)
|
|
|
|
fs_type = lines.split('\n')[0]
|
|
except putils.ProcessExecutionError:
|
|
return None
|
|
|
|
return fs_type
|