448 lines
18 KiB
Python
448 lines
18 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import tempfile
|
|
import typing
|
|
from typing import Any, Optional, Union # noqa: H301
|
|
|
|
if typing.TYPE_CHECKING:
|
|
import io
|
|
|
|
from oslo_concurrency import processutils as putils
|
|
from oslo_log import log as logging
|
|
from oslo_serialization import jsonutils
|
|
from oslo_utils import excutils
|
|
from oslo_utils import fileutils
|
|
|
|
from os_brick import exception
|
|
from os_brick.i18n import _
|
|
from os_brick import initiator
|
|
from os_brick.initiator.connectors import base
|
|
from os_brick.initiator.connectors import base_rbd
|
|
from os_brick.initiator import linuxrbd
|
|
from os_brick.privileged import rbd as rbd_privsep
|
|
from os_brick import utils
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
class RBDConnector(base_rbd.RBDConnectorMixin, base.BaseLinuxConnector):
|
|
""""Connector class to attach/detach RBD volumes."""
|
|
|
|
def __init__(self, root_helper, driver=None, use_multipath=False,
|
|
device_scan_attempts=initiator.DEVICE_SCAN_ATTEMPTS_DEFAULT,
|
|
*args, **kwargs):
|
|
|
|
super(RBDConnector, self).__init__(root_helper,
|
|
driver=driver, # type: ignore
|
|
device_scan_attempts=
|
|
device_scan_attempts,
|
|
*args, **kwargs)
|
|
self.do_local_attach = kwargs.get('do_local_attach', False)
|
|
|
|
@staticmethod
|
|
def get_connector_properties(root_helper, *args, **kwargs):
|
|
"""The RBD connector properties."""
|
|
return {'do_local_attach': kwargs.get('do_local_attach', False)}
|
|
|
|
def get_volume_paths(self,
|
|
connection_properties: dict[str, Any]) -> list[str]:
|
|
# TODO(e0ne): Implement this for local volume.
|
|
return []
|
|
|
|
def get_search_path(self) -> None:
|
|
# TODO(walter-boring): don't know where the connector
|
|
# looks for RBD volumes.
|
|
return None
|
|
|
|
def get_all_available_volumes(
|
|
self,
|
|
connection_properties: Optional[dict[str, Any]] = None) -> \
|
|
list[str]:
|
|
# TODO(e0ne): Implement this for local volume.
|
|
return []
|
|
|
|
@staticmethod
|
|
def _check_or_get_keyring_contents(keyring: Optional[str],
|
|
cluster_name: str,
|
|
user: str) -> str:
|
|
try:
|
|
if keyring is None:
|
|
if user:
|
|
keyring_path = ("/etc/ceph/%s.client.%s.keyring" %
|
|
(cluster_name, user))
|
|
with open(keyring_path, 'r') as keyring_file:
|
|
keyring = keyring_file.read()
|
|
else:
|
|
keyring = ''
|
|
return keyring
|
|
except IOError:
|
|
msg = (_("Keyring path %s is not readable.") % (keyring_path))
|
|
raise exception.BrickException(msg=msg)
|
|
|
|
@classmethod
|
|
def _create_ceph_conf(cls,
|
|
monitor_ips: list[str],
|
|
monitor_ports: list[str],
|
|
cluster_name: str,
|
|
user: str,
|
|
keyring) -> str:
|
|
monitors = ["%s:%s" % (ip, port) for ip, port in
|
|
zip(cls._sanitize_mon_hosts(monitor_ips), monitor_ports)]
|
|
mon_hosts = "mon_host = %s" % (','.join(monitors))
|
|
|
|
keyring = cls._check_or_get_keyring_contents(keyring, cluster_name,
|
|
user)
|
|
|
|
try:
|
|
fd, ceph_conf_path = tempfile.mkstemp(prefix="brickrbd_")
|
|
with os.fdopen(fd, 'w') as conf_file:
|
|
# Bug #1865754 - '[global]' has been the appropriate
|
|
# place for this stuff since at least Hammer, but in
|
|
# Octopus (15.2.0+), Ceph began enforcing this.
|
|
conf_file.writelines(["[global]", "\n",
|
|
mon_hosts, "\n",
|
|
keyring, "\n"])
|
|
return ceph_conf_path
|
|
except IOError:
|
|
msg = (_("Failed to write data to %s.") % (ceph_conf_path))
|
|
raise exception.BrickException(msg=msg)
|
|
|
|
def _get_rbd_handle(self, connection_properties: dict[str, Any]) -> \
|
|
linuxrbd.RBDVolumeIOWrapper:
|
|
try:
|
|
user = connection_properties['auth_username']
|
|
pool, volume = connection_properties['name'].split('/')
|
|
cluster_name = connection_properties['cluster_name']
|
|
monitor_ips = connection_properties['hosts']
|
|
monitor_ports = connection_properties['ports']
|
|
# NOTE: cinder no longer passes keyring data in the connection
|
|
# properties as of the victoria release. See OSSN-0085. But
|
|
# cinderlib does, so we must keep the code related to the keyring.
|
|
keyring = connection_properties.get('keyring')
|
|
except (KeyError, ValueError):
|
|
msg = _("Connect volume failed, malformed connection properties.")
|
|
raise exception.BrickException(msg=msg)
|
|
|
|
conf = self._create_ceph_conf(monitor_ips, monitor_ports,
|
|
str(cluster_name), user,
|
|
keyring)
|
|
try:
|
|
rbd_client = linuxrbd.RBDClient(user, pool, conffile=conf,
|
|
rbd_cluster_name=str(cluster_name))
|
|
rbd_volume = linuxrbd.RBDVolume(rbd_client, volume)
|
|
rbd_handle = linuxrbd.RBDVolumeIOWrapper(
|
|
linuxrbd.RBDImageMetadata(rbd_volume, pool, user, conf))
|
|
except Exception:
|
|
fileutils.delete_if_exists(conf)
|
|
raise
|
|
|
|
return rbd_handle
|
|
|
|
@staticmethod
|
|
def get_rbd_device_name(pool: str, volume: str) -> str:
|
|
"""Return device name which will be generated by RBD kernel module.
|
|
|
|
:param pool: RBD pool name.
|
|
:type pool: string
|
|
:param volume: RBD image name.
|
|
:type volume: string
|
|
"""
|
|
return '/dev/rbd/{pool}/{volume}'.format(pool=pool, volume=volume)
|
|
|
|
@classmethod
|
|
def create_non_openstack_config(
|
|
cls,
|
|
connection_properties: dict[str, Any]):
|
|
"""Get root owned Ceph's .conf file for non OpenStack usage."""
|
|
# If keyring info is missing then we are in OpenStack, nothing to do
|
|
keyring = connection_properties.get('keyring')
|
|
if not keyring:
|
|
return None
|
|
|
|
try:
|
|
user = connection_properties['auth_username']
|
|
pool, volume = connection_properties['name'].split('/')
|
|
cluster_name = connection_properties['cluster_name']
|
|
monitor_ips = connection_properties['hosts']
|
|
monitor_ports = connection_properties['ports']
|
|
keyring = connection_properties.get('keyring')
|
|
except (KeyError, ValueError):
|
|
msg = _("Connect volume failed, malformed connection properties.")
|
|
raise exception.BrickException(msg=msg)
|
|
|
|
conf = rbd_privsep.root_create_ceph_conf(monitor_ips, monitor_ports,
|
|
str(cluster_name), user,
|
|
keyring)
|
|
return conf
|
|
|
|
def _local_attach_volume(
|
|
self,
|
|
connection_properties: dict[str, Any]) -> \
|
|
dict[str, Union[str, linuxrbd.RBDVolumeIOWrapper]]:
|
|
# NOTE(e0ne): sanity check if ceph-common is installed.
|
|
try:
|
|
self._execute('which', 'rbd')
|
|
except putils.ProcessExecutionError:
|
|
msg = _("ceph-common package is not installed.")
|
|
LOG.error(msg)
|
|
raise exception.BrickException(message=msg)
|
|
|
|
# NOTE(e0ne): map volume to a block device
|
|
# via the rbd kernel module.
|
|
pool, volume = connection_properties['name'].split('/')
|
|
rbd_dev_path = self.get_rbd_device_name(pool, volume)
|
|
# If we are not running on OpenStack, create config file
|
|
conf = self.create_non_openstack_config(connection_properties)
|
|
try:
|
|
if (
|
|
not os.path.islink(rbd_dev_path) or
|
|
not os.path.exists(os.path.realpath(rbd_dev_path))
|
|
):
|
|
# TODO(stephenfin): Update to the unified 'rbd device map'
|
|
# command introduced in ceph 13.0 (commit 6a57358add1157629a6d)
|
|
# when we drop support earlier versions
|
|
cmd = ['rbd', 'map', volume, '--pool', pool]
|
|
cmd += self._get_rbd_args(connection_properties, conf)
|
|
self._execute(*cmd, root_helper=self._root_helper,
|
|
run_as_root=True)
|
|
else:
|
|
LOG.debug(
|
|
'Volume %(vol)s is already mapped to local device %(dev)s',
|
|
{'vol': volume, 'dev': os.path.realpath(rbd_dev_path)}
|
|
)
|
|
|
|
if (
|
|
not os.path.islink(rbd_dev_path) or
|
|
not os.path.exists(os.path.realpath(rbd_dev_path))
|
|
):
|
|
LOG.warning(
|
|
'Volume %(vol)s has not been mapped to local device '
|
|
'%(dev)s; is the udev daemon running and are the '
|
|
'ceph-renamer udev rules configured? See bug #1884114 for '
|
|
'more information.',
|
|
{'vol': volume, 'dev': rbd_dev_path},
|
|
)
|
|
except Exception:
|
|
# Cleanup conf file on failure
|
|
with excutils.save_and_reraise_exception():
|
|
if conf:
|
|
rbd_privsep.delete_if_exists(conf)
|
|
|
|
res: dict[str, Union[str, linuxrbd.RBDVolumeIOWrapper]]
|
|
res = {'path': rbd_dev_path,
|
|
'type': 'block'}
|
|
if conf:
|
|
res['conf'] = conf
|
|
return res
|
|
|
|
@utils.trace
|
|
@utils.connect_volume_prepare_result
|
|
def connect_volume(self,
|
|
connection_properties: dict[str, Any]) -> \
|
|
dict[str, Union[linuxrbd.RBDVolumeIOWrapper, str]]:
|
|
"""Connect to a volume.
|
|
|
|
:param connection_properties: The dictionary that describes all
|
|
of the target volume attributes.
|
|
:type connection_properties: dict
|
|
:returns: dict
|
|
"""
|
|
do_local_attach = connection_properties.get('do_local_attach',
|
|
self.do_local_attach)
|
|
|
|
if do_local_attach:
|
|
return self._local_attach_volume(connection_properties)
|
|
|
|
rbd_handle = self._get_rbd_handle(connection_properties)
|
|
return {'path': rbd_handle}
|
|
|
|
def _find_root_device(self,
|
|
connection_properties: dict[str, Any],
|
|
conf) -> Optional[str]:
|
|
"""Find the underlying /dev/rbd* device for a mapping.
|
|
|
|
Use the showmapped command to list all acive mappings and find the
|
|
underlying /dev/rbd* device that corresponds to our pool and volume.
|
|
|
|
:param connection_properties: The dictionary that describes all
|
|
of the target volume attributes.
|
|
:type connection_properties: dict
|
|
:returns: '/dev/rbd*' or None if no active mapping is found.
|
|
"""
|
|
__, volume = connection_properties['name'].split('/')
|
|
# TODO(stephenfin): Update to the unified 'rbd device list'
|
|
# command introduced in ceph 13.0 (commit 6a57358add1157629a6d)
|
|
# when we drop support earlier versions
|
|
cmd = ['rbd', 'showmapped', '--format=json']
|
|
cmd += self._get_rbd_args(connection_properties, conf)
|
|
(out, err) = self._execute(*cmd, root_helper=self._root_helper,
|
|
run_as_root=True)
|
|
|
|
# ceph v13.2.0 (Mimic) changed the output format of 'rbd showmapped'
|
|
# from a dict of mappings keyed by ID to a simple list of mappings
|
|
# https://docs.ceph.com/docs/master/releases/mimic/
|
|
#
|
|
# before:
|
|
#
|
|
# {
|
|
# "0": {
|
|
# "pool":"volumes",
|
|
# "namespace":"",
|
|
# "name":"volume-6d54cb90-a5d1-40d8-9cb2-c6adf43a02af",
|
|
# "snap":"-",
|
|
# "device":"/dev/rbd0"
|
|
# }
|
|
# }
|
|
#
|
|
# after:
|
|
#
|
|
# [
|
|
# {
|
|
# "id":"0",
|
|
# "pool":"volumes",
|
|
# "namespace":"",
|
|
# "name":"volume-6d54cb90-a5d1-40d8-9cb2-c6adf43a02af",
|
|
# "snap":"-",
|
|
# "device":"/dev/rbd0"
|
|
# }
|
|
# ]
|
|
#
|
|
# TODO(stephenfin): Drop when we drop support for ceph 13.2.0
|
|
mappings = jsonutils.loads(out)
|
|
if isinstance(mappings, dict):
|
|
# yes, we're losing the ID field but we don't need it here
|
|
mappings = mappings.values()
|
|
|
|
for mapping in mappings:
|
|
if mapping['name'] == volume:
|
|
return mapping['device']
|
|
return None
|
|
|
|
@utils.trace
|
|
@utils.connect_volume_undo_prepare_result(unlink_after=True)
|
|
def disconnect_volume(self,
|
|
connection_properties: dict[str, Any],
|
|
device_info: dict,
|
|
force: bool = False,
|
|
ignore_errors: bool = False) -> None:
|
|
"""Disconnect a volume.
|
|
|
|
:param connection_properties: The dictionary that describes all
|
|
of the target volume attributes.
|
|
:type connection_properties: dict
|
|
:param device_info: historical difference, but same as connection_props
|
|
:type device_info: dict
|
|
"""
|
|
do_local_attach = connection_properties.get('do_local_attach',
|
|
self.do_local_attach)
|
|
if do_local_attach:
|
|
conf = device_info.get('conf') if device_info else None
|
|
root_device = self._find_root_device(connection_properties, conf)
|
|
if root_device:
|
|
# TODO(stephenfin): Update to the unified 'rbd device unmap'
|
|
# command introduced in ceph 13.0 (commit 6a57358add1157629a6d)
|
|
# when we drop support earlier versions
|
|
cmd = ['rbd', 'unmap', root_device]
|
|
cmd += self._get_rbd_args(connection_properties, conf)
|
|
self._execute(*cmd, root_helper=self._root_helper,
|
|
run_as_root=True)
|
|
if conf:
|
|
rbd_privsep.delete_if_exists(conf)
|
|
else:
|
|
if device_info:
|
|
rbd_handle = device_info.get('path', None)
|
|
if rbd_handle is not None:
|
|
fileutils.delete_if_exists(rbd_handle.rbd_conf)
|
|
rbd_handle.close()
|
|
|
|
@staticmethod
|
|
def _check_valid_device(rbd_handle: 'io.BufferedReader') -> bool:
|
|
original_offset = rbd_handle.tell()
|
|
|
|
try:
|
|
rbd_handle.read(4096)
|
|
except Exception as e:
|
|
LOG.error("Failed to access RBD device handle: %(error)s",
|
|
{"error": e})
|
|
return False
|
|
finally:
|
|
rbd_handle.seek(original_offset, 0)
|
|
|
|
return True
|
|
|
|
def check_valid_device(self,
|
|
path: Optional[str],
|
|
run_as_root: bool = True) -> bool:
|
|
"""Verify an existing RBD handle is connected and valid."""
|
|
if not path:
|
|
return False
|
|
|
|
# We can receive a file handle or a path to a device
|
|
if isinstance(path, str):
|
|
if run_as_root:
|
|
return rbd_privsep.check_valid_path(path)
|
|
else:
|
|
with open(path, 'rb') as rbd_handle:
|
|
return self._check_valid_device(rbd_handle)
|
|
|
|
# For backward compatibility ignore run_as_root param with handles
|
|
return self._check_valid_device(path)
|
|
|
|
@utils.connect_volume_undo_prepare_result
|
|
def extend_volume(self, connection_properties: dict[str, Any]) -> int:
|
|
"""Refresh local volume view and return current size in bytes."""
|
|
# Nothing to do, RBD attached volumes are automatically refreshed, but
|
|
# we need to return the new size for compatibility
|
|
do_local_attach = connection_properties.get('do_local_attach',
|
|
self.do_local_attach)
|
|
|
|
if not do_local_attach:
|
|
handle = self._get_rbd_handle(connection_properties)
|
|
try:
|
|
# Handles should return absolute position on seek, but the RBD
|
|
# wrapper doesn't, so we need to call tell afterwards
|
|
handle.seek(0, 2)
|
|
return handle.tell()
|
|
finally:
|
|
fileutils.delete_if_exists(handle.rbd_conf)
|
|
handle.close()
|
|
|
|
# Create config file when we do the attach on the host and not the VM
|
|
conf = self.create_non_openstack_config(connection_properties)
|
|
|
|
try:
|
|
device_path = self._find_root_device(connection_properties, conf)
|
|
finally:
|
|
# If we have generated the config file we need to remove it
|
|
if conf:
|
|
try:
|
|
rbd_privsep.delete_if_exists(conf)
|
|
except Exception as exc:
|
|
LOG.warning(_('Could not remove config file %(filename)s: '
|
|
'%(exc)s'), {'filename': conf, 'exc': exc})
|
|
|
|
if not device_path:
|
|
msg = _('Cannot extend non mapped device.')
|
|
raise exception.BrickException(msg=msg)
|
|
|
|
device_name = os.path.basename(device_path) # ie: rbd0
|
|
device_number = device_name[3:] # ie: 0
|
|
# Get size from /sys/devices/rbd/0/size instead of
|
|
# /sys/class/block/rbd0/size because the latter isn't updated
|
|
with open('/sys/devices/rbd/' + device_number + '/size') as f:
|
|
size_bytes = f.read().strip()
|
|
return int(size_bytes)
|