RBD: Support non OpenStack usage

The RBD connector can get a handle to a volume when we have the config
and keyring file in the host and when all the information is provided by
the Cinder driver (including the keyring).

The code for the local attach (using krbd) does not work if all the
information is provided by the Cinder driver, which is the case of
cinderlib.

This patch ensures that the local attach/detach behaves in the same way
as when we request a handle.

For security reasons, when doing a local attach for non-OpenStack usage
the configuration file which contains the keyring will be owned by root
instead of the normal user.

Related-Bug: #1883720
Change-Id: I315f474979ea5e797e3551cdc56ef136fa7a67c0
This commit is contained in:
Gorka Eguileor 2020-06-18 20:45:02 +02:00
parent 460e92c4ce
commit e67b780db1
5 changed files with 350 additions and 39 deletions

View File

@ -19,6 +19,7 @@ import tempfile
from oslo_concurrency import processutils as putils
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import excutils
from oslo_utils import fileutils
from oslo_utils import netutils
@ -27,6 +28,7 @@ from os_brick.i18n import _
from os_brick import initiator
from os_brick.initiator.connectors import base
from os_brick.initiator import linuxrbd
from os_brick.privileged import rbd as rbd_privsep
from os_brick import utils
LOG = logging.getLogger(__name__)
@ -63,14 +65,16 @@ class RBDConnector(base.BaseLinuxConnector):
# TODO(e0ne): Implement this for local volume.
return []
def _sanitize_mon_hosts(self, hosts):
@staticmethod
def _sanitize_mon_hosts(hosts):
def _sanitize_host(host):
if netutils.is_valid_ipv6(host):
host = '[%s]' % host
return host
return list(map(_sanitize_host, hosts))
def _check_or_get_keyring_contents(self, keyring, cluster_name, user):
@staticmethod
def _check_or_get_keyring_contents(keyring, cluster_name, user):
try:
if keyring is None:
if user:
@ -85,14 +89,15 @@ class RBDConnector(base.BaseLinuxConnector):
msg = (_("Keyring path %s is not readable.") % (keyring_path))
raise exception.BrickException(msg=msg)
def _create_ceph_conf(self, monitor_ips, monitor_ports,
@classmethod
def _create_ceph_conf(cls, monitor_ips, monitor_ports,
cluster_name, user, keyring):
monitors = ["%s:%s" % (ip, port) for ip, port in
zip(self._sanitize_mon_hosts(monitor_ips), monitor_ports)]
zip(cls._sanitize_mon_hosts(monitor_ips), monitor_ports)]
mon_hosts = "mon_host = %s" % (','.join(monitors))
keyring = self._check_or_get_keyring_contents(keyring, cluster_name,
user)
keyring = cls._check_or_get_keyring_contents(keyring, cluster_name,
user)
try:
fd, ceph_conf_path = tempfile.mkstemp(prefix="brickrbd_")
@ -130,7 +135,8 @@ class RBDConnector(base.BaseLinuxConnector):
return rbd_handle
def _get_rbd_args(self, connection_properties):
@classmethod
def _get_rbd_args(cls, connection_properties, conf=None):
try:
user = connection_properties['auth_username']
monitor_ips = connection_properties.get('hosts')
@ -143,10 +149,14 @@ class RBDConnector(base.BaseLinuxConnector):
if monitor_ips and monitor_ports:
monitors = ["%s:%s" % (ip, port) for ip, port in
zip(
self._sanitize_mon_hosts(monitor_ips),
cls._sanitize_mon_hosts(monitor_ips),
monitor_ports)]
for monitor in monitors:
args += ['--mon_host', monitor]
if conf:
args += ['--conf', conf]
return args
@staticmethod
@ -160,39 +170,52 @@ class RBDConnector(base.BaseLinuxConnector):
"""
return '/dev/rbd/{pool}/{volume}'.format(pool=pool, volume=volume)
@utils.trace
def connect_volume(self, connection_properties):
"""Connect to a volume.
@classmethod
def create_non_openstack_config(cls, connection_properties):
"""Get root owned Ceph's .conf file for non OpenStack usage."""
# If keyring info is missing then we are in OpenStack, nothing to do
keyring = connection_properties.get('keyring')
if not keyring:
return None
:param connection_properties: The dictionary that describes all
of the target volume attributes.
:type connection_properties: dict
:returns: dict
"""
do_local_attach = connection_properties.get('do_local_attach',
self.do_local_attach)
if do_local_attach:
# NOTE(e0ne): sanity check if ceph-common is installed.
cmd = ['which', 'rbd']
try:
self._execute(*cmd)
except putils.ProcessExecutionError:
msg = _("ceph-common package is not installed.")
LOG.error(msg)
raise exception.BrickException(message=msg)
# NOTE(e0ne): map volume to a block device
# via the rbd kernel module.
try:
user = connection_properties['auth_username']
pool, volume = connection_properties['name'].split('/')
rbd_dev_path = RBDConnector.get_rbd_device_name(pool, volume)
cluster_name = connection_properties['cluster_name']
monitor_ips = connection_properties['hosts']
monitor_ports = connection_properties['ports']
keyring = connection_properties.get('keyring')
except (KeyError, ValueError):
msg = _("Connect volume failed, malformed connection properties.")
raise exception.BrickException(msg=msg)
conf = rbd_privsep.root_create_ceph_conf(monitor_ips, monitor_ports,
str(cluster_name), user,
keyring)
return conf
def _local_attach_volume(self, connection_properties):
# NOTE(e0ne): sanity check if ceph-common is installed.
try:
self._execute('which', 'rbd')
except putils.ProcessExecutionError:
msg = _("ceph-common package is not installed.")
LOG.error(msg)
raise exception.BrickException(message=msg)
# NOTE(e0ne): map volume to a block device
# via the rbd kernel module.
pool, volume = connection_properties['name'].split('/')
rbd_dev_path = self.get_rbd_device_name(pool, volume)
# If we are not running on OpenStack, create config file
conf = self.create_non_openstack_config(connection_properties)
try:
if (
not os.path.islink(rbd_dev_path) or
not os.path.exists(os.path.realpath(rbd_dev_path))
):
cmd = ['rbd', 'map', volume, '--pool', pool]
cmd += self._get_rbd_args(connection_properties)
cmd += self._get_rbd_args(connection_properties, conf)
self._execute(*cmd, root_helper=self._root_helper,
run_as_root=True)
else:
@ -212,13 +235,37 @@ class RBDConnector(base.BaseLinuxConnector):
'more information.',
{'vol': volume, 'dev': rbd_dev_path},
)
except Exception:
# Cleanup conf file on failure
with excutils.save_and_reraise_exception():
if conf:
rbd_privsep.delete_if_exists(conf)
return {'path': rbd_dev_path, 'type': 'block'}
res = {'path': rbd_dev_path,
'type': 'block'}
if conf:
res['conf'] = conf
return res
@utils.trace
def connect_volume(self, connection_properties):
"""Connect to a volume.
:param connection_properties: The dictionary that describes all
of the target volume attributes.
:type connection_properties: dict
:returns: dict
"""
do_local_attach = connection_properties.get('do_local_attach',
self.do_local_attach)
if do_local_attach:
return self._local_attach_volume(connection_properties)
rbd_handle = self._get_rbd_handle(connection_properties)
return {'path': rbd_handle}
def _find_root_device(self, connection_properties):
def _find_root_device(self, connection_properties, conf):
"""Find the underlying /dev/rbd* device for a mapping.
Use the showmapped command to list all acive mappings and find the
@ -231,7 +278,7 @@ class RBDConnector(base.BaseLinuxConnector):
"""
__, volume = connection_properties['name'].split('/')
cmd = ['rbd', 'showmapped', '--format=json']
cmd += self._get_rbd_args(connection_properties)
cmd += self._get_rbd_args(connection_properties, conf)
(out, err) = self._execute(*cmd, root_helper=self._root_helper,
run_as_root=True)
@ -289,12 +336,15 @@ class RBDConnector(base.BaseLinuxConnector):
do_local_attach = connection_properties.get('do_local_attach',
self.do_local_attach)
if do_local_attach:
root_device = self._find_root_device(connection_properties)
conf = device_info.get('conf') if device_info else None
root_device = self._find_root_device(connection_properties, conf)
if root_device:
cmd = ['rbd', 'unmap', root_device]
cmd += self._get_rbd_args(connection_properties)
cmd += self._get_rbd_args(connection_properties, conf)
self._execute(*cmd, root_helper=self._root_helper,
run_as_root=True)
if conf:
rbd_privsep.delete_if_exists(conf)
else:
if device_info:
rbd_handle = device_info.get('path', None)

View File

@ -0,0 +1,52 @@
# Copyright (c) 2020, Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_utils import fileutils
from oslo_utils import importutils
import os_brick.privileged
# Lazy load the rbd module to avoid circular references
RBDConnector = None
def _get_rbd_class():
global RBDConnector
global get_rbd_class
# Lazy load the class
if not RBDConnector:
rbd_class_route = 'os_brick.initiator.connectors.rbd.RBDConnector'
RBDConnector = importutils.import_class(rbd_class_route)
# Job is done, following calls don't need to do anything
get_rbd_class = lambda: None # noqa
get_rbd_class = _get_rbd_class
@os_brick.privileged.default.entrypoint
def delete_if_exists(path):
return fileutils.delete_if_exists(path)
@os_brick.privileged.default.entrypoint
def root_create_ceph_conf(monitor_ips, monitor_ports, cluster_name, user,
keyring):
"""Create a .conf file for Ceph cluster only accessible by root."""
get_rbd_class()
return RBDConnector._create_ceph_conf(monitor_ips, monitor_ports,
cluster_name, user, keyring)

View File

@ -173,6 +173,24 @@ class RBDConnectorTestCase(test_connector.ConnectorTestCase):
self.assertEqual(conf_path, tmpfile)
mock_mkstemp.assert_called_once_with(prefix='brickrbd_')
@mock.patch('os_brick.privileged.rbd.root_create_ceph_conf')
def test_create_non_openstack_config(self, mock_priv_create):
res = rbd.RBDConnector.create_non_openstack_config(
self.connection_properties)
mock_priv_create.assert_called_once_with(self.hosts, self.ports,
self.clustername, self.user,
self.keyring)
self.assertIs(mock_priv_create.return_value, res)
@mock.patch('os_brick.privileged.rbd.root_create_ceph_conf')
def test_create_non_openstack_config_in_openstack(self, mock_priv_create):
connection_properties = self.connection_properties.copy()
del connection_properties['keyring']
res = rbd.RBDConnector.create_non_openstack_config(
connection_properties)
mock_priv_create.assert_not_called()
self.assertIsNone(res)
@mock.patch.object(priv_rootwrap, 'execute', return_value=None)
def test_connect_local_volume(self, mock_execute):
rbd_connector = rbd.RBDConnector(None, do_local_attach=True)
@ -239,6 +257,67 @@ class RBDConnectorTestCase(test_connector.ConnectorTestCase):
rbd_connector.connect_volume,
conn)
@mock.patch('os_brick.initiator.connectors.rbd.'
'RBDConnector._local_attach_volume')
def test_connect_volume_local(self, mock_local_attach):
connector = rbd.RBDConnector(None, do_local_attach=True)
res = connector.connect_volume(self.connection_properties)
mock_local_attach.assert_called_once_with(self.connection_properties)
self.assertIs(mock_local_attach.return_value, res)
@mock.patch.object(rbd.RBDConnector, '_get_rbd_args')
@mock.patch.object(rbd.RBDConnector, 'create_non_openstack_config')
@mock.patch.object(rbd.RBDConnector, '_execute')
def test__local_attach_volume_non_openstack(self, mock_execute,
mock_rbd_cfg, mock_args):
mock_args.return_value = [mock.sentinel.rbd_args]
connector = rbd.RBDConnector(None, do_local_attach=True)
res = connector._local_attach_volume(self.connection_properties)
mock_rbd_cfg.assert_called_once_with(self.connection_properties)
mock_args.assert_called_once_with(self.connection_properties,
mock_rbd_cfg.return_value)
self.assertEqual(2, mock_execute.call_count)
mock_execute.assert_has_calls([
mock.call('which', 'rbd'),
mock.call('rbd', 'map', 'fake_volume', '--pool', 'fake_pool',
mock.sentinel.rbd_args,
root_helper=connector._root_helper, run_as_root=True)
])
expected = {'path': '/dev/rbd/fake_pool/fake_volume',
'type': 'block',
'conf': mock_rbd_cfg.return_value}
self.assertEqual(expected, res)
@mock.patch('os_brick.privileged.rbd.delete_if_exists')
@mock.patch.object(rbd.RBDConnector, '_get_rbd_args')
@mock.patch.object(rbd.RBDConnector, 'create_non_openstack_config')
@mock.patch.object(rbd.RBDConnector, '_execute')
def test__local_attach_volume_fail_non_openstack(self, mock_execute,
mock_rbd_cfg, mock_args,
mock_delete):
mock_args.return_value = [mock.sentinel.rbd_args]
mock_execute.side_effect = [None, ValueError]
connector = rbd.RBDConnector(None, do_local_attach=True)
self.assertRaises(ValueError, connector._local_attach_volume,
self.connection_properties)
mock_rbd_cfg.assert_called_once_with(self.connection_properties)
mock_args.assert_called_once_with(self.connection_properties,
mock_rbd_cfg.return_value)
self.assertEqual(2, mock_execute.call_count)
mock_execute.assert_has_calls([
mock.call('which', 'rbd'),
mock.call('rbd', 'map', 'fake_volume', '--pool', 'fake_pool',
mock.sentinel.rbd_args,
root_helper=connector._root_helper, run_as_root=True)
])
mock_delete.assert_called_once_with(mock_rbd_cfg.return_value)
@mock.patch('os_brick.initiator.linuxrbd.rbd')
@mock.patch('os_brick.initiator.linuxrbd.rados')
@mock.patch.object(linuxrbd.RBDVolumeIOWrapper, 'close')
@ -261,8 +340,10 @@ class RBDConnectorTestCase(test_connector.ConnectorTestCase):
"1":{"pool":"pool","device":"/dev/rdb1","name":"image_2"}}
""", # old-style output
)
@mock.patch('os_brick.privileged.rbd.delete_if_exists')
@mock.patch.object(priv_rootwrap, 'execute', return_value=None)
def test_disconnect_local_volume(self, rbd_map_out, mock_execute):
def test_disconnect_local_volume(self, rbd_map_out, mock_execute,
mock_delete):
"""Test the disconnect volume case with local attach."""
rbd_connector = rbd.RBDConnector(None, do_local_attach=True)
conn = {'name': 'pool/image',
@ -282,6 +363,28 @@ class RBDConnectorTestCase(test_connector.ConnectorTestCase):
mock.call(*show_cmd, root_helper=None, run_as_root=True),
mock.call(*unmap_cmd, root_helper=None, run_as_root=True)])
mock_delete.assert_not_called()
@mock.patch('os_brick.privileged.rbd.delete_if_exists')
@mock.patch.object(rbd.RBDConnector, '_find_root_device')
@mock.patch.object(rbd.RBDConnector, '_execute')
def test_disconnect_local_volume_non_openstack(self, mock_execute,
mock_find, mock_delete):
connector = rbd.RBDConnector(None, do_local_attach=True)
mock_find.return_value = '/dev/rbd0'
connector.disconnect_volume(self.connection_properties,
{'conf': mock.sentinel.conf})
mock_find.assert_called_once_with(self.connection_properties,
mock.sentinel.conf)
mock_execute.assert_called_once_with(
'rbd', 'unmap', '/dev/rbd0', '--id', 'fake_user', '--mon_host',
'192.168.10.2:6789', '--conf', mock.sentinel.conf,
root_helper=connector._root_helper, run_as_root=True)
mock_delete.assert_called_once_with(mock.sentinel.conf)
@mock.patch.object(priv_rootwrap, 'execute', return_value=None)
def test_disconnect_local_volume_no_mapping(self, mock_execute):
rbd_connector = rbd.RBDConnector(None, do_local_attach=True)
@ -321,3 +424,37 @@ class RBDConnectorTestCase(test_connector.ConnectorTestCase):
self.assertRaises(NotImplementedError,
rbd_connector.extend_volume,
self.connection_properties)
def test__get_rbd_args(self):
res = rbd.RBDConnector._get_rbd_args(self.connection_properties, None)
expected = ['--id', self.user,
'--mon_host', self.hosts[0] + ':' + self.ports[0]]
self.assertEqual(expected, res)
def test__get_rbd_args_with_conf(self):
res = rbd.RBDConnector._get_rbd_args(self.connection_properties,
mock.sentinel.conf_path)
expected = ['--id', self.user,
'--mon_host', self.hosts[0] + ':' + self.ports[0],
'--conf', mock.sentinel.conf_path]
self.assertEqual(expected, res)
@mock.patch.object(rbd.RBDConnector, '_get_rbd_args')
@mock.patch.object(rbd.RBDConnector, '_execute')
def test_find_root_device(self, mock_execute, mock_args):
mock_args.return_value = [mock.sentinel.rbd_args]
mock_execute.return_value = (
'{"0":{"pool":"pool","device":"/dev/rdb0","name":"image"},'
'"1":{"pool":"pool","device":"/dev/rbd1","name":"fake_volume"}}',
'stderr')
connector = rbd.RBDConnector(None)
res = connector._find_root_device(self.connection_properties,
mock.sentinel.conf)
mock_args.assert_called_once_with(self.connection_properties,
mock.sentinel.conf)
mock_execute.assert_called_once_with(
'rbd', 'showmapped', '--format=json', mock.sentinel.rbd_args,
root_helper=connector._root_helper, run_as_root=True)
self.assertEqual('/dev/rbd1', res)

View File

@ -0,0 +1,67 @@
# Copyright (c) 2020, Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from unittest import mock
import os_brick.privileged as privsep_brick
import os_brick.privileged.rbd as privsep_rbd
from os_brick.tests import base
class PrivRBDTestCase(base.TestCase):
def setUp(self):
super(PrivRBDTestCase, self).setUp()
# Disable privsep server/client mode
privsep_brick.default.set_client_mode(False)
self.addCleanup(privsep_brick.default.set_client_mode, True)
@mock.patch('oslo_utils.importutils.import_class')
def test__get_rbd_class(self, mock_import):
self.assertIsNone(privsep_rbd.RBDConnector)
self.assertIs(privsep_rbd._get_rbd_class, privsep_rbd.get_rbd_class)
self.addCleanup(setattr, privsep_rbd, 'RBDConnector', None)
self.addCleanup(setattr, privsep_rbd, 'get_rbd_class',
privsep_rbd._get_rbd_class)
privsep_rbd._get_rbd_class()
mock_import.assert_called_once_with(
'os_brick.initiator.connectors.rbd.RBDConnector')
self.assertEqual(mock_import.return_value, privsep_rbd.RBDConnector)
self.assertIsNot(privsep_rbd._get_rbd_class, privsep_rbd.get_rbd_class)
@mock.patch.object(privsep_rbd, 'get_rbd_class')
@mock.patch('oslo_utils.fileutils.delete_if_exists')
def test_delete_if_exists(self, mock_delete, mock_get_class):
res = privsep_rbd.delete_if_exists(mock.sentinel.path)
mock_get_class.assert_not_called()
mock_delete.assert_called_once_with(mock.sentinel.path)
self.assertIs(mock_delete.return_value, res)
@mock.patch.object(privsep_rbd, 'get_rbd_class')
@mock.patch.object(privsep_rbd, 'RBDConnector')
def test_root_create_ceph_conf(self, mock_connector, mock_get_class):
s = mock.sentinel
res = privsep_rbd.root_create_ceph_conf(s.monitor_ips,
s.monitor_ports,
s.cluster_name, s.user,
s.keyring)
mock_get_class.assert_called_once_with()
mock_connector._create_ceph_conf.assert_called_once_with(
s.monitor_ips, s.monitor_ports, s.cluster_name, s.user, s.keyring)
self.assertIs(mock_connector._create_ceph_conf.return_value, res)

View File

@ -0,0 +1,5 @@
---
features:
- |
Add support for RBD non OpenStack (cinderlib) attach/detach.