diff --git a/manila/api/middleware/auth.py b/manila/api/middleware/auth.py index 07b3f42796..258483a199 100644 --- a/manila/api/middleware/auth.py +++ b/manila/api/middleware/auth.py @@ -26,6 +26,7 @@ import webob.exc from manila.api.openstack import wsgi from manila import context +from manila.openstack.common import jsonutils from manila.openstack.common import log as logging from manila import wsgi as base_wsgi @@ -96,11 +97,22 @@ class ManilaKeystoneContext(base_wsgi.Middleware): remote_address = req.remote_addr if CONF.use_forwarded_for: remote_address = req.headers.get('X-Forwarded-For', remote_address) + + service_catalog = None + if req.headers.get('X_SERVICE_CATALOG') is not None: + try: + catalog_header = req.headers.get('X_SERVICE_CATALOG') + service_catalog = jsonutils.loads(catalog_header) + except ValueError: + raise webob.exc.HTTPInternalServerError( + _('Invalid service catalog json.')) + ctx = context.RequestContext(user_id, project_id, roles=roles, auth_token=auth_token, - remote_address=remote_address) + remote_address=remote_address, + service_catalog=service_catalog) req.environ['manila.context'] = ctx return self.application diff --git a/manila/context.py b/manila/context.py index 1bc4e0fcc4..b798dbba01 100644 --- a/manila/context.py +++ b/manila/context.py @@ -45,7 +45,7 @@ class RequestContext(object): def __init__(self, user_id, project_id, is_admin=None, read_deleted="no", roles=None, remote_address=None, timestamp=None, request_id=None, auth_token=None, overwrite=True, - quota_class=None, **kwargs): + quota_class=None, service_catalog=None, **kwargs): """ :param read_deleted: 'no' indicates deleted records are hidden, 'yes' indicates deleted records are visible, 'only' indicates that @@ -76,6 +76,12 @@ class RequestContext(object): if isinstance(timestamp, basestring): timestamp = timeutils.parse_strtime(timestamp) self.timestamp = timestamp + if service_catalog: + self.service_catalog = [s for s in service_catalog + if s.get('type') in ('compute', 'volume')] + else: + self.service_catalog = [] + if not request_id: request_id = generate_request_id() self.request_id = request_id @@ -114,6 +120,7 @@ class RequestContext(object): 'auth_token': self.auth_token, 'quota_class': self.quota_class, 'tenant': self.tenant, + 'service_catalog': self.service_catalog, 'user': self.user} @classmethod diff --git a/manila/exception.py b/manila/exception.py index 5273415f6b..34faab0e76 100644 --- a/manila/exception.py +++ b/manila/exception.py @@ -525,3 +525,15 @@ class ShareNetworkSecurityServiceDissociationError(ManilaException): class InvalidShareNetwork(ManilaException): message = _("Invalid share network: %(reason)s") + + +class InvalidVolume(Invalid): + message = _("Invalid volume.") + + +class VolumeNotFound(NotFound): + message = _("Volume %(volume_id)s could not be found.") + + +class VolumeSnapshotNotFound(NotFound): + message = _("Snapshot %(snapshot_id)s could not be found.") diff --git a/manila/test.py b/manila/test.py index 337aa00900..b4ffba30b1 100644 --- a/manila/test.py +++ b/manila/test.py @@ -293,3 +293,12 @@ class TestCase(unittest.TestCase): self.assertTrue(isinstance(a, b)) else: f(a, b, *args, **kwargs) + + def assertIsNone(self, a, *args, **kwargs): + """Python < v2.7 compatibility.""" + try: + f = super(TestCase, self).assertIsNone + except AttributeError: + self.assertTrue(a is None) + else: + f(a, *args, **kwargs) diff --git a/manila/tests/volume/__init__.py b/manila/tests/volume/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/manila/tests/volume/test_cinder.py b/manila/tests/volume/test_cinder.py new file mode 100644 index 0000000000..3b658cd52a --- /dev/null +++ b/manila/tests/volume/test_cinder.py @@ -0,0 +1,213 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from cinderclient import exceptions as cinder_exception + +from manila import context +from manila import exception +from manila import test +from manila.volume import cinder + + +class FakeCinderClient(object): + class Volumes(object): + def get(self, volume_id): + return {'id': volume_id} + + def list(self, detailed, search_opts={}): + return [{'id': 'id1'}, {'id': 'id2'}] + + def create(self, *args, **kwargs): + return {'id': 'created_id'} + + def __getattr__(self, item): + return None + + def __init__(self): + self.volumes = self.Volumes() + self.volume_snapshots = self.volumes + + +class CinderApiTestCase(test.TestCase): + def setUp(self): + super(CinderApiTestCase, self).setUp() + + self.api = cinder.API() + self.cinderclient = FakeCinderClient() + self.ctx = context.get_admin_context() + self.stubs.Set(cinder, 'cinderclient', + mock.Mock(return_value=self.cinderclient)) + self.stubs.Set(cinder, '_untranslate_volume_summary_view', + lambda ctx, vol: vol) + self.stubs.Set(cinder, '_untranslate_snapshot_summary_view', + lambda ctx, snap: snap) + + def test_get(self): + volume_id = 'volume_id1' + result = self.api.get(self.ctx, volume_id) + self.assertEqual(result['id'], volume_id) + + def test_get_failed(self): + cinder.cinderclient.side_effect = cinder_exception.NotFound(404) + volume_id = 'volume_id' + self.assertRaises(exception.VolumeNotFound, + self.api.get, self.ctx, volume_id) + + def test_create(self): + result = self.api.create(self.ctx, 1, '', '') + self.assertEqual(result['id'], 'created_id') + + def test_create_failed(self): + cinder.cinderclient.side_effect = cinder_exception.BadRequest(400) + self.assertRaises(exception.InvalidInput, + self.api.create, self.ctx, 1, '', '') + + def test_get_all(self): + cinder._untranslate_volume_summary_view.return_value = ['id1', 'id2'] + self.assertEqual([{'id': 'id1'}, {'id': 'id2'}], + self.api.get_all(self.ctx)) + + def test_check_attach_volume_status_error(self): + volume = {'status': 'error'} + self.assertRaises(exception.InvalidVolume, + self.api.check_attach, self.ctx, volume) + + def test_check_attach_volume_already_attached(self): + volume = {'status': 'available'} + volume['attach_status'] = "attached" + self.assertRaises(exception.InvalidVolume, + self.api.check_attach, self.ctx, volume) + + def test_check_attach_availability_zone_differs(self): + volume = {'status': 'available'} + volume['attach_status'] = "detached" + instance = {'availability_zone': 'zone1'} + volume['availability_zone'] = 'zone2' + cinder.CONF.set_override('cinder_cross_az_attach', False) + self.assertRaises(exception.InvalidVolume, + self.api.check_attach, self.ctx, volume, instance) + volume['availability_zone'] = 'zone1' + self.assertIsNone(self.api.check_attach(self.ctx, volume, instance)) + cinder.CONF.reset() + + def test_check_attach(self): + volume = {'status': 'available'} + volume['attach_status'] = "detached" + volume['availability_zone'] = 'zone1' + instance = {'availability_zone': 'zone1'} + cinder.CONF.set_override('cinder_cross_az_attach', False) + self.assertIsNone(self.api.check_attach(self.ctx, volume, instance)) + cinder.CONF.reset() + + def test_check_detach(self): + volume = {'status': 'available'} + self.assertRaises(exception.InvalidVolume, + self.api.check_detach, self.ctx, volume) + volume['status'] = 'non-available' + self.assertIsNone(self.api.check_detach(self.ctx, volume)) + + def test_update(self): + self.assertRaises(NotImplementedError, + self.api.update, self.ctx, '', '') + + def test_reserve_volume(self): + self.stubs.Set(self.cinderclient.volumes, 'reserve', mock.Mock()) + self.api.reserve_volume(self.ctx, 'id1') + self.cinderclient.volumes.reserve.assert_called_once_with('id1') + + def test_unreserve_volume(self): + self.stubs.Set(self.cinderclient.volumes, 'unreserve', mock.Mock()) + self.api.unreserve_volume(self.ctx, 'id1') + self.cinderclient.volumes.unreserve.assert_called_once_with('id1') + + def test_begin_detaching(self): + self.stubs.Set(self.cinderclient.volumes, 'begin_detaching', + mock.Mock()) + self.api.begin_detaching(self.ctx, 'id1') + self.cinderclient.volumes.begin_detaching.\ + assert_called_once_with('id1') + + def test_roll_detaching(self): + self.stubs.Set(self.cinderclient.volumes, 'roll_detaching', + mock.Mock()) + self.api.roll_detaching(self.ctx, 'id1') + self.cinderclient.volumes.roll_detaching.\ + assert_called_once_with('id1') + + def test_attach(self): + self.stubs.Set(self.cinderclient.volumes, 'attach', mock.Mock()) + self.api.attach(self.ctx, 'id1', 'uuid', 'point') + self.cinderclient.volumes.attach.assert_called_once_with('id1', + 'uuid', + 'point') + + def test_detach(self): + self.stubs.Set(self.cinderclient.volumes, 'detach', mock.Mock()) + self.api.detach(self.ctx, 'id1') + self.cinderclient.volumes.detach.assert_called_once_with('id1') + + def test_initialize_connection(self): + self.stubs.Set(self.cinderclient.volumes, 'initialize_connection', + mock.Mock()) + self.api.initialize_connection(self.ctx, 'id1', 'connector') + self.cinderclient.volumes.initialize_connection.\ + assert_called_once_with('id1', 'connector') + + def test_terminate_connection(self): + self.stubs.Set(self.cinderclient.volumes, 'terminate_connection', + mock.Mock()) + self.api.terminate_connection(self.ctx, 'id1', 'connector') + self.cinderclient.volumes.terminate_connection.\ + assert_called_once_with('id1', 'connector') + + def test_delete(self): + self.stubs.Set(self.cinderclient.volumes, 'delete', mock.Mock()) + self.api.delete(self.ctx, 'id1') + self.cinderclient.volumes.delete.assert_called_once_with('id1') + + def test_get_snapshot(self): + snapshot_id = 'snapshot_id1' + result = self.api.get_snapshot(self.ctx, snapshot_id) + self.assertEqual(result['id'], snapshot_id) + + def test_get_snapshot_failed(self): + cinder.cinderclient.side_effect = cinder_exception.NotFound(404) + snapshot_id = 'snapshot_id' + self.assertRaises(exception.VolumeSnapshotNotFound, + self.api.get_snapshot, self.ctx, snapshot_id) + + def test_get_all_snapshots(self): + cinder._untranslate_snapshot_summary_view.return_value = ['id1', 'id2'] + self.assertEqual([{'id': 'id1'}, {'id': 'id2'}], + self.api.get_all_snapshots(self.ctx)) + + def test_create_snapshot(self): + result = self.api.create_snapshot(self.ctx, {'id': 'id1'}, '', '') + self.assertEqual(result['id'], 'created_id') + + def test_create_force(self): + result = self.api.create_snapshot_force(self.ctx, + {'id': 'id1'}, '', '') + self.assertEqual(result['id'], 'created_id') + + def test_delete_snapshot(self): + self.stubs.Set(self.cinderclient.volume_snapshots, + 'delete', mock.Mock()) + self.api.delete_snapshot(self.ctx, 'id1') + self.cinderclient.volume_snapshots.delete.\ + assert_called_once_with('id1') diff --git a/manila/volume/__init__.py b/manila/volume/__init__.py new file mode 100644 index 0000000000..36fdfd17f1 --- /dev/null +++ b/manila/volume/__init__.py @@ -0,0 +1,36 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2014 Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import oslo.config.cfg + +import manila.openstack.common.importutils + +_volume_opts = [ + oslo.config.cfg.StrOpt('volume_api_class', + default='manila.volume.cinder.API', + help='The full class name of the ' + 'volume API class to use'), +] + +oslo.config.cfg.CONF.register_opts(_volume_opts) + + +def API(): + importutils = manila.openstack.common.importutils + volume_api_class = oslo.config.cfg.CONF.volume_api_class + cls = importutils.import_class(volume_api_class) + return cls() diff --git a/manila/volume/cinder.py b/manila/volume/cinder.py new file mode 100644 index 0000000000..1693ccb7d2 --- /dev/null +++ b/manila/volume/cinder.py @@ -0,0 +1,353 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2014 Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Handles all requests relating to volumes + cinder. +""" + +import copy +import sys + +from cinderclient import exceptions as cinder_exception +from cinderclient import service_catalog +from cinderclient.v1 import client as cinder_client +from oslo.config import cfg + +from manila.db import base +from manila import exception +from manila.openstack.common.gettextutils import _ +from manila.openstack.common import log as logging + + +cinder_opts = [ + cfg.StrOpt('cinder_catalog_info', + default='volume:cinder:publicURL', + help='Info to match when looking for cinder in the service ' + 'catalog. Format is : separated values of the form: ' + '::'), + cfg.StrOpt('os_region_name', + help='region name of this node'), + cfg.StrOpt('cinder_ca_certificates_file', + help='Location of ca certificates file to use for cinder ' + 'client requests.'), + cfg.IntOpt('cinder_http_retries', + default=3, + help='Number of cinderclient retries on failed http calls'), + cfg.BoolOpt('cinder_api_insecure', + default=False, + help='Allow to perform insecure SSL requests to cinder'), + cfg.BoolOpt('cinder_cross_az_attach', + default=True, + help='Allow attach between instance and volume in different ' + 'availability zones.'), + cfg.StrOpt('cinder_admin_username', + default='cinder', + help='Cinder admin username'), + cfg.StrOpt('cinder_admin_password', + help='Cinder admin password'), + cfg.StrOpt('cinder_admin_tenant_name', + default='service', + help='Cinder admin tenant name'), + cfg.StrOpt('cinder_admin_auth_url', + default='http://localhost:5000/v2.0', + help='Identity service url') +] + +CONF = cfg.CONF +CONF.register_opts(cinder_opts) + +LOG = logging.getLogger(__name__) + + +def cinderclient(context): + if context.is_admin and context.project_id is None: + c = cinder_client.Client(CONF.cinder_admin_username, + CONF.cinder_admin_password, + CONF.cinder_admin_tenant_name, + CONF.cinder_admin_auth_url) + c.authenticate() + return c + + compat_catalog = { + 'access': {'serviceCatalog': context.service_catalog or []} + } + sc = service_catalog.ServiceCatalog(compat_catalog) + info = CONF.cinder_catalog_info + service_type, service_name, endpoint_type = info.split(':') + # extract the region if set in configuration + if CONF.os_region_name: + attr = 'region' + filter_value = CONF.os_region_name + else: + attr = None + filter_value = None + url = sc.url_for(attr=attr, + filter_value=filter_value, + service_type=service_type, + service_name=service_name, + endpoint_type=endpoint_type) + + LOG.debug(_('Cinderclient connection created using URL: %s') % url) + + c = cinder_client.Client(context.user_id, + context.auth_token, + project_id=context.project_id, + auth_url=url, + insecure=CONF.cinder_api_insecure, + retries=CONF.cinder_http_retries, + cacert=CONF.cinder_ca_certificates_file) + # noauth extracts user_id:project_id from auth_token + c.client.auth_token = context.auth_token or '%s:%s' % (context.user_id, + context.project_id) + c.client.management_url = url + return c + + +def _untranslate_volume_summary_view(context, vol): + """Maps keys for volumes summary view.""" + d = {} + d['id'] = vol.id + d['status'] = vol.status + d['size'] = vol.size + d['availability_zone'] = vol.availability_zone + d['created_at'] = vol.created_at + + d['attach_time'] = "" + d['mountpoint'] = "" + + if vol.attachments: + att = vol.attachments[0] + d['attach_status'] = 'attached' + d['instance_uuid'] = att['server_id'] + d['mountpoint'] = att['device'] + else: + d['attach_status'] = 'detached' + + d['display_name'] = vol.display_name + d['display_description'] = vol.display_description + + d['volume_type_id'] = vol.volume_type + d['snapshot_id'] = vol.snapshot_id + + d['volume_metadata'] = {} + for key, value in vol.metadata.items(): + d['volume_metadata'][key] = value + + if hasattr(vol, 'volume_image_metadata'): + d['volume_image_metadata'] = copy.deepcopy(vol.volume_image_metadata) + + return d + + +def _untranslate_snapshot_summary_view(context, snapshot): + """Maps keys for snapshots summary view.""" + d = {} + + d['id'] = snapshot.id + d['status'] = snapshot.status + d['progress'] = snapshot.progress + d['size'] = snapshot.size + d['created_at'] = snapshot.created_at + d['display_name'] = snapshot.display_name + d['display_description'] = snapshot.display_description + d['volume_id'] = snapshot.volume_id + d['project_id'] = snapshot.project_id + d['volume_size'] = snapshot.size + + return d + + +def translate_volume_exception(method): + """Transforms the exception for the volume but keeps its traceback intact. + """ + def wrapper(self, ctx, volume_id, *args, **kwargs): + try: + res = method(self, ctx, volume_id, *args, **kwargs) + except cinder_exception.ClientException: + exc_type, exc_value, exc_trace = sys.exc_info() + if isinstance(exc_value, cinder_exception.NotFound): + exc_value = exception.VolumeNotFound(volume_id=volume_id) + elif isinstance(exc_value, cinder_exception.BadRequest): + exc_value = exception.InvalidInput(reason=exc_value.message) + raise exc_value, None, exc_trace + return res + return wrapper + + +def translate_snapshot_exception(method): + """Transforms the exception for the snapshot but keeps its traceback + intact. + """ + def wrapper(self, ctx, snapshot_id, *args, **kwargs): + try: + res = method(self, ctx, snapshot_id, *args, **kwargs) + except cinder_exception.ClientException: + exc_type, exc_value, exc_trace = sys.exc_info() + if isinstance(exc_value, cinder_exception.NotFound): + exc_value = exception.\ + VolumeSnapshotNotFound(snapshot_id=snapshot_id) + raise exc_value, None, exc_trace + return res + return wrapper + + +class API(base.Base): + """API for interacting with the volume manager.""" + @translate_volume_exception + def get(self, context, volume_id): + item = cinderclient(context).volumes.get(volume_id) + return _untranslate_volume_summary_view(context, item) + + def get_all(self, context, search_opts={}): + items = cinderclient(context).volumes.list(detailed=True, + search_opts=search_opts) + rval = [] + + for item in items: + rval.append(_untranslate_volume_summary_view(context, item)) + + return rval + + def check_attached(self, context, volume): + """Raise exception if volume in use.""" + if volume['status'] != "in-use": + msg = _("status must be 'in-use'") + raise exception.InvalidVolume(reason=msg) + + def check_attach(self, context, volume, instance=None): + if volume['status'] != "available": + msg = _("status must be 'available'") + raise exception.InvalidVolume(reason=msg) + if volume['attach_status'] == "attached": + msg = _("already attached") + raise exception.InvalidVolume(reason=msg) + if instance and not CONF.cinder_cross_az_attach: + if instance['availability_zone'] != volume['availability_zone']: + msg = _("Instance and volume not in same availability_zone") + raise exception.InvalidVolume(reason=msg) + + def check_detach(self, context, volume): + if volume['status'] == "available": + msg = _("already detached") + raise exception.InvalidVolume(reason=msg) + + @translate_volume_exception + def reserve_volume(self, context, volume_id): + cinderclient(context).volumes.reserve(volume_id) + + @translate_volume_exception + def unreserve_volume(self, context, volume_id): + cinderclient(context).volumes.unreserve(volume_id) + + @translate_volume_exception + def begin_detaching(self, context, volume_id): + cinderclient(context).volumes.begin_detaching(volume_id) + + @translate_volume_exception + def roll_detaching(self, context, volume_id): + cinderclient(context).volumes.roll_detaching(volume_id) + + @translate_volume_exception + def attach(self, context, volume_id, instance_uuid, mountpoint): + cinderclient(context).volumes.attach(volume_id, instance_uuid, + mountpoint) + + @translate_volume_exception + def detach(self, context, volume_id): + cinderclient(context).volumes.detach(volume_id) + + @translate_volume_exception + def initialize_connection(self, context, volume_id, connector): + return cinderclient(context).volumes.initialize_connection(volume_id, + connector) + + @translate_volume_exception + def terminate_connection(self, context, volume_id, connector): + return cinderclient(context).volumes.terminate_connection(volume_id, + connector) + + def create(self, context, size, name, description, snapshot=None, + image_id=None, volume_type=None, metadata=None, + availability_zone=None): + + if snapshot is not None: + snapshot_id = snapshot['id'] + else: + snapshot_id = None + + kwargs = dict(snapshot_id=snapshot_id, + display_name=name, + display_description=description, + volume_type=volume_type, + user_id=context.user_id, + project_id=context.project_id, + availability_zone=availability_zone, + metadata=metadata, + imageRef=image_id) + + try: + item = cinderclient(context).volumes.create(size, **kwargs) + return _untranslate_volume_summary_view(context, item) + except cinder_exception.BadRequest as e: + raise exception.InvalidInput(reason=e.message) + + @translate_volume_exception + def delete(self, context, volume_id): + cinderclient(context).volumes.delete(volume_id) + + @translate_volume_exception + def update(self, context, volume_id, fields): + raise NotImplementedError() + + def get_volume_encryption_metadata(self, context, volume_id): + return cinderclient(context).volumes.get_encryption_metadata(volume_id) + + @translate_snapshot_exception + def get_snapshot(self, context, snapshot_id): + item = cinderclient(context).volume_snapshots.get(snapshot_id) + return _untranslate_snapshot_summary_view(context, item) + + def get_all_snapshots(self, context, search_opts=None): + items = cinderclient(context).volume_snapshots.list(detailed=True, + search_opts=search_opts) + rvals = [] + + for item in items: + rvals.append(_untranslate_snapshot_summary_view(context, item)) + + return rvals + + @translate_volume_exception + def create_snapshot(self, context, volume_id, name, description): + item = cinderclient(context).volume_snapshots.create(volume_id, + False, + name, + description) + return _untranslate_snapshot_summary_view(context, item) + + @translate_volume_exception + def create_snapshot_force(self, context, volume_id, name, description): + item = cinderclient(context).volume_snapshots.create(volume_id, + True, + name, + description) + + return _untranslate_snapshot_summary_view(context, item) + + @translate_snapshot_exception + def delete_snapshot(self, context, snapshot_id): + cinderclient(context).volume_snapshots.delete(snapshot_id) diff --git a/requirements.txt b/requirements.txt index 00a7ba1c21..ec9ecc2deb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ Routes>=1.12.3 SQLAlchemy>=0.7.8,<=0.7.99 sqlalchemy-migrate>=0.7.2 stevedore>=0.10 +python-cinderclient>=1.0.6 suds>=0.4 WebOb>=1.2.3,<1.3 wsgiref>=0.1.2