openstack/cinder
Code Issues Proposed changes

Schedule request to scheduler when create group from resource

Browse Source 
Pass the request to scheduler rather than volume service in
order to check the backend's capacity.

Change-Id: Ie4c157f11e5fde0c2dd1d3e06feb0caa9d2d9ede
Partial-Implements: bp inspection-mechanism-for-capacity-limited-host
This commit is contained in:
TommyLike 2017-11-03 14:57:03 +08:00
parent 1491af0f95
commit d812f5705f
7 changed files with 118 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
cinder/group/api.py
View File

@ -170,15 +170,22 @@ class API(base.Base):
# Populate group_type_id and volume_type_ids
group_type_id = None
volume_type_ids = []
size = 0
if group_snapshot_id:
grp_snap = self.get_group_snapshot(context, group_snapshot_id)
group_type_id = grp_snap.group_type_id
grp_snap_src_grp = self.get(context, grp_snap.group_id)
volume_type_ids = [vt.id for vt in grp_snap_src_grp.volume_types]
snapshots = objects.SnapshotList.get_all_for_group_snapshot(
context, group_snapshot_id)
size = sum(s.volume.size for s in snapshots)
elif source_group_id:
source_group = self.get(context, source_group_id)
group_type_id = source_group.group_type_id
volume_type_ids = [vt.id for vt in source_group.volume_types]
source_vols = objects.VolumeList.get_all_by_generic_group(
context, source_group.id)
size = sum(v.size for v in source_vols)
kwargs = {
'user_id': context.user_id,
@ -226,8 +233,15 @@ class API(base.Base):
# Update quota for groups
GROUP_QUOTAS.commit(context, reservations)
if not group.host:
msg = _("No host to create group %s.") % group.id
# NOTE(tommylikehu): We wrap the size inside of the attribute
# 'volume_properties' as scheduler's filter logic are all designed
# based on this attribute.
kwargs = {'group_id': group.id,
'volume_properties': objects.VolumeProperties(size=size)}
if not group.host or not self.scheduler_rpcapi.validate_host_capacity(
context, group.host, objects.RequestSpec(**kwargs)):
msg = _("No valid host to create group %s.") % group.id
LOG.error(msg)
raise exception.InvalidGroup(reason=msg)

15
cinder/scheduler/filter_scheduler.py
View File

@ -123,11 +123,16 @@ class FilterScheduler(driver.Scheduler):
if backend_id == backend:
return weighed_backend.obj
volume_id = request_spec.get('volume_id', '??volume_id missing??')
raise exception.NoValidBackend(reason=_('Cannot place volume %(id)s '
'on %(backend)s') %
{'id': volume_id,
'backend': backend})
reason_param = {'resource': 'volume',
'id': '??id missing??',
'backend': backend}
for resource in ['volume', 'group']:
resource_id = request_spec.get('%s_id' % resource, None)
if resource_id:
reason_param.update({'resource': resource, 'id': resource_id})
break
raise exception.NoValidBackend(_('Cannot place %(resource)s %(id)s '
'on %(backend)s.') % reason_param)
def find_retype_backend(self, context, request_spec,
filter_properties=None, migration_policy='never'):

15
cinder/scheduler/manager.py
View File

@ -335,6 +335,21 @@ class SchedulerManager(manager.CleanableManager, manager.Manager):
"""
return self.driver.get_pools(context, filters)
def validate_host_capacity(self, context, backend, request_spec,
filter_properties):
try:
backend_state = self.driver.backend_passes_filters(
context,
backend,
request_spec, filter_properties)
backend_state.consume_from_volume(
{'size': request_spec['volume_properties']['size']})
except exception.NoValidBackend:
LOG.error("Desired host %(host)s does not have enough "
"capacity.", {'host': backend})
return False
return True
def extend_volume(self, context, volume, new_size, reservations,
request_spec=None, filter_properties=None):

11
cinder/scheduler/rpcapi.py
View File

@ -68,9 +68,10 @@ class SchedulerAPI(rpc.RPCAPI):
3.5 - Make notify_service_capabilities support A/A
3.6 - Removed create_consistencygroup method
3.7 - Adds set_log_levels and get_log_levels
3.8 - Addds ``valid_host_capacity`` method
"""
RPC_API_VERSION = '3.7'
RPC_API_VERSION = '3.8'
RPC_DEFAULT_VERSION = '3.0'
TOPIC = constants.SCHEDULER_TOPIC
BINARY = 'cinder-scheduler'
@ -100,6 +101,14 @@ class SchedulerAPI(rpc.RPCAPI):
'filter_properties': filter_properties, 'volume': volume}
return cctxt.cast(ctxt, 'create_volume', **msg_args)
@rpc.assert_min_rpc_version('3.8')
def validate_host_capacity(self, ctxt, backend, request_spec,
filter_properties=None):
msg_args = {'request_spec': request_spec,
'filter_properties': filter_properties, 'backend': backend}
cctxt = self._get_cctxt()
return cctxt.call(ctxt, 'validate_host_capacity', **msg_args)
def migrate_volume(self, ctxt, volume, backend, force_copy=False,
request_spec=None, filter_properties=None):
request_spec_p = jsonutils.to_primitive(request_spec)

29
cinder/tests/unit/api/contrib/test_consistencygroups.py
View File

@ -1143,7 +1143,9 @@ class ConsistencyGroupsAPITestCase(test.TestCase):
@mock.patch(
'cinder.api.openstack.wsgi.Controller.validate_name_and_description')
def test_create_consistencygroup_from_src_snap(self, mock_validate):
@mock.patch('cinder.scheduler.rpcapi.SchedulerAPI.validate_host_capacity')
def test_create_consistencygroup_from_src_snap(self, mock_validate_host,
mock_validate):
self.mock_object(volume_api.API, "create", v2_fakes.fake_volume_create)
consistencygroup = utils.create_group(
@ -1161,6 +1163,7 @@ class ConsistencyGroupsAPITestCase(test.TestCase):
volume_id,
group_snapshot_id=cgsnapshot.id,
status=fields.SnapshotStatus.AVAILABLE)
mock_validate_host.return_value = True
test_cg_name = 'test cg'
body = {"consistencygroup-from-src": {"name": test_cg_name,
@ -1190,7 +1193,8 @@ class ConsistencyGroupsAPITestCase(test.TestCase):
consistencygroup.destroy()
cgsnapshot.destroy()
def test_create_consistencygroup_from_src_cg(self):
@mock.patch('cinder.scheduler.rpcapi.SchedulerAPI.validate_host_capacity')
def test_create_consistencygroup_from_src_cg(self, mock_validate):
self.mock_object(volume_api.API, "create", v2_fakes.fake_volume_create)
source_cg = utils.create_group(
@ -1200,6 +1204,7 @@ class ConsistencyGroupsAPITestCase(test.TestCase):
volume_id = utils.create_volume(
self.ctxt,
group_id=source_cg.id)['id']
mock_validate.return_value = True
test_cg_name = 'test cg'
body = {"consistencygroup-from-src": {"name": test_cg_name,
@ -1343,7 +1348,7 @@ class ConsistencyGroupsAPITestCase(test.TestCase):
self.assertEqual(http_client.BAD_REQUEST, res.status_int)
self.assertEqual(http_client.BAD_REQUEST,
res_dict['badRequest']['code'])
msg = _('Invalid Group: No host to create group')
msg = _('Invalid Group: No valid host to create group')
self.assertIn(msg, res_dict['badRequest']['message'])
snapshot.destroy()
@ -1351,7 +1356,9 @@ class ConsistencyGroupsAPITestCase(test.TestCase):
consistencygroup.destroy()
cgsnapshot.destroy()
def test_create_consistencygroup_from_src_cgsnapshot_empty(self):
@mock.patch('cinder.scheduler.rpcapi.SchedulerAPI.validate_host_capacity')
def test_create_consistencygroup_from_src_cgsnapshot_empty(self,
mock_validate):
consistencygroup = utils.create_group(
self.ctxt, group_type_id=fake.GROUP_TYPE_ID,
volume_type_ids=[fake.VOLUME_TYPE_ID],)
@ -1361,6 +1368,7 @@ class ConsistencyGroupsAPITestCase(test.TestCase):
cgsnapshot = utils.create_group_snapshot(
self.ctxt, group_id=consistencygroup.id,
group_type_id=fake.GROUP_TYPE_ID,)
mock_validate.return_value = True
test_cg_name = 'test cg'
body = {"consistencygroup-from-src": {"name": test_cg_name,
@ -1385,10 +1393,13 @@ class ConsistencyGroupsAPITestCase(test.TestCase):
consistencygroup.destroy()
cgsnapshot.destroy()
def test_create_consistencygroup_from_src_source_cg_empty(self):
@mock.patch('cinder.scheduler.rpcapi.SchedulerAPI.validate_host_capacity')
def test_create_consistencygroup_from_src_source_cg_empty(self,
mock_validate):
source_cg = utils.create_group(
self.ctxt, group_type_id=fake.GROUP_TYPE_ID,
volume_type_ids=[fake.VOLUME_TYPE_ID],)
mock_validate.return_value = True
test_cg_name = 'test cg'
body = {"consistencygroup-from-src": {"name": test_cg_name,
@ -1472,8 +1483,9 @@ class ConsistencyGroupsAPITestCase(test.TestCase):
@mock.patch.object(volume_api.API, 'create',
side_effect=exception.CinderException(
'Create volume failed.'))
@mock.patch('cinder.scheduler.rpcapi.SchedulerAPI.validate_host_capacity')
def test_create_consistencygroup_from_src_cgsnapshot_create_volume_failed(
self, mock_create):
self, mock_validate, mock_create):
consistencygroup = utils.create_group(
self.ctxt, group_type_id=fake.GROUP_TYPE_ID,
volume_type_ids=[fake.VOLUME_TYPE_ID],)
@ -1488,6 +1500,7 @@ class ConsistencyGroupsAPITestCase(test.TestCase):
volume_id,
group_snapshot_id=cgsnapshot.id,
status=fields.SnapshotStatus.AVAILABLE)
mock_validate.return_value = True
test_cg_name = 'test cg'
body = {"consistencygroup-from-src": {"name": test_cg_name,
@ -1517,14 +1530,16 @@ class ConsistencyGroupsAPITestCase(test.TestCase):
@mock.patch.object(volume_api.API, 'create',
side_effect=exception.CinderException(
'Create volume failed.'))
@mock.patch('cinder.scheduler.rpcapi.SchedulerAPI.validate_host_capacity')
def test_create_consistencygroup_from_src_cg_create_volume_failed(
self, mock_create):
self, mock_validate, mock_create):
source_cg = utils.create_group(
self.ctxt, group_type_id=fake.GROUP_TYPE_ID,
volume_type_ids=[fake.VOLUME_TYPE_ID],)
volume_id = utils.create_volume(
self.ctxt,
group_id=source_cg.id)['id']
mock_validate.return_value = True
test_cg_name = 'test cg'
body = {"consistencygroup-from-src": {"name": test_cg_name,

56
cinder/tests/unit/api/v3/test_groups.py
View File

@ -1043,9 +1043,12 @@ class GroupsAPITestCase(test.TestCase):
self.assertEqual(http_client.ACCEPTED, response.status_int)
self.assertEqual(fields.GroupStatus.AVAILABLE, group.status)
@ddt.data(True, False)
@mock.patch(
'cinder.api.openstack.wsgi.Controller.validate_name_and_description')
def test_create_group_from_src_snap(self, mock_validate):
@mock.patch('cinder.scheduler.rpcapi.SchedulerAPI.validate_host_capacity')
def test_create_group_from_src_snap(self, valid_host, mock_validate_host,
mock_validate):
self.mock_object(volume_api.API, "create", v3_fakes.fake_volume_create)
group = utils.create_group(self.ctxt,
@ -1064,6 +1067,7 @@ class GroupsAPITestCase(test.TestCase):
group_snapshot_id=group_snapshot.id,
status=fields.SnapshotStatus.AVAILABLE,
volume_type_id=volume.volume_type_id)
mock_validate_host.return_value = valid_host
test_grp_name = 'test grp'
body = {"create-from-src": {"name": test_grp_name,
@ -1072,22 +1076,30 @@ class GroupsAPITestCase(test.TestCase):
req = fakes.HTTPRequest.blank('/v3/%s/groups/action' %
fake.PROJECT_ID,
version=mv.GROUP_SNAPSHOTS)
res_dict = self.controller.create_from_src(req, body)
self.assertIn('id', res_dict['group'])
self.assertEqual(test_grp_name, res_dict['group']['name'])
self.assertTrue(mock_validate.called)
grp_ref = objects.Group.get_by_id(
self.ctxt.elevated(), res_dict['group']['id'])
if valid_host:
res_dict = self.controller.create_from_src(req, body)
self.assertIn('id', res_dict['group'])
self.assertEqual(test_grp_name, res_dict['group']['name'])
self.assertTrue(mock_validate.called)
grp_ref = objects.Group.get_by_id(
self.ctxt.elevated(), res_dict['group']['id'])
else:
self.assertRaises(webob.exc.HTTPBadRequest,
self.controller.create_from_src, req, body)
groups = objects.GroupList.get_all_by_project(self.ctxt,
fake.PROJECT_ID)
grp_ref = objects.Group.get_by_id(
self.ctxt.elevated(), groups[0]['id'])
grp_ref.destroy()
snapshot.destroy()
volume.destroy()
group.destroy()
group_snapshot.destroy()
def test_create_group_from_src_grp(self):
@ddt.data(True, False)
@mock.patch('cinder.scheduler.rpcapi.SchedulerAPI.validate_host_capacity')
def test_create_group_from_src_grp(self, host_valid, mock_validate_host):
self.mock_object(volume_api.API, "create", v3_fakes.fake_volume_create)
source_grp = utils.create_group(self.ctxt,
@ -1097,6 +1109,7 @@ class GroupsAPITestCase(test.TestCase):
self.ctxt,
group_id=source_grp.id,
volume_type_id=fake.VOLUME_TYPE_ID)
mock_validate_host.return_value = host_valid
test_grp_name = 'test cg'
body = {"create-from-src": {"name": test_grp_name,
@ -1105,13 +1118,22 @@ class GroupsAPITestCase(test.TestCase):
req = fakes.HTTPRequest.blank('/v3/%s/groups/action' %
fake.PROJECT_ID,
version=mv.GROUP_SNAPSHOTS)
res_dict = self.controller.create_from_src(req, body)
self.assertIn('id', res_dict['group'])
self.assertEqual(test_grp_name, res_dict['group']['name'])
grp = objects.Group.get_by_id(
self.ctxt, res_dict['group']['id'])
if host_valid:
res_dict = self.controller.create_from_src(req, body)
self.assertIn('id', res_dict['group'])
self.assertEqual(test_grp_name, res_dict['group']['name'])
grp = objects.Group.get_by_id(
self.ctxt, res_dict['group']['id'])
grp.destroy()
volume.destroy()
source_grp.destroy()
else:
self.assertRaises(webob.exc.HTTPBadRequest,
self.controller.create_from_src, req, body)
groups = objects.GroupList.get_all_by_project(self.ctxt,
fake.PROJECT_ID)
grp = objects.Group.get_by_id(
self.ctxt.elevated(), groups[0]['id'])
grp.destroy()
volume.destroy()
source_grp.destroy()

8
cinder/tests/unit/group/test_groups_api.py
View File

@ -515,7 +515,8 @@ class GroupAPITestCase(test.TestCase):
@mock.patch('cinder.group.api.API.update_quota')
@mock.patch('cinder.objects.GroupSnapshot.get_by_id')
@mock.patch('cinder.objects.SnapshotList.get_all_for_group_snapshot')
def test_create_from_src(self, mock_snap_get_all,
@mock.patch('cinder.scheduler.rpcapi.SchedulerAPI.validate_host_capacity')
def test_create_from_src(self, mock_validate_host, mock_snap_get_all,
mock_group_snap_get, mock_update_quota,
mock_create_from_group,
mock_create_from_snap):
@ -537,6 +538,7 @@ class GroupAPITestCase(test.TestCase):
volume_type_id=fake.VOLUME_TYPE_ID,
status=fields.SnapshotStatus.AVAILABLE)
mock_snap_get_all.return_value = [snap]
mock_validate_host.return_host = True
grp_snap = utils.create_group_snapshot(
self.ctxt, grp.id,
@ -584,10 +586,12 @@ class GroupAPITestCase(test.TestCase):
mock_group_snapshot.update.assert_called_once_with(update_field)
mock_group_snapshot.save.assert_called_once_with()
def test_create_group_from_src_frozen(self):
@mock.patch('cinder.scheduler.rpcapi.SchedulerAPI.validate_host_capacity')
def test_create_group_from_src_frozen(self, mock_validate_host):
service = utils.create_service(self.ctxt, {'frozen': True})
group = utils.create_group(self.ctxt, host=service.host,
group_type_id='gt')
mock_validate_host.return_value = True
group_api = cinder.group.api.API()
self.assertRaises(exception.InvalidInput,
group_api.create_from_src,