openstack/nova
Code Issues Proposed changes

Add request_spec.RequestGroup versioned object

Browse Source 
Later patches will introduce a field in RequestSpec using this type as
the field type to store the resource requests coming from outside of
Nova like the bandwidth request coming from the Neutron ports.

This patch refactors the usage of placement.lib.RequestGroup. Until now
this class was used both by placement and nova services and they used
it only as a util class. However after this series the nova services
would like to use such a class via RPC which requires an OVO. This
patch makes sure that the new OVO is used by nova and the old plain
object is used by placement. This way placement is not forced to use
an OVO where no OVO functionality is required.

The minimum required version of oslo.versionedobjects is updated to
1.33.3 to include the fix for bug 1771804.

Change-Id: I46c97d2641d9685ef59771314665a17a5236097d
blueprint: bandwidth-resource-provider
This commit is contained in:
Balazs Gibizer 2018-05-16 11:53:06 +02:00 committed by Matt Riedemann
parent c64b03d218
commit d4f8974f87
12 changed files with 122 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lower-constraints.txt
View File

@ -91,7 +91,7 @@ oslo.rootwrap==5.8.0
oslo.serialization==2.18.0
oslo.service==1.24.0
oslo.utils==3.37.0
oslo.versionedobjects==1.31.2
oslo.versionedobjects==1.33.3
oslo.vmware==2.17.0
oslotest==3.2.0
osprofiler==1.4.0

2
nova/api/openstack/placement/lib.py
View File

@ -45,7 +45,7 @@ class RequestGroup(object):
for rc, amount in sorted(list(self.resources.items())))
ret += ', traits=[%s]' % ', '.join(
sorted(self.required_traits) +
['!%s' % ft for ft in self.forbidden_traits])
['!%s' % ft for ft in sorted(self.forbidden_traits)])
ret += ', aggregates=[%s]' % ', '.join(
sorted('[%s]' % ', '.join(agglist)
for agglist in sorted(self.member_of)))

4
nova/objects/fields.py
View File

@ -1240,3 +1240,7 @@ class InstanceTaskStateField(BaseEnumField):
class InstancePowerStateField(BaseEnumField):
AUTO_TYPE = InstancePowerState()
class ListOfListsOfStringsField(fields.AutoTypedField):
AUTO_TYPE = List(List(fields.String()))

26
nova/objects/request_spec.py
View File

@ -850,3 +850,29 @@ class SchedulerLimits(base.NovaObject):
if getattr(self, field) is not None:
limits[field] = getattr(self, field)
return limits
@base.NovaObjectRegistry.register
class RequestGroup(base.NovaObject):
"""Versioned object based on the unversioned
nova.api.openstack.placement.lib.RequestGroup object.
"""
VERSION = '1.0'
fields = {
'use_same_provider': fields.BooleanField(default=True),
'resources': fields.DictOfIntegersField(default={}),
'required_traits': fields.SetOfStringsField(default=set()),
'forbidden_traits': fields.SetOfStringsField(default=set()),
# The aggregates field has a form of
# [[aggregate_UUID1],
# [aggregate_UUID2, aggregate_UUID3]]
# meaning that the request should be fulfilled from an RP that is a
# member of the aggregate aggregate_UUID1 and member of the aggregate
# aggregate_UUID2 or aggregate_UUID3 .
'aggregates': fields.ListOfListsOfStringsField(default=[]),
}
def __init__(self, context=None, **kwargs):
super(RequestGroup, self).__init__(context=context, **kwargs)
self.obj_set_defaults()

11
nova/scheduler/utils.py
View File

@ -24,7 +24,6 @@ import oslo_messaging as messaging
from oslo_serialization import jsonutils
from six.moves.urllib import parse
from nova.api.openstack.placement import lib as placement_lib
from nova.compute import flavors
from nova.compute import utils as compute_utils
import nova.conf
@ -69,7 +68,7 @@ class ResourceRequest(object):
def get_request_group(self, ident):
if ident not in self._rg_by_id:
rq_grp = placement_lib.RequestGroup(use_same_provider=bool(ident))
rq_grp = objects.RequestGroup(use_same_provider=bool(ident))
self._rg_by_id[ident] = rq_grp
return self._rg_by_id[ident]
@ -245,13 +244,15 @@ class ResourceRequest(object):
"""Produce a querystring of the form expected by
GET /allocation_candidates.
"""
# TODO(gibi): We have a RequestGroup OVO so we can move this to that
# class as a member function.
# NOTE(efried): The sorting herein is not necessary for the API; it is
# to make testing easier and logging/debugging predictable.
def to_queryparams(request_group, suffix):
res = request_group.resources
required_traits = request_group.required_traits
forbidden_traits = request_group.forbidden_traits
aggregates = request_group.member_of
aggregates = request_group.aggregates
resource_query = ",".join(
sorted("%s:%s" % (rc, amount)
@ -457,8 +458,8 @@ def resources_from_request_spec(spec_obj):
destination = spec_obj.requested_destination
if destination and destination.aggregates:
grp = res_req.get_request_group(None)
grp.member_of = [tuple(ored.split(','))
for ored in destination.aggregates]
grp.aggregates = [ored.split(',')
for ored in destination.aggregates]
# Don't limit allocation candidates when using force_hosts or force_nodes.
if 'force_hosts' in spec_obj and spec_obj.force_hosts:

4
nova/tests/functional/api/openstack/placement/db/test_allocation_candidates.py
View File

@ -1993,7 +1993,7 @@ class AllocationCandidatesTestCase(tb.PlacementDbBaseTestCase):
fields.ResourceClass.MEMORY_MB: 256,
fields.ResourceClass.SRIOV_NET_VF: 1,
},
required_traits=[os_traits.HW_NIC_OFFLOAD_GENEVE],
required_traits=set([os_traits.HW_NIC_OFFLOAD_GENEVE]),
)}
)
@ -2128,7 +2128,7 @@ class AllocationCandidatesTestCase(tb.PlacementDbBaseTestCase):
fields.ResourceClass.MEMORY_MB: 256,
fields.ResourceClass.SRIOV_NET_VF: 1,
},
required_traits=[os_traits.HW_NIC_OFFLOAD_GENEVE],
required_traits=set([os_traits.HW_NIC_OFFLOAD_GENEVE]),
)}
)

29
nova/tests/unit/api/openstack/placement/test_util.py
View File

@ -904,6 +904,35 @@ class TestPickLastModified(testtools.TestCase):
mock_utc.assert_called_once_with(with_timezone=True)
class TestRequestGroup(testtools.TestCase):
def test_stringification(self):
grp = pl.RequestGroup(
resources={
'VCPU': 2,
'CUSTOM_MAGIC': 1,
},
required_traits={
'CUSTOM_VNIC_TYPE_NORMAL',
'CUSTOM_PHYSNET1',
},
forbidden_traits={
'CUSTOM_PHYSNET2',
'CUSTOM_VNIC_TYPE_DIRECT'
},
member_of=[
['baz'],
['foo', 'bar']
]
)
self.assertEqual(
'RequestGroup(use_same_provider=True, '
'resources={CUSTOM_MAGIC:1, VCPU:2}, '
'traits=[CUSTOM_PHYSNET1, CUSTOM_VNIC_TYPE_NORMAL, '
'!CUSTOM_PHYSNET2, !CUSTOM_VNIC_TYPE_DIRECT], '
'aggregates=[[baz], [foo, bar]])',
str(grp))
class TestEnsureConsumer(testtools.TestCase):
def setUp(self):
super(TestEnsureConsumer, self).setUp()

1
nova/tests/unit/objects/test_objects.py
View File

@ -1142,6 +1142,7 @@ object_data = {
'PowerVMLiveMigrateData': '1.4-a745f4eda16b45e1bc5686a0c498f27e',
'Quotas': '1.3-40fcefe522111dddd3e5e6155702cf4e',
'QuotasNoOp': '1.3-347a039fc7cfee7b225b68b5181e0733',
'RequestGroup': '1.0-5f694d4237c00c7b01136a4e4bcacd6d',
'RequestSpec': '1.11-851a690dbf116fb5165cc94ea6c85629',
'S3ImageMapping': '1.0-7dd7366a890d82660ed121de9092276e',
'SchedulerLimits': '1.0-249c4bd8e62a9b327b7026b7f19cc641',

17
nova/tests/unit/objects/test_request_spec.py
View File

@ -24,6 +24,7 @@ from nova.network import model as network_model
from nova import objects
from nova.objects import base
from nova.objects import request_spec
from nova import test
from nova.tests.unit.api.openstack import fakes
from nova.tests.unit import fake_flavor
from nova.tests.unit import fake_instance
@ -753,3 +754,19 @@ class TestRequestSpecObject(test_objects._LocalTest,
class TestRemoteRequestSpecObject(test_objects._RemoteTest,
_TestRequestSpecObject):
pass
class TestRequestGroupObject(test.TestCase):
def setUp(self):
super(TestRequestGroupObject, self).setUp()
self.user_id = uuids.user_id
self.project_id = uuids.project_id
self.context = context.RequestContext(uuids.user_id, uuids.project_id)
def test_fields_defaulted_at_create(self):
rg = request_spec.RequestGroup(self.context)
self.assertTrue(True, rg.use_same_provider)
self.assertEqual({}, rg.resources)
self.assertEqual(set(), rg.required_traits)
self.assertEqual(set(), rg.forbidden_traits)
self.assertEqual([], rg.aggregates)

4
nova/tests/unit/scheduler/client/test_report.py
View File

@ -1993,8 +1993,8 @@ class TestProviderOperations(SchedulerReportClientTestCase):
'trait:HW_NIC_OFFLOAD_LRO': 'preferred',
'group_policy3': 'none',
})
resources.get_request_group(None).member_of = [
('agg1', 'agg2', 'agg3'), ('agg1', 'agg2')]
resources.get_request_group(None).aggregates = [
['agg1', 'agg2', 'agg3'], ['agg1', 'agg2']]
expected_path = '/allocation_candidates'
expected_query = [
('group_policy', 'isolate'),

93
nova/tests/unit/scheduler/test_utils.py
View File

@ -13,7 +13,6 @@
import mock
from oslo_utils.fixture import uuidsentinel as uuids
from nova.api.openstack.placement import lib as plib
from nova import context as nova_context
from nova import exception
from nova import objects
@ -64,7 +63,7 @@ class TestUtils(test.NoDBTestCase):
ephemeral_gb=5,
swap=0)
expected_resources = utils.ResourceRequest()
expected_resources._rg_by_id[None] = plib.RequestGroup(
expected_resources._rg_by_id[None] = objects.RequestGroup(
use_same_provider=False,
resources={
'VCPU': 1,
@ -85,7 +84,7 @@ class TestUtils(test.NoDBTestCase):
'trait:CUSTOM_FLAVOR_TRAIT': 'required',
'trait:CUSTOM_IMAGE_TRAIT2': 'required'})
expected_resources = utils.ResourceRequest()
expected_resources._rg_by_id[None] = plib.RequestGroup(
expected_resources._rg_by_id[None] = objects.RequestGroup(
use_same_provider=False,
resources={
'VCPU': 1,
@ -110,7 +109,7 @@ class TestUtils(test.NoDBTestCase):
ephemeral_gb=0,
swap=0)
expected_resources = utils.ResourceRequest()
expected_resources._rg_by_id[None] = plib.RequestGroup(
expected_resources._rg_by_id[None] = objects.RequestGroup(
use_same_provider=False,
resources={
'VCPU': 1,
@ -127,7 +126,7 @@ class TestUtils(test.NoDBTestCase):
swap=0,
extra_specs={"resources:CUSTOM_TEST_CLASS": 1})
expected_resources = utils.ResourceRequest()
expected_resources._rg_by_id[None] = plib.RequestGroup(
expected_resources._rg_by_id[None] = objects.RequestGroup(
use_same_provider=False,
resources={
"VCPU": 1,
@ -149,7 +148,7 @@ class TestUtils(test.NoDBTestCase):
"resources:MEMORY_MB": 99,
"resources:DISK_GB": 99})
expected_resources = utils.ResourceRequest()
expected_resources._rg_by_id[None] = plib.RequestGroup(
expected_resources._rg_by_id[None] = objects.RequestGroup(
use_same_provider=False,
resources={
"VCPU": 99,
@ -169,7 +168,7 @@ class TestUtils(test.NoDBTestCase):
"resources:VCPU": 0,
"resources:DISK_GB": 0})
expected_resources = utils.ResourceRequest()
expected_resources._rg_by_id[None] = plib.RequestGroup(
expected_resources._rg_by_id[None] = objects.RequestGroup(
use_same_provider=False,
resources={
"MEMORY_MB": 1024,
@ -187,7 +186,7 @@ class TestUtils(test.NoDBTestCase):
"resources:VGPU": 1,
"resources:VGPU_DISPLAY_HEAD": 1})
expected_resources = utils.ResourceRequest()
expected_resources._rg_by_id[None] = plib.RequestGroup(
expected_resources._rg_by_id[None] = objects.RequestGroup(
use_same_provider=False,
resources={
"VCPU": 1,
@ -243,20 +242,20 @@ class TestUtils(test.NoDBTestCase):
'group_policy': 'none'})
expected_resources = utils.ResourceRequest()
expected_resources._group_policy = 'none'
expected_resources._rg_by_id[None] = plib.RequestGroup(
expected_resources._rg_by_id[None] = objects.RequestGroup(
use_same_provider=False,
resources={
'DISK_GB': 10,
'CUSTOM_THING': 123,
}
)
expected_resources._rg_by_id['1'] = plib.RequestGroup(
expected_resources._rg_by_id['1'] = objects.RequestGroup(
resources={
'VGPU': 1,
'VGPU_DISPLAY_HEAD': 2,
}
)
expected_resources._rg_by_id['3'] = plib.RequestGroup(
expected_resources._rg_by_id['3'] = objects.RequestGroup(
resources={
'VCPU': 2,
},
@ -265,12 +264,12 @@ class TestUtils(test.NoDBTestCase):
'CUSTOM_SILVER',
}
)
expected_resources._rg_by_id['24'] = plib.RequestGroup(
expected_resources._rg_by_id['24'] = objects.RequestGroup(
resources={
'SRIOV_NET_VF': 2,
},
)
expected_resources._rg_by_id['42'] = plib.RequestGroup(
expected_resources._rg_by_id['42'] = objects.RequestGroup(
resources={
'SRIOV_NET_VF': 1,
}
@ -299,21 +298,13 @@ class TestUtils(test.NoDBTestCase):
destination.require_aggregates(['foo', 'bar'])
req = utils.resources_from_request_spec(reqspec)
self.assertEqual([('foo', 'bar',)],
req.get_request_group(None).member_of)
self.assertEqual([['foo', 'bar']],
req.get_request_group(None).aggregates)
destination.require_aggregates(['baz'])
req = utils.resources_from_request_spec(reqspec)
self.assertEqual([('foo', 'bar'), ('baz',)],
req.get_request_group(None).member_of)
# Test stringification
self.assertEqual(
'RequestGroup(use_same_provider=False, '
'resources={DISK_GB:1, MEMORY_MB:1024, VCPU:1}, '
'traits=[], '
'aggregates=[[baz], [foo, bar]])',
str(req))
self.assertEqual([['foo', 'bar'], ['baz']],
req.get_request_group(None).aggregates)
def test_resources_from_request_spec_no_aggregates(self):
flavor = objects.Flavor(vcpus=1, memory_mb=1024,
@ -322,19 +313,19 @@ class TestUtils(test.NoDBTestCase):
reqspec = objects.RequestSpec(flavor=flavor)
req = utils.resources_from_request_spec(reqspec)
self.assertEqual([], req.get_request_group(None).member_of)
self.assertEqual([], req.get_request_group(None).aggregates)
reqspec.requested_destination = None
req = utils.resources_from_request_spec(reqspec)
self.assertEqual([], req.get_request_group(None).member_of)
self.assertEqual([], req.get_request_group(None).aggregates)
reqspec.requested_destination = objects.Destination()
req = utils.resources_from_request_spec(reqspec)
self.assertEqual([], req.get_request_group(None).member_of)
self.assertEqual([], req.get_request_group(None).aggregates)
reqspec.requested_destination.aggregates = None
req = utils.resources_from_request_spec(reqspec)
self.assertEqual([], req.get_request_group(None).member_of)
self.assertEqual([], req.get_request_group(None).aggregates)
@mock.patch("nova.scheduler.utils.ResourceRequest.from_extra_specs")
def test_process_extra_specs_granular_called(self, mock_proc):
@ -377,7 +368,7 @@ class TestUtils(test.NoDBTestCase):
ephemeral_gb=0,
swap=0)
expected = utils.ResourceRequest()
expected._rg_by_id[None] = plib.RequestGroup(
expected._rg_by_id[None] = objects.RequestGroup(
use_same_provider=False,
resources={
'VCPU': 1,
@ -400,7 +391,7 @@ class TestUtils(test.NoDBTestCase):
swap=0)
fake_spec = objects.RequestSpec(flavor=flavor, force_nodes=['test'])
expected = utils.ResourceRequest()
expected._rg_by_id[None] = plib.RequestGroup(
expected._rg_by_id[None] = objects.RequestGroup(
use_same_provider=False,
resources={
'VCPU': 1,
@ -424,7 +415,7 @@ class TestUtils(test.NoDBTestCase):
swap=0)
fake_spec = objects.RequestSpec(flavor=flavor, force_hosts=['test'])
expected = utils.ResourceRequest()
expected._rg_by_id[None] = plib.RequestGroup(
expected._rg_by_id[None] = objects.RequestGroup(
use_same_provider=False,
resources={
'VCPU': 1,
@ -538,7 +529,7 @@ class TestUtils(test.NoDBTestCase):
# Build up a ResourceRequest from the inside to compare against.
expected = utils.ResourceRequest()
expected._group_policy = 'isolate'
expected._rg_by_id[None] = plib.RequestGroup(
expected._rg_by_id[None] = objects.RequestGroup(
use_same_provider=False,
resources={
'VCPU': 2,
@ -552,7 +543,7 @@ class TestUtils(test.NoDBTestCase):
'CUSTOM_BRONZE',
},
)
expected._rg_by_id['1'] = plib.RequestGroup(
expected._rg_by_id['1'] = objects.RequestGroup(
resources={
'SRIOV_NET_VF': 1,
'IPV4_ADDRESS': 1,
@ -564,7 +555,7 @@ class TestUtils(test.NoDBTestCase):
'CUSTOM_PHYSNET_NET2',
},
)
expected._rg_by_id['2'] = plib.RequestGroup(
expected._rg_by_id['2'] = objects.RequestGroup(
resources={
'SRIOV_NET_VF': 1,
'IPV4_ADDRESS': 2,
@ -574,7 +565,7 @@ class TestUtils(test.NoDBTestCase):
'HW_NIC_ACCEL_SSL',
}
)
expected._rg_by_id['3'] = plib.RequestGroup(
expected._rg_by_id['3'] = objects.RequestGroup(
resources={
'DISK_GB': 5,
}
@ -595,26 +586,6 @@ class TestUtils(test.NoDBTestCase):
)
self.assertEqual(expected_querystring, rr.to_querystring())
# Test stringification
self.assertEqual(
'RequestGroup(use_same_provider=False, '
'resources={MEMORY_MB:2048, VCPU:2}, '
'traits=[CUSTOM_MAGIC, HW_CPU_X86_AVX, !CUSTOM_BRONZE], '
'aggregates=[]), '
'RequestGroup(use_same_provider=True, '
'resources={DISK_GB:5}, '
'traits=[], '
'aggregates=[]), '
'RequestGroup(use_same_provider=True, '
'resources={IPV4_ADDRESS:1, SRIOV_NET_VF:1}, '
'traits=[CUSTOM_PHYSNET_NET1, !CUSTOM_PHYSNET_NET2], '
'aggregates=[]), '
'RequestGroup(use_same_provider=True, '
'resources={IPV4_ADDRESS:2, SRIOV_NET_VF:1}, '
'traits=[CUSTOM_PHYSNET_NET2, HW_NIC_ACCEL_SSL], '
'aggregates=[])',
str(rr))
def test_resource_request_from_extra_specs_append_request(self):
extra_specs = {
'resources:VCPU': '2',
@ -622,7 +593,7 @@ class TestUtils(test.NoDBTestCase):
'trait:HW_CPU_X86_AVX': 'required',
}
existing_req = utils.ResourceRequest()
existing_req._rg_by_id[None] = plib.RequestGroup(
existing_req._rg_by_id[None] = objects.RequestGroup(
use_same_provider=False,
required_traits={
'CUSTOM_MAGIC',
@ -630,7 +601,7 @@ class TestUtils(test.NoDBTestCase):
)
# Build up a ResourceRequest from the inside to compare against.
expected = utils.ResourceRequest()
expected._rg_by_id[None] = plib.RequestGroup(
expected._rg_by_id[None] = objects.RequestGroup(
use_same_provider=False,
resources={
'VCPU': 2,
@ -653,7 +624,7 @@ class TestUtils(test.NoDBTestCase):
# Build up a ResourceRequest from the inside to compare against.
expected = utils.ResourceRequest()
expected._rg_by_id[None] = plib.RequestGroup(
expected._rg_by_id[None] = objects.RequestGroup(
use_same_provider=False,
required_traits={
'CUSTOM_TRUSTED',
@ -667,7 +638,7 @@ class TestUtils(test.NoDBTestCase):
image_meta_props = objects.ImageMetaProps.from_dict(props)
existing_req = utils.ResourceRequest()
existing_req._rg_by_id[None] = plib.RequestGroup(
existing_req._rg_by_id[None] = objects.RequestGroup(
use_same_provider=False,
resources={
'VCPU': 2,
@ -679,7 +650,7 @@ class TestUtils(test.NoDBTestCase):
)
# Build up a ResourceRequest from the inside to compare against.
expected = utils.ResourceRequest()
expected._rg_by_id[None] = plib.RequestGroup(
expected._rg_by_id[None] = objects.RequestGroup(
use_same_provider=False,
resources={
'VCPU': 2,

2
requirements.txt
View File

@ -53,7 +53,7 @@ oslo.service!=1.28.1,>=1.24.0 # Apache-2.0
rfc3986>=0.3.1 # Apache-2.0
oslo.middleware>=3.31.0 # Apache-2.0
psutil>=3.2.2 # BSD
oslo.versionedobjects>=1.31.2 # Apache-2.0
oslo.versionedobjects>=1.33.3 # Apache-2.0
os-brick>=2.6.1 # Apache-2.0
os-traits>=0.4.0 # Apache-2.0
os-vif!=1.8.0,>=1.7.0 # Apache-2.0