openstack/placement
Code Issues Proposed changes

Merge "placement: Parse granular resources & traits"

Browse Source
This commit is contained in:
Zuul 2017-11-03 17:52:00 +00:00 committed by Gerrit Code Review
commit 1b7b6f5026
2 changed files with 369 additions and 0 deletions
nova
api/openstack/placement
util.py
tests/unit/api/openstack/placement
test_util.py

138
nova/api/openstack/placement/util.py

@ -12,6 +12,7 @@
"""Utility methods for placement API."""
import functools
import re
import jsonschema
from oslo_middleware import request_id
@ -19,12 +20,20 @@ from oslo_serialization import jsonutils
from oslo_utils import uuidutils
import webob
from nova.api.openstack.placement import lib as placement_lib
# NOTE(cdent): avoid cyclical import conflict between util and
# microversion
import nova.api.openstack.placement.microversion
from nova.i18n import _
# Querystring-related constants
_QS_RESOURCES = 'resources'
_QS_REQUIRED = 'required'
_QS_KEY_PATTERN = re.compile(
r"^(%s)([1-9][0-9]*)?$" % '|'.join((_QS_RESOURCES, _QS_REQUIRED)))
# NOTE(cdent): This registers a FormatChecker on the jsonschema
# module. Do not delete this code! Although it appears that nothing
# is using the decorated method it is being used in JSON schema
@ -257,3 +266,132 @@ def normalize_resources_qs_param(qs):
raise webob.exc.HTTPBadRequest(msg)
result[rc_name] = amount
return result
def normalize_traits_qs_param(val):
"""Parse a traits query string parameter value.
Note that this method doesn't know or care about the query parameter key,
which may currently be of the form `required`, `required123`, etc., but
which may someday also include `preferred`, etc.
This method currently does no format validation of trait strings, other
than to ensure they're not zero-length.
:param val: A traits query parameter value: a comma-separated string of
trait names.
:return: A set of trait names.
:raises `webob.exc.HTTPBadRequest` if the val parameter is not in the
expected format.
"""
ret = set(substr.strip() for substr in val.split(','))
if not all(trait for trait in ret):
msg = _('Malformed traits parameter. Expected query string value '
'of the form: HW_CPU_X86_VMX,CUSTOM_MAGIC. '
'Got: "%s"') % val
raise webob.exc.HTTPBadRequest(msg)
return ret
def parse_qs_request_groups(qsdict):
"""Parse numbered resources and traits groupings out of a querystring dict.
The input qsdict represents a query string of the form:
?resources=$RESOURCE_CLASS_NAME:$AMOUNT,$RESOURCE_CLASS_NAME:$AMOUNT
&required=$TRAIT_NAME,$TRAIT_NAME
&resources1=$RESOURCE_CLASS_NAME:$AMOUNT,RESOURCE_CLASS_NAME:$AMOUNT
&required1=$TRAIT_NAME,$TRAIT_NAME
&resources2=$RESOURCE_CLASS_NAME:$AMOUNT,RESOURCE_CLASS_NAME:$AMOUNT
&required2=$TRAIT_NAME,$TRAIT_NAME
These are parsed in groups according to the numeric suffix of the key.
For each group, a RequestGroup instance is created containing that group's
resources and required traits. For the (single) group with no suffix, the
RequestGroup.use_same_provider attribute is False; for the numbered groups
it is True.
The return is a list of these RequestGroup instances.
As an example, if qsdict represents the query string:
?resources=VCPU:2,MEMORY_MB:1024,DISK_GB=50
&required=HW_CPU_X86_VMX,CUSTOM_STORAGE_RAID
&resources1=SRIOV_NET_VF:2
&required1=CUSTOM_PHYSNET_PUBLIC,CUSTOM_SWITCH_A
&resources2=SRIOV_NET_VF:1
&required2=CUSTOM_PHYSNET_PRIVATE
...the return value will be:
[ RequestGroup(
use_same_provider=False,
resources={
"VCPU": 2,
"MEMORY_MB": 1024,
"DISK_GB" 50,
},
required_traits=[
"HW_CPU_X86_VMX",
"CUSTOM_STORAGE_RAID",
],
),
RequestGroup(
use_same_provider=True,
resources={
"SRIOV_NET_VF": 2,
},
required_traits=[
"CUSTOM_PHYSNET_PUBLIC",
"CUSTOM_SWITCH_A",
],
),
RequestGroup(
use_same_provider=True,
resources={
"SRIOV_NET_VF": 1,
},
required_traits=[
"CUSTOM_PHYSNET_PRIVATE",
],
),
]
:param qsdict: The MultiDict representing the querystring on a GET.
:return: A list of RequestGroup instances.
:raises `webob.exc.HTTPBadRequest` if any value is malformed, or if a
trait list is given without corresponding resources.
"""
# Temporary dict of the form: { suffix: RequestGroup }
by_suffix = {}
def get_request_group(suffix):
if suffix not in by_suffix:
rq_grp = placement_lib.RequestGroup(use_same_provider=bool(suffix))
by_suffix[suffix] = rq_grp
return by_suffix[suffix]
for key, val in qsdict.items():
match = _QS_KEY_PATTERN.match(key)
if not match:
continue
# `prefix` is 'resources' or 'required'
# `suffix` is an integer string, or None
prefix, suffix = match.groups()
request_group = get_request_group(suffix or '')
if prefix == _QS_RESOURCES:
request_group.resources = normalize_resources_qs_param(val)
elif prefix == _QS_REQUIRED:
request_group.required_traits = normalize_traits_qs_param(val)
# Ensure any group with 'required' also has 'resources'.
orphans = [('required%s' % suff) for suff, group in by_suffix.items()
if group.required_traits and not group.resources]
if orphans:
msg = _('All traits parameters must be associated with resources. '
'Found the following orphaned traits keys: %s')
raise webob.exc.HTTPBadRequest(msg % ', '.join(orphans))
# NOTE(efried): The sorting is not necessary for the API, but it makes
# testing easier.
return [by_suffix[suff] for suff in sorted(by_suffix)]

231
nova/tests/unit/api/openstack/placement/test_util.py

@ -17,6 +17,9 @@ import fixtures
from oslo_middleware import request_id
import webob
import six.moves.urllib.parse as urlparse
from nova.api.openstack.placement import lib as pl
from nova.api.openstack.placement import microversion
from nova.api.openstack.placement import util
from nova.objects import resource_provider as rp_obj
@ -363,3 +366,231 @@ class TestNormalizeResourceQsParam(test.NoDBTestCase):
util.normalize_resources_qs_param,
qs,
)
class TestNormalizeTraitsQsParam(test.NoDBTestCase):
def test_one(self):
trait = 'HW_CPU_X86_VMX'
# Various whitespace permutations
for fmt in ('%s', ' %s', '%s ', ' %s ', ' %s '):
self.assertEqual(set([trait]),
util.normalize_traits_qs_param(fmt % trait))
def test_multiple(self):
traits = (
'HW_CPU_X86_VMX',
'HW_GPU_API_DIRECT3D_V12_0',
'HW_NIC_OFFLOAD_RX',
'CUSTOM_GOLD',
'STORAGE_DISK_SSD',
)
self.assertEqual(
set(traits),
util.normalize_traits_qs_param('%s, %s,%s , %s , %s ' % traits))
def test_400_all_empty(self):
for qs in ('', ' ', ' ', ',', ' , , '):
self.assertRaises(
webob.exc.HTTPBadRequest, util.normalize_traits_qs_param, qs)
def test_400_some_empty(self):
traits = (
'HW_NIC_OFFLOAD_RX',
'CUSTOM_GOLD',
'STORAGE_DISK_SSD',
)
for fmt in ('%s,,%s,%s', ',%s,%s,%s', '%s,%s,%s,', ' %s , %s , , %s'):
self.assertRaises(webob.exc.HTTPBadRequest,
util.normalize_traits_qs_param, fmt % traits)
class TestParseQsResourcesAndTraits(test.NoDBTestCase):
@staticmethod
def do_parse(qstring):
"""Converts a querystring to a MultiDict, mimicking request.GET, and
runs parse_qs_request_groups on it.
"""
return util.parse_qs_request_groups(webob.multidict.MultiDict(
urlparse.parse_qsl(qstring)))
def assertRequestGroupsEqual(self, expected, observed):
self.assertEqual(len(expected), len(observed))
for exp, obs in zip(expected, observed):
self.assertEqual(vars(exp), vars(obs))
def test_empty(self):
self.assertRequestGroupsEqual([], self.do_parse(''))
def test_unnumbered_only(self):
"""Unnumbered resources & traits - no numbered groupings."""
qs = ('resources=VCPU:2,MEMORY_MB:2048'
'&required=HW_CPU_X86_VMX,CUSTOM_GOLD')
expected = [
pl.RequestGroup(
use_same_provider=False,
resources={
'VCPU': 2,
'MEMORY_MB': 2048,
},
required_traits={
'HW_CPU_X86_VMX',
'CUSTOM_GOLD',
},
),
]
self.assertRequestGroupsEqual(expected, self.do_parse(qs))
def test_unnumbered_resources_only(self):
"""Validate the bit that can be used for 1.10 and earlier."""
qs = 'resources=VCPU:2,MEMORY_MB:2048,DISK_GB:5,CUSTOM_MAGIC:123'
expected = [
pl.RequestGroup(
use_same_provider=False,
resources={
'VCPU': 2,
'MEMORY_MB': 2048,
'DISK_GB': 5,
'CUSTOM_MAGIC': 123,
},
),
]
self.assertRequestGroupsEqual(expected, self.do_parse(qs))
def test_numbered_only(self):
# Crazy ordering and nonsequential numbers don't matter.
# It's okay to have a 'resources' without a 'required'.
# A trait that's repeated shows up in both spots.
qs = ('resources1=VCPU:2,MEMORY_MB:2048'
'&required42=CUSTOM_GOLD'
'&resources99=DISK_GB:5'
'&resources42=CUSTOM_MAGIC:123'
'&required1=HW_CPU_X86_VMX,CUSTOM_GOLD')
expected = [
pl.RequestGroup(
resources={
'VCPU': 2,
'MEMORY_MB': 2048,
},
required_traits={
'HW_CPU_X86_VMX',
'CUSTOM_GOLD',
},
),
pl.RequestGroup(
resources={
'CUSTOM_MAGIC': 123,
},
required_traits={
'CUSTOM_GOLD',
},
),
pl.RequestGroup(
resources={
'DISK_GB': 5,
},
),
]
self.assertRequestGroupsEqual(expected, self.do_parse(qs))
def test_numbered_and_unnumbered(self):
qs = ('resources=VCPU:3,MEMORY_MB:4096,DISK_GB:10'
'&required=HW_CPU_X86_VMX,CUSTOM_MEM_FLASH,STORAGE_DISK_SSD'
'&resources1=SRIOV_NET_VF:2'
'&required1=CUSTOM_PHYSNET_PRIVATE'
'&resources2=SRIOV_NET_VF:1,NET_INGRESS_BYTES_SEC:20000'
',NET_EGRESS_BYTES_SEC:10000'
'&required2=CUSTOM_SWITCH_BIG,CUSTOM_PHYSNET_PROD'
'&resources3=CUSTOM_MAGIC:123')
expected = [
pl.RequestGroup(
use_same_provider=False,
resources={
'VCPU': 3,
'MEMORY_MB': 4096,
'DISK_GB': 10,
},
required_traits={
'HW_CPU_X86_VMX',
'CUSTOM_MEM_FLASH',
'STORAGE_DISK_SSD',
},
),
pl.RequestGroup(
resources={
'SRIOV_NET_VF': 2,
},
required_traits={
'CUSTOM_PHYSNET_PRIVATE',
},
),
pl.RequestGroup(
resources={
'SRIOV_NET_VF': 1,
'NET_INGRESS_BYTES_SEC': 20000,
'NET_EGRESS_BYTES_SEC': 10000,
},
required_traits={
'CUSTOM_SWITCH_BIG',
'CUSTOM_PHYSNET_PROD',
},
),
pl.RequestGroup(
resources={
'CUSTOM_MAGIC': 123,
},
),
]
self.assertRequestGroupsEqual(expected, self.do_parse(qs))
def test_400_malformed_resources(self):
# Somewhat duplicates TestNormalizeResourceQsParam.test_400*.
qs = ('resources=VCPU:0,MEMORY_MB:4096,DISK_GB:10'
# Bad ----------^
'&required=HW_CPU_X86_VMX,CUSTOM_MEM_FLASH,STORAGE_DISK_SSD'
'&resources1=SRIOV_NET_VF:2'
'&required1=CUSTOM_PHYSNET_PRIVATE'
'&resources2=SRIOV_NET_VF:1,NET_INGRESS_BYTES_SEC:20000'
',NET_EGRESS_BYTES_SEC:10000'
'&required2=CUSTOM_SWITCH_BIG,CUSTOM_PHYSNET_PROD'
'&resources3=CUSTOM_MAGIC:123')
self.assertRaises(webob.exc.HTTPBadRequest, self.do_parse, qs)
def test_400_malformed_traits(self):
# Somewhat duplicates TestNormalizeResourceQsParam.test_400*.
qs = ('resources=VCPU:7,MEMORY_MB:4096,DISK_GB:10'
'&required=HW_CPU_X86_VMX,CUSTOM_MEM_FLASH,STORAGE_DISK_SSD'
'&resources1=SRIOV_NET_VF:2'
'&required1=CUSTOM_PHYSNET_PRIVATE'
'&resources2=SRIOV_NET_VF:1,NET_INGRESS_BYTES_SEC:20000'
',NET_EGRESS_BYTES_SEC:10000'
'&required2=CUSTOM_SWITCH_BIG,CUSTOM_PHYSNET_PROD,'
# Bad -------------------------------------------^
'&resources3=CUSTOM_MAGIC:123')
self.assertRaises(webob.exc.HTTPBadRequest, self.do_parse, qs)
def test_400_traits_no_resources_unnumbered(self):
qs = ('resources9=VCPU:7,MEMORY_MB:4096,DISK_GB:10'
# Oops ---^
'&required=HW_CPU_X86_VMX,CUSTOM_MEM_FLASH,STORAGE_DISK_SSD'
'&resources1=SRIOV_NET_VF:2'
'&required1=CUSTOM_PHYSNET_PRIVATE'
'&resources2=SRIOV_NET_VF:1,NET_INGRESS_BYTES_SEC:20000'
',NET_EGRESS_BYTES_SEC:10000'
'&required2=CUSTOM_SWITCH_BIG,CUSTOM_PHYSNET_PROD'
'&resources3=CUSTOM_MAGIC:123')
self.assertRaises(webob.exc.HTTPBadRequest, self.do_parse, qs)
def test_400_traits_no_resources_numbered(self):
qs = ('resources=VCPU:7,MEMORY_MB:4096,DISK_GB:10'
'&required=HW_CPU_X86_VMX,CUSTOM_MEM_FLASH,STORAGE_DISK_SSD'
'&resources11=SRIOV_NET_VF:2'
# Oops ----^^
'&required1=CUSTOM_PHYSNET_PRIVATE'
'&resources20=SRIOV_NET_VF:1,NET_INGRESS_BYTES_SEC:20000'
# Oops ----^^
',NET_EGRESS_BYTES_SEC:10000'
'&required2=CUSTOM_SWITCH_BIG,CUSTOM_PHYSNET_PROD'
'&resources3=CUSTOM_MAGIC:123')
self.assertRaises(webob.exc.HTTPBadRequest, self.do_parse, qs)