openstack/nova
Code Issues Proposed changes

Added new scheduler filter: AggregateTypeExtraSpecsAffinityFilter

Browse Source 
The flavor_extra_spec metadata pair will be consumed by the
AggregateTypeExtraSpecsAfinityFilter to allow operators to define a set of
extra specs key value pairs that are required to schedule to the aggregate,
e.g.:
standard memory backing aggregate:
flavor_extra_spec: "mem_page_size=small,mem_page_size=any"
high bandwidth memory backing aggregate:
flavor_extra_spec: "mem_page_size=2M,mem_page_size=1G,mem_page_size=large"

Added unit test for this feature.

DocImpact
Implements: blueprint aggregate-extra-specs-filter
Change-Id: Id3a9918cf9f83b2a9b1dfbcd91803b5b1b2bcc78
This commit is contained in:
Rodolfo Alonso Hernandez
2015-05-14 03:04:06 +01:00
parent 5ea471ed88
commit 4b142e53e4
3 changed files with 254 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
doc/source/filter_scheduler.rst
View File

@@ -166,6 +166,10 @@ There are some standard filter classes to use (:mod:`nova.scheduler.filters`):
the available metrics are passed.
* |NUMATopologyFilter| - filters hosts based on the NUMA topology requested by the
instance, if any.
* |AggregateTypeExtraSpecsAffinityFilter| - filters only hosts aggregated inside
host aggregates containing "flavor_extra_spec" metadata. Keys inside this
variable must match with the instance extra specs. Eg. 'flavor_extra_spec=\
mem_page_size=any,mem_page_size=~,mem_page_size=small'
Now we can focus on these standard filter classes in details. I will pass the
simplest ones, such as |AllHostsFilter|, |CoreFilter| and |RamFilter| are,
@@ -409,6 +413,7 @@ in :mod:``nova.tests.scheduler``.
.. |TrustedFilter| replace:: :class:`TrustedFilter <nova.scheduler.filters.trusted_filter.TrustedFilter>`
.. |TypeAffinityFilter| replace:: :class:`TypeAffinityFilter <nova.scheduler.filters.type_filter.TypeAffinityFilter>`
.. |AggregateTypeAffinityFilter| replace:: :class:`AggregateTypeAffinityFilter <nova.scheduler.filters.type_filter.AggregateTypeAffinityFilter>`
.. |AggregateTypeExtraSpecsAffinityFilter| replace:: :class:`AggregateTypeAffinityFilter <nova.scheduler.filters.type_filter.AggregateTypeExtraSpecsAffinityFilter>`
.. |ServerGroupAntiAffinityFilter| replace:: :class:`ServerGroupAntiAffinityFilter <nova.scheduler.filters.affinity_filter.ServerGroupAntiAffinityFilter>`
.. |ServerGroupAffinityFilter| replace:: :class:`ServerGroupAffinityFilter <nova.scheduler.filters.affinity_filter.ServerGroupAffinityFilter>`
.. |AggregateInstanceExtraSpecsFilter| replace:: :class:`AggregateInstanceExtraSpecsFilter <nova.scheduler.filters.aggregate_instance_extra_specs.AggregateInstanceExtraSpecsFilter>`

101
nova/scheduler/filters/type_filter.py
View File

@@ -13,9 +13,13 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
from nova.scheduler import filters
from nova.scheduler.filters import utils
from oslo_log import log as logging
LOG = logging.getLogger(__name__)
class TypeAffinityFilter(filters.BaseHostFilter):
@@ -59,3 +63,100 @@ class AggregateTypeAffinityFilter(filters.BaseHostFilter):
[x.strip() for x in val.split(',')]):
return True
return not aggregate_vals
class AggregateTypeExtraSpecsAffinityFilter(filters.BaseHostFilter):
"""AggregateTypeExtraSpecsAffinityFilter filters only hosts aggregated
inside host aggregates containing "flavor_extra_spec" metadata. Keys inside
this variable must match with the instance extra specs.
"""
def _group_extra_specs(self, aggregate_extraspec):
"""Group all key/data from aggregate_extraspec.
:param aggregate_extraspec: string containing a list of required
instance flavor extra specs, separated by commas, with format
"key=value"
:type aggregate: basestring.
:return: dictionary with the values parsed and grouped by keys.
:type: dict.
"""
extra_specs = collections.defaultdict(set)
kv_list = aggregate_extraspec.split(',')
# Create an entry in a dict for every new key/value.
for kv_element in kv_list:
key, value = kv_element.split('=')[:2]
extra_specs[key].add(value)
return extra_specs
def _test_extra_specs_elements(self,
aggregate_extra_spec,
instance_extra_specs):
"""Analyze and execute all aggregate_extra_spec parsed.
:param aggregate_extra_spec: dictionary with the values parsed and
grouped by keys
:type aggregate_extra_spec: dict.
:return: True if all parameters executed correctly; False is there
is any error.
:type: boolean.
"""
for key, value in aggregate_extra_spec.iteritems():
if '*' in value:
if key not in instance_extra_specs:
LOG.debug("'flavor_extra_spec' key: %(key)s "
"is not present", {'key': key})
return False
else:
continue
if '!' in value:
if key in instance_extra_specs:
LOG.debug("'flavor_extra_spec' key: %(key)s "
"is present", {'key': key})
return False
else:
continue
if '~' in value:
if key not in instance_extra_specs:
continue
else:
value.discard('~')
for element in value:
if key not in instance_extra_specs:
LOG.debug("'flavor_extra_spec' key: %(key)s "
"is not present", {'key': key})
return False
if instance_extra_specs[key] not in value:
LOG.debug("The following 'flavor_extra_spec' "
"key/value: %(key)s/%(value)s doesn't "
"match", {'key': key, 'value': value})
return False
return True
def host_passes(self, host_state, filter_properties):
# Read extra specs needed by instance.
instance_type = filter_properties.get('instance_type')
instance_extra_specs = instance_type.get('extra_specs')
# Read metadata from host: find "flavor_extra_spec" in host
# aggregates metadata, each separately.
aggregate_extra_spec_list = utils.aggregate_values_from_key(host_state,
'flavor_extra_spec')
for aggregate_extra_spec in aggregate_extra_spec_list:
# Recover and group all key/data from aggregate_extraspec.
aggregate_extra_spec = \
self._group_extra_specs(aggregate_extra_spec)
# Loop all aggregate flavor extra specs and compare to
# instance extra specs.
if self._test_extra_specs_elements(aggregate_extra_spec,
instance_extra_specs) is False:
return False
return True

148
nova/tests/unit/scheduler/filters/test_type_filters.py
View File

@@ -119,3 +119,151 @@ class TestTypeFilter(test.NoDBTestCase):
self.assertTrue(self.filt_cls.host_passes(host, filter2_properties))
# False as instance_type is not allowed for aggregate
self.assertFalse(self.filt_cls.host_passes(host, filter3_properties))
class TestAggregateTypeExtraSpecsAffinityFilter(test.NoDBTestCase):
def setUp(self):
super(TestAggregateTypeExtraSpecsAffinityFilter, self).setUp()
self.filt_cls = type_filter.AggregateTypeExtraSpecsAffinityFilter()
self.aggr_no_extraspecs = objects.Aggregate(context='test')
self.aggr_no_extraspecs.metadata = {}
self.aggr_invalid_extraspecs = objects.Aggregate(context='test')
self.aggr_invalid_extraspecs.metadata = {'flavor_extra_spec':
'mem_page_size=large'}
self.aggr_valid_extraspecs = objects.Aggregate(context='test')
self.aggr_valid_extraspecs.metadata = {'flavor_extra_spec':
'mem_page_size=small'}
self.aggr_optional_large = objects.Aggregate(context='test')
self.aggr_optional_large.metadata = {'flavor_extra_spec':
'mem_page_size=large,mem_page_size=~'}
self.aggr_optional_small = objects.Aggregate(context='test')
self.aggr_optional_small.metadata = {'flavor_extra_spec':
'mem_page_size=small,mem_page_size=~'}
self.aggr_optional_first = objects.Aggregate(context='test')
self.aggr_optional_first.metadata = {'flavor_extra_spec':
'mem_page_size=~,mem_page_size=small,mem_page_size=any'}
self.aggr_optional_last = objects.Aggregate(context='test')
self.aggr_optional_last.metadata = {'flavor_extra_spec':
'mem_page_size=any,mem_page_size=small,mem_page_size=~'}
self.aggr_optional_middle = objects.Aggregate(context='test')
self.aggr_optional_middle.metadata = {'flavor_extra_spec':
'mem_page_size=any,mem_page_size=~,mem_page_size=small'}
self.aggr_any = objects.Aggregate(context='test')
self.aggr_any.metadata = {'flavor_extra_spec': 'mem_page_size=*'}
self.aggr_nopresent = objects.Aggregate(context='test')
self.aggr_nopresent.metadata = {'flavor_extra_spec':
'mem_page_size=!'}
self.filter_properties = {'instance_type': {'extra_specs':
{'mem_page_size': 'small'}}}
self.filter_properties_noextraspecs = {'instance_type':
{'extra_specs': {}}}
def _do_test_compute_filter_extra_specs(self, host_state,
filter_properties, passes):
assertion = self.assertTrue if passes else self.assertFalse
assertion(self.filt_cls.host_passes(host_state, filter_properties))
def test_aggregate_type_extra_specs_ha_no_extraspecs(self):
host_state = fakes.FakeHostState('host1', 'compute', {})
host_state.aggregates.append(self.aggr_no_extraspecs)
self.assertTrue(self.filt_cls.host_passes(host_state,
self.filter_properties))
def test_aggregate_type_extra_specs_ha_wrong_extraspecs(self):
host_state = fakes.FakeHostState('host1', 'compute', {})
host_state.aggregates.append(self.aggr_invalid_extraspecs)
self.assertFalse(self.filt_cls.host_passes(host_state,
self.filter_properties))
def test_aggregate_type_extra_specs_ha_right_extraspecs(self):
host_state = fakes.FakeHostState('host1', 'compute', {})
host_state.aggregates.append(self.aggr_valid_extraspecs)
self.assertTrue(self.filt_cls.host_passes(host_state,
self.filter_properties))
def test_aggregate_type_extra_specs_2ha_noextra_wrong(self):
host_state = fakes.FakeHostState('host1', 'compute', {})
host_state.aggregates.append(self.aggr_no_extraspecs)
host_state.aggregates.append(self.aggr_invalid_extraspecs)
self.assertFalse(self.filt_cls.host_passes(host_state,
self.filter_properties))
def test_aggregate_type_extra_specs_2ha_noextra_right(self):
host_state = fakes.FakeHostState('host1', 'compute', {})
host_state.aggregates.append(self.aggr_no_extraspecs)
host_state.aggregates.append(self.aggr_valid_extraspecs)
self.assertTrue(self.filt_cls.host_passes(host_state,
self.filter_properties))
def test_aggregate_type_extra_specs_2ha_wrong_right(self):
host_state = fakes.FakeHostState('host1', 'compute', {})
host_state.aggregates.append(self.aggr_invalid_extraspecs)
host_state.aggregates.append(self.aggr_valid_extraspecs)
self.assertFalse(self.filt_cls.host_passes(host_state,
self.filter_properties))
def test_aggregate_type_extra_specs_any_and_more_large(self):
host_state = fakes.FakeHostState('host1', 'compute', {})
host_state.aggregates.append(self.aggr_optional_large)
self.assertFalse(self.filt_cls.host_passes(host_state,
self.filter_properties))
def test_aggregate_type_extra_specs_any_and_more_small(self):
host_state = fakes.FakeHostState('host1', 'compute', {})
host_state.aggregates.append(self.aggr_optional_small)
self.assertTrue(self.filt_cls.host_passes(host_state,
self.filter_properties))
def test_aggregate_type_extra_specs_any_and_more_no_extraspecs(self):
host_state = fakes.FakeHostState('host1', 'compute', {})
host_state.aggregates.append(self.aggr_optional_large)
self._do_test_compute_filter_extra_specs(host_state,
self.filter_properties_noextraspecs, True)
def test_aggregate_type_extra_specs_any_order(self):
host_state = fakes.FakeHostState('host1', 'compute', {})
host_state.aggregates.append(self.aggr_optional_first)
self.assertTrue(self.filt_cls.host_passes(host_state,
self.filter_properties))
host_state.aggregates = [self.aggr_optional_last]
self.assertTrue(self.filt_cls.host_passes(host_state,
self.filter_properties))
host_state.aggregates = [self.aggr_optional_middle]
self.assertTrue(self.filt_cls.host_passes(host_state,
self.filter_properties))
def test_aggregate_type_extra_specs_kleene(self):
host_state = fakes.FakeHostState('host1', 'compute', {})
host_state.aggregates.append(self.aggr_any)
self.assertTrue(self.filt_cls.host_passes(host_state,
self.filter_properties))
def test_aggregate_type_extra_specs_kleene_no_extraspecs(self):
host_state = fakes.FakeHostState('host1', 'compute', {})
host_state.aggregates.append(self.aggr_any)
self.assertFalse(self.filt_cls.host_passes(host_state,
self.filter_properties_noextraspecs))
def test_aggregate_type_extra_specs_nospec(self):
host_state = fakes.FakeHostState('host1', 'compute', {})
host_state.aggregates.append(self.aggr_nopresent)
self.assertFalse(self.filt_cls.host_passes(host_state,
self.filter_properties))
def test_aggregate_type_extra_specs_nospec_no_extraspecs(self):
host_state = fakes.FakeHostState('host1', 'compute', {})
host_state.aggregates.append(self.aggr_nopresent)
self.assertTrue(self.filt_cls.host_passes(host_state,
self.filter_properties_noextraspecs))