From e7000b1f7d234cc4036bf4bed840173a9cc6a243 Mon Sep 17 00:00:00 2001 From: shilpa Date: Mon, 15 Jul 2019 15:25:40 +0530 Subject: [PATCH] DB API changes to get non-matching aggregates from metadata Added a new DB API to return list of aggregates that don't match metadata keys that start with key_prefix with the list of required traits set on the flavor and image. Change-Id: I7dd54e11288a0ad2569abfc351258e97b5f5803c Implements: blueprint placement-req-filter-forbidden-aggregates --- nova/objects/aggregate.py | 70 ++++++++++++ nova/tests/functional/db/test_aggregate.py | 118 +++++++++++++++++++++ nova/tests/unit/objects/test_aggregate.py | 18 ++++ 3 files changed, 206 insertions(+) diff --git a/nova/objects/aggregate.py b/nova/objects/aggregate.py index 399837a53811..548fc038e598 100644 --- a/nova/objects/aggregate.py +++ b/nova/objects/aggregate.py @@ -451,6 +451,38 @@ def _get_by_metadata_from_db(context, key=None, value=None): return query.all() +@db_api.api_context_manager.reader +def _get_non_matching_by_metadata_keys_from_db(context, ignored_keys, + key_prefix, value): + """Filter aggregates based on non matching metadata. + + Find aggregates with at least one ${key_prefix}*[=${value}] metadata where + the metadata key are not in the ignored_keys list. + + :return: Aggregates with any metadata entry: + - whose key starts with `key_prefix`; and + - whose value is `value` and + - whose key is *not* in the `ignored_keys` list. + """ + + if not key_prefix: + raise ValueError(_('key_prefix mandatory field.')) + + query = context.session.query(api_models.Aggregate) + query = query.join("_metadata") + query = query.filter(api_models.AggregateMetadata.value == value) + query = query.filter(api_models.AggregateMetadata.key.like( + key_prefix + '%')) + if len(ignored_keys) > 0: + query = query.filter(~api_models.AggregateMetadata.key.in_( + ignored_keys)) + + query = query.options(contains_eager("_metadata")) + query = query.options(joinedload("_hosts")) + + return query.all() + + @base.NovaObjectRegistry.register class AggregateList(base.ObjectListBase, base.NovaObject): # Version 1.0: Initial version @@ -507,3 +539,41 @@ class AggregateList(base.ObjectListBase, base.NovaObject): db_aggregates = _get_by_metadata_from_db(context, key=key, value=value) return base.obj_make_list(context, cls(context), objects.Aggregate, db_aggregates) + + @classmethod + def get_non_matching_by_metadata_keys(cls, context, ignored_keys, + key_prefix, value): + """Return aggregates that are not matching with metadata. + + For example, we have aggregates with metadata as below: + + 'agg1' with trait:HW_CPU_X86_MMX="required" + 'agg2' with trait:HW_CPU_X86_SGX="required" + 'agg3' with trait:HW_CPU_X86_MMX="required" + 'agg3' with trait:HW_CPU_X86_SGX="required" + + Assume below request: + + aggregate_obj.AggregateList.get_non_matching_by_metadata_keys( + self.context, + ['trait:HW_CPU_X86_MMX'], + 'trait:', + value='required') + + It will return 'agg2' and 'agg3' as aggregates that are not matching + with metadata. + + :param context: The security context + :param ignored_keys: List of keys to match with the aggregate metadata + keys that starts with key_prefix. + :param key_prefix: Only compares metadata keys that starts with the + key_prefix + :param value: Value of metadata + + :returns: List of aggregates that doesn't match metadata keys that + starts with key_prefix with the supplied keys. + """ + db_aggregates = _get_non_matching_by_metadata_keys_from_db( + context, ignored_keys, key_prefix, value) + return base.obj_make_list(context, objects.AggregateList(context), + objects.Aggregate, db_aggregates) diff --git a/nova/tests/functional/db/test_aggregate.py b/nova/tests/functional/db/test_aggregate.py index ffc08cfec03c..9300baa678d2 100644 --- a/nova/tests/functional/db/test_aggregate.py +++ b/nova/tests/functional/db/test_aggregate.py @@ -561,3 +561,121 @@ class AggregateObjectTestCase(test.TestCase): self.assertRaises(AssertionError, aggregate_obj._get_by_metadata_from_db, self.context) + + def test_get_non_matching_by_metadata_keys(self): + """Test aggregates that are not matching with metadata.""" + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 1) + agg.update_metadata({'trait:HW_CPU_X86_MMX': 'required'}) + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 2) + agg.update_metadata({'trait:HW_CPU_X86_SGX': 'required'}) + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 3) + agg.update_metadata({'trait:HW_CPU_X86_MMX': 'required', + 'trait:HW_CPU_X86_SGX': 'required'}) + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 4) + agg.update_metadata({'trait:HW_CPU_X86_MMX': 'required', + 'trait:HW_CPU_X86_SGX': 'just_for_marking'}) + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 5) + agg.update_metadata({'trait:HW_CPU_X86_MMX': 'just_for_marking'}) + + aggs = aggregate_obj.AggregateList.get_non_matching_by_metadata_keys( + self.context, ['trait:HW_CPU_X86_MMX'], 'trait:', + value='required') + + self.assertEqual(2, len(aggs)) + self.assertItemsEqual([2, 3], [a.id for a in aggs]) + + def test_matching_aggregates_multiple_keys(self): + """All matching aggregates for multiple keys.""" + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 1) + agg.update_metadata({'trait:HW_CPU_X86_MMX': 'required'}) + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 2) + agg.update_metadata({'trait:HW_CPU_X86_SGX': 'required'}) + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 3) + agg.update_metadata({'trait:HW_CPU_X86_MMX': 'required', + 'trait:HW_CPU_X86_SGX': 'required'}) + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 4) + agg.update_metadata({'trait:HW_CPU_X86_MMX': 'required', + 'trait:HW_CPU_X86_SGX': 'just_for_marking'}) + + aggs = aggregate_obj.AggregateList.get_non_matching_by_metadata_keys( + self.context, ['trait:HW_CPU_X86_MMX', 'trait:HW_CPU_X86_SGX'], + 'trait:', value='required') + + self.assertEqual(0, len(aggs)) + + def test_get_non_matching_aggregates_multiple_keys(self): + """Return non matching aggregates for multiple keys.""" + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 1) + agg.update_metadata({'trait:HW_CPU_X86_MMX': 'required'}) + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 2) + agg.update_metadata({'trait:HW_CPU_X86_MMX': 'required', + 'trait:HW_CPU_X86_SGX': 'required', + 'trait:HW_CPU_API_DXVA': 'required'}) + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 3) + agg.update_metadata({'trait:HW_CPU_X86_MMX': 'required', + 'trait:HW_CPU_X86_SGX': 'required'}) + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 4) + agg.update_metadata({'trait:HW_CPU_X86_MMX': 'required', + 'trait:HW_CPU_X86_SGX': 'just_for_marking'}) + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 5) + agg.update_metadata({'trait:HW_CPU_X86_MMX': 'required', + 'trait:HW_CPU_X86_SGX': 'just_for_marking', + 'trait:HW_CPU_X86_SSE': 'required'}) + + aggs = aggregate_obj.AggregateList.get_non_matching_by_metadata_keys( + self.context, ['trait:HW_CPU_X86_MMX', 'trait:HW_CPU_X86_SGX'], + 'trait:', value='required') + + self.assertEqual(2, len(aggs)) + self.assertItemsEqual([2, 5], [a.id for a in aggs]) + + def test_get_non_matching_by_metadata_keys_empty_keys(self): + """Test aggregates non matching by metadata with empty keys.""" + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 1) + agg.update_metadata({'trait:HW_CPU_X86_MMX': 'required'}) + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 2) + agg.update_metadata({'trait:HW_CPU_X86_SGX': 'required'}) + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 3) + agg.update_metadata({'trait:HW_CPU_X86_MMX': 'required', + 'trait:HW_CPU_X86_SGX': 'required'}) + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 4) + agg.update_metadata({'trait:HW_CPU_X86_MMX': 'required', + 'trait:HW_CPU_X86_SGX': 'just_for_marking'}) + + agg = aggregate_obj.Aggregate.get_by_id(self.context, 5) + agg.update_metadata({'trait:HW_CPU_X86_MMX': 'required', + 'trait:HW_CPU_X86_SGX': 'just_for_marking', + 'trait:HW_CPU_X86_SSE': 'required'}) + + aggs = aggregate_obj.AggregateList.get_non_matching_by_metadata_keys( + self.context, [], 'trait:', value='required') + + self.assertEqual(5, len(aggs)) + self.assertItemsEqual([1, 2, 3, 4, 5], [a.id for a in aggs]) + + def test_get_non_matching_by_metadata_keys_empty_key_prefix(self): + """Test aggregates non matching by metadata with empty key_prefix.""" + + self.assertRaises( + ValueError, + aggregate_obj.AggregateList.get_non_matching_by_metadata_keys, + self.context, ['trait:HW_CPU_X86_MMX'], '', + value='required') diff --git a/nova/tests/unit/objects/test_aggregate.py b/nova/tests/unit/objects/test_aggregate.py index e0e189050f97..c0e148b61926 100644 --- a/nova/tests/unit/objects/test_aggregate.py +++ b/nova/tests/unit/objects/test_aggregate.py @@ -239,6 +239,24 @@ class _TestAggregateObject(object): self.assertEqual(1, len(aggs)) self.compare_obj(aggs[0], fake_aggregate, subs=SUBS) + @mock.patch('nova.objects.aggregate.' + '_get_non_matching_by_metadata_keys_from_db') + def test_get_non_matching_by_metadata_keys( + self, get_non_matching_by_metadata_keys): + get_non_matching_by_metadata_keys.return_value = [fake_aggregate] + aggs = aggregate.AggregateList.get_non_matching_by_metadata_keys( + self.context, ['abc'], 'th', value='that') + self.assertEqual('that', aggs[0].metadata['this']) + + @mock.patch('nova.objects.aggregate.' + '_get_non_matching_by_metadata_keys_from_db') + def test_get_non_matching_by_metadata_keys_and_hosts_no_match( + self, get_non_matching_by_metadata_keys): + get_non_matching_by_metadata_keys.return_value = [] + aggs = aggregate.AggregateList.get_non_matching_by_metadata_keys( + self.context, ['this'], 'th', value='that') + self.assertEqual(0, len(aggs)) + class TestAggregateObject(test_objects._LocalTest, _TestAggregateObject):