Implements "not" operator for complex query

Change-Id: Idf17b35c5f4267b9254a64e37b0d8b1b0dcbca89
Implements: blueprint complex-filter-expressions-in-api-queries
This commit is contained in:
Balazs Gibizer 2014-01-03 16:26:50 +01:00
parent 861d83cad3
commit c5670978d9
7 changed files with 294 additions and 74 deletions

View File

@ -1115,15 +1115,24 @@ class ValidatedComplexQuery(object):
"minProperties": 1,
"maxProperties": 1}
schema_not = {
"type": "object",
"patternProperties": {"(?i)^not$": {"$ref": "#"}},
"additionalProperties": False,
"minProperties": 1,
"maxProperties": 1}
self.schema = {
"oneOf": [{"$ref": "#/definitions/leaf_simple_ops"},
{"$ref": "#/definitions/leaf_in"},
{"$ref": "#/definitions/and_or"}],
{"$ref": "#/definitions/and_or"},
{"$ref": "#/definitions/not"}],
"minProperties": 1,
"maxProperties": 1,
"definitions": {"leaf_simple_ops": schema_leaf_simple_ops,
"leaf_in": schema_leaf_in,
"and_or": schema_and_or}}
"and_or": schema_and_or,
"not": schema_not}}
self.orderby_schema = {
"type": "array",
@ -1184,6 +1193,8 @@ class ValidatedComplexQuery(object):
if op.lower() in self.complex_operators:
for i, operand in enumerate(tree[op]):
self._traverse_postorder(operand, visitor)
if op.lower() == "not":
self._traverse_postorder(tree[op], visitor)
visitor(tree)

View File

@ -27,6 +27,7 @@ from sqlalchemy import and_
from sqlalchemy import asc
from sqlalchemy import desc
from sqlalchemy import func
from sqlalchemy import not_
from sqlalchemy import or_
from sqlalchemy.orm import aliased
@ -1207,7 +1208,8 @@ class QueryTransformer(object):
"in": lambda field_name, values: field_name.in_(values)}
complex_operators = {"or": or_,
"and": and_}
"and": and_,
"not": not_}
ordering_functions = {"asc": asc,
"desc": desc}
@ -1218,6 +1220,8 @@ class QueryTransformer(object):
def _handle_complex_op(self, complex_op, nodes):
op = self.complex_operators[complex_op]
if op == not_:
nodes = [nodes]
element_list = []
for node in nodes:
element = self._transform(node)

View File

@ -130,21 +130,6 @@ class Connection(base.Connection):
"""Base Connection class for MongoDB and DB2 drivers.
"""
operators = {"<": "$lt",
">": "$gt",
"<=": "$lte",
"=<": "$lte",
">=": "$gte",
"=>": "$gte",
"!=": "$ne",
"in": "$in"}
complex_operators = {"or": "$or",
"and": "$and"}
ordering_functions = {"asc": pymongo.ASCENDING,
"desc": pymongo.DESCENDING}
def get_users(self, source=None):
"""Return an iterable of user id strings.
@ -288,10 +273,11 @@ class Connection(base.Connection):
return []
query_filter = {}
orderby_filter = [("timestamp", pymongo.DESCENDING)]
transformer = QueryTransformer()
if orderby is not None:
orderby_filter = self._transform_orderby(orderby)
orderby_filter = transformer.transform_orderby(orderby)
if filter_expr is not None:
query_filter = self._transform_filter(filter_expr)
query_filter = transformer.transform_filter(filter_expr)
retrieve = {models.Meter: self._retrieve_samples,
models.Alarm: self._retrieve_alarms,
@ -348,45 +334,6 @@ class Connection(base.Connection):
del ah['_id']
yield models.AlarmChange(**ah)
def _transform_orderby(self, orderby):
orderby_filter = []
for field in orderby:
field_name = field.keys()[0]
ordering = self.ordering_functions[field.values()[0]]
orderby_filter.append((field_name, ordering))
return orderby_filter
def _transform_filter(self, condition):
def process_json_tree(condition_tree):
operator_node = condition_tree.keys()[0]
nodes = condition_tree.values()[0]
if operator_node in self.complex_operators:
element_list = []
for node in nodes:
element = process_json_tree(node)
element_list.append(element)
complex_operator = self.complex_operators[operator_node]
op = {complex_operator: element_list}
return op
else:
field_name = nodes.keys()[0]
field_value = nodes.values()[0]
# no operator for equal in Mongo
if operator_node == "=":
op = {field_name: field_value}
return op
if operator_node in self.operators:
operator = self.operators[operator_node]
op = {
field_name: {
operator: field_value}}
return op
return process_json_tree(condition)
@classmethod
def _ensure_encapsulated_rule_format(cls, alarm):
"""This ensure the alarm returned by the storage have the correct
@ -449,3 +396,124 @@ class Connection(base.Connection):
for elem in matching_metadata:
new_matching_metadata[elem['key']] = elem['value']
return new_matching_metadata
class QueryTransformer(object):
operators = {"<": "$lt",
">": "$gt",
"<=": "$lte",
"=<": "$lte",
">=": "$gte",
"=>": "$gte",
"!=": "$ne",
"in": "$in"}
complex_operators = {"or": "$or",
"and": "$and"}
ordering_functions = {"asc": pymongo.ASCENDING,
"desc": pymongo.DESCENDING}
def transform_orderby(self, orderby):
orderby_filter = []
for field in orderby:
field_name = field.keys()[0]
ordering = self.ordering_functions[field.values()[0]]
orderby_filter.append((field_name, ordering))
return orderby_filter
@staticmethod
def _move_negation_to_leaf(condition):
"""Moves every not operator to the leafs by
applying the De Morgan rules and anihilating
double negations
"""
def _apply_de_morgan(tree, negated_subtree, negated_op):
if negated_op == "and":
new_op = "or"
else:
new_op = "and"
tree[new_op] = [{"not": child}
for child in negated_subtree[negated_op]]
del tree["not"]
def transform(subtree):
op = subtree.keys()[0]
if op in ["and", "or"]:
[transform(child) for child in subtree[op]]
elif op == "not":
negated_tree = subtree[op]
negated_op = negated_tree.keys()[0]
if negated_op == "and":
_apply_de_morgan(subtree, negated_tree, negated_op)
transform(subtree)
elif negated_op == "or":
_apply_de_morgan(subtree, negated_tree, negated_op)
transform(subtree)
elif negated_op == "not":
# two consecutive not annihilates theirselves
new_op = negated_tree.values()[0].keys()[0]
subtree[new_op] = negated_tree[negated_op][new_op]
del subtree["not"]
transform(subtree)
transform(condition)
def transform_filter(self, condition):
# in Mongo not operator can only be applied to
# simple expressions so we have to move every
# not operator to the leafs of the expression tree
self._move_negation_to_leaf(condition)
return self._process_json_tree(condition)
def _handle_complex_op(self, complex_op, nodes):
element_list = []
for node in nodes:
element = self._process_json_tree(node)
element_list.append(element)
complex_operator = self.complex_operators[complex_op]
op = {complex_operator: element_list}
return op
def _handle_not_op(self, negated_tree):
# assumes that not is moved to the leaf already
# so we are next to a leaf
negated_op = negated_tree.keys()[0]
negated_field = negated_tree[negated_op].keys()[0]
value = negated_tree[negated_op][negated_field]
if negated_op == "=":
return {negated_field: {"$ne": value}}
elif negated_op == "!=":
return {negated_field: value}
else:
return {negated_field: {"$not":
{self.operators[negated_op]: value}}}
def _handle_simple_op(self, simple_op, nodes):
field_name = nodes.keys()[0]
field_value = nodes.values()[0]
# no operator for equal in Mongo
if simple_op == "=":
op = {field_name: field_value}
return op
operator = self.operators[simple_op]
op = {field_name: {operator: field_value}}
return op
def _process_json_tree(self, condition_tree):
operator_node = condition_tree.keys()[0]
nodes = condition_tree.values()[0]
if operator_node in self.complex_operators:
return self._handle_complex_op(operator_node, nodes)
if operator_node == "not":
negated_tree = condition_tree[operator_node]
return self._handle_not_op(negated_tree)
return self._handle_simple_op(operator_node, nodes)

View File

@ -360,3 +360,30 @@ class TestFilterSyntaxValidation(test.BaseTestCase):
self.assertRaises(jsonschema.ValidationError,
self.query._validate_filter,
filter)
def test_not(self):
filter = {"not": {"=": {"project_id": "value"}}}
self.query._validate_filter(filter)
filter = {
"not":
{"or":
[{"and":
[{"=": {"project_id": "string_value"}},
{"=": {"resource_id": "value"}},
{"<": {"counter_name": 42}}]},
{"=": {"counter_name": "value"}}]}}
self.query._validate_filter(filter)
def test_not_with_zero_child_is_invalid(self):
filter = {"not": {}}
self.assertRaises(jsonschema.ValidationError,
self.query._validate_filter,
filter)
def test_not_with_more_than_one_child_is_invalid(self):
filter = {"not": {"=": {"project_id": "value"},
"!=": {"resource_id": "value"}}}
self.assertRaises(jsonschema.ValidationError,
self.query._validate_filter,
filter)

View File

@ -256,6 +256,15 @@ class TestQueryMetersController(tests_api.FunctionalTest,
for sample in data.json:
self.assertTrue(sample["metadata"]["util"] >= 0.5)
def test_filter_with_negation(self):
filter_expr = '{"not": {">=": {"metadata.util": 0.5}}}'
data = self.post_json(self.url,
params={"filter": filter_expr})
self.assertEqual(1, len(data.json))
for sample in data.json:
self.assertTrue(float(sample["metadata"]["util"]) < 0.5)
def test_limit_should_be_positive(self):
data = self.post_json(self.url,
params={"limit": 0},

View File

@ -728,20 +728,21 @@ class ComplexSampleQueryTest(DBTestBase,
tests_db.MixinTestsWithBackendScenarios):
def setUp(self):
super(ComplexSampleQueryTest, self).setUp()
self.complex_filter = {"and":
[{"or":
[{"=": {"resource_id": "resource-id-42"}},
{"=": {"resource_id": "resource-id-44"}}]},
{"and":
[{"=": {"counter_name": "cpu_util"}},
{"and":
[{">": {"counter_volume": 0.4}},
{"<=": {"counter_volume": 0.8}}]}]}]}
self.complex_filter = {
"and":
[{"or":
[{"=": {"resource_id": "resource-id-42"}},
{"=": {"resource_id": "resource-id-44"}}]},
{"and":
[{"=": {"counter_name": "cpu_util"}},
{"and":
[{">": {"counter_volume": 0.4}},
{"not": {">": {"counter_volume": 0.8}}}]}]}]}
or_expression = [{"=": {"resource_id": "resource-id-42"}},
{"=": {"resource_id": "resource-id-43"}},
{"=": {"resource_id": "resource-id-44"}}]
and_expression = [{">": {"counter_volume": 0.4}},
{"<=": {"counter_volume": 0.8}}]
{"not": {">": {"counter_volume": 0.8}}}]
self.complex_filter_list = {"and":
[{"or": or_expression},
{"and":
@ -1017,6 +1018,95 @@ class ComplexSampleQueryTest(DBTestBase,
results = list(self.conn.query_samples(filter_expr=filter_expr))
self.assertEqual(len(results), 0)
def test_query_negated_metadata(self):
self._create_samples()
filter_expr = {
"and": [{"=": {"resource_id": "resource-id-42"}},
{"not": {"or": [{">": {"resource_metadata.an_int_key":
43}},
{"<=": {"resource_metadata.a_float_key":
0.41}}]}}]}
results = list(self.conn.query_samples(filter_expr=filter_expr))
self.assertEqual(len(results), 3)
for sample in results:
self.assertEqual(sample.resource_id, "resource-id-42")
self.assertTrue(sample.resource_metadata["an_int_key"] <= 43)
self.assertTrue(sample.resource_metadata["a_float_key"] > 0.41)
def test_query_negated_complex_expression(self):
self._create_samples()
filter_expr = {
"and":
[{"=": {"counter_name": "cpu_util"}},
{"not":
{"or":
[{"or":
[{"=": {"resource_id": "resource-id-42"}},
{"=": {"resource_id": "resource-id-44"}}]},
{"and":
[{">": {"counter_volume": 0.4}},
{"<": {"counter_volume": 0.8}}]}]}}]}
results = list(self.conn.query_samples(filter_expr=filter_expr))
self.assertEqual(len(results), 4)
for sample in results:
self.assertEqual(sample.resource_id,
"resource-id-43")
self.assertIn(sample.counter_volume, [0.39, 0.4, 0.8, 0.81])
self.assertEqual(sample.counter_name,
"cpu_util")
def test_query_with_double_negation(self):
self._create_samples()
filter_expr = {
"and":
[{"=": {"counter_name": "cpu_util"}},
{"not":
{"or":
[{"or":
[{"=": {"resource_id": "resource-id-42"}},
{"=": {"resource_id": "resource-id-44"}}]},
{"and": [{"not": {"<=": {"counter_volume": 0.4}}},
{"<": {"counter_volume": 0.8}}]}]}}]}
results = list(self.conn.query_samples(filter_expr=filter_expr))
self.assertEqual(len(results), 4)
for sample in results:
self.assertEqual(sample.resource_id,
"resource-id-43")
self.assertIn(sample.counter_volume, [0.39, 0.4, 0.8, 0.81])
self.assertEqual(sample.counter_name,
"cpu_util")
def test_query_negate_not_equal(self):
self._create_samples()
filter_expr = {"not": {"!=": {"resource_id": "resource-id-43"}}}
results = list(self.conn.query_samples(filter_expr=filter_expr))
self.assertEqual(len(results), 6)
for sample in results:
self.assertEqual(sample.resource_id,
"resource-id-43")
def test_query_negated_in_op(self):
self._create_samples()
filter_expr = {
"and": [{"not": {"in": {"counter_volume": [0.39, 0.4, 0.79]}}},
{"=": {"resource_id": "resource-id-42"}}]}
results = list(self.conn.query_samples(filter_expr=filter_expr))
self.assertEqual(len(results), 3)
for sample in results:
self.assertIn(sample.counter_volume,
[0.41, 0.8, 0.81])
class StatisticsTest(DBTestBase,
tests_db.MixinTestsWithBackendScenarios):

View File

@ -96,8 +96,18 @@ Complex Query
The filter expressions of the Complex Query feature operate on the fields
of *Sample*, *Alarm* and *AlarmChange*. The following comparison operators are
supported: *=*, *!=*, *<*, *<=*, *>*, *>=* and *in*; and the following logical
operators can be used: *and* and *or*. The field names are validated against
the database models.
operators can be used: *and* *or* and *not*. The field names are validated
against the database models.
.. note:: The *not* operator has different meaning in Mongo DB and in SQL DB engine.
If the *not* operator is applied on a non existent metadata field then
the result depends on the DB engine. For example if
{"not": {"metadata.nonexistent_field" : "some value"}} filter is used in a query
the Mongo DB will return every Sample object as *not* operator evaluated true
for every Sample where the given field does not exists. See more in the Mongod DB doc.
In the other hand SQL based DB engine will return empty result as the join operation
on the metadata table will return zero row as the on clause of the join which
tries to match on the metadata field name is never fulfilled.
Complex Query supports defining the list of orderby expressions in the form
of [{"field_name": "asc"}, {"field_name2": "desc"}, ...].
@ -418,13 +428,14 @@ to the /v2/query/samples endpoint of Ceilometer API using POST request.
To check for *cpu_util* samples reported between 18:00-18:15 or between 18:30 - 18:45
on a particular date (2013-12-01), where the utilization is between 23 and 26 percent,
the following filter expression can be created::
but not exactly 25.12 percent, the following filter expression can be created::
{"and":
[{"and":
[{"=": {"counter_name": "cpu_util"}},
{">": {"counter_volume": 0.23}},
{"<": {"counter_volume": 0.26}}]},
{"<": {"counter_volume": 0.26}},
{"not": {"=": {"counter_volume": 0.2512}}}]},
{"or":
[{"and":
[{">": {"timestamp": "2013-12-01T18:00:00"}},
@ -446,7 +457,7 @@ By adding a limit criteria to the request, which maximizes the number of returne
to four, the query looks like the following::
{
"filter" : "{\"and\":[{\"and\": [{\"=\": {\"counter_name\": \"cpu_util\"}}, {\">\": {\"counter_volume\": 0.23}}, {\"<\": {\"counter_volume\": 0.26}}]}, {\"or\": [{\"and\": [{\">\": {\"timestamp\": \"2013-12-01T18:00:00\"}}, {\"<\": {\"timestamp\": \"2013-12-01T18:15:00\"}}]}, {\"and\": [{\">\": {\"timestamp\": \"2013-12-01T18:30:00\"}}, {\"<\": {\"timestamp\": \"2013-12-01T18:45:00\"}}]}]}]}",
"filter" : "{\"and\":[{\"and\": [{\"=\": {\"counter_name\": \"cpu_util\"}}, {\">\": {\"counter_volume\": 0.23}}, {\"<\": {\"counter_volume\": 0.26}}, {\"not\": {\"=\": {\"counter_volume\": 0.2512}}}]}, {\"or\": [{\"and\": [{\">\": {\"timestamp\": \"2013-12-01T18:00:00\"}}, {\"<\": {\"timestamp\": \"2013-12-01T18:15:00\"}}]}, {\"and\": [{\">\": {\"timestamp\": \"2013-12-01T18:30:00\"}}, {\"<\": {\"timestamp\": \"2013-12-01T18:45:00\"}}]}]}]}",
"orderby" : "[{\"counter_volume\": \"ASC\"}, {\"timestamp\": \"DESC\"}]",
"limit" : 4
}