openstack/deb-ceilometer
Code Issues Proposed changes

Implements metadata query for complex query feature

Browse Source 
A subfield in the resource_metadata field of the Sample
can be used in filter expression as
  metadata.<subfield_name>

Validation of the names of the metadata is not implemented

Implements: blueprint complex-filter-expressions-in-api-queries
Change-Id: Icf198f740191b5fff0008e9aa7b054c1cc33d75d
This commit is contained in:
Balazs Gibizer
2014-01-02 19:42:09 +01:00
parent 2c0bfc7009
commit 861d83cad3
5 changed files with 332 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

172
ceilometer/api/controllers/v2.py
View File

@@ -1047,105 +1047,95 @@ class ValidatedComplexQuery(object):
simple_ops = _list_to_regexp(simple_ops, regexp_prefix)
order_directions = _list_to_regexp(order_directions, regexp_prefix)
schema_value = {
"oneOf": [{"type": "string"},
{"type": "number"}],
"minProperties": 1,
"maxProperties": 1}
schema_value_in = {
"type": "array",
"items": {"oneOf": [{"type": "string"},
{"type": "number"}]}}
schema_field = {
"type": "object",
"patternProperties": {"[\S]+": schema_value},
"additionalProperties": False,
"minProperties": 1,
"maxProperties": 1}
schema_field_in = {
"type": "object",
"patternProperties": {"[\S]+": schema_value_in},
"additionalProperties": False,
"minProperties": 1,
"maxProperties": 1}
schema_leaf = {
"type": "object",
"patternProperties": {simple_ops: schema_field},
"additionalProperties": False,
"minProperties": 1,
"maxProperties": 1}
schema_leaf_in = {
"type": "object",
"patternProperties": {"(?i)^in$": schema_field_in},
"additionalProperties": False,
"minProperties": 1,
"maxProperties": 1}
schema_leaf_simple_ops = {
"type": "object",
"patternProperties": {simple_ops: schema_field},
"additionalProperties": False,
"minProperties": 1,
"maxProperties": 1}
schema_and_or_array = {
"type": "array",
"items": {"$ref": "#"},
"minItems": 2}
schema_and_or = {
"type": "object",
"patternProperties": {complex_ops: schema_and_or_array},
"additionalProperties": False,
"minProperties": 1,
"maxProperties": 1}
schema = {
"oneOf": [{"$ref": "#/definitions/leaf_simple_ops"},
{"$ref": "#/definitions/leaf_in"},
{"$ref": "#/definitions/and_or"}],
"minProperties": 1,
"maxProperties": 1,
"definitions": {"leaf_simple_ops": schema_leaf_simple_ops,
"leaf_in": schema_leaf_in,
"and_or": schema_and_or}}
orderby_schema = {
"type": "array",
"items": {
"type": "object",
"patternProperties":
{"[\S]+":
{"type": "string",
"pattern": order_directions}},
"additionalProperties": False,
"minProperties": 1,
"maxProperties": 1}}
timestamp_fields = ["timestamp", "state_timestamp"]
name_mapping = {"user": "user_id",
"project": "project_id",
"resource": "resource_id"}
def __init__(self, query, db_model, additional_valid_keys):
def __init__(self, query, db_model, additional_valid_keys,
metadata_allowed=False):
valid_keys = db_model.get_field_names()
valid_keys = list(valid_keys) + additional_valid_keys
valid_fields = _list_to_regexp(valid_keys)
self.schema_field["patternProperties"] = {
valid_fields: self.schema_value}
if metadata_allowed:
valid_filter_fields = valid_fields + "|^metadata\.[\S]+$"
else:
valid_filter_fields = valid_fields
self.schema_field_in["patternProperties"] = {
valid_fields: self.schema_value_in}
schema_value = {
"oneOf": [{"type": "string"},
{"type": "number"},
{"type": "boolean"}],
"minProperties": 1,
"maxProperties": 1}
self.orderby_schema["items"]["patternProperties"] = {
valid_fields: {"type": "string",
"pattern": self.order_directions}}
schema_value_in = {
"type": "array",
"items": {"oneOf": [{"type": "string"},
{"type": "number"}]}}
schema_field = {
"type": "object",
"patternProperties": {valid_filter_fields: schema_value},
"additionalProperties": False,
"minProperties": 1,
"maxProperties": 1}
schema_field_in = {
"type": "object",
"patternProperties": {valid_filter_fields: schema_value_in},
"additionalProperties": False,
"minProperties": 1,
"maxProperties": 1}
schema_leaf_in = {
"type": "object",
"patternProperties": {"(?i)^in$": schema_field_in},
"additionalProperties": False,
"minProperties": 1,
"maxProperties": 1}
schema_leaf_simple_ops = {
"type": "object",
"patternProperties": {self.simple_ops: schema_field},
"additionalProperties": False,
"minProperties": 1,
"maxProperties": 1}
schema_and_or_array = {
"type": "array",
"items": {"$ref": "#"},
"minItems": 2}
schema_and_or = {
"type": "object",
"patternProperties": {self.complex_ops: schema_and_or_array},
"additionalProperties": False,
"minProperties": 1,
"maxProperties": 1}
self.schema = {
"oneOf": [{"$ref": "#/definitions/leaf_simple_ops"},
{"$ref": "#/definitions/leaf_in"},
{"$ref": "#/definitions/and_or"}],
"minProperties": 1,
"maxProperties": 1,
"definitions": {"leaf_simple_ops": schema_leaf_simple_ops,
"leaf_in": schema_leaf_in,
"and_or": schema_and_or}}
self.orderby_schema = {
"type": "array",
"items": {
"type": "object",
"patternProperties":
{valid_fields:
{"type": "string",
"pattern": self.order_directions}},
"additionalProperties": False,
"minProperties": 1,
"maxProperties": 1}}
self.original_query = query
@@ -1253,6 +1243,9 @@ class ValidatedComplexQuery(object):
if field in self.name_mapping:
del subfilter[field]
subfilter[self.name_mapping[field]] = value
if field.startswith("metadata."):
del subfilter[field]
subfilter["resource_" + field] = value
def _convert_operator_to_lower_case(self, filter_expr):
self._traverse_postorder(filter_expr, utils.lowercase_keys)
@@ -2151,7 +2144,8 @@ class QuerySamplesController(rest.RestController):
"""
query = ValidatedComplexQuery(body,
storage.models.Sample,
["user", "project", "resource"])
["user", "project", "resource"],
metadata_allowed=True)
query.validate(visibility_field="project_id")
conn = pecan.request.storage_conn
return [Sample.from_db_model(s)

156
ceilometer/storage/impl_sqlalchemy.py
View File

@@ -180,27 +180,9 @@ def make_query_from_filter(session, query, sample_filter, require_meter=True):
return query
def operator_in(field_name, field_value):
return field_name.in_(field_value)
class Connection(base.Connection):
"""SqlAlchemy connection."""
operators = {"=": operator.eq,
"<": operator.lt,
">": operator.gt,
"<=": operator.le,
"=<": operator.le,
">=": operator.ge,
"=>": operator.ge,
"!=": operator.ne,
"in": operator_in}
complex_operators = {"or": or_,
"and": and_}
ordering_functions = {"asc": asc,
"desc": desc}
def __init__(self, conf):
url = conf.database.connection
if url == 'sqlite://':
@@ -622,12 +604,10 @@ class Connection(base.Connection):
query = session.query(table)
query = make_query_from_filter(session, query, sample_filter,
require_meter=False)
query = self._apply_options(query,
None,
limit,
table)
return self._retrieve_samples(query)
transformer = QueryTransformer(table, query)
transformer.apply_options(None,
limit)
return self._retrieve_samples(transformer.get_query())
def _retrieve_data(self, filter_expr, orderby, limit, table):
if limit == 0:
@@ -635,21 +615,17 @@ class Connection(base.Connection):
session = self._get_db_session()
query = session.query(table)
transformer = QueryTransformer(table, query)
if filter_expr is not None:
sql_condition = self._transform_expression(filter_expr,
table)
query = query.filter(sql_condition)
transformer.apply_filter(filter_expr)
query = self._apply_options(query,
orderby,
limit,
table)
transformer.apply_options(orderby,
limit)
retrieve = {models.MeterSample: self._retrieve_samples,
models.Alarm: self._retrieve_alarms,
models.AlarmChange: self._retrieve_alarm_history}
return retrieve[table](query)
return retrieve[table](transformer.get_query())
def query_samples(self, filter_expr=None, orderby=None, limit=None):
return self._retrieve_data(filter_expr,
@@ -657,35 +633,6 @@ class Connection(base.Connection):
limit,
models.MeterSample)
def _transform_expression(self, expression_tree, table):
def transform(sub_tree):
operator = sub_tree.keys()[0]
nodes = sub_tree.values()[0]
if operator in self.complex_operators:
op = self.complex_operators[operator]
element_list = []
for node in nodes:
element = transform(node)
element_list.append(element)
return op(*element_list)
else:
op = self.operators[operator]
return op(getattr(table, nodes.keys()[0]), nodes.values()[0])
return transform(expression_tree)
def _apply_order_by(self, query, orderby, table):
if orderby is not None:
for field in orderby:
ordering_function = self.ordering_functions[field.values()[0]]
query = query.order_by(ordering_function(
getattr(table, field.keys()[0])))
else:
query = query.order_by(desc(table.timestamp))
return query
def _make_stats_query(self, sample_filter, groupby):
select = [
models.Meter.unit,
@@ -1246,3 +1193,88 @@ class Connection(base.Connection):
yield api_models.Trait(name=type.desc,
dtype=type.data_type,
value=trait.get_value())
class QueryTransformer(object):
operators = {"=": operator.eq,
"<": operator.lt,
">": operator.gt,
"<=": operator.le,
"=<": operator.le,
">=": operator.ge,
"=>": operator.ge,
"!=": operator.ne,
"in": lambda field_name, values: field_name.in_(values)}
complex_operators = {"or": or_,
"and": and_}
ordering_functions = {"asc": asc,
"desc": desc}
def __init__(self, table, query):
self.table = table
self.query = query
def _handle_complex_op(self, complex_op, nodes):
op = self.complex_operators[complex_op]
element_list = []
for node in nodes:
element = self._transform(node)
element_list.append(element)
return op(*element_list)
def _handle_simple_op(self, simple_op, nodes):
op = self.operators[simple_op]
field_name = nodes.keys()[0]
value = nodes.values()[0]
if field_name.startswith('resource_metadata.'):
return self._handle_metadata(op, field_name, value)
else:
return op(getattr(self.table, field_name), value)
def _handle_metadata(self, op, field_name, value):
if op == self.operators["in"]:
raise NotImplementedError(
_("Metadata query with in operator is not implemented"))
field_name = field_name[len('resource_metadata.'):]
meta_table = META_TYPE_MAP[type(value)]
meta_alias = aliased(meta_table)
on_clause = and_(self.table.id == meta_alias.id,
meta_alias.meta_key == field_name)
# outer join is needed to support metaquery
# with or operator on non existent metadata field
# see: test_query_non_existing_metadata_with_result
# test case.
self.query = self.query.outerjoin(meta_alias, on_clause)
return op(meta_alias.value, value)
def _transform(self, sub_tree):
operator = sub_tree.keys()[0]
nodes = sub_tree.values()[0]
if operator in self.complex_operators:
return self._handle_complex_op(operator, nodes)
else:
return self._handle_simple_op(operator, nodes)
def apply_filter(self, expression_tree):
condition = self._transform(expression_tree)
self.query = self.query.filter(condition)
def apply_options(self, orderby, limit):
self._apply_order_by(orderby)
if limit is not None:
self.query = self.query.limit(limit)
def _apply_order_by(self, orderby):
if orderby is not None:
for field in orderby:
ordering_function = self.ordering_functions[field.values()[0]]
self.query = self.query.order_by(ordering_function(
getattr(self.table, field.keys()[0])))
else:
self.query = self.query.order_by(desc(self.table.timestamp))
def get_query(self):
return self.query

33
ceilometer/tests/api/v2/test_complex_query.py
View File

@@ -29,11 +29,12 @@ from ceilometer import storage as storage
class FakeComplexQuery(api.ValidatedComplexQuery):
def __init__(self, db_model, additional_valid_keys):
def __init__(self, db_model, additional_valid_keys, metadata=False):
super(FakeComplexQuery, self).__init__(query=None,
db_model=db_model,
additional_valid_keys=
additional_valid_keys)
additional_valid_keys,
metadata_allowed=metadata)
class TestComplexQuery(test.BaseTestCase):
@@ -42,7 +43,8 @@ class TestComplexQuery(test.BaseTestCase):
self.useFixture(fixtures.MonkeyPatch(
'pecan.response', mock.MagicMock()))
self.query = FakeComplexQuery(storage.models.Sample,
["user", "project", "resource"])
["user", "project", "resource"],
True)
self.query_alarm = FakeComplexQuery(storage.models.Alarm,
["user", "project"])
self.query_alarmchange = FakeComplexQuery(
@@ -218,12 +220,19 @@ class TestComplexQuery(test.BaseTestCase):
self.query._validate_orderby,
orderby)
def test_validate_orderby_metadata_is_not_allowed(self):
orderby = [{"metadata.display_name": "asc"}]
self.assertRaises(jsonschema.ValidationError,
self.query._validate_orderby,
orderby)
class TestFilterSyntaxValidation(test.BaseTestCase):
def setUp(self):
super(TestFilterSyntaxValidation, self).setUp()
self.query = FakeComplexQuery(storage.models.Sample,
["user", "project", "resource"])
["user", "project", "resource"],
True)
def test_simple_operator(self):
filter = {"=": {"project_id": "string_value"}}
@@ -232,6 +241,22 @@ class TestFilterSyntaxValidation(test.BaseTestCase):
filter = {"=>": {"project_id": "string_value"}}
self.query._validate_filter(filter)
def test_valid_value_types(self):
filter = {"=": {"project_id": "string_value"}}
self.query._validate_filter(filter)
filter = {"=": {"project_id": 42}}
self.query._validate_filter(filter)
filter = {"=": {"project_id": 3.14}}
self.query._validate_filter(filter)
filter = {"=": {"project_id": True}}
self.query._validate_filter(filter)
filter = {"=": {"project_id": False}}
self.query._validate_filter(filter)
def test_invalid_simple_operator(self):
filter = {"==": {"project_id": "string_value"}}
self.assertRaises(jsonschema.ValidationError,

51
ceilometer/tests/api/v2/test_complex_query_scenarios.py
View File

@@ -58,10 +58,10 @@ class TestQueryMetersController(tests_api.FunctionalTest,
'project-id1',
'resource-id1',
timestamp=datetime.datetime(2012, 7, 2, 10, 40),
resource_metadata={'display_name': 'test-server',
resource_metadata={'display_name': 'test-server1',
'tag': 'self.sample',
'size': 123,
'util': 0.75,
'size': 456,
'util': 0.25,
'is_public': True},
source='test_source'),
sample.Sample('meter.test',
@@ -72,11 +72,25 @@ class TestQueryMetersController(tests_api.FunctionalTest,
'project-id2',
'resource-id2',
timestamp=datetime.datetime(2012, 7, 2, 10, 41),
resource_metadata={'display_name': 'test-server',
resource_metadata={'display_name': 'test-server2',
'tag': 'self.sample',
'size': 123,
'util': 0.75,
'is_public': True},
source='test_source'),
sample.Sample('meter.test',
'cumulative',
'',
1,
'user-id3',
'project-id3',
'resource-id3',
timestamp=datetime.datetime(2012, 7, 2, 10, 42),
resource_metadata={'display_name': 'test-server3',
'tag': 'self.sample',
'size': 789,
'util': 0.95,
'is_public': True},
source='test_source')]:
msg = utils.meter_message_from_counter(
@@ -86,7 +100,7 @@ class TestQueryMetersController(tests_api.FunctionalTest,
def test_query_fields_are_optional(self):
data = self.post_json(self.url, params={})
self.assertEqual(2, len(data.json))
self.assertEqual(3, len(data.json))
def test_query_with_isotime(self):
date_time = datetime.datetime(2012, 7, 2, 10, 41)
@@ -97,7 +111,7 @@ class TestQueryMetersController(tests_api.FunctionalTest,
'{">=": {"timestamp": "'
+ isotime + '"}}'})
self.assertEqual(1, len(data.json))
self.assertEqual(2, len(data.json))
for sample in data.json:
result_time = timeutils.parse_isotime(sample['timestamp'])
result_time = result_time.replace(tzinfo=None)
@@ -135,10 +149,10 @@ class TestQueryMetersController(tests_api.FunctionalTest,
params={},
headers=admin_header)
self.assertEqual(2, len(data.json))
self.assertEqual(3, len(data.json))
for sample in data.json:
self.assertIn(sample['project_id'],
(["project-id1", "project-id2"]))
(["project-id1", "project-id2", "project-id3"]))
def test_admin_tenant_sees_every_project_with_complex_filter(self):
filter = ('{"OR": ' +
@@ -179,8 +193,8 @@ class TestQueryMetersController(tests_api.FunctionalTest,
data = self.post_json(self.url,
params={"orderby": '[{"project_id": "DESC"}]'})
self.assertEqual(2, len(data.json))
self.assertEqual(["project-id2", "project-id1"],
self.assertEqual(3, len(data.json))
self.assertEqual(["project-id3", "project-id2", "project-id1"],
[s["project_id"] for s in data.json])
def test_query_with_field_name_project(self):
@@ -214,16 +228,16 @@ class TestQueryMetersController(tests_api.FunctionalTest,
data = self.post_json(self.url,
params={"orderby": '[{"project_id": "DeSc"}]'})
self.assertEqual(2, len(data.json))
self.assertEqual(["project-id2", "project-id1"],
self.assertEqual(3, len(data.json))
self.assertEqual(["project-id3", "project-id2", "project-id1"],
[s["project_id"] for s in data.json])
def test_query_with_user_field_name_orderby(self):
data = self.post_json(self.url,
params={"orderby": '[{"user": "aSc"}]'})
self.assertEqual(2, len(data.json))
self.assertEqual(["user-id1", "user-id2"],
self.assertEqual(3, len(data.json))
self.assertEqual(["user-id1", "user-id2", "user-id3"],
[s["user_id"] for s in data.json])
def test_query_with_missing_order_in_orderby(self):
@@ -233,6 +247,15 @@ class TestQueryMetersController(tests_api.FunctionalTest,
self.assertEqual(500, data.status_int)
def test_filter_with_metadata(self):
data = self.post_json(self.url,
params={"filter":
'{">=": {"metadata.util": 0.5}}'})
self.assertEqual(2, len(data.json))
for sample in data.json:
self.assertTrue(sample["metadata"]["util"] >= 0.5)
def test_limit_should_be_positive(self):
data = self.post_json(self.url,
params={"limit": 0},

89
ceilometer/tests/storage/test_storage_scenarios.py
View File

@@ -759,8 +759,14 @@ class ComplexSampleQueryTest(DBTestBase,
def _create_samples(self):
for resource in range(42, 45):
for volume in [0.79, 0.41, 0.4, 0.8, 0.39, 0.81]:
metadata = {'a_string_key': "meta-value" + str(volume),
'a_float_key': volume,
'an_int_key': resource,
'a_bool_key': (resource == 43)}
self.create_and_store_sample(resource_id="resource-id-%s"
% resource,
metadata=metadata,
name="cpu_util",
volume=volume)
@@ -928,6 +934,89 @@ class ComplexSampleQueryTest(DBTestBase,
self.conn.query_samples(filter_expr={"in": {"resource_id": []}}))
self.assertEqual(len(results), 0)
def test_query_simple_metadata_filter(self):
self._create_samples()
filter_expr = {"=": {"resource_metadata.a_bool_key": True}}
results = list(self.conn.query_samples(filter_expr=filter_expr))
self.assertEqual(len(results), 6)
for sample in results:
self.assertTrue(sample.resource_metadata["a_bool_key"])
def test_query_simple_metadata_with_in_op(self):
self._create_samples()
filter_expr = {"in": {"resource_metadata.an_int_key": [42, 43]}}
results = list(self.conn.query_samples(filter_expr=filter_expr))
self.assertEqual(len(results), 12)
for sample in results:
self.assertIn(sample.resource_metadata["an_int_key"], [42, 43])
def test_query_complex_metadata_filter(self):
self._create_samples()
subfilter = {"or": [{"=": {"resource_metadata.a_string_key":
"meta-value0.81"}},
{"<=": {"resource_metadata.a_float_key": 0.41}}]}
filter_expr = {"and": [{">": {"resource_metadata.an_int_key": 42}},
subfilter]}
results = list(self.conn.query_samples(filter_expr=filter_expr))
self.assertEqual(len(results), 8)
for sample in results:
self.assertTrue((sample.resource_metadata["a_string_key"] ==
"meta-value0.81" or
sample.resource_metadata["a_float_key"] <= 0.41))
self.assertTrue(sample.resource_metadata["an_int_key"] > 42)
def test_query_mixed_data_and_metadata_filter(self):
self._create_samples()
subfilter = {"or": [{"=": {"resource_metadata.a_string_key":
"meta-value0.81"}},
{"<=": {"resource_metadata.a_float_key": 0.41}}]}
filter_expr = {"and": [{"=": {"resource_id": "resource-id-42"}},
subfilter]}
results = list(self.conn.query_samples(filter_expr=filter_expr))
self.assertEqual(len(results), 4)
for sample in results:
self.assertTrue((sample.resource_metadata["a_string_key"] ==
"meta-value0.81" or
sample.resource_metadata["a_float_key"] <= 0.41))
self.assertEqual(sample.resource_id, "resource-id-42")
def test_query_non_existing_metadata_with_result(self):
self._create_samples()
filter_expr = {
"or": [{"=": {"resource_metadata.a_string_key":
"meta-value0.81"}},
{"<=": {"resource_metadata.key_not_exists": 0.41}}]}
results = list(self.conn.query_samples(filter_expr=filter_expr))
self.assertEqual(len(results), 3)
for sample in results:
self.assertEqual(sample.resource_metadata["a_string_key"],
"meta-value0.81")
def test_query_non_existing_metadata_without_result(self):
self._create_samples()
filter_expr = {
"or": [{"=": {"resource_metadata.key_not_exists":
"meta-value0.81"}},
{"<=": {"resource_metadata.key_not_exists": 0.41}}]}
results = list(self.conn.query_samples(filter_expr=filter_expr))
self.assertEqual(len(results), 0)
class StatisticsTest(DBTestBase,
tests_db.MixinTestsWithBackendScenarios):