Merge "Add support for the range_function field to the Prometheus collector"
This commit is contained in:
commit
4e8cfe1d82
|
@ -77,6 +77,12 @@ PROMETHEUS_EXTRA_SCHEMA = {
|
||||||
'abs', 'ceil', 'exp',
|
'abs', 'ceil', 'exp',
|
||||||
'floor', 'ln', 'log2',
|
'floor', 'ln', 'log2',
|
||||||
'log10', 'round', 'sqrt'
|
'log10', 'round', 'sqrt'
|
||||||
|
]),
|
||||||
|
Optional('range_function'):
|
||||||
|
In([
|
||||||
|
'changes', 'delta', 'deriv',
|
||||||
|
'idelta', 'irange', 'irate',
|
||||||
|
'rate'
|
||||||
])
|
])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -148,6 +154,8 @@ class PrometheusCollector(collector.BaseCollector):
|
||||||
method = self.conf[metric_name]['extra_args']['aggregation_method']
|
method = self.conf[metric_name]['extra_args']['aggregation_method']
|
||||||
query_function = self.conf[metric_name]['extra_args'].get(
|
query_function = self.conf[metric_name]['extra_args'].get(
|
||||||
'query_function')
|
'query_function')
|
||||||
|
range_function = self.conf[metric_name]['extra_args'].get(
|
||||||
|
'range_function')
|
||||||
groupby = self.conf[metric_name].get('groupby', [])
|
groupby = self.conf[metric_name].get('groupby', [])
|
||||||
metadata = self.conf[metric_name].get('metadata', [])
|
metadata = self.conf[metric_name].get('metadata', [])
|
||||||
period = tzutils.diff_seconds(end, start)
|
period = tzutils.diff_seconds(end, start)
|
||||||
|
@ -160,11 +168,18 @@ class PrometheusCollector(collector.BaseCollector):
|
||||||
scope_id,
|
scope_id,
|
||||||
period
|
period
|
||||||
)
|
)
|
||||||
# Applying the aggregation_method on a Range Vector
|
# Applying the aggregation_method or the range_function on
|
||||||
query = "{0}_over_time({1})".format(
|
# a Range Vector
|
||||||
method,
|
if range_function is not None:
|
||||||
query
|
query = "{0}({1})".format(
|
||||||
)
|
range_function,
|
||||||
|
query
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
query = "{0}_over_time({1})".format(
|
||||||
|
method,
|
||||||
|
query
|
||||||
|
)
|
||||||
# Applying the query_function
|
# Applying the query_function
|
||||||
if query_function is not None:
|
if query_function is not None:
|
||||||
query = "{0}({1})".format(
|
query = "{0}({1})".format(
|
||||||
|
|
|
@ -46,16 +46,111 @@ class PrometheusCollectorTest(tests.TestCase):
|
||||||
'instance',
|
'instance',
|
||||||
],
|
],
|
||||||
'extra_args': {
|
'extra_args': {
|
||||||
'aggregation_method': 'avg',
|
'aggregation_method': 'avg'
|
||||||
'query_function': 'abs'
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
self.collector = prometheus.PrometheusCollector(**args)
|
args_range_function = {
|
||||||
|
'period': 3600,
|
||||||
|
'scope_key': 'namespace',
|
||||||
|
'conf': {
|
||||||
|
'metrics': {
|
||||||
|
'http_requests_total': {
|
||||||
|
'unit': 'instance',
|
||||||
|
'groupby': [
|
||||||
|
'foo',
|
||||||
|
'bar',
|
||||||
|
],
|
||||||
|
'metadata': [
|
||||||
|
'code',
|
||||||
|
'instance',
|
||||||
|
],
|
||||||
|
'extra_args': {
|
||||||
|
'aggregation_method': 'avg',
|
||||||
|
'query_function': 'abs',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
args_query_function = {
|
||||||
|
'period': 3600,
|
||||||
|
'scope_key': 'namespace',
|
||||||
|
'conf': {
|
||||||
|
'metrics': {
|
||||||
|
'http_requests_total': {
|
||||||
|
'unit': 'instance',
|
||||||
|
'groupby': [
|
||||||
|
'foo',
|
||||||
|
'bar',
|
||||||
|
],
|
||||||
|
'metadata': [
|
||||||
|
'code',
|
||||||
|
'instance',
|
||||||
|
],
|
||||||
|
'extra_args': {
|
||||||
|
'aggregation_method': 'avg',
|
||||||
|
'range_function': 'delta',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
args_all = {
|
||||||
|
'period': 3600,
|
||||||
|
'scope_key': 'namespace',
|
||||||
|
'conf': {
|
||||||
|
'metrics': {
|
||||||
|
'http_requests_total': {
|
||||||
|
'unit': 'instance',
|
||||||
|
'groupby': [
|
||||||
|
'foo',
|
||||||
|
'bar',
|
||||||
|
],
|
||||||
|
'metadata': [
|
||||||
|
'code',
|
||||||
|
'instance',
|
||||||
|
],
|
||||||
|
'extra_args': {
|
||||||
|
'aggregation_method': 'avg',
|
||||||
|
'range_function': 'delta',
|
||||||
|
'query_function': 'abs',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.collector_mandatory = prometheus.PrometheusCollector(**args)
|
||||||
|
self.collector_without_range_function = prometheus.PrometheusCollector(
|
||||||
|
**args_range_function)
|
||||||
|
self.collector_without_query_function = prometheus.PrometheusCollector(
|
||||||
|
**args_query_function)
|
||||||
|
self.collector_all = prometheus.PrometheusCollector(**args_all)
|
||||||
|
|
||||||
def test_fetch_all_build_query(self):
|
def test_fetch_all_build_query_only_mandatory(self):
|
||||||
|
query = (
|
||||||
|
'avg(avg_over_time(http_requests_total'
|
||||||
|
'{project_id="f266f30b11f246b589fd266f85eeec39"}[3600s]'
|
||||||
|
')) by (foo, bar, project_id, code, instance)'
|
||||||
|
)
|
||||||
|
|
||||||
|
with mock.patch.object(
|
||||||
|
prometheus.PrometheusClient, 'get_instant',
|
||||||
|
) as mock_get:
|
||||||
|
self.collector_mandatory.fetch_all(
|
||||||
|
'http_requests_total',
|
||||||
|
samples.FIRST_PERIOD_BEGIN,
|
||||||
|
samples.FIRST_PERIOD_END,
|
||||||
|
self._tenant_id,
|
||||||
|
)
|
||||||
|
mock_get.assert_called_once_with(
|
||||||
|
query,
|
||||||
|
samples.FIRST_PERIOD_END.isoformat(),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_fetch_all_build_query_without_range_function(self):
|
||||||
query = (
|
query = (
|
||||||
'avg(abs(avg_over_time(http_requests_total'
|
'avg(abs(avg_over_time(http_requests_total'
|
||||||
'{project_id="f266f30b11f246b589fd266f85eeec39"}[3600s]'
|
'{project_id="f266f30b11f246b589fd266f85eeec39"}[3600s]'
|
||||||
|
@ -65,7 +160,49 @@ class PrometheusCollectorTest(tests.TestCase):
|
||||||
with mock.patch.object(
|
with mock.patch.object(
|
||||||
prometheus.PrometheusClient, 'get_instant',
|
prometheus.PrometheusClient, 'get_instant',
|
||||||
) as mock_get:
|
) as mock_get:
|
||||||
self.collector.fetch_all(
|
self.collector_without_range_function.fetch_all(
|
||||||
|
'http_requests_total',
|
||||||
|
samples.FIRST_PERIOD_BEGIN,
|
||||||
|
samples.FIRST_PERIOD_END,
|
||||||
|
self._tenant_id,
|
||||||
|
)
|
||||||
|
mock_get.assert_called_once_with(
|
||||||
|
query,
|
||||||
|
samples.FIRST_PERIOD_END.isoformat(),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_fetch_all_build_query_without_query_function(self):
|
||||||
|
query = (
|
||||||
|
'avg(delta(http_requests_total'
|
||||||
|
'{project_id="f266f30b11f246b589fd266f85eeec39"}[3600s]'
|
||||||
|
')) by (foo, bar, project_id, code, instance)'
|
||||||
|
)
|
||||||
|
|
||||||
|
with mock.patch.object(
|
||||||
|
prometheus.PrometheusClient, 'get_instant',
|
||||||
|
) as mock_get:
|
||||||
|
self.collector_without_query_function.fetch_all(
|
||||||
|
'http_requests_total',
|
||||||
|
samples.FIRST_PERIOD_BEGIN,
|
||||||
|
samples.FIRST_PERIOD_END,
|
||||||
|
self._tenant_id,
|
||||||
|
)
|
||||||
|
mock_get.assert_called_once_with(
|
||||||
|
query,
|
||||||
|
samples.FIRST_PERIOD_END.isoformat(),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_fetch_all_build_query_all(self):
|
||||||
|
query = (
|
||||||
|
'avg(abs(delta(http_requests_total'
|
||||||
|
'{project_id="f266f30b11f246b589fd266f85eeec39"}[3600s]'
|
||||||
|
'))) by (foo, bar, project_id, code, instance)'
|
||||||
|
)
|
||||||
|
|
||||||
|
with mock.patch.object(
|
||||||
|
prometheus.PrometheusClient, 'get_instant',
|
||||||
|
) as mock_get:
|
||||||
|
self.collector_all.fetch_all(
|
||||||
'http_requests_total',
|
'http_requests_total',
|
||||||
samples.FIRST_PERIOD_BEGIN,
|
samples.FIRST_PERIOD_BEGIN,
|
||||||
samples.FIRST_PERIOD_END,
|
samples.FIRST_PERIOD_END,
|
||||||
|
@ -94,7 +231,7 @@ class PrometheusCollectorTest(tests.TestCase):
|
||||||
'end': samples.FIRST_PERIOD_END,
|
'end': samples.FIRST_PERIOD_END,
|
||||||
'data': samples.PROMETHEUS_RESP_INSTANT_QUERY['data']['result'][0],
|
'data': samples.PROMETHEUS_RESP_INSTANT_QUERY['data']['result'][0],
|
||||||
}
|
}
|
||||||
actual = self.collector._format_data(**params)
|
actual = self.collector_mandatory._format_data(**params)
|
||||||
self.assertEqual(expected, actual)
|
self.assertEqual(expected, actual)
|
||||||
|
|
||||||
def test_format_data_instant_query_2(self):
|
def test_format_data_instant_query_2(self):
|
||||||
|
@ -115,7 +252,7 @@ class PrometheusCollectorTest(tests.TestCase):
|
||||||
'end': samples.FIRST_PERIOD_END,
|
'end': samples.FIRST_PERIOD_END,
|
||||||
'data': samples.PROMETHEUS_RESP_INSTANT_QUERY['data']['result'][1],
|
'data': samples.PROMETHEUS_RESP_INSTANT_QUERY['data']['result'][1],
|
||||||
}
|
}
|
||||||
actual = self.collector._format_data(**params)
|
actual = self.collector_mandatory._format_data(**params)
|
||||||
self.assertEqual(expected, actual)
|
self.assertEqual(expected, actual)
|
||||||
|
|
||||||
def test_format_retrieve(self):
|
def test_format_retrieve(self):
|
||||||
|
@ -137,7 +274,7 @@ class PrometheusCollectorTest(tests.TestCase):
|
||||||
)
|
)
|
||||||
|
|
||||||
with no_response:
|
with no_response:
|
||||||
actual_name, actual_data = self.collector.retrieve(
|
actual_name, actual_data = self.collector_mandatory.retrieve(
|
||||||
metric_name='http_requests_total',
|
metric_name='http_requests_total',
|
||||||
start=samples.FIRST_PERIOD_BEGIN,
|
start=samples.FIRST_PERIOD_BEGIN,
|
||||||
end=samples.FIRST_PERIOD_END,
|
end=samples.FIRST_PERIOD_END,
|
||||||
|
@ -157,7 +294,7 @@ class PrometheusCollectorTest(tests.TestCase):
|
||||||
with no_response:
|
with no_response:
|
||||||
self.assertRaises(
|
self.assertRaises(
|
||||||
collector.NoDataCollected,
|
collector.NoDataCollected,
|
||||||
self.collector.retrieve,
|
self.collector_mandatory.retrieve,
|
||||||
metric_name='http_requests_total',
|
metric_name='http_requests_total',
|
||||||
start=samples.FIRST_PERIOD_BEGIN,
|
start=samples.FIRST_PERIOD_BEGIN,
|
||||||
end=samples.FIRST_PERIOD_END,
|
end=samples.FIRST_PERIOD_END,
|
||||||
|
@ -174,7 +311,7 @@ class PrometheusCollectorTest(tests.TestCase):
|
||||||
with invalid_response:
|
with invalid_response:
|
||||||
self.assertRaises(
|
self.assertRaises(
|
||||||
exceptions.CollectError,
|
exceptions.CollectError,
|
||||||
self.collector.retrieve,
|
self.collector_mandatory.retrieve,
|
||||||
metric_name='http_requests_total',
|
metric_name='http_requests_total',
|
||||||
start=samples.FIRST_PERIOD_BEGIN,
|
start=samples.FIRST_PERIOD_BEGIN,
|
||||||
end=samples.FIRST_PERIOD_END,
|
end=samples.FIRST_PERIOD_END,
|
||||||
|
|
|
@ -154,12 +154,14 @@ class MetricConfigValidationTest(tests.TestCase):
|
||||||
data['metrics']['metric_one']['extra_args'] = {
|
data['metrics']['metric_one']['extra_args'] = {
|
||||||
'aggregation_method': 'max',
|
'aggregation_method': 'max',
|
||||||
'query_function': 'abs',
|
'query_function': 'abs',
|
||||||
|
'range_function': 'delta',
|
||||||
}
|
}
|
||||||
expected_output = copy.deepcopy(self.base_output)
|
expected_output = copy.deepcopy(self.base_output)
|
||||||
expected_output['metric_one']['groupby'].append('project_id')
|
expected_output['metric_one']['groupby'].append('project_id')
|
||||||
expected_output['metric_one']['extra_args'] = {
|
expected_output['metric_one']['extra_args'] = {
|
||||||
'aggregation_method': 'max',
|
'aggregation_method': 'max',
|
||||||
'query_function': 'abs',
|
'query_function': 'abs',
|
||||||
|
'range_function': 'delta',
|
||||||
}
|
}
|
||||||
|
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
|
|
|
@ -309,4 +309,9 @@ Prometheus
|
||||||
``log10``, ``round``, ``sqrt``. For more information on these functions,
|
``log10``, ``round``, ``sqrt``. For more information on these functions,
|
||||||
you can check `this page`_
|
you can check `this page`_
|
||||||
|
|
||||||
|
* ``range_function``: Optional argument. The function to apply instead of the
|
||||||
|
implicit ``{aggregation_method}_over_time``. Must be one of ``changes``,
|
||||||
|
``delta``, ``deriv``, ``idelta``, ``irange``, ``irate``, ``rate``. For more
|
||||||
|
information on these functions, you can check `this page`_
|
||||||
|
|
||||||
.. _this page: https://prometheus.io/docs/prometheus/latest/querying/basics/
|
.. _this page: https://prometheus.io/docs/prometheus/latest/querying/basics/
|
||||||
|
|
Loading…
Reference in New Issue