x
/
kiloeyes
Code Issues Proposed changes
kiloeyes/kiloeyes/tests/v2/elasticsearch/test_metrics.py

349 lines
15 KiB
Python
Executable File
Raw Blame History

# Copyright 2013 IBM Corp
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import falcon
import mock
from oslo_config import fixture as fixture_config
from oslotest import base
import requests
from kiloeyes.common import kafka_conn
from kiloeyes.v2.elasticsearch import metrics
try:
import ujson as json
except ImportError:
import json
class TestParamUtil(base.BaseTestCase):
def setUp(self):
super(TestParamUtil, self).setUp()
self.req = mock.Mock()
self.req.get_param.side_effect = self._side_effect
def _side_effect(self, arg):
if arg == 'name':
return 'tongli'
elif arg == 'dimensions':
return 'key1:100, key2:200'
elif arg == 'start_time':
return '2015-01-31T13:35:00Z'
elif arg == 'end_time':
return '2015-11-30T14:05:00Z'
elif arg == 'period':
return '200'
elif arg == 'statistics':
return 'sum, avg'
def test_common(self):
result = [{'match': {'name': 'tongli'}},
{'range': {'timestamp': {'gte': 1422711300.0,
'lt': 1448892300.0}}},
{'match': {'dimensions.key1': 100.0}},
{'match': {'dimensions.key2': 200.0}}]
q = []
ret = metrics.ParamUtil.common(self.req, q)
self.assertEqual(q, result)
self.assertTrue(ret)
def test_period(self):
ret = metrics.ParamUtil.period(self.req)
self.assertEqual(ret, '200s')
def test_stats(self):
ret = metrics.ParamUtil.stats(self.req)
self.assertEqual(ret, ['sum', 'avg'])
class TestMetricDispatcher(base.BaseTestCase):
def setUp(self):
super(TestMetricDispatcher, self).setUp()
self.CONF = self.useFixture(fixture_config.Config()).conf
self.CONF.set_override('uri', 'fake_url', group='kafka_opts')
self.CONF.set_override('topic', 'fake', group='metrics')
self.CONF.set_override('doc_type', 'fake', group='metrics')
self.CONF.set_override('index_prefix', 'also_fake', group='metrics')
self.CONF.set_override('index_template', 'etc/metrics.template',
group='metrics')
self.CONF.set_override('uri', 'http://fake_es_uri', group='es_conn')
res = mock.Mock()
res.status_code = 200
res.json.return_value = {"data": {"mappings": {"fake": {
"properties": {
"dimensions": {"properties": {
"key1": {"type": "long"}, "key2": {"type": "long"},
"rkey0": {"type": "long"}, "rkey1": {"type": "long"},
"rkey2": {"type": "long"}, "rkey3": {"type": "long"}}},
"name": {"type": "string", "index": "not_analyzed"},
"timestamp": {"type": "string", "index": "not_analyzed"},
"value": {"type": "double"}}}}}}
put_res = mock.Mock()
put_res.status_code = '200'
with mock.patch.object(requests, 'get',
return_value=res):
with mock.patch.object(requests, 'put', return_value=put_res):
self.dispatcher = metrics.MetricDispatcher({})
def test_initialization(self):
# test that the kafka connection uri should be 'fake' as it was passed
# in from configuration
self.assertEqual(self.dispatcher._kafka_conn.uri, 'fake_url')
# test that the topic is metrics as it was passed into dispatcher
self.assertEqual(self.dispatcher._kafka_conn.topic, 'fake')
# test that the doc type of the es connection is fake
self.assertEqual(self.dispatcher._es_conn.doc_type, 'fake')
self.assertEqual(self.dispatcher._es_conn.uri, 'http://fake_es_uri/')
# test that the query url is correctly formed
self.assertEqual(self.dispatcher._query_url, (
'http://fake_es_uri/also_fake*/fake/_search?search_type=count'))
def test_post_data(self):
with mock.patch.object(kafka_conn.KafkaConnection, 'send_messages',
return_value=204):
res = mock.Mock()
self.dispatcher.post_data(mock.Mock(), res)
# test that the response code is 204
self.assertEqual(getattr(falcon, 'HTTP_204'), res.status)
with mock.patch.object(kafka_conn.KafkaConnection, 'send_messages',
return_value=400):
res = mock.Mock()
self.dispatcher.post_data(mock.Mock(), res)
# test that the response code is 204
self.assertEqual(getattr(falcon, 'HTTP_400'), res.status)
def test_do_get_metrics(self):
res = mock.Mock()
req = mock.Mock()
def _side_effect(arg):
if arg == 'name':
return 'tongli'
elif arg == 'dimensions':
return 'key1:100, key2:200'
req.get_param.side_effect = _side_effect
req_result = mock.Mock()
response_str = """
{"aggregations":{"by_name":{"doc_count_error_upper_bound":0,
"sum_other_doc_count":0,"buckets":[{"key":"BABMGD","doc_count":300,
"by_dim":{"buckets":[{"key": "64e6ce08b3b8547b7c32e5cfa5b7d81f",
"doc_count":300,"metrics":{"hits":{"hits":[{
"_source":{"name":"BABMGD","dimensions":{"key2":"NVITDU",
"key1":"FUFMPY","key_43":"ROQBZM"}}}]}}}]}},
{"key":"BABSYZ","doc_count":300,"by_dim":{"buckets":[{
"key":"84863c7cfee6837a77eb476ea9f35f87","doc_count":300,
"metrics":{"hits":{"hits":[{"_source":{"name":"BABSYZ",
"dimensions":{"key2":"UIPAJD","key1":"DKPNKA",
"key_10": "RADAJP"}}}]}}}]}}]}}}
"""
req_result.json.return_value = json.loads(response_str)
req_result.status_code = 200
with mock.patch.object(requests, 'post', return_value=req_result):
self.dispatcher.do_get_metrics(req, res)
# test that the response code is 200
self.assertEqual(res.status, getattr(falcon, 'HTTP_200'))
obj = json.loads(res.body)
self.assertEqual(obj[0]['name'], 'BABMGD')
self.assertEqual(obj[0]['dimensions']['key2'], 'NVITDU')
self.assertEqual(len(obj), 2)
def test_do_post_metrics(self):
with mock.patch.object(kafka_conn.KafkaConnection, 'send_messages',
return_value=204):
res = mock.Mock()
self.dispatcher.do_post_metrics(mock.Mock(), res)
self.assertEqual(getattr(falcon, 'HTTP_204'), res.status)
def test_do_get_measurements(self):
res = mock.Mock()
req = mock.Mock()
def _side_effect(arg):
if arg == 'name':
return 'tongli'
elif arg == 'dimensions':
return 'key1:100, key2:200'
elif arg == 'start_time':
return '2014-01-01'
req.get_param.side_effect = _side_effect
req_result = mock.Mock()
response_str = """
{"took":226,"timed_out":false,"_shards":{"total":5,"successful":5,
"failed":0},"hits":{"total":6600,"max_score":0.0,"hits":[]},
"aggregations":{"by_name":{"doc_count_error_upper_bound":293,
"sum_other_doc_count":5791,"buckets":[{"key":"ABYTPK",
"doc_count":300,"by_dim":{"doc_count_error_upper_bound":0,
"sum_other_doc_count":0,
"buckets":[{"key":"e62ef04ee44abcccdd177087d159c1e3","doc_count":300,
"dimension":{"hits":{"total":300,"max_score":1.4142135,
"hits":[{"_index":"data_20150121","_type":"metrics",
"_id":"AUsShaLKTZaMxA7_0_Hj","_score":1.4142135,
"_source":{"name":"ABYTPK","dimensions":{"key_81":"MKKNSA",
"key2":"TJJQGE","key1":"GYYLEG"}}}]}},
"measures":{"hits":{"total":300,"max_score":null,
"hits":[{"_index":"data_20150121","_type":"metrics",
"_id":"AUsShaKuTZaMxA7_0_Hd","_score":null,
"_source":{"timestamp":1.421944922765286E9,"value":0.0},
"sort":[1.421944922765286E9]},{"_index":"data_20150121",
"_type":"metrics","_id":"AUsShaM8TZaMxA7_0_H7",
"_score":null,"_source":{"timestamp":1.421944922907783E9,
"value":0.0},"sort":[1.421944922907783E9]},{"_index":"data_20150121",
"_type":"metrics","_id":"AUsShaR2TZaMxA7_0_IZ","_score":null,
"_source":{"timestamp":1.421944923222439E9,"value":0.0},
"sort":[1.421944923222439E9]}]}}}]}},{"key":"ABUYPI","doc_count":256,
"by_dim":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,
"buckets":[{"key":"3dba425d350f6f46f8eda8a883231e58",
"doc_count":256,"dimension":{"hits":{"total":256,
"max_score":1.4142135,"hits":[{"_index":"data_20150121",
"_type":"metrics","_id":"AUsSaTfYTZaMxA7_zaxn","_score":1.4142135,
"_source":{"name":"ABUYPI","dimensions":{"key2":"BEBGIY",
"key1":"JZAZQS","key_67":"EAJWVV"}}}]}},
"measures":{"hits":{"total":256,"max_score":null,
"hits":[{"_index":"data_20150121","_type":"metrics",
"_id":"AUsSaTfQTZaMxA7_zaxl","_score":null,
"_source":{"timestamp":1.421943060399819E9,"value":0.0},
"sort":[1.421943060399819E9]},{"_index":"data_20150121",
"_type":"metrics","_id":"AUsSaThJTZaMxA7_zayD","_score":null,
"_source":{"timestamp":1.421943060519964E9,"value":0.0},
"sort":[1.421943060519964E9]},{"_index":"data_20150121",
"_type":"metrics","_id":"AUsSaTjKTZaMxA7_zayh","_score":null,
"_source":{"timestamp":1.421943060648909E9,"value":0.0},
"sort":[1.421943060648909E9]}]}}}]}},
{"key":"ABEPJR","doc_count":253,
"by_dim":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,
"buckets":[{"key":"6d6bbdda3ed7f14c76e746e2fbd52a37",
"doc_count":253,"dimension":{"hits":{"total":253,
"max_score":1.4142135,"hits":[{"_index":"data_20150121",
"_type":"metrics","_id":"AUsR6STQTZaMxA7_sjp6",
"_score":1.4142135,"_source":{"name":"ABEPJR",
"dimensions":{"key_7":"ZAUVQN","key2":"NSXBUW","key1":"UXTDSW"}}}]}},
"measures":{"hits":{"total":253,"max_score":null,
"hits":[{"_index":"data_20150121","_type":"metrics",
"_id":"AUsR6SItTZaMxA7_sjnV","_score":null,
"_source":{"timestamp":1.421934666252589E9,"value":0.0},
"sort":[1.421934666252589E9]},{"_index":"data_20150121",
"_type":"metrics","_id":"AUsR6SKqTZaMxA7_sjnz","_score":null,
"_source":{"timestamp":1.421934666377047E9,"value":0.0},
"sort":[1.421934666377047E9]},{"_index":"data_20150121",
"_type":"metrics","_id":"AUsR6SMiTZaMxA7_sjoR","_score":null,
"_source":{"timestamp":1.421934666497888E9,"value":0.0},
"sort":[1.421934666497888E9]}]}}}]}}]}}}
"""
req_result.json.return_value = json.loads(response_str)
req_result.status_code = 200
with mock.patch.object(requests, 'post', return_value=req_result):
self.dispatcher.do_get_measurements(req, res)
# test that the response code is 200
self.assertEqual(res.status, getattr(falcon, 'HTTP_200'))
obj = json.loads(res.body)
# there should be total of 3 objects
self.assertEqual(len(obj), 3)
self.assertIsNotNone(obj[0]['name'])
self.assertIsNotNone(obj[0]['dimensions'])
self.assertIsNotNone(obj[0]['columns'])
self.assertIsNotNone(obj[0]['measurements'])
def test_do_get_statistics(self):
res = mock.Mock()
req = mock.Mock()
def _side_effect(arg):
if arg == 'name':
return 'tongli'
elif arg == 'dimensions':
return 'key1:100, key2:200'
elif arg == 'start_time':
return '2014-01-01'
elif arg == 'end_time':
return None
elif arg == 'period':
return None
elif arg == 'statistics':
return 'avg, sum, max'
req.get_param.side_effect = _side_effect
req_result = mock.Mock()
response_str = """
{"took":2006,"timed_out":false,"_shards":{"total":5,"successful":5,
"failed":0},"hits":{"total":600,"max_score":0.0,"hits":[]},
"aggregations":{"by_name":{"doc_count_error_upper_bound":0,
"sum_other_doc_count":0,"buckets":[{"key":"BABMGD","doc_count":300,
"by_dim":{"doc_count_error_upper_bound":0,"sum_other_doc_count":0,
"buckets":[{"key":"64e6ce08b3b8547b7c32e5cfa5b7d81f","doc_count":300,
"periods":{"buckets":[{"key":1421700000,"doc_count":130,
"statistics":{"count":130,"min":0.0,"max":595.0274095324651,
"avg":91.83085293930924,"sum":11938.0108821102}},
{"key":1422000000,"doc_count":170,"statistics":{"count":170,
"min":0.0,"max":1623.511307756313,"avg":324.69434786459897,
"sum":55198.039136981824}}]},"dimension":{"hits":{"total":300,
"max_score":1.4142135,"hits":[{"_index":"data_20150121",
"_type":"metrics","_id":"AUsSNF5mTZaMxA7_wmFx","_score":1.4142135,
"_source":{"name":"BABMGD","dimensions":{"key2":"NVITDU",
"key1":"FUFMPY","key_43":"ROQBZM"}}}]}}}]}},{"key":"BABSYZ",
"doc_count":300,"by_dim":{"doc_count_error_upper_bound":0,
"sum_other_doc_count":0,
"buckets":[{"key":"84863c7cfee6837a77eb476ea9f35f87","doc_count":300,
"periods":{"buckets":[{"key":1421700000,"doc_count":130,
"statistics":{"count":130,"min":0.0,"max":588.7273873368565,
"avg":100.45023098906705,"sum":13058.530028578716}},
{"key":1422000000,"doc_count":170,"statistics":{"count":170,
"min":0.0,"max":1515.5538517109185,"avg":332.5777043693029,
"sum":56538.209742781495}}]},"dimension":{"hits":{"total":300,
"max_score":1.4142135,"hits":[{"_index":"data_20150121",
"_type":"metrics","_id":"AUsR7oGETZaMxA7_s0Y0","_score":1.4142135,
"_source":{"name":"BABSYZ","dimensions":{"key2":"UIPAJD",
"key1":"DKPNKA","key_10":"RADAJP"}}}]}}}]}}]}}}
"""
req_result.json.return_value = json.loads(response_str)
req_result.status_code = 200
with mock.patch.object(requests, 'post', return_value=req_result):
self.dispatcher.do_get_statistics(req, res)
# test that the response code is 200
self.assertEqual(res.status, getattr(falcon, 'HTTP_200'))
obj = json.loads(res.body)
# there should be total of 2 objects
self.assertEqual(len(obj), 2)
self.assertIsNotNone(obj[0]['name'])
self.assertIsNotNone(obj[0]['dimensions'])
self.assertIsNotNone(obj[0]['columns'])
self.assertEqual(obj[0]['columns'],
["timestamp", "avg", "sum", "max"])
self.assertIsNotNone(obj[0]['statistics'])
View Git Blame Copy Permalink