Add group by multiple dimensions
Added influx group by for multiple dimensions Added vertica group by for multiple dimensions Added tempest tests for group by with one, multiple, and all dimensions Change-Id: I69c27198ab180e17b7636bbd0f26fc1bd5292f3b
This commit is contained in:
parent
6908e62886
commit
d00cb15861
@ -127,3 +127,43 @@ def get_query_param(uri, query_param_name):
|
|||||||
if query_param_name == parsed_query_name:
|
if query_param_name == parsed_query_name:
|
||||||
query_param_val = parsed_query_val
|
query_param_val = parsed_query_val
|
||||||
return query_param_val
|
return query_param_val
|
||||||
|
|
||||||
|
|
||||||
|
def get_expected_elements_inner_offset_limit(all_elements, offset, limit, inner_key):
|
||||||
|
expected_elements = []
|
||||||
|
total_statistics = 0
|
||||||
|
|
||||||
|
if offset is None:
|
||||||
|
offset_id = 0
|
||||||
|
offset_time = ""
|
||||||
|
else:
|
||||||
|
offset_tuple = offset.split('_')
|
||||||
|
offset_id = int(offset_tuple[0]) if len(offset_tuple) > 1 else 0
|
||||||
|
offset_time = offset_tuple[1] if len(offset_tuple) > 1 else offset_tuple[0]
|
||||||
|
|
||||||
|
for element in all_elements:
|
||||||
|
element_id = int(element['id'])
|
||||||
|
if offset_id is not None and element_id < offset_id:
|
||||||
|
continue
|
||||||
|
next_element = None
|
||||||
|
for value in element[inner_key]:
|
||||||
|
if (element_id == offset_id and value[0] > offset_time) or \
|
||||||
|
element_id > offset_id:
|
||||||
|
|
||||||
|
if not next_element:
|
||||||
|
next_element = element.copy()
|
||||||
|
next_element[inner_key] = [value]
|
||||||
|
else:
|
||||||
|
next_element[inner_key].append(value)
|
||||||
|
total_statistics += 1
|
||||||
|
if total_statistics >= limit:
|
||||||
|
break
|
||||||
|
if next_element:
|
||||||
|
expected_elements.append(next_element)
|
||||||
|
if total_statistics >= limit:
|
||||||
|
break
|
||||||
|
|
||||||
|
for i in xrange(len(expected_elements)):
|
||||||
|
expected_elements[i]['id'] = str(i)
|
||||||
|
|
||||||
|
return expected_elements
|
||||||
|
@ -61,16 +61,16 @@ class TestMeasurements(base.BaseMonascaTest):
|
|||||||
metric3 = [
|
metric3 = [
|
||||||
helpers.create_metric(
|
helpers.create_metric(
|
||||||
name=name2, timestamp=start_timestamp + ONE_SECOND * 3,
|
name=name2, timestamp=start_timestamp + ONE_SECOND * 3,
|
||||||
dimensions={'key1': 'value1', 'key2': 'value1'}),
|
dimensions={'key1': 'value1', 'key2': 'value5', 'key3': 'value7'}),
|
||||||
helpers.create_metric(
|
helpers.create_metric(
|
||||||
name=name2, timestamp=start_timestamp + ONE_SECOND * 3 + 10,
|
name=name2, timestamp=start_timestamp + ONE_SECOND * 3 + 10,
|
||||||
dimensions={'key1': 'value2', 'key2': 'value2'}),
|
dimensions={'key1': 'value2', 'key2': 'value5', 'key3': 'value7'}),
|
||||||
helpers.create_metric(
|
helpers.create_metric(
|
||||||
name=name2, timestamp=start_timestamp + ONE_SECOND * 3 + 20,
|
name=name2, timestamp=start_timestamp + ONE_SECOND * 3 + 20,
|
||||||
dimensions={'key1': 'value3', 'key2': 'value3'}),
|
dimensions={'key1': 'value3', 'key2': 'value6', 'key3': 'value7'}),
|
||||||
helpers.create_metric(
|
helpers.create_metric(
|
||||||
name=name2, timestamp=start_timestamp + ONE_SECOND * 3 + 30,
|
name=name2, timestamp=start_timestamp + ONE_SECOND * 3 + 30,
|
||||||
dimensions={'key1': 'value4', 'key2': 'value4'})
|
dimensions={'key1': 'value4', 'key2': 'value6', 'key3': 'value8'})
|
||||||
]
|
]
|
||||||
cls.monasca_client.create_metrics(metric3)
|
cls.monasca_client.create_metrics(metric3)
|
||||||
|
|
||||||
@ -194,7 +194,6 @@ class TestMeasurements(base.BaseMonascaTest):
|
|||||||
self.assertRaises(exceptions.BadRequest,
|
self.assertRaises(exceptions.BadRequest,
|
||||||
self.monasca_client.list_measurements, query_parms)
|
self.monasca_client.list_measurements, query_parms)
|
||||||
|
|
||||||
|
|
||||||
@test.attr(type="gate")
|
@test.attr(type="gate")
|
||||||
def test_list_measurements_with_offset_limit(self):
|
def test_list_measurements_with_offset_limit(self):
|
||||||
query_parms = '?name=' + str(self._names_list[1]) + \
|
query_parms = '?name=' + str(self._names_list[1]) + \
|
||||||
@ -259,7 +258,39 @@ class TestMeasurements(base.BaseMonascaTest):
|
|||||||
self.assertEqual(200, resp.status)
|
self.assertEqual(200, resp.status)
|
||||||
|
|
||||||
@test.attr(type="gate")
|
@test.attr(type="gate")
|
||||||
def test_list_measurements_with_group_by(self):
|
def test_list_measurements_with_group_by_one(self):
|
||||||
|
query_parms = '?name=' + str(self._names_list[1]) + \
|
||||||
|
'&group_by=key2' + \
|
||||||
|
'&start_time=' + str(self._start_time) + \
|
||||||
|
'&end_time=' + str(self._end_time)
|
||||||
|
resp, response_body = self.monasca_client.list_measurements(
|
||||||
|
query_parms)
|
||||||
|
self.assertEqual(200, resp.status)
|
||||||
|
elements = response_body['elements']
|
||||||
|
self.assertEqual(len(elements), 2)
|
||||||
|
self._verify_list_measurements_elements(elements, None, None)
|
||||||
|
for measurements in elements:
|
||||||
|
self.assertEqual(1, len(measurements['dimensions'].keys()))
|
||||||
|
self.assertEqual([u'key2'], measurements['dimensions'].keys())
|
||||||
|
|
||||||
|
@test.attr(type="gate")
|
||||||
|
def test_list_measurements_with_group_by_multiple(self):
|
||||||
|
query_parms = '?name=' + str(self._names_list[1]) + \
|
||||||
|
'&group_by=key2,key3' + \
|
||||||
|
'&start_time=' + str(self._start_time) + \
|
||||||
|
'&end_time=' + str(self._end_time)
|
||||||
|
resp, response_body = self.monasca_client.list_measurements(
|
||||||
|
query_parms)
|
||||||
|
self.assertEqual(200, resp.status)
|
||||||
|
elements = response_body['elements']
|
||||||
|
self.assertEqual(len(elements), 3)
|
||||||
|
self._verify_list_measurements_elements(elements, None, None)
|
||||||
|
for measurements in elements:
|
||||||
|
self.assertEqual(2, len(measurements['dimensions'].keys()))
|
||||||
|
self.assertEqual({u'key2', u'key3'}, set(measurements['dimensions'].keys()))
|
||||||
|
|
||||||
|
@test.attr(type="gate")
|
||||||
|
def test_list_measurements_with_group_by_all(self):
|
||||||
query_parms = '?name=' + str(self._names_list[1]) + \
|
query_parms = '?name=' + str(self._names_list[1]) + \
|
||||||
'&group_by=*' + \
|
'&group_by=*' + \
|
||||||
'&start_time=' + str(self._start_time) + \
|
'&start_time=' + str(self._start_time) + \
|
||||||
|
@ -27,8 +27,6 @@ from urllib import urlencode
|
|||||||
NUM_MEASUREMENTS = 100
|
NUM_MEASUREMENTS = 100
|
||||||
MIN_REQUIRED_MEASUREMENTS = 2
|
MIN_REQUIRED_MEASUREMENTS = 2
|
||||||
WAIT_TIME = 30
|
WAIT_TIME = 30
|
||||||
metric_value1 = 1.23
|
|
||||||
metric_value2 = 4.56
|
|
||||||
|
|
||||||
|
|
||||||
class TestStatistics(base.BaseMonascaTest):
|
class TestStatistics(base.BaseMonascaTest):
|
||||||
@ -48,12 +46,13 @@ class TestStatistics(base.BaseMonascaTest):
|
|||||||
helpers.create_metric(name=name,
|
helpers.create_metric(name=name,
|
||||||
dimensions={key: value1},
|
dimensions={key: value1},
|
||||||
timestamp=cls._start_timestamp,
|
timestamp=cls._start_timestamp,
|
||||||
value=metric_value1),
|
value=1.23),
|
||||||
helpers.create_metric(name=name,
|
helpers.create_metric(name=name,
|
||||||
dimensions={key: value2},
|
dimensions={key: value2},
|
||||||
timestamp=cls._start_timestamp + 1000,
|
timestamp=cls._start_timestamp + 1000,
|
||||||
value=metric_value2)
|
value=4.56)
|
||||||
]
|
]
|
||||||
|
cls.metric_values = [m['value'] for m in metrics]
|
||||||
cls.monasca_client.create_metrics(metrics)
|
cls.monasca_client.create_metrics(metrics)
|
||||||
start_time_iso = helpers.timestamp_to_iso(cls._start_timestamp)
|
start_time_iso = helpers.timestamp_to_iso(cls._start_timestamp)
|
||||||
query_param = '?name=' + str(name) + '&start_time=' + \
|
query_param = '?name=' + str(name) + '&start_time=' + \
|
||||||
@ -67,19 +66,53 @@ class TestStatistics(base.BaseMonascaTest):
|
|||||||
resp, response_body = cls.monasca_client.\
|
resp, response_body = cls.monasca_client.\
|
||||||
list_measurements(query_param)
|
list_measurements(query_param)
|
||||||
elements = response_body['elements']
|
elements = response_body['elements']
|
||||||
for element in elements:
|
if len(elements) > 0:
|
||||||
if str(element['name']) == name:
|
num_measurements = len(elements[0]['measurements'])
|
||||||
if len(element['measurements']) >= MIN_REQUIRED_MEASUREMENTS:
|
if num_measurements >= MIN_REQUIRED_MEASUREMENTS:
|
||||||
cls._end_timestamp = cls._start_timestamp + 1000 * 3
|
break
|
||||||
cls._end_time_iso = helpers.timestamp_to_iso(
|
|
||||||
cls._end_timestamp)
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
num_measurements = len(element['measurements'])
|
|
||||||
break
|
|
||||||
time.sleep(constants.RETRY_WAIT_SECS)
|
time.sleep(constants.RETRY_WAIT_SECS)
|
||||||
|
|
||||||
assert False, "Required {} measurements, found {}".format(MIN_REQUIRED_MEASUREMENTS, num_measurements)
|
if num_measurements < MIN_REQUIRED_MEASUREMENTS:
|
||||||
|
assert False, "Required {} measurements, found {}".format(MIN_REQUIRED_MEASUREMENTS, num_measurements)
|
||||||
|
|
||||||
|
cls._end_timestamp = cls._start_timestamp + 3000
|
||||||
|
cls._end_time_iso = helpers.timestamp_to_iso(cls._end_timestamp)
|
||||||
|
|
||||||
|
name2 = data_utils.rand_name("group-by")
|
||||||
|
cls._group_by_metric_name = name2
|
||||||
|
cls._group_by_end_time_iso = helpers.timestamp_to_iso(cls._start_timestamp + 4000)
|
||||||
|
|
||||||
|
group_by_metrics = [
|
||||||
|
helpers.create_metric(name=name2, dimensions={'key1': 'value1', 'key2': 'value5', 'key3': 'value7'},
|
||||||
|
timestamp=cls._start_timestamp + 1, value=2),
|
||||||
|
helpers.create_metric(name=name2, dimensions={'key1': 'value2', 'key2': 'value5', 'key3': 'value7'},
|
||||||
|
timestamp=cls._start_timestamp + 1001, value=3),
|
||||||
|
helpers.create_metric(name=name2, dimensions={'key1': 'value3', 'key2': 'value6', 'key3': 'value7'},
|
||||||
|
timestamp=cls._start_timestamp + 2001, value=5),
|
||||||
|
helpers.create_metric(name=name2, dimensions={'key1': 'value4', 'key2': 'value6', 'key3': 'value8'},
|
||||||
|
timestamp=cls._start_timestamp + 3001, value=7),
|
||||||
|
]
|
||||||
|
|
||||||
|
cls.monasca_client.create_metrics(group_by_metrics)
|
||||||
|
query_param = '?name=' + str(name2) + \
|
||||||
|
'&start_time=' + start_time_iso + \
|
||||||
|
'&merge_metrics=true' + \
|
||||||
|
'&end_time=' + cls._group_by_end_time_iso
|
||||||
|
|
||||||
|
num_measurements = 0
|
||||||
|
for i in xrange(constants.MAX_RETRIES):
|
||||||
|
resp, response_body = cls.monasca_client. \
|
||||||
|
list_measurements(query_param)
|
||||||
|
elements = response_body['elements']
|
||||||
|
if len(elements) > 0:
|
||||||
|
num_measurements = len(elements[0]['measurements'])
|
||||||
|
if num_measurements >= len(group_by_metrics):
|
||||||
|
break
|
||||||
|
time.sleep(constants.RETRY_WAIT_SECS)
|
||||||
|
|
||||||
|
if num_measurements < len(group_by_metrics):
|
||||||
|
assert False, "Required {} measurements, found {}".format(len(group_by_metrics),
|
||||||
|
response_body)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def resource_cleanup(cls):
|
def resource_cleanup(cls):
|
||||||
@ -102,8 +135,7 @@ class TestStatistics(base.BaseMonascaTest):
|
|||||||
num_statistics_method = 5
|
num_statistics_method = 5
|
||||||
statistics = element['statistics'][0]
|
statistics = element['statistics'][0]
|
||||||
self._verify_column_and_statistics(
|
self._verify_column_and_statistics(
|
||||||
column, num_statistics_method, statistics, metric_value1,
|
column, num_statistics_method, statistics, self.metric_values)
|
||||||
metric_value2)
|
|
||||||
|
|
||||||
@test.attr(type="gate")
|
@test.attr(type="gate")
|
||||||
@test.attr(type=['negative'])
|
@test.attr(type=['negative'])
|
||||||
@ -270,6 +302,87 @@ class TestStatistics(base.BaseMonascaTest):
|
|||||||
# Get the next set
|
# Get the next set
|
||||||
offset = self._get_offset(response_body)
|
offset = self._get_offset(response_body)
|
||||||
|
|
||||||
|
@test.attr(type="gate")
|
||||||
|
def test_list_statistics_with_group_by_one(self):
|
||||||
|
query_parms = '?name=' + self._group_by_metric_name + \
|
||||||
|
'&group_by=key2' + \
|
||||||
|
'&statistics=max,avg,min' + \
|
||||||
|
'&start_time=' + str(self._start_time_iso) + \
|
||||||
|
'&end_time=' + str(self._group_by_end_time_iso)
|
||||||
|
resp, response_body = self.monasca_client.list_statistics(
|
||||||
|
query_parms)
|
||||||
|
self.assertEqual(200, resp.status)
|
||||||
|
elements = response_body['elements']
|
||||||
|
self.assertEqual(len(elements), 2)
|
||||||
|
for statistics in elements:
|
||||||
|
self.assertEqual(1, len(statistics['dimensions'].keys()))
|
||||||
|
self.assertEqual([u'key2'], statistics['dimensions'].keys())
|
||||||
|
|
||||||
|
@test.attr(type="gate")
|
||||||
|
def test_list_statistics_with_group_by_multiple(self):
|
||||||
|
query_parms = '?name=' + self._group_by_metric_name + \
|
||||||
|
'&group_by=key2,key3' + \
|
||||||
|
'&statistics=max,avg,min' + \
|
||||||
|
'&start_time=' + str(self._start_time_iso) + \
|
||||||
|
'&end_time=' + str(self._group_by_end_time_iso)
|
||||||
|
resp, response_body = self.monasca_client.list_statistics(
|
||||||
|
query_parms)
|
||||||
|
self.assertEqual(200, resp.status)
|
||||||
|
elements = response_body['elements']
|
||||||
|
self.assertEqual(len(elements), 3)
|
||||||
|
for statistics in elements:
|
||||||
|
self.assertEqual(2, len(statistics['dimensions'].keys()))
|
||||||
|
self.assertEqual({u'key2', u'key3'}, set(statistics['dimensions'].keys()))
|
||||||
|
|
||||||
|
@test.attr(type="gate")
|
||||||
|
def test_list_statistics_with_group_by_all(self):
|
||||||
|
query_parms = '?name=' + self._group_by_metric_name + \
|
||||||
|
'&group_by=*' + \
|
||||||
|
'&statistics=max,avg,min' + \
|
||||||
|
'&start_time=' + str(self._start_time_iso) + \
|
||||||
|
'&end_time=' + str(self._group_by_end_time_iso)
|
||||||
|
resp, response_body = self.monasca_client.list_statistics(
|
||||||
|
query_parms)
|
||||||
|
self.assertEqual(200, resp.status)
|
||||||
|
elements = response_body['elements']
|
||||||
|
self.assertEqual(len(elements), 4)
|
||||||
|
|
||||||
|
@test.attr(type="gate")
|
||||||
|
def test_list_statistics_with_group_by_offset_limit(self):
|
||||||
|
query_parms = '?name=' + str(self._group_by_metric_name) + \
|
||||||
|
'&group_by=key2' + \
|
||||||
|
'&statistics=avg,max' + \
|
||||||
|
'&start_time=' + str(self._start_time_iso) + \
|
||||||
|
'&end_time=' + str(self._group_by_end_time_iso) + \
|
||||||
|
'&period=1'
|
||||||
|
resp, response_body = self.monasca_client.list_statistics(query_parms)
|
||||||
|
self.assertEqual(200, resp.status)
|
||||||
|
all_expected_elements = response_body['elements']
|
||||||
|
|
||||||
|
for limit in xrange(1, 4):
|
||||||
|
offset = None
|
||||||
|
for i in xrange(4 - limit):
|
||||||
|
query_parms = '?name=' + str(self._group_by_metric_name) + \
|
||||||
|
'&group_by=key2' + \
|
||||||
|
'&statistics=avg,max' + \
|
||||||
|
'&start_time=' + str(self._start_time_iso) + \
|
||||||
|
'&end_time=' + str(self._group_by_end_time_iso) + \
|
||||||
|
'&period=1' + \
|
||||||
|
'&limit=' + str(limit)
|
||||||
|
if i > 0:
|
||||||
|
offset = self._get_offset(response_body)
|
||||||
|
query_parms += "&offset=" + offset
|
||||||
|
|
||||||
|
expected_elements = helpers.get_expected_elements_inner_offset_limit(
|
||||||
|
all_expected_elements,
|
||||||
|
offset,
|
||||||
|
limit,
|
||||||
|
'statistics')
|
||||||
|
|
||||||
|
resp, response_body = self.monasca_client.list_statistics(query_parms)
|
||||||
|
self.assertEqual(200, resp.status)
|
||||||
|
self.assertEqual(expected_elements, response_body['elements'])
|
||||||
|
|
||||||
@test.attr(type="gate")
|
@test.attr(type="gate")
|
||||||
@test.attr(type=['negative'])
|
@test.attr(type=['negative'])
|
||||||
def test_list_statistics_with_no_merge_metrics(self):
|
def test_list_statistics_with_no_merge_metrics(self):
|
||||||
@ -342,22 +455,22 @@ class TestStatistics(base.BaseMonascaTest):
|
|||||||
self.assertEqual(element['name'], self._test_name)
|
self.assertEqual(element['name'], self._test_name)
|
||||||
|
|
||||||
def _verify_column_and_statistics(
|
def _verify_column_and_statistics(
|
||||||
self, column, num_statistics_method, statistics, num1, num2):
|
self, column, num_statistics_method, statistics, values):
|
||||||
self.assertTrue(type(column) is list)
|
self.assertTrue(type(column) is list)
|
||||||
self.assertTrue(type(statistics) is list)
|
self.assertTrue(type(statistics) is list)
|
||||||
self.assertEqual(len(column), num_statistics_method + 1)
|
self.assertEqual(len(column), num_statistics_method + 1)
|
||||||
self.assertEqual(column[0], 'timestamp')
|
self.assertEqual(column[0], 'timestamp')
|
||||||
for i, method in enumerate(column):
|
for i, method in enumerate(column):
|
||||||
if method == 'avg':
|
if method == 'avg':
|
||||||
self.assertAlmostEqual(statistics[i], (num1 + num2) / 2)
|
self.assertAlmostEqual(statistics[i], float(sum(values) / len(values)))
|
||||||
elif method == 'max':
|
elif method == 'max':
|
||||||
self.assertEqual(statistics[i], max(num1, num2))
|
self.assertEqual(statistics[i], max(values))
|
||||||
elif method == 'min':
|
elif method == 'min':
|
||||||
self.assertEqual(statistics[i], min(num1, num2))
|
self.assertEqual(statistics[i], min(values))
|
||||||
elif method == 'sum':
|
elif method == 'sum':
|
||||||
self.assertAlmostEqual(statistics[i], num1 + num2)
|
self.assertAlmostEqual(statistics[i], sum(values))
|
||||||
elif method == 'count':
|
elif method == 'count':
|
||||||
self.assertEqual(statistics[i], 2)
|
self.assertEqual(statistics[i], len(values))
|
||||||
|
|
||||||
def _check_timeout(self, timer, max_retries, elements,
|
def _check_timeout(self, timer, max_retries, elements,
|
||||||
expect_num_elements):
|
expect_num_elements):
|
||||||
|
Loading…
Reference in New Issue
Block a user