Merge "fix prepare for numeric data"
This commit is contained in:
commit
f2591e1061
|
@ -381,7 +381,10 @@ def get_runs_by_run_metadata_key(run_metadata_key, value):
|
||||||
continue
|
continue
|
||||||
build_name = run['metadata']['build_name']
|
build_name = run['metadata']['build_name']
|
||||||
if run_at in run_times:
|
if run_at in run_times:
|
||||||
run_times[run_at][build_name].append(run['run_time'])
|
if build_name in run_times[run_at]:
|
||||||
|
run_times[run_at][build_name].append(run['run_time'])
|
||||||
|
else:
|
||||||
|
run_times[run_at][build_name] = [run['run_time']]
|
||||||
else:
|
else:
|
||||||
run_times[run_at] = {build_name: [run['run_time']]}
|
run_times[run_at] = {build_name: [run['run_time']]}
|
||||||
# if there is more than one run with the same run_at time
|
# if there is more than one run with the same run_at time
|
||||||
|
|
|
@ -35,11 +35,11 @@ def get_numeric_data(run_times_time_series, sample_rate):
|
||||||
temp_dict = {}
|
temp_dict = {}
|
||||||
sample_rate = base.resample_matrix[sample_rate]
|
sample_rate = base.resample_matrix[sample_rate]
|
||||||
for run_at, run in run_times_time_series.items():
|
for run_at, run in run_times_time_series.items():
|
||||||
build_name, run_time = list(run.items())[0]
|
for build_name, run_time in list(run.items()):
|
||||||
if build_name in temp_dict:
|
if build_name in temp_dict:
|
||||||
temp_dict[build_name][run_at] = run_time
|
temp_dict[build_name][run_at] = run_time
|
||||||
else:
|
else:
|
||||||
temp_dict[build_name] = {run_at: run_time}
|
temp_dict[build_name] = {run_at: run_time}
|
||||||
df = pd.DataFrame(temp_dict)
|
df = pd.DataFrame(temp_dict)
|
||||||
numeric_df = df.resample(sample_rate).mean()
|
numeric_df = df.resample(sample_rate).mean()
|
||||||
temp_numeric_df = numeric_df.interpolate(method='time', limit=10)
|
temp_numeric_df = numeric_df.interpolate(method='time', limit=10)
|
||||||
|
|
|
@ -614,13 +614,13 @@ class TestRestAPI(base.TestCase):
|
||||||
response_data = json.loads(res.data.decode('utf-8'))
|
response_data = json.loads(res.data.decode('utf-8'))
|
||||||
# numpy.NaN == numpy.NaN result is False, a key error here means the
|
# numpy.NaN == numpy.NaN result is False, a key error here means the
|
||||||
# dicts are not equal
|
# dicts are not equal
|
||||||
for project, item in list(expected_response_data['numeric'].items()):
|
for project, item in expected_response_data['numeric'].items():
|
||||||
for date, run_time in list(item.items()):
|
for date, run_time in list(item.items()):
|
||||||
if (numpy.isnan(run_time) and
|
if (numpy.isnan(run_time) and
|
||||||
numpy.isnan(response_data['numeric'][project][date])):
|
numpy.isnan(response_data['numeric'][project][date])):
|
||||||
del expected_response_data['numeric'][project][date]
|
del expected_response_data['numeric'][project][date]
|
||||||
del response_data['numeric'][project][date]
|
del response_data['numeric'][project][date]
|
||||||
self.assertEqual(expected_response_data, response_data)
|
self.assertDictEqual(expected_response_data, response_data)
|
||||||
api_mock.assert_called_once_with('project',
|
api_mock.assert_called_once_with('project',
|
||||||
'openstack/trove',
|
'openstack/trove',
|
||||||
None,
|
None,
|
||||||
|
@ -770,6 +770,85 @@ class TestRestAPI(base.TestCase):
|
||||||
self.maxDiff = None
|
self.maxDiff = None
|
||||||
self.assertDictEqual(expected_response_data, response_data)
|
self.assertDictEqual(expected_response_data, response_data)
|
||||||
|
|
||||||
|
@mock.patch('subunit2sql.db.api.get_time_series_runs_by_key_value',
|
||||||
|
return_value={
|
||||||
|
timestamp_d1: [{'pass': 1,
|
||||||
|
'fail': 0,
|
||||||
|
'skip': 0,
|
||||||
|
'id': 'abc1',
|
||||||
|
'run_time': 4.0,
|
||||||
|
'metadata': {
|
||||||
|
'build_name':
|
||||||
|
'tempest-dsvm-neutron-full'}},
|
||||||
|
{'pass': 10,
|
||||||
|
'fail': 1,
|
||||||
|
'skip': 0,
|
||||||
|
'id': 'abc1',
|
||||||
|
'run_time': 9.0,
|
||||||
|
'metadata': {
|
||||||
|
'build_name':
|
||||||
|
'tempest-dsvm-neutron-full'}},
|
||||||
|
{'pass': 2,
|
||||||
|
'fail': 0,
|
||||||
|
'skip': 0,
|
||||||
|
'id': 'abc2',
|
||||||
|
'run_time': 2.0,
|
||||||
|
'metadata': {
|
||||||
|
'build_name':
|
||||||
|
'openstack-tox-py27-trove'}}],
|
||||||
|
timestamp_d2: [{'pass': 100,
|
||||||
|
'fail': 0,
|
||||||
|
'skip': 0,
|
||||||
|
'id': 'abc3',
|
||||||
|
'run_time': 20.0,
|
||||||
|
'metadata': {
|
||||||
|
'build_name':
|
||||||
|
'tempest-dsvm-neutron-full'}}]
|
||||||
|
})
|
||||||
|
def test_get_runs_by_project_diff_build_and_same_run_at(self, api_mock):
|
||||||
|
start_date = timestamp_d1.date().isoformat()
|
||||||
|
stop_date = timestamp_d2.date().isoformat()
|
||||||
|
query = ('datetime_resolution=day&start_date={0}&stop_date={1}'
|
||||||
|
.format(start_date, stop_date))
|
||||||
|
res = self.app.get('/runs/key/project/trove?{0}'
|
||||||
|
.format(query))
|
||||||
|
self.assertEqual(200, res.status_code)
|
||||||
|
expected_response_data = {
|
||||||
|
u'data': {
|
||||||
|
u'timedelta': [
|
||||||
|
{u'datetime': u'%s' % timestamp_d1.date().isoformat(),
|
||||||
|
u'job_data': [{u'pass': 1,
|
||||||
|
u'fail': 0,
|
||||||
|
u'job_name': u'openstack-tox-py27-trove',
|
||||||
|
u'mean_run_time': 2.0},
|
||||||
|
{u'pass': 1,
|
||||||
|
u'fail': 1,
|
||||||
|
u'job_name': u'tempest-dsvm-neutron-full',
|
||||||
|
u'mean_run_time': 6.5}]},
|
||||||
|
{u'datetime': u'%s' % timestamp_d2.date().isoformat(),
|
||||||
|
u'job_data': [{u'pass': 1,
|
||||||
|
u'fail': 0,
|
||||||
|
u'job_name': u'tempest-dsvm-neutron-full',
|
||||||
|
u'mean_run_time': 20.0}]}]},
|
||||||
|
u'numeric': {
|
||||||
|
u'tempest-dsvm-neutron-full': {
|
||||||
|
u'%s' % timestamp_d1.isoformat(): 4.0,
|
||||||
|
u'%s' % timestamp_d2.isoformat(): 20.0},
|
||||||
|
u'openstack-tox-py27-trove': {
|
||||||
|
u'%s' % timestamp_d1.isoformat(): 2.0,
|
||||||
|
u'%s' % timestamp_d2.isoformat(): numpy.NaN}}}
|
||||||
|
response_data = json.loads(res.data.decode('utf-8'))
|
||||||
|
self.maxDiff = None
|
||||||
|
# numpy.NaN == numpy.NaN result is False, a key error here means the
|
||||||
|
# dicts are not equal
|
||||||
|
for project, item in expected_response_data['numeric'].items():
|
||||||
|
for date, run_time in list(item.items()):
|
||||||
|
if (numpy.isnan(run_time) and
|
||||||
|
numpy.isnan(response_data['numeric'][project][date])):
|
||||||
|
del expected_response_data['numeric'][project][date]
|
||||||
|
del response_data['numeric'][project][date]
|
||||||
|
self.assertDictEqual(expected_response_data, response_data)
|
||||||
|
|
||||||
@mock.patch('openstack_health.api._check_db_availability',
|
@mock.patch('openstack_health.api._check_db_availability',
|
||||||
return_value=False)
|
return_value=False)
|
||||||
@mock.patch('openstack_health.api._check_er_availability',
|
@mock.patch('openstack_health.api._check_er_availability',
|
||||||
|
|
|
@ -63,8 +63,90 @@ class TestRunAggregatorGetNumericData(base.TestCase):
|
||||||
actual = run_aggregator.get_numeric_data({}, 'day')
|
actual = run_aggregator.get_numeric_data({}, 'day')
|
||||||
self.assertEqual(expected, actual)
|
self.assertEqual(expected, actual)
|
||||||
|
|
||||||
|
def test_get_numeric_data_diff_build_name(self):
|
||||||
|
self.runs[datetime.datetime(2018, 6, 14, 3, 52, 24)][
|
||||||
|
'openstack-tox-py27-trove'] = 321.304
|
||||||
|
expected = {
|
||||||
|
'tempest-dsvm-neutron-full': {
|
||||||
|
'2018-06-13T00:00:00': 5391.195,
|
||||||
|
'2018-06-14T00:00:00': 4768.1,
|
||||||
|
'2018-06-15T00:00:00': np.nan,
|
||||||
|
'2018-06-16T00:00:00': np.nan,
|
||||||
|
'2018-06-17T00:00:00': np.nan,
|
||||||
|
'2018-06-18T00:00:00': 4183.85,
|
||||||
|
'2018-06-19T00:00:00': 4545.41,
|
||||||
|
'2018-06-20T00:00:00': 4133.03,
|
||||||
|
'2018-06-21T00:00:00': np.nan,
|
||||||
|
'2018-06-22T00:00:00': 5592.295,
|
||||||
|
'2018-06-23T00:00:00': 6150.95,
|
||||||
|
'2018-06-24T00:00:00': np.nan,
|
||||||
|
'2018-06-25T00:00:00': 6047.95
|
||||||
|
},
|
||||||
|
'tempest-dsvm-neutron-full-avg': {
|
||||||
|
'2018-06-13T00:00:00': np.nan,
|
||||||
|
'2018-06-14T00:00:00': np.nan,
|
||||||
|
'2018-06-15T00:00:00': np.nan,
|
||||||
|
'2018-06-16T00:00:00': np.nan,
|
||||||
|
'2018-06-17T00:00:00': np.nan,
|
||||||
|
'2018-06-18T00:00:00': np.nan,
|
||||||
|
'2018-06-19T00:00:00': np.nan,
|
||||||
|
'2018-06-20T00:00:00': np.nan,
|
||||||
|
'2018-06-21T00:00:00': np.nan,
|
||||||
|
'2018-06-22T00:00:00': 4690.44675,
|
||||||
|
'2018-06-23T00:00:00': 4766.42225,
|
||||||
|
'2018-06-24T00:00:00': 4899.55725,
|
||||||
|
'2018-06-25T00:00:00': 5042.148499999999
|
||||||
|
},
|
||||||
|
'openstack-tox-py27-trove': {
|
||||||
|
'2018-06-13T00:00:00': np.nan,
|
||||||
|
'2018-06-14T00:00:00': 321.304,
|
||||||
|
'2018-06-15T00:00:00': np.nan,
|
||||||
|
'2018-06-16T00:00:00': np.nan,
|
||||||
|
'2018-06-17T00:00:00': np.nan,
|
||||||
|
'2018-06-18T00:00:00': np.nan,
|
||||||
|
'2018-06-19T00:00:00': np.nan,
|
||||||
|
'2018-06-20T00:00:00': np.nan,
|
||||||
|
'2018-06-21T00:00:00': np.nan,
|
||||||
|
'2018-06-22T00:00:00': np.nan,
|
||||||
|
'2018-06-23T00:00:00': np.nan,
|
||||||
|
'2018-06-24T00:00:00': np.nan,
|
||||||
|
'2018-06-25T00:00:00': np.nan
|
||||||
|
},
|
||||||
|
'openstack-tox-py27-trove-avg': {
|
||||||
|
'2018-06-13T00:00:00': np.nan,
|
||||||
|
'2018-06-14T00:00:00': np.nan,
|
||||||
|
'2018-06-15T00:00:00': np.nan,
|
||||||
|
'2018-06-16T00:00:00': np.nan,
|
||||||
|
'2018-06-17T00:00:00': np.nan,
|
||||||
|
'2018-06-18T00:00:00': np.nan,
|
||||||
|
'2018-06-19T00:00:00': np.nan,
|
||||||
|
'2018-06-20T00:00:00': np.nan,
|
||||||
|
'2018-06-21T00:00:00': np.nan,
|
||||||
|
'2018-06-22T00:00:00': np.nan,
|
||||||
|
'2018-06-23T00:00:00': 321.30400000000003,
|
||||||
|
'2018-06-24T00:00:00': 321.30400000000003,
|
||||||
|
'2018-06-25T00:00:00': np.nan
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
actual = run_aggregator.get_numeric_data(self.runs, 'day')
|
||||||
|
self.assertItemsEqual(expected, actual)
|
||||||
|
self.assertItemsEqual(
|
||||||
|
expected['tempest-dsvm-neutron-full'].keys(),
|
||||||
|
actual['tempest-dsvm-neutron-full'].keys())
|
||||||
|
self.assertItemsEqual(
|
||||||
|
expected['tempest-dsvm-neutron-full-avg'].keys(),
|
||||||
|
actual['tempest-dsvm-neutron-full-avg'].keys())
|
||||||
|
# np.nan == np.nan is False, remove the key entries with np.nan values,
|
||||||
|
# if a key error is thrown then expected does not equal actual.
|
||||||
|
for key in expected:
|
||||||
|
for date, run_time in list(expected[key].items()):
|
||||||
|
if np.isnan(run_time) and np.isnan(actual[key][date]):
|
||||||
|
del actual[key][date]
|
||||||
|
del expected[key][date]
|
||||||
|
self.assertDictEqual(expected, actual)
|
||||||
|
|
||||||
def test_get_numeric_data(self):
|
def test_get_numeric_data(self):
|
||||||
self.maxDiff = None
|
|
||||||
expected = {
|
expected = {
|
||||||
'tempest-dsvm-neutron-full': {
|
'tempest-dsvm-neutron-full': {
|
||||||
'2018-06-13T00:00:00': 5391.195,
|
'2018-06-13T00:00:00': 5391.195,
|
||||||
|
|
Loading…
Reference in New Issue