Merge "fix prepare for numeric data"
This commit is contained in:
commit
f2591e1061
|
@ -381,7 +381,10 @@ def get_runs_by_run_metadata_key(run_metadata_key, value):
|
|||
continue
|
||||
build_name = run['metadata']['build_name']
|
||||
if run_at in run_times:
|
||||
run_times[run_at][build_name].append(run['run_time'])
|
||||
if build_name in run_times[run_at]:
|
||||
run_times[run_at][build_name].append(run['run_time'])
|
||||
else:
|
||||
run_times[run_at][build_name] = [run['run_time']]
|
||||
else:
|
||||
run_times[run_at] = {build_name: [run['run_time']]}
|
||||
# if there is more than one run with the same run_at time
|
||||
|
|
|
@ -35,11 +35,11 @@ def get_numeric_data(run_times_time_series, sample_rate):
|
|||
temp_dict = {}
|
||||
sample_rate = base.resample_matrix[sample_rate]
|
||||
for run_at, run in run_times_time_series.items():
|
||||
build_name, run_time = list(run.items())[0]
|
||||
if build_name in temp_dict:
|
||||
temp_dict[build_name][run_at] = run_time
|
||||
else:
|
||||
temp_dict[build_name] = {run_at: run_time}
|
||||
for build_name, run_time in list(run.items()):
|
||||
if build_name in temp_dict:
|
||||
temp_dict[build_name][run_at] = run_time
|
||||
else:
|
||||
temp_dict[build_name] = {run_at: run_time}
|
||||
df = pd.DataFrame(temp_dict)
|
||||
numeric_df = df.resample(sample_rate).mean()
|
||||
temp_numeric_df = numeric_df.interpolate(method='time', limit=10)
|
||||
|
|
|
@ -614,13 +614,13 @@ class TestRestAPI(base.TestCase):
|
|||
response_data = json.loads(res.data.decode('utf-8'))
|
||||
# numpy.NaN == numpy.NaN result is False, a key error here means the
|
||||
# dicts are not equal
|
||||
for project, item in list(expected_response_data['numeric'].items()):
|
||||
for project, item in expected_response_data['numeric'].items():
|
||||
for date, run_time in list(item.items()):
|
||||
if (numpy.isnan(run_time) and
|
||||
numpy.isnan(response_data['numeric'][project][date])):
|
||||
del expected_response_data['numeric'][project][date]
|
||||
del response_data['numeric'][project][date]
|
||||
self.assertEqual(expected_response_data, response_data)
|
||||
self.assertDictEqual(expected_response_data, response_data)
|
||||
api_mock.assert_called_once_with('project',
|
||||
'openstack/trove',
|
||||
None,
|
||||
|
@ -770,6 +770,85 @@ class TestRestAPI(base.TestCase):
|
|||
self.maxDiff = None
|
||||
self.assertDictEqual(expected_response_data, response_data)
|
||||
|
||||
@mock.patch('subunit2sql.db.api.get_time_series_runs_by_key_value',
|
||||
return_value={
|
||||
timestamp_d1: [{'pass': 1,
|
||||
'fail': 0,
|
||||
'skip': 0,
|
||||
'id': 'abc1',
|
||||
'run_time': 4.0,
|
||||
'metadata': {
|
||||
'build_name':
|
||||
'tempest-dsvm-neutron-full'}},
|
||||
{'pass': 10,
|
||||
'fail': 1,
|
||||
'skip': 0,
|
||||
'id': 'abc1',
|
||||
'run_time': 9.0,
|
||||
'metadata': {
|
||||
'build_name':
|
||||
'tempest-dsvm-neutron-full'}},
|
||||
{'pass': 2,
|
||||
'fail': 0,
|
||||
'skip': 0,
|
||||
'id': 'abc2',
|
||||
'run_time': 2.0,
|
||||
'metadata': {
|
||||
'build_name':
|
||||
'openstack-tox-py27-trove'}}],
|
||||
timestamp_d2: [{'pass': 100,
|
||||
'fail': 0,
|
||||
'skip': 0,
|
||||
'id': 'abc3',
|
||||
'run_time': 20.0,
|
||||
'metadata': {
|
||||
'build_name':
|
||||
'tempest-dsvm-neutron-full'}}]
|
||||
})
|
||||
def test_get_runs_by_project_diff_build_and_same_run_at(self, api_mock):
|
||||
start_date = timestamp_d1.date().isoformat()
|
||||
stop_date = timestamp_d2.date().isoformat()
|
||||
query = ('datetime_resolution=day&start_date={0}&stop_date={1}'
|
||||
.format(start_date, stop_date))
|
||||
res = self.app.get('/runs/key/project/trove?{0}'
|
||||
.format(query))
|
||||
self.assertEqual(200, res.status_code)
|
||||
expected_response_data = {
|
||||
u'data': {
|
||||
u'timedelta': [
|
||||
{u'datetime': u'%s' % timestamp_d1.date().isoformat(),
|
||||
u'job_data': [{u'pass': 1,
|
||||
u'fail': 0,
|
||||
u'job_name': u'openstack-tox-py27-trove',
|
||||
u'mean_run_time': 2.0},
|
||||
{u'pass': 1,
|
||||
u'fail': 1,
|
||||
u'job_name': u'tempest-dsvm-neutron-full',
|
||||
u'mean_run_time': 6.5}]},
|
||||
{u'datetime': u'%s' % timestamp_d2.date().isoformat(),
|
||||
u'job_data': [{u'pass': 1,
|
||||
u'fail': 0,
|
||||
u'job_name': u'tempest-dsvm-neutron-full',
|
||||
u'mean_run_time': 20.0}]}]},
|
||||
u'numeric': {
|
||||
u'tempest-dsvm-neutron-full': {
|
||||
u'%s' % timestamp_d1.isoformat(): 4.0,
|
||||
u'%s' % timestamp_d2.isoformat(): 20.0},
|
||||
u'openstack-tox-py27-trove': {
|
||||
u'%s' % timestamp_d1.isoformat(): 2.0,
|
||||
u'%s' % timestamp_d2.isoformat(): numpy.NaN}}}
|
||||
response_data = json.loads(res.data.decode('utf-8'))
|
||||
self.maxDiff = None
|
||||
# numpy.NaN == numpy.NaN result is False, a key error here means the
|
||||
# dicts are not equal
|
||||
for project, item in expected_response_data['numeric'].items():
|
||||
for date, run_time in list(item.items()):
|
||||
if (numpy.isnan(run_time) and
|
||||
numpy.isnan(response_data['numeric'][project][date])):
|
||||
del expected_response_data['numeric'][project][date]
|
||||
del response_data['numeric'][project][date]
|
||||
self.assertDictEqual(expected_response_data, response_data)
|
||||
|
||||
@mock.patch('openstack_health.api._check_db_availability',
|
||||
return_value=False)
|
||||
@mock.patch('openstack_health.api._check_er_availability',
|
||||
|
|
|
@ -63,8 +63,90 @@ class TestRunAggregatorGetNumericData(base.TestCase):
|
|||
actual = run_aggregator.get_numeric_data({}, 'day')
|
||||
self.assertEqual(expected, actual)
|
||||
|
||||
def test_get_numeric_data_diff_build_name(self):
|
||||
self.runs[datetime.datetime(2018, 6, 14, 3, 52, 24)][
|
||||
'openstack-tox-py27-trove'] = 321.304
|
||||
expected = {
|
||||
'tempest-dsvm-neutron-full': {
|
||||
'2018-06-13T00:00:00': 5391.195,
|
||||
'2018-06-14T00:00:00': 4768.1,
|
||||
'2018-06-15T00:00:00': np.nan,
|
||||
'2018-06-16T00:00:00': np.nan,
|
||||
'2018-06-17T00:00:00': np.nan,
|
||||
'2018-06-18T00:00:00': 4183.85,
|
||||
'2018-06-19T00:00:00': 4545.41,
|
||||
'2018-06-20T00:00:00': 4133.03,
|
||||
'2018-06-21T00:00:00': np.nan,
|
||||
'2018-06-22T00:00:00': 5592.295,
|
||||
'2018-06-23T00:00:00': 6150.95,
|
||||
'2018-06-24T00:00:00': np.nan,
|
||||
'2018-06-25T00:00:00': 6047.95
|
||||
},
|
||||
'tempest-dsvm-neutron-full-avg': {
|
||||
'2018-06-13T00:00:00': np.nan,
|
||||
'2018-06-14T00:00:00': np.nan,
|
||||
'2018-06-15T00:00:00': np.nan,
|
||||
'2018-06-16T00:00:00': np.nan,
|
||||
'2018-06-17T00:00:00': np.nan,
|
||||
'2018-06-18T00:00:00': np.nan,
|
||||
'2018-06-19T00:00:00': np.nan,
|
||||
'2018-06-20T00:00:00': np.nan,
|
||||
'2018-06-21T00:00:00': np.nan,
|
||||
'2018-06-22T00:00:00': 4690.44675,
|
||||
'2018-06-23T00:00:00': 4766.42225,
|
||||
'2018-06-24T00:00:00': 4899.55725,
|
||||
'2018-06-25T00:00:00': 5042.148499999999
|
||||
},
|
||||
'openstack-tox-py27-trove': {
|
||||
'2018-06-13T00:00:00': np.nan,
|
||||
'2018-06-14T00:00:00': 321.304,
|
||||
'2018-06-15T00:00:00': np.nan,
|
||||
'2018-06-16T00:00:00': np.nan,
|
||||
'2018-06-17T00:00:00': np.nan,
|
||||
'2018-06-18T00:00:00': np.nan,
|
||||
'2018-06-19T00:00:00': np.nan,
|
||||
'2018-06-20T00:00:00': np.nan,
|
||||
'2018-06-21T00:00:00': np.nan,
|
||||
'2018-06-22T00:00:00': np.nan,
|
||||
'2018-06-23T00:00:00': np.nan,
|
||||
'2018-06-24T00:00:00': np.nan,
|
||||
'2018-06-25T00:00:00': np.nan
|
||||
},
|
||||
'openstack-tox-py27-trove-avg': {
|
||||
'2018-06-13T00:00:00': np.nan,
|
||||
'2018-06-14T00:00:00': np.nan,
|
||||
'2018-06-15T00:00:00': np.nan,
|
||||
'2018-06-16T00:00:00': np.nan,
|
||||
'2018-06-17T00:00:00': np.nan,
|
||||
'2018-06-18T00:00:00': np.nan,
|
||||
'2018-06-19T00:00:00': np.nan,
|
||||
'2018-06-20T00:00:00': np.nan,
|
||||
'2018-06-21T00:00:00': np.nan,
|
||||
'2018-06-22T00:00:00': np.nan,
|
||||
'2018-06-23T00:00:00': 321.30400000000003,
|
||||
'2018-06-24T00:00:00': 321.30400000000003,
|
||||
'2018-06-25T00:00:00': np.nan
|
||||
}
|
||||
|
||||
}
|
||||
actual = run_aggregator.get_numeric_data(self.runs, 'day')
|
||||
self.assertItemsEqual(expected, actual)
|
||||
self.assertItemsEqual(
|
||||
expected['tempest-dsvm-neutron-full'].keys(),
|
||||
actual['tempest-dsvm-neutron-full'].keys())
|
||||
self.assertItemsEqual(
|
||||
expected['tempest-dsvm-neutron-full-avg'].keys(),
|
||||
actual['tempest-dsvm-neutron-full-avg'].keys())
|
||||
# np.nan == np.nan is False, remove the key entries with np.nan values,
|
||||
# if a key error is thrown then expected does not equal actual.
|
||||
for key in expected:
|
||||
for date, run_time in list(expected[key].items()):
|
||||
if np.isnan(run_time) and np.isnan(actual[key][date]):
|
||||
del actual[key][date]
|
||||
del expected[key][date]
|
||||
self.assertDictEqual(expected, actual)
|
||||
|
||||
def test_get_numeric_data(self):
|
||||
self.maxDiff = None
|
||||
expected = {
|
||||
'tempest-dsvm-neutron-full': {
|
||||
'2018-06-13T00:00:00': 5391.195,
|
||||
|
|
Loading…
Reference in New Issue