Browse Source

Merge "fix prepare for numeric data"

changes/06/602406/1
Zuul 9 months ago
parent
commit
f2591e1061

+ 4
- 1
openstack_health/api.py View File

@@ -381,7 +381,10 @@ def get_runs_by_run_metadata_key(run_metadata_key, value):
381 381
                     continue
382 382
                 build_name = run['metadata']['build_name']
383 383
                 if run_at in run_times:
384
-                    run_times[run_at][build_name].append(run['run_time'])
384
+                    if build_name in run_times[run_at]:
385
+                        run_times[run_at][build_name].append(run['run_time'])
386
+                    else:
387
+                        run_times[run_at][build_name] = [run['run_time']]
385 388
                 else:
386 389
                     run_times[run_at] = {build_name: [run['run_time']]}
387 390
         # if there is more than one run with the same run_at time

+ 5
- 5
openstack_health/run_aggregator.py View File

@@ -35,11 +35,11 @@ def get_numeric_data(run_times_time_series, sample_rate):
35 35
     temp_dict = {}
36 36
     sample_rate = base.resample_matrix[sample_rate]
37 37
     for run_at, run in run_times_time_series.items():
38
-        build_name, run_time = list(run.items())[0]
39
-        if build_name in temp_dict:
40
-            temp_dict[build_name][run_at] = run_time
41
-        else:
42
-            temp_dict[build_name] = {run_at: run_time}
38
+        for build_name, run_time in list(run.items()):
39
+            if build_name in temp_dict:
40
+                temp_dict[build_name][run_at] = run_time
41
+            else:
42
+                temp_dict[build_name] = {run_at: run_time}
43 43
     df = pd.DataFrame(temp_dict)
44 44
     numeric_df = df.resample(sample_rate).mean()
45 45
     temp_numeric_df = numeric_df.interpolate(method='time', limit=10)

+ 81
- 2
openstack_health/tests/test_api.py View File

@@ -614,13 +614,13 @@ class TestRestAPI(base.TestCase):
614 614
         response_data = json.loads(res.data.decode('utf-8'))
615 615
         # numpy.NaN == numpy.NaN result is False, a key error here means the
616 616
         # dicts are not equal
617
-        for project, item in list(expected_response_data['numeric'].items()):
617
+        for project, item in expected_response_data['numeric'].items():
618 618
             for date, run_time in list(item.items()):
619 619
                 if (numpy.isnan(run_time) and
620 620
                     numpy.isnan(response_data['numeric'][project][date])):
621 621
                     del expected_response_data['numeric'][project][date]
622 622
                     del response_data['numeric'][project][date]
623
-        self.assertEqual(expected_response_data, response_data)
623
+        self.assertDictEqual(expected_response_data, response_data)
624 624
         api_mock.assert_called_once_with('project',
625 625
                                          'openstack/trove',
626 626
                                          None,
@@ -770,6 +770,85 @@ class TestRestAPI(base.TestCase):
770 770
         self.maxDiff = None
771 771
         self.assertDictEqual(expected_response_data, response_data)
772 772
 
773
+    @mock.patch('subunit2sql.db.api.get_time_series_runs_by_key_value',
774
+                return_value={
775
+                    timestamp_d1: [{'pass': 1,
776
+                                    'fail': 0,
777
+                                    'skip': 0,
778
+                                    'id': 'abc1',
779
+                                    'run_time': 4.0,
780
+                                    'metadata': {
781
+                                        'build_name':
782
+                                            'tempest-dsvm-neutron-full'}},
783
+                                   {'pass': 10,
784
+                                    'fail': 1,
785
+                                    'skip': 0,
786
+                                    'id': 'abc1',
787
+                                    'run_time': 9.0,
788
+                                    'metadata': {
789
+                                        'build_name':
790
+                                            'tempest-dsvm-neutron-full'}},
791
+                                   {'pass': 2,
792
+                                    'fail': 0,
793
+                                    'skip': 0,
794
+                                    'id': 'abc2',
795
+                                    'run_time': 2.0,
796
+                                    'metadata': {
797
+                                        'build_name':
798
+                                            'openstack-tox-py27-trove'}}],
799
+                    timestamp_d2: [{'pass': 100,
800
+                                    'fail': 0,
801
+                                    'skip': 0,
802
+                                    'id': 'abc3',
803
+                                    'run_time': 20.0,
804
+                                    'metadata': {
805
+                                        'build_name':
806
+                                            'tempest-dsvm-neutron-full'}}]
807
+                })
808
+    def test_get_runs_by_project_diff_build_and_same_run_at(self, api_mock):
809
+        start_date = timestamp_d1.date().isoformat()
810
+        stop_date = timestamp_d2.date().isoformat()
811
+        query = ('datetime_resolution=day&start_date={0}&stop_date={1}'
812
+                 .format(start_date, stop_date))
813
+        res = self.app.get('/runs/key/project/trove?{0}'
814
+                           .format(query))
815
+        self.assertEqual(200, res.status_code)
816
+        expected_response_data = {
817
+            u'data': {
818
+                u'timedelta': [
819
+                    {u'datetime': u'%s' % timestamp_d1.date().isoformat(),
820
+                     u'job_data': [{u'pass': 1,
821
+                                    u'fail': 0,
822
+                                    u'job_name': u'openstack-tox-py27-trove',
823
+                                    u'mean_run_time': 2.0},
824
+                                   {u'pass': 1,
825
+                                    u'fail': 1,
826
+                                    u'job_name': u'tempest-dsvm-neutron-full',
827
+                                    u'mean_run_time': 6.5}]},
828
+                    {u'datetime': u'%s' % timestamp_d2.date().isoformat(),
829
+                     u'job_data': [{u'pass': 1,
830
+                                    u'fail': 0,
831
+                                    u'job_name': u'tempest-dsvm-neutron-full',
832
+                                    u'mean_run_time': 20.0}]}]},
833
+            u'numeric': {
834
+                u'tempest-dsvm-neutron-full': {
835
+                    u'%s' % timestamp_d1.isoformat(): 4.0,
836
+                    u'%s' % timestamp_d2.isoformat(): 20.0},
837
+                u'openstack-tox-py27-trove': {
838
+                    u'%s' % timestamp_d1.isoformat(): 2.0,
839
+                    u'%s' % timestamp_d2.isoformat(): numpy.NaN}}}
840
+        response_data = json.loads(res.data.decode('utf-8'))
841
+        self.maxDiff = None
842
+        # numpy.NaN == numpy.NaN result is False, a key error here means the
843
+        # dicts are not equal
844
+        for project, item in expected_response_data['numeric'].items():
845
+            for date, run_time in list(item.items()):
846
+                if (numpy.isnan(run_time) and
847
+                    numpy.isnan(response_data['numeric'][project][date])):
848
+                    del expected_response_data['numeric'][project][date]
849
+                    del response_data['numeric'][project][date]
850
+        self.assertDictEqual(expected_response_data, response_data)
851
+
773 852
     @mock.patch('openstack_health.api._check_db_availability',
774 853
                 return_value=False)
775 854
     @mock.patch('openstack_health.api._check_er_availability',

+ 83
- 1
openstack_health/tests/test_run_aggregator.py View File

@@ -63,8 +63,90 @@ class TestRunAggregatorGetNumericData(base.TestCase):
63 63
         actual = run_aggregator.get_numeric_data({}, 'day')
64 64
         self.assertEqual(expected, actual)
65 65
 
66
+    def test_get_numeric_data_diff_build_name(self):
67
+        self.runs[datetime.datetime(2018, 6, 14, 3, 52, 24)][
68
+            'openstack-tox-py27-trove'] = 321.304
69
+        expected = {
70
+            'tempest-dsvm-neutron-full': {
71
+                '2018-06-13T00:00:00': 5391.195,
72
+                '2018-06-14T00:00:00': 4768.1,
73
+                '2018-06-15T00:00:00': np.nan,
74
+                '2018-06-16T00:00:00': np.nan,
75
+                '2018-06-17T00:00:00': np.nan,
76
+                '2018-06-18T00:00:00': 4183.85,
77
+                '2018-06-19T00:00:00': 4545.41,
78
+                '2018-06-20T00:00:00': 4133.03,
79
+                '2018-06-21T00:00:00': np.nan,
80
+                '2018-06-22T00:00:00': 5592.295,
81
+                '2018-06-23T00:00:00': 6150.95,
82
+                '2018-06-24T00:00:00': np.nan,
83
+                '2018-06-25T00:00:00': 6047.95
84
+            },
85
+            'tempest-dsvm-neutron-full-avg': {
86
+                '2018-06-13T00:00:00': np.nan,
87
+                '2018-06-14T00:00:00': np.nan,
88
+                '2018-06-15T00:00:00': np.nan,
89
+                '2018-06-16T00:00:00': np.nan,
90
+                '2018-06-17T00:00:00': np.nan,
91
+                '2018-06-18T00:00:00': np.nan,
92
+                '2018-06-19T00:00:00': np.nan,
93
+                '2018-06-20T00:00:00': np.nan,
94
+                '2018-06-21T00:00:00': np.nan,
95
+                '2018-06-22T00:00:00': 4690.44675,
96
+                '2018-06-23T00:00:00': 4766.42225,
97
+                '2018-06-24T00:00:00': 4899.55725,
98
+                '2018-06-25T00:00:00': 5042.148499999999
99
+            },
100
+            'openstack-tox-py27-trove': {
101
+                '2018-06-13T00:00:00': np.nan,
102
+                '2018-06-14T00:00:00': 321.304,
103
+                '2018-06-15T00:00:00': np.nan,
104
+                '2018-06-16T00:00:00': np.nan,
105
+                '2018-06-17T00:00:00': np.nan,
106
+                '2018-06-18T00:00:00': np.nan,
107
+                '2018-06-19T00:00:00': np.nan,
108
+                '2018-06-20T00:00:00': np.nan,
109
+                '2018-06-21T00:00:00': np.nan,
110
+                '2018-06-22T00:00:00': np.nan,
111
+                '2018-06-23T00:00:00': np.nan,
112
+                '2018-06-24T00:00:00': np.nan,
113
+                '2018-06-25T00:00:00': np.nan
114
+            },
115
+            'openstack-tox-py27-trove-avg': {
116
+                '2018-06-13T00:00:00': np.nan,
117
+                '2018-06-14T00:00:00': np.nan,
118
+                '2018-06-15T00:00:00': np.nan,
119
+                '2018-06-16T00:00:00': np.nan,
120
+                '2018-06-17T00:00:00': np.nan,
121
+                '2018-06-18T00:00:00': np.nan,
122
+                '2018-06-19T00:00:00': np.nan,
123
+                '2018-06-20T00:00:00': np.nan,
124
+                '2018-06-21T00:00:00': np.nan,
125
+                '2018-06-22T00:00:00': np.nan,
126
+                '2018-06-23T00:00:00': 321.30400000000003,
127
+                '2018-06-24T00:00:00': 321.30400000000003,
128
+                '2018-06-25T00:00:00': np.nan
129
+            }
130
+
131
+        }
132
+        actual = run_aggregator.get_numeric_data(self.runs, 'day')
133
+        self.assertItemsEqual(expected, actual)
134
+        self.assertItemsEqual(
135
+            expected['tempest-dsvm-neutron-full'].keys(),
136
+            actual['tempest-dsvm-neutron-full'].keys())
137
+        self.assertItemsEqual(
138
+            expected['tempest-dsvm-neutron-full-avg'].keys(),
139
+            actual['tempest-dsvm-neutron-full-avg'].keys())
140
+        # np.nan == np.nan is False, remove the key entries with np.nan values,
141
+        # if a key error is thrown then expected does not equal actual.
142
+        for key in expected:
143
+            for date, run_time in list(expected[key].items()):
144
+                if np.isnan(run_time) and np.isnan(actual[key][date]):
145
+                    del actual[key][date]
146
+                    del expected[key][date]
147
+        self.assertDictEqual(expected, actual)
148
+
66 149
     def test_get_numeric_data(self):
67
-        self.maxDiff = None
68 150
         expected = {
69 151
             'tempest-dsvm-neutron-full': {
70 152
                 '2018-06-13T00:00:00': 5391.195,

Loading…
Cancel
Save