Browse Source

Merge "Store collect period in InfluxDB driver datapoints"

changes/07/679807/4
Zuul 4 days ago
parent
commit
3c28781888

+ 28
- 20
cloudkitty/storage/v2/influx.py View File

@@ -62,6 +62,9 @@ influx_storage_opts = [
62 62
 CONF.register_opts(influx_storage_opts, INFLUX_STORAGE_GROUP)
63 63
 
64 64
 
65
+PERIOD_FIELD_NAME = '__ck_collect_period'
66
+
67
+
65 68
 class InfluxClient(object):
66 69
     """Classe used to ease interaction with InfluxDB"""
67 70
 
@@ -110,14 +113,17 @@ class InfluxClient(object):
110 113
 
111 114
     def append_point(self,
112 115
                      metric_type,
113
-                     timestamp,
116
+                     start,
117
+                     period,
114 118
                      point):
115 119
         """Adds a point to commit to InfluxDB.
116 120
 
117 121
         :param metric_type: Name of the metric type
118 122
         :type metric_type: str
119
-        :param timestamp: Timestamp of the time
120
-        :type timestamp: datetime.datetime
123
+        :param start: Start of the period the point applies to
124
+        :type start: datetime.datetime
125
+        :param period: length of the period the point applies to (in seconds)
126
+        :type period: int
121 127
         :param point: Point to push
122 128
         :type point: dataframe.DataPoint
123 129
         """
@@ -131,6 +137,8 @@ class InfluxClient(object):
131 137
         measurement_fields['groupby'] = '|'.join(point.groupby.keys())
132 138
         measurement_fields['metadata'] = '|'.join(point.metadata.keys())
133 139
 
140
+        measurement_fields[PERIOD_FIELD_NAME] = period
141
+
134 142
         measurement_tags = dict(point.groupby)
135 143
         measurement_tags['type'] = metric_type
136 144
 
@@ -138,7 +146,7 @@ class InfluxClient(object):
138 146
             'measurement': 'dataframes',
139 147
             'tags': measurement_tags,
140 148
             'fields': measurement_fields,
141
-            'time': timestamp,
149
+            'time': start,
142 150
         })
143 151
         if self._autocommit and len(self._points) >= self._chunk_size:
144 152
             self.commit()
@@ -234,8 +242,8 @@ class InfluxStorage(v2_storage.BaseStorage):
234 242
 
235 243
     def __init__(self, *args, **kwargs):
236 244
         super(InfluxStorage, self).__init__(*args, **kwargs)
245
+        self._default_period = kwargs.get('period') or CONF.collect.period
237 246
         self._conn = InfluxClient()
238
-        self._period = kwargs.get('period', None) or CONF.collect.period
239 247
 
240 248
     def init(self):
241 249
         policy = CONF.storage_influxdb.retention_policy
@@ -249,9 +257,9 @@ class InfluxStorage(v2_storage.BaseStorage):
249 257
     def push(self, dataframes, scope_id=None):
250 258
 
251 259
         for frame in dataframes:
252
-            timestamp = frame.start
260
+            period = tzutils.diff_seconds(frame.end, frame.start)
253 261
             for type_, point in frame.iterpoints():
254
-                self._conn.append_point(type_, timestamp, point)
262
+                self._conn.append_point(type_, frame.start, period, point)
255 263
 
256 264
         self._conn.commit()
257 265
 
@@ -265,10 +273,8 @@ class InfluxStorage(v2_storage.BaseStorage):
265 273
 
266 274
     @staticmethod
267 275
     def _point_to_dataframe_entry(point):
268
-        groupby = filter(lambda x: bool(x),
269
-                         (point.pop('groupby', None) or '').split('|'))
270
-        metadata = filter(lambda x: bool(x),
271
-                          (point.pop('metadata', None) or '').split('|'))
276
+        groupby = filter(bool, (point.pop('groupby', None) or '').split('|'))
277
+        metadata = filter(bool, (point.pop('metadata', None) or '').split('|'))
272 278
         return dataframe.DataPoint(
273 279
             point['unit'],
274 280
             point['qty'],
@@ -282,18 +288,20 @@ class InfluxStorage(v2_storage.BaseStorage):
282 288
         for point in points:
283 289
             point_type = point['type']
284 290
             time = tzutils.dt_from_iso(point['time'])
285
-            if time not in dataframes.keys():
286
-                dataframes[time] = dataframe.DataFrame(
287
-                    start=time,
288
-                    end=tzutils.add_delta(
289
-                        time, datetime.timedelta(seconds=self._period)),
290
-                )
291
-
292
-            dataframes[time].add_point(
291
+            period = point.get(PERIOD_FIELD_NAME) or self._default_period
292
+            timekey = (
293
+                time,
294
+                tzutils.add_delta(time, datetime.timedelta(seconds=period)))
295
+            if timekey not in dataframes.keys():
296
+                dataframes[timekey] = dataframe.DataFrame(
297
+                    start=timekey[0],
298
+                    end=timekey[1])
299
+
300
+            dataframes[timekey].add_point(
293 301
                 self._point_to_dataframe_entry(point), point_type)
294 302
 
295 303
         output = list(dataframes.values())
296
-        output.sort(key=lambda frame: frame.start)
304
+        output.sort(key=lambda frame: (frame.start, frame.end))
297 305
         return output
298 306
 
299 307
     def retrieve(self, begin=None, end=None,

+ 28
- 3
cloudkitty/tests/storage/v2/test_influxdb.py View File

@@ -13,9 +13,12 @@
13 13
 #    under the License.
14 14
 #
15 15
 import collections
16
+import copy
16 17
 from datetime import datetime
18
+from datetime import timedelta
17 19
 import unittest
18 20
 
21
+from dateutil import tz
19 22
 import mock
20 23
 
21 24
 from cloudkitty import dataframe
@@ -29,6 +32,7 @@ class TestInfluxDBStorage(TestCase):
29 32
     def setUp(self):
30 33
         super(TestInfluxDBStorage, self).setUp()
31 34
         self.point = {
35
+            'type': 'amazing_type',
32 36
             'unit': 'banana',
33 37
             'qty': 42,
34 38
             'price': 1.0,
@@ -38,6 +42,7 @@ class TestInfluxDBStorage(TestCase):
38 42
             'two': '2',
39 43
             '1': 'one',
40 44
             '2': 'two',
45
+            'time': datetime(2019, 1, 1, tzinfo=tz.UTC).isoformat(),
41 46
         }
42 47
 
43 48
     def test_point_to_dataframe_entry_valid_point(self):
@@ -53,10 +58,11 @@ class TestInfluxDBStorage(TestCase):
53 58
         )
54 59
 
55 60
     def test_point_to_dataframe_entry_invalid_groupby_metadata(self):
56
-        self.point['groupby'] = 'a'
57
-        self.point['metadata'] = None
61
+        point = copy.deepcopy(self.point)
62
+        point['groupby'] = 'a'
63
+        point['metadata'] = None
58 64
         self.assertEqual(
59
-            influx.InfluxStorage._point_to_dataframe_entry(self.point),
65
+            influx.InfluxStorage._point_to_dataframe_entry(point),
60 66
             dataframe.DataPoint(
61 67
                 'banana',
62 68
                 42,
@@ -66,6 +72,25 @@ class TestInfluxDBStorage(TestCase):
66 72
             ),
67 73
         )
68 74
 
75
+    def test_build_dataframes_differenciates_periods(self):
76
+        points = [copy.deepcopy(self.point) for _ in range(3)]
77
+        for idx, point in enumerate(points):
78
+            point[influx.PERIOD_FIELD_NAME] = 100 * (idx + 1)
79
+
80
+        dataframes = influx.InfluxStorage()._build_dataframes(points)
81
+        self.assertEqual(len(dataframes), 3)
82
+
83
+        for idx, frame in enumerate(dataframes):
84
+            self.assertEqual(frame.start, datetime(2019, 1, 1, tzinfo=tz.UTC))
85
+            delta = timedelta(seconds=(idx + 1) * 100)
86
+            self.assertEqual(frame.end,
87
+                             datetime(2019, 1, 1, tzinfo=tz.UTC) + delta)
88
+            typelist = list(frame.itertypes())
89
+            self.assertEqual(len(typelist), 1)
90
+            type_, points = typelist[0]
91
+            self.assertEqual(len(points), 1)
92
+            self.assertEqual(type_, 'amazing_type')
93
+
69 94
 
70 95
 class FakeResultSet(object):
71 96
     def __init__(self, points=[], items=[]):

+ 27
- 0
cloudkitty/tests/test_tzutils.py View File

@@ -101,3 +101,30 @@ class TestTZUtils(unittest.TestCase):
101 101
         month_start = tzutils.get_month_start(param, naive=True)
102 102
         self.assertIsNone(month_start.tzinfo)
103 103
         self.assertEqual(month_start, datetime.datetime(2019, 1, 1))
104
+
105
+    def test_diff_seconds_positive_arg_naive_objects(self):
106
+        one = datetime.datetime(2019, 1, 1, 1, 1, 30)
107
+        two = datetime.datetime(2019, 1, 1, 1, 1)
108
+        self.assertEqual(tzutils.diff_seconds(one, two), 30)
109
+
110
+    def test_diff_seconds_negative_arg_naive_objects(self):
111
+        one = datetime.datetime(2019, 1, 1, 1, 1, 30)
112
+        two = datetime.datetime(2019, 1, 1, 1, 1)
113
+        self.assertEqual(tzutils.diff_seconds(two, one), 30)
114
+
115
+    def test_diff_seconds_positive_arg_aware_objects(self):
116
+        one = datetime.datetime(2019, 1, 1, 1, 1, 30, tzinfo=tz.UTC)
117
+        two = datetime.datetime(2019, 1, 1, 1, 1, tzinfo=tz.UTC)
118
+        self.assertEqual(tzutils.diff_seconds(one, two), 30)
119
+
120
+    def test_diff_seconds_negative_arg_aware_objects(self):
121
+        one = datetime.datetime(2019, 1, 1, 1, 1, 30, tzinfo=tz.UTC)
122
+        two = datetime.datetime(2019, 1, 1, 1, 1, tzinfo=tz.UTC)
123
+        self.assertEqual(tzutils.diff_seconds(two, one), 30)
124
+
125
+    def test_diff_seconds_negative_arg_aware_objects_on_summer_change(self):
126
+        one = datetime.datetime(2019, 3, 31, 1,
127
+                                tzinfo=tz.gettz('Europe/Paris'))
128
+        two = datetime.datetime(2019, 3, 31, 3,
129
+                                tzinfo=tz.gettz('Europe/Paris'))
130
+        self.assertEqual(tzutils.diff_seconds(two, one), 3600)

+ 16
- 0
cloudkitty/tzutils.py View File

@@ -143,3 +143,19 @@ def get_next_month(dt=None, naive=False):
143 143
     start = get_month_start(dt, naive=naive)
144 144
     month_days = calendar.monthrange(start.year, start.month)[1]
145 145
     return add_delta(start, datetime.timedelta(days=month_days))
146
+
147
+
148
+def diff_seconds(one, two):
149
+    """Returns the difference in seconds between two datetime objects.
150
+
151
+    Objects will be converted to naive UTC objects before calculating the
152
+    difference. The return value is the absolute value of the difference.
153
+
154
+    :param one: First datetime object
155
+    :type one: datetime.datetime
156
+    :param two: datetime object to substract from the first one
157
+    :type two: datetime.datetime
158
+    :rtype: int
159
+    """
160
+    return abs(int((local_to_utc(one, naive=True)
161
+                    - local_to_utc(two, naive=True)).total_seconds()))

Loading…
Cancel
Save