Browse Source

enable bulk message post on persister

Change-Id: Ifbf2c5e1517c0e7a042b241657725640e8cdebec
changes/43/279243/2
Tong Li 3 years ago
parent
commit
b93ded41ed

+ 2
- 11
kiloeyes/common/es_conn.py View File

@@ -67,20 +67,11 @@ class ESConnection(object):
67 67
         if self.drop_data:
68 68
             return
69 69
         else:
70
-            # figure out id situation
71
-            _id = ''
72
-            if self.id_field:
73
-                obj = json.loads(msg)
74
-                _id = obj.get(self.id_field)
75
-                if not _id:
76
-                    LOG.error('Msg does not have required id field %s' %
77
-                              self.id_field)
78
-                    return 400
79 70
             # index may change over the time, it has to be called for each
80 71
             # request
81 72
             index = self.index_strategy.get_index()
82
-            path = '%s%s%s/%s/%s' % (self.uri, self.index_prefix,
83
-                                     index, self.doc_type, _id)
73
+            path = '%s%s%s/%s/_bulk' % (self.uri, self.index_prefix,
74
+                                        index, self.doc_type)
84 75
             res = requests.post(path, data=msg)
85 76
             LOG.debug('Msg post target=%s' % path)
86 77
             LOG.debug('Msg posted with response code: %s' % res.status_code)

+ 8
- 1
kiloeyes/microservice/metrics_fixer.py View File

@@ -45,7 +45,14 @@ class MetricsFixer(object):
45 45
 
46 46
     def process_msg(self, msg):
47 47
         try:
48
-            return MetricsFixer._add_hash(json.loads(msg))
48
+            data = json.loads(msg)
49
+            if not isinstance(data, list):
50
+                data = [data]
51
+            result = ''
52
+            for item in data:
53
+                result += '{"index":{}}\n' + MetricsFixer._add_hash(item)
54
+                result += '\n'
55
+            return result
49 56
         except Exception:
50 57
             LOG.exception('')
51 58
             return ''

+ 0
- 14
kiloeyes/tests/common/test_es_conn.py View File

@@ -43,17 +43,3 @@ class TestESConnection(tests.BaseTestCase):
43 43
         with mock.patch.object(requests, 'post', return_value=req_result):
44 44
             conn.send_messages(json.dumps(msg))
45 45
             self.assertTrue(requests.post.called)
46
-
47
-    def test_send_messages_without_id(self):
48
-        self.CONF.set_override('id_field', 'id', group='es_conn')
49
-        self.CONF.set_override('uri', 'http://fake', group='es_conn')
50
-        self.CONF.set_override('time_unit', 'h', group='timed_strategy')
51
-        strategy = timed_strategy.TimedStrategy()
52
-        conn = es_conn.ESConnection('alarms', strategy, 'pre_')
53
-        req_result = mock.Mock()
54
-        req_result.status_code = 204
55
-        msg = {'not_id': 'whatever'}
56
-        with mock.patch.object(requests, 'post', return_value=req_result):
57
-            res = conn.send_messages(json.dumps(msg))
58
-            self.assertFalse(requests.post.called)
59
-            self.assertEqual(res, 400)

+ 1
- 1
kiloeyes/tests/microservice/test_metrics_fixer.py View File

@@ -48,7 +48,7 @@ class TestMetricsFixer(tests.BaseTestCase):
48 48
         fixer = metrics_fixer.MetricsFixer()
49 49
         result = fixer.process_msg(json.dumps(items))
50 50
         self.assertTrue(isinstance(result, str))
51
-        self.assertFalse(result.startswith('{"index":{}}'))
51
+        self.assertTrue(result.startswith('{"index":{}}'))
52 52
 
53 53
     def test_process_msg_multiple(self):
54 54
         items = [{'name': 'name1', 'dimensions': {'name1': 'value1'},

Loading…
Cancel
Save