Browse Source

Make sure that the index field mapping is correct

This patch set added initialization code to make sure that before
an index gets created, index template is added to ElasticSearch so
field such as timestamp will be correctly indexed as date. This
patch set also updated the dependency.

Change-Id: I8a5a77674fb172ad5c1b73a940477fa6acd30059
Tong Li 3 years ago
parent
commit
a731b9f649
7 changed files with 89 additions and 17 deletions
  1. 4
    12
      AUTHORS
  2. 18
    0
      ChangeLog
  3. 35
    0
      etc/metrics.template
  4. 9
    4
      kiloeyes/tests/v2/elasticsearch/test_metrics.py
  5. 21
    0
      kiloeyes/v2/elasticsearch/metrics.py
  6. 1
    1
      requirements.txt
  7. 1
    0
      setup.cfg

+ 4
- 12
AUTHORS View File

@@ -1,12 +1,4 @@
1
-Maintainer
2
-----------
3
-OpenStack Foundation
4
-IRC: #openstack-kiloeyes on irc.freenode.net
5
-
6
-Original Authors
7
-----------------
8
-Tong Li (litong01@us.ibm.com)
9
-
10
-Contributors
11
-------------
12
-Tong Li (litong01@us.ibm.com)
1
+Chang-Yi Lee <cy.l@inwinstack.com>
2
+Jiaming Lin <robin890650@gmail.com>
3
+Tong Li <litong01@us.ibm.com>
4
+spzala <spzala@us.ibm.com>

+ 18
- 0
ChangeLog View File

@@ -0,0 +1,18 @@
1
+CHANGES
2
+=======
3
+
4
+* ES now returns timestamp as milliseconds vs seconds
5
+* enable bulk message post on persister
6
+* Added more instructions
7
+* bulk insert can not be done implicitly
8
+* fix the partitions data type error
9
+* Updated the installation instructions
10
+* added more instruction on how to create an all-in-one kiloeyes
11
+* unit test passed for py27
12
+* Add Vagrant sample file to ease development environment bootstrap
13
+* Make the server more flexible with configuration files
14
+* remove old openstack incubator project reference
15
+* remove old oslo.config and use new oslo_config
16
+* Make minor modifications in the README
17
+* seeding the project
18
+* Added .gitreview

+ 35
- 0
etc/metrics.template View File

@@ -0,0 +1,35 @@
1
+{
2
+   "template":"data_*",
3
+   "order":100,
4
+   "settings":{
5
+      "number_of_shards":5
6
+   },
7
+   "mappings":{
8
+      "_default_":{
9
+         "_all":{
10
+            "enabled":false
11
+         },
12
+         "dynamic_templates":[
13
+            {
14
+               "date_template":{
15
+                  "match":"timestamp",
16
+                  "match_mapping_type":"date",
17
+                  "mapping":{
18
+                     "type":"date"
19
+                  }
20
+               }
21
+            },
22
+            {
23
+               "string_template":{
24
+                  "match":"*",
25
+                  "match_mapping_type":"string",
26
+                  "mapping":{
27
+                     "type":"string",
28
+                     "index":"not_analyzed"
29
+                  }
30
+               }
31
+            }
32
+         ]
33
+      }
34
+   }
35
+}

+ 9
- 4
kiloeyes/tests/v2/elasticsearch/test_metrics.py View File

@@ -78,7 +78,9 @@ class TestMetricDispatcher(base.BaseTestCase):
78 78
         self.CONF.set_override('topic', 'fake', group='metrics')
79 79
         self.CONF.set_override('doc_type', 'fake', group='metrics')
80 80
         self.CONF.set_override('index_prefix', 'also_fake', group='metrics')
81
-        self.CONF.set_override('uri', 'fake_es_uri', group='es_conn')
81
+        self.CONF.set_override('index_template', 'etc/metrics.template',
82
+                               group='metrics')
83
+        self.CONF.set_override('uri', 'http://fake_es_uri', group='es_conn')
82 84
 
83 85
         res = mock.Mock()
84 86
         res.status_code = 200
@@ -91,9 +93,12 @@ class TestMetricDispatcher(base.BaseTestCase):
91 93
                 "name": {"type": "string", "index": "not_analyzed"},
92 94
                 "timestamp": {"type": "string", "index": "not_analyzed"},
93 95
                 "value": {"type": "double"}}}}}}
96
+        put_res = mock.Mock()
97
+        put_res.status_code = '200'
94 98
         with mock.patch.object(requests, 'get',
95 99
                                return_value=res):
96
-            self.dispatcher = metrics.MetricDispatcher({})
100
+            with mock.patch.object(requests, 'put', return_value=put_res):
101
+                self.dispatcher = metrics.MetricDispatcher({})
97 102
 
98 103
     def test_initialization(self):
99 104
         # test that the kafka connection uri should be 'fake' as it was passed
@@ -106,11 +111,11 @@ class TestMetricDispatcher(base.BaseTestCase):
106 111
         # test that the doc type of the es connection is fake
107 112
         self.assertEqual(self.dispatcher._es_conn.doc_type, 'fake')
108 113
 
109
-        self.assertEqual(self.dispatcher._es_conn.uri, 'fake_es_uri/')
114
+        self.assertEqual(self.dispatcher._es_conn.uri, 'http://fake_es_uri/')
110 115
 
111 116
         # test that the query url is correctly formed
112 117
         self.assertEqual(self.dispatcher._query_url, (
113
-            'fake_es_uri/also_fake*/fake/_search?search_type=count'))
118
+            'http://fake_es_uri/also_fake*/fake/_search?search_type=count'))
114 119
 
115 120
     def test_post_data(self):
116 121
         with mock.patch.object(kafka_conn.KafkaConnection, 'send_messages',

+ 21
- 0
kiloeyes/v2/elasticsearch/metrics.py View File

@@ -42,6 +42,8 @@ METRICS_OPTS = [
42 42
                help='The index strategy used to create index name.'),
43 43
     cfg.StrOpt('index_prefix', default='data_',
44 44
                help='The index prefix where metrics were saved to.'),
45
+    cfg.StrOpt('index_template', default='/etc/kiloeyes/metrics.template',
46
+               help='The index template which metrics index should use.'),
45 47
     cfg.IntOpt('size', default=10000,
46 48
                help=('The query result limit. Any result set more than '
47 49
                      'the limit will be discarded. To see all the matching '
@@ -135,6 +137,7 @@ class MetricDispatcher(object):
135 137
         super(MetricDispatcher, self).__init__()
136 138
         self.topic = cfg.CONF.metrics.topic
137 139
         self.doc_type = cfg.CONF.metrics.doc_type
140
+        self.index_template = cfg.CONF.metrics.index_template
138 141
         self.size = cfg.CONF.metrics.size
139 142
         self._kafka_conn = kafka_conn.KafkaConnection(self.topic)
140 143
 
@@ -207,6 +210,24 @@ class MetricDispatcher(object):
207 210
         {"field":"value"}}}}}}}}}
208 211
         """
209 212
 
213
+        # Setup index template
214
+        self.setup_index_template()
215
+
216
+    def setup_index_template(self):
217
+        status = '400'
218
+        with open(self.index_template) as template_file:
219
+            template_path = ''.join([self._es_conn.uri,
220
+                                     '/_template/metrics'])
221
+            es_res = requests.put(template_path, data=template_file.read())
222
+            status = getattr(falcon, 'HTTP_%s' % es_res.status_code)
223
+
224
+        if status == '400':
225
+            LOG.error('Metrics template can not be created. Status code %s'
226
+                      % status)
227
+            exit(1)
228
+        else:
229
+            LOG.debug('Index template set successfully! Status %s' % status)
230
+
210 231
     def post_data(self, req, res):
211 232
         LOG.debug('Getting the call.')
212 233
         msg = req.stream.read()

+ 1
- 1
requirements.txt View File

@@ -12,7 +12,7 @@ oslo.i18n>=1.5.0
12 12
 oslo.log>=1.0.0
13 13
 oslo.service>=0.1.0
14 14
 pastedeploy>=1.3.3
15
-pbr>=0.6,!=0.7,<1.0
15
+pbr>=1.6
16 16
 python-dateutil>=1.5
17 17
 six>=1.7.0
18 18
 stevedore>=0.14

+ 1
- 0
setup.cfg View File

@@ -33,6 +33,7 @@ data_files =
33 33
         etc/alarms-persister.conf
34 34
         etc/kiloeyes-notification-engine.conf
35 35
         etc/kiloeyes-threshold-engine.conf
36
+        etc/metrics.template
36 37
 
37 38
 [entry_points]
38 39
 console_scripts =

Loading…
Cancel
Save