Browse Source

Merge "Sample Get and Get-by-id implemented"

Jenkins 2 years ago
parent
commit
18ad81cb4b

+ 1
- 0
AUTHORS View File

@@ -4,3 +4,4 @@ Jiaming Lin <robin890650@gmail.com>
4 4
 Tong Li <litong01@us.ibm.com>
5 5
 Xiao Tan <xt85@cornell.edu>
6 6
 spzala <spzala@us.ibm.com>
7
+xiaotan2 <xiaotan2@uw.edu>

+ 2
- 0
ChangeLog View File

@@ -1,6 +1,8 @@
1 1
 CHANGES
2 2
 =======
3 3
 
4
+* Sample Get and Get-by-id implemented
5
+* Meter Get Statistics by name implemented
4 6
 * Meter Get_Meter_Byname implemented
5 7
 * Meters GET request implemented
6 8
 * Added more instructions on how to configure keystone middleware

+ 1
- 0
etc/kiloeyes.conf View File

@@ -12,6 +12,7 @@ dispatcher = alarmdefinitions
12 12
 dispatcher = notificationmethods
13 13
 dispatcher = alarms
14 14
 dispatcher = meters
15
+dispatcher = samples
15 16
 
16 17
 [metrics]
17 18
 topic = metrics

+ 8
- 0
kiloeyes/api/ceilometer_api_v2.py View File

@@ -40,3 +40,11 @@ class V2API(object):
40 40
     @resource_api.Restify('/v2.0/meters/{meter_name}/statistics', method='get')
41 41
     def get_meter_statistics(self, req, res, meter_name):
42 42
         res.status = '501 Not Implemented'
43
+
44
+    @resource_api.Restify('/v2.0/samples', method='get')
45
+    def get_samples(self, req, res):
46
+        res.status = '501 Not Implemented'
47
+
48
+    @resource_api.Restify('/v2.0/samples/{sample_id}', method='get')
49
+    def get_sample_byid(self, req, res, sample_id):
50
+        res.status = '501 Not Implemented'

+ 157
- 0
kiloeyes/tests/v2/elasticsearch/test_samples.py View File

@@ -0,0 +1,157 @@
1
+# Copyright 2013 IBM Corp
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+# not use this file except in compliance with the License. You may obtain
5
+# a copy of the License at
6
+#
7
+#      http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+# License for the specific language governing permissions and limitations
13
+# under the License.
14
+
15
+import falcon
16
+import mock
17
+from oslo_config import fixture as fixture_config
18
+from oslotest import base
19
+import requests
20
+
21
+from kiloeyes.v2.elasticsearch import samples
22
+
23
+try:
24
+    import ujson as json
25
+except ImportError:
26
+    import json
27
+
28
+
29
+class TestCeilometerSampleDispatcher(base.BaseTestCase):
30
+
31
+    def setUp(self):
32
+        super(TestCeilometerSampleDispatcher, self).setUp()
33
+        self.CONF = self.useFixture(fixture_config.Config()).conf
34
+        self.CONF.set_override('uri', 'fake_url', group='kafka_opts')
35
+        self.CONF.set_override('topic', 'fake', group='samples')
36
+        self.CONF.set_override('doc_type', 'fake', group='samples')
37
+        self.CONF.set_override('index_prefix', 'also_fake', group='samples')
38
+        self.CONF.set_override('index_template', 'etc/metrics.template',
39
+                               group='samples')
40
+        self.CONF.set_override('uri', 'http://fake_es_uri', group='es_conn')
41
+
42
+        res = mock.Mock()
43
+        res.status_code = 200
44
+        res.json.return_value = {"data": {"mappings": {"fake": {
45
+            "properties": {
46
+                "dimensions": {"properties": {
47
+                    "key1": {"type": "long"}, "key2": {"type": "long"},
48
+                    "rkey0": {"type": "long"}, "rkey1": {"type": "long"},
49
+                    "rkey2": {"type": "long"}, "rkey3": {"type": "long"}}},
50
+                "name": {"type": "string", "index": "not_analyzed"},
51
+                "timestamp": {"type": "string", "index": "not_analyzed"},
52
+                "value": {"type": "double"}}}}}}
53
+        put_res = mock.Mock()
54
+        put_res.status_code = '200'
55
+        with mock.patch.object(requests, 'get',
56
+                               return_value=res):
57
+            with mock.patch.object(requests, 'put', return_value=put_res):
58
+                self.dispatcher = samples.CeilometerSampleDispatcher({})
59
+
60
+        self.response_str = """
61
+        {"aggregations":{"by_name":{"doc_count_error_upper_bound":0,
62
+        "sum_other_doc_count":0,"buckets":[{"key":"BABMGD","doc_count":300,
63
+        "by_dim":{"buckets":[{"key": "64e6ce08b3b8547b7c32e5cfa5b7d81f",
64
+        "doc_count":300,"samples":{"hits":{"hits":[{ "_type": "metrics",
65
+        "_id": "AVOziWmP6-pxt0dRmr7j", "_index": "data_20160401000000",
66
+        "_source":{"name":"BABMGD", "value": 4,
67
+        "timestamp": 1461337094000,
68
+        "dimensions_hash": "0afdb86f508962bb5d8af52df07ef35a",
69
+        "project_id": "35b17138-b364-4e6a-a131-8f3099c5be68",
70
+        "tenant_id": "bd9431c1-8d69-4ad3-803a-8d4a6b89fd36",
71
+        "user_agent": "openstack", "dimensions": null,
72
+        "user": "admin", "value_meta": null, "tenant": "admin",
73
+        "user_id": "efd87807-12d2-4b38-9c70-5f5c2ac427ff"}}]}}}]}}]}}}
74
+        """
75
+
76
+    def test_initialization(self):
77
+        # test that the kafka connection uri should be 'fake' as it was passed
78
+        # in from configuration
79
+        self.assertEqual(self.dispatcher._kafka_conn.uri, 'fake_url')
80
+
81
+        # test that the topic is samples as it was passed into dispatcher
82
+        self.assertEqual(self.dispatcher._kafka_conn.topic, 'fake')
83
+
84
+        # test that the doc type of the es connection is fake
85
+        self.assertEqual(self.dispatcher._es_conn.doc_type, 'fake')
86
+
87
+        self.assertEqual(self.dispatcher._es_conn.uri, 'http://fake_es_uri/')
88
+
89
+        # test that the query url is correctly formed
90
+        self.assertEqual(self.dispatcher._query_url, (
91
+            'http://fake_es_uri/also_fake*/fake/_search?search_type=count'))
92
+
93
+    def test_get_samples(self):
94
+        res = mock.Mock()
95
+        req = mock.Mock()
96
+
97
+        def _side_effect(arg):
98
+            if arg == 'name':
99
+                return 'tongli'
100
+            elif arg == 'dimensions':
101
+                return 'key1:100, key2:200'
102
+        req.get_param.side_effect = _side_effect
103
+
104
+        req_result = mock.Mock()
105
+
106
+        req_result.json.return_value = json.loads(self.response_str)
107
+        req_result.status_code = 200
108
+
109
+        with mock.patch.object(requests, 'post', return_value=req_result):
110
+            self.dispatcher.get_samples(req, res)
111
+
112
+        # test that the response code is 200
113
+        self.assertEqual(res.status, getattr(falcon, 'HTTP_200'))
114
+        obj = json.loads(res.body)
115
+        self.assertEqual(obj[0]['meter'], 'BABMGD')
116
+        self.assertEqual(obj[0]['id'], 'AVOziWmP6-pxt0dRmr7j')
117
+        self.assertEqual(obj[0]['type'], 'metrics')
118
+        self.assertEqual(obj[0]['user_id'],
119
+                         'efd87807-12d2-4b38-9c70-5f5c2ac427ff')
120
+        self.assertEqual(obj[0]['project_id'],
121
+                         '35b17138-b364-4e6a-a131-8f3099c5be68')
122
+        self.assertEqual(obj[0]['timestamp'], 1461337094000)
123
+        self.assertEqual(obj[0]['volume'], 4)
124
+        self.assertEqual(len(obj), 1)
125
+
126
+    def test_get_sample_byid(self):
127
+        res = mock.Mock()
128
+        req = mock.Mock()
129
+
130
+        def _side_effect(arg):
131
+            if arg == 'name':
132
+                return 'tongli'
133
+            elif arg == 'dimensions':
134
+                return 'key1:100, key2:200'
135
+        req.get_param.side_effect = _side_effect
136
+
137
+        req_result = mock.Mock()
138
+
139
+        req_result.json.return_value = json.loads(self.response_str)
140
+        req_result.status_code = 200
141
+
142
+        with mock.patch.object(requests, 'post', return_value=req_result):
143
+            self.dispatcher.get_sample_byid(req, res, "AVOziWmP6-pxt0dRmr7j")
144
+
145
+        # test that the response code is 200
146
+        self.assertEqual(res.status, getattr(falcon, 'HTTP_200'))
147
+        obj = json.loads(res.body)
148
+        self.assertEqual(obj[0]['meter'], 'BABMGD')
149
+        self.assertEqual(obj[0]['id'], 'AVOziWmP6-pxt0dRmr7j')
150
+        self.assertEqual(obj[0]['type'], 'metrics')
151
+        self.assertEqual(obj[0]['user_id'],
152
+                         'efd87807-12d2-4b38-9c70-5f5c2ac427ff')
153
+        self.assertEqual(obj[0]['project_id'],
154
+                         '35b17138-b364-4e6a-a131-8f3099c5be68')
155
+        self.assertEqual(obj[0]['timestamp'], 1461337094000)
156
+        self.assertEqual(obj[0]['volume'], 4)
157
+        self.assertEqual(len(obj), 1)

+ 238
- 0
kiloeyes/v2/elasticsearch/samples.py View File

@@ -0,0 +1,238 @@
1
+# Copyright 2013 IBM Corp
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+# not use this file except in compliance with the License. You may obtain
5
+# a copy of the License at
6
+#
7
+#      http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+# License for the specific language governing permissions and limitations
13
+# under the License.
14
+
15
+import datetime
16
+import falcon
17
+from oslo_config import cfg
18
+from oslo_log import log
19
+import requests
20
+from stevedore import driver
21
+
22
+from kiloeyes.common import es_conn
23
+from kiloeyes.common import kafka_conn
24
+from kiloeyes.common import namespace
25
+from kiloeyes.common import resource_api
26
+from kiloeyes.v2.elasticsearch import metrics
27
+
28
+try:
29
+    import ujson as json
30
+except ImportError:
31
+    import json
32
+
33
+SAMPLES_OPTS = [
34
+    cfg.StrOpt('topic', default='metrics',
35
+               help='The topic that samples will be published to.'),
36
+    cfg.StrOpt('doc_type', default='metrics',
37
+               help='The doc type that samples will be saved into.'),
38
+    cfg.StrOpt('index_strategy', default='fixed',
39
+               help='The index strategy used to create index name.'),
40
+    cfg.StrOpt('index_prefix', default='data_',
41
+               help='The index prefix where samples were saved to.'),
42
+    cfg.StrOpt('index_template', default='/etc/kiloeyes/metrics.template',
43
+               help='The index template which samples index should use.'),
44
+    cfg.IntOpt('size', default=10000,
45
+               help=('The query result limit. Any result set more than '
46
+                     'the limit will be discarded. To see all the matching '
47
+                     'result, narrow your search by using a small time '
48
+                     'window or strong matching name')),
49
+]
50
+
51
+cfg.CONF.register_opts(SAMPLES_OPTS, group="samples")
52
+
53
+LOG = log.getLogger(__name__)
54
+
55
+UPDATED = str(datetime.datetime(2014, 1, 1, 0, 0, 0))
56
+
57
+
58
+class CeilometerSampleDispatcher(object):
59
+    def __init__(self, global_conf):
60
+        LOG.debug('initializing V2API!')
61
+        super(CeilometerSampleDispatcher, self).__init__()
62
+        self.topic = cfg.CONF.samples.topic
63
+        self.doc_type = cfg.CONF.samples.doc_type
64
+        self.index_template = cfg.CONF.samples.index_template
65
+        self.size = cfg.CONF.samples.size
66
+        self._kafka_conn = kafka_conn.KafkaConnection(self.topic)
67
+
68
+        # load index strategy
69
+        if cfg.CONF.samples.index_strategy:
70
+            self.index_strategy = driver.DriverManager(
71
+                namespace.STRATEGY_NS,
72
+                cfg.CONF.samples.index_strategy,
73
+                invoke_on_load=True,
74
+                invoke_kwds={}).driver
75
+            LOG.debug(dir(self.index_strategy))
76
+        else:
77
+            self.index_strategy = None
78
+
79
+        self.index_prefix = cfg.CONF.samples.index_prefix
80
+
81
+        self._es_conn = es_conn.ESConnection(
82
+            self.doc_type, self.index_strategy, self.index_prefix)
83
+
84
+        # Setup the get samples query body pattern
85
+        self._query_body = {
86
+            "query": {"bool": {"must": []}},
87
+            "size": self.size}
88
+
89
+        self._aggs_body = {}
90
+        self._stats_body = {}
91
+        self._sort_clause = []
92
+
93
+        # Setup the get samples query url, the url should be similar to this:
94
+        # http://host:port/data_20141201/metrics/_search
95
+        # the url should be made of es_conn uri, the index prefix, samples
96
+        # dispatcher topic, then add the key word _search.
97
+        self._query_url = ''.join([self._es_conn.uri,
98
+                                  self._es_conn.index_prefix, '*/',
99
+                                  cfg.CONF.samples.topic,
100
+                                  '/_search?search_type=count'])
101
+
102
+        # Setup sample query aggregation command. To see the structure of
103
+        # the aggregation, copy and paste it to a json formatter.
104
+        self._sample_agg = """
105
+        {"by_name":{"terms":{"field":"name","size":%(size)d},
106
+        "aggs":{"by_dim":{"terms":{"field":"dimensions_hash","size":%(size)d},
107
+        "aggs":{"samples":{"top_hits":{"_source":{"exclude":
108
+        ["dimensions_hash"]},"size":1}}}}}}}
109
+        """
110
+
111
+        self.setup_index_template()
112
+
113
+    def setup_index_template(self):
114
+        status = '400'
115
+        with open(self.index_template) as template_file:
116
+            template_path = ''.join([self._es_conn.uri,
117
+                                     '/_template/metrics'])
118
+            es_res = requests.put(template_path, data=template_file.read())
119
+            status = getattr(falcon, 'HTTP_%s' % es_res.status_code)
120
+
121
+        if status == '400':
122
+            LOG.error('Metrics template can not be created. Status code %s'
123
+                      % status)
124
+            exit(1)
125
+        else:
126
+            LOG.debug('Index template set successfully! Status %s' % status)
127
+
128
+    def _get_agg_response(self, res):
129
+        if res and res.status_code == 200:
130
+            obj = res.json()
131
+            if obj:
132
+                return obj.get('aggregations')
133
+            return None
134
+        else:
135
+            return None
136
+
137
+    def _render_hits(self, item, flag):
138
+        _id = item['samples']['hits']['hits'][0]['_id']
139
+        _type = item['samples']['hits']['hits'][0]['_type']
140
+        _source = item['samples']['hits']['hits'][0]['_source']
141
+        rslt = ('{"id":' + json.dumps(_id) + ','
142
+                '"metadata":' + json.dumps(_source['dimensions']) + ','
143
+                '"meter":' + json.dumps(_source['name']) + ','
144
+                '"project_id":' +
145
+                json.dumps(_source['project_id']) + ','
146
+                '"recorded_at":' +
147
+                json.dumps(_source['timestamp']) + ','
148
+                '"resource_id":' +
149
+                json.dumps(_source['tenant_id']) + ','
150
+                '"source":' + json.dumps(_source['user_agent']) + ','
151
+                '"timestamp":' + json.dumps(_source['timestamp']) + ','
152
+                '"type":' + json.dumps(_type) + ','
153
+                '"unit":null,'
154
+                '"user_id":' + json.dumps(_source['user_id']) + ','
155
+                '"volume":' + json.dumps(_source['value']) + '}')
156
+        if flag['is_first']:
157
+            flag['is_first'] = False
158
+            return rslt
159
+        else:
160
+            return ',' + rslt
161
+
162
+    def _make_body(self, buckets):
163
+        flag = {'is_first': True}
164
+        yield '['
165
+        for by_name in buckets:
166
+            if by_name['by_dim']:
167
+                for by_dim in by_name['by_dim']['buckets']:
168
+                    yield self._render_hits(by_dim, flag)
169
+        yield ']'
170
+
171
+    @resource_api.Restify('/v2.0/samples', method='get')
172
+    def get_samples(self, req, res):
173
+        LOG.debug('The samples GET request is received')
174
+
175
+        # process query condition
176
+        query = []
177
+        metrics.ParamUtil.common(req, query)
178
+        _samples_ag = self._sample_agg % {"size": self.size}
179
+        if query:
180
+            body = ('{"query":{"bool":{"must":' + json.dumps(query) + '}},'
181
+                    '"size":' + str(self.size) + ','
182
+                    '"aggs":' + _samples_ag + '}')
183
+        else:
184
+            body = '{"aggs":' + _samples_ag + '}'
185
+
186
+        LOG.debug('Request body:' + body)
187
+        LOG.debug('Request url:' + self._query_url)
188
+        es_res = requests.post(self._query_url, data=body)
189
+        res.status = getattr(falcon, 'HTTP_%s' % es_res.status_code)
190
+
191
+        LOG.debug('Query to ElasticSearch returned: %s' % es_res.status_code)
192
+        res_data = self._get_agg_response(es_res)
193
+        if res_data:
194
+            # convert the response into ceilometer sample format
195
+            aggs = res_data['by_name']['buckets']
196
+
197
+            res.body = ''.join(self._make_body(aggs))
198
+            res.content_type = 'application/json;charset=utf-8'
199
+        else:
200
+            res.body = ''
201
+
202
+    @resource_api.Restify('/v2.0/samples/{sample_id}', method='get')
203
+    def get_sample_byid(self, req, res, sample_id):
204
+        LOG.debug('The sample %s GET request is received' % sample_id)
205
+
206
+        # process query condition
207
+        query = []
208
+        metrics.ParamUtil.common(req, query)
209
+        _sample_ag = self._sample_agg % {"size": self.size}
210
+        if query:
211
+            body = ('{"query":{"bool":{"must":' + json.dumps(query) + '}},'
212
+                    '"size":' + str(self.size) + ','
213
+                    '"aggs":' + _sample_ag + '}')
214
+        else:
215
+            body = '{"aggs":' + _sample_ag + '}'
216
+
217
+        # modify the query url to filter out name
218
+        query_url = []
219
+        if sample_id:
220
+            query_url = self._query_url + '&q=_id:' + sample_id
221
+        else:
222
+            query_url = self._query_url
223
+        LOG.debug('Request body:' + body)
224
+        LOG.debug('Request url:' + query_url)
225
+        es_res = requests.post(query_url, data=body)
226
+        res.status = getattr(falcon, 'HTTP_%s' % es_res.status_code)
227
+
228
+        LOG.debug('Query to ElasticSearch returned: %s' % es_res.status_code)
229
+        res_data = self._get_agg_response(es_res)
230
+        LOG.debug('@$Result data is %s\n' % res_data)
231
+        if res_data:
232
+            # convert the response into ceilometer sample format
233
+            aggs = res_data['by_name']['buckets']
234
+
235
+            res.body = ''.join(self._make_body(aggs))
236
+            res.content_type = 'application/json;charset=utf-8'
237
+        else:
238
+            res.body = ''

+ 1
- 0
setup.cfg View File

@@ -51,6 +51,7 @@ kiloeyes.dispatcher =
51 51
     notificationmethods = kiloeyes.v2.elasticsearch.notificationmethods:NotificationMethodDispatcher
52 52
     alarms = kiloeyes.v2.elasticsearch.alarms:AlarmDispatcher
53 53
     meters = kiloeyes.v2.elasticsearch.meters:MeterDispatcher
54
+    samples = kiloeyes.v2.elasticsearch.samples:CeilometerSampleDispatcher
54 55
 
55 56
 kiloeyes.index.strategy =
56 57
     timed = kiloeyes.microservice.timed_strategy:TimedStrategy

Loading…
Cancel
Save