Meters GET request implemented.

Meters GET request implemented and its unit test added.

Added ceilometer API file.

Copy the Meter POST request code from metrics. Ceilometer specific
feature not implemented yet.

Added meter validator to validate Ceilometer meter format for POST.

Change-Id: I4e095348729f32e63c478559a9bc5a90a29663a5
This commit is contained in:
xiaotan2 2016-04-05 12:46:50 +00:00 committed by Xiao Tan
parent b9131a526a
commit 7c0766d455
10 changed files with 560 additions and 12 deletions

View File

@ -1,4 +1,6 @@
Andreas Jaeger <aj@suse.com>
Chang-Yi Lee <cy.l@inwinstack.com>
Jiaming Lin <robin890650@gmail.com>
Tong Li <litong01@us.ibm.com>
spzala <spzala@us.ibm.com>
xiaotan2 <xiaotan2@uw.edu>

View File

@ -1,7 +1,29 @@
kiloeyes (1.0)
CHANGES
=======
* Initial project setup
Choose framework of wsgiref, pasteDeploy, falcon.
The server will be wsgiref server like any other OpenStack server
Use PasteDeploy to allow WSGI pipelines
Use Falcon framework to implement ReSTful API services
* Meters GET request implemented
* Added more instructions on how to configure keystone middleware
* move up one more dependencies
* Move to the same level of dependencies as other openstack project
* Added more functions for the vagrant sub project
* Move out the binary files folder out of the project
* Restructure the vagrant scripts and fix a dependency error
* Changed vagrant file so that installation can be easier
* Remove sphinx requires from test-requirements
* Reformat the readme.MD file and update the listOpt in kafka_conn.py
* Make sure that the index field mapping is correct
* ES now returns timestamp as milliseconds vs seconds
* enable bulk message post on persister
* Added more instructions
* bulk insert can not be done implicitly
* fix the partitions data type error
* Updated the installation instructions
* added more instruction on how to create an all-in-one kiloeyes
* unit test passed for py27
* Add Vagrant sample file to ease development environment bootstrap
* Make the server more flexible with configuration files
* remove old openstack incubator project reference
* remove old oslo.config and use new oslo_config
* Make minor modifications in the README
* seeding the project
* Added .gitreview

View File

@ -11,6 +11,7 @@ dispatcher = versions
dispatcher = alarmdefinitions
dispatcher = notificationmethods
dispatcher = alarms
dispatcher = meters
[metrics]
topic = metrics
@ -78,6 +79,6 @@ compact = False
partitions = 0
[es_conn]
uri = http://localhost:9200
uri = http://128.84.105.102:9200
time_id = timestamp
drop_data = False

View File

@ -14,9 +14,12 @@ use = egg: kiloeyes#login
[filter:inspector]
use = egg: kiloeyes#inspector
[filter:validator]
[filter:metric_validator]
use = egg: kiloeyes#metric_validator
[filter:meter_validator]
use = egg: kiloeyes#meter_validator
[server:main]
use = egg:gunicorn#main
host = 0.0.0.0

View File

@ -0,0 +1,34 @@
# Copyright 2013 IBM Corp
##
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from kiloeyes.common import resource_api
from oslo_log import log
LOG = log.getLogger(__name__)
# Ceilometer V2 API
class V2API(object):
def __init__(self, global_conf):
LOG.debug('initializing V2API!')
self.global_conf = global_conf
# Meter APIs
@resource_api.Restify('/v2.0/meters', method='get')
def get_meters(self, req, res):
res.status = '501 Not Implemented'
@resource_api.Restify('/v2.0/meters', method='post')
def post_meters(self, req, res):
res.status = '501 Not Implemented'

View File

@ -0,0 +1,133 @@
# Copyright 2013 IBM Corp
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import StringIO
try:
import ujson as json
except ImportError:
import json
class MeterValidator(object):
"""middleware that validate the meter input stream.
This middleware checks if the input stream actually follows meter spec
and all the messages in the request has valid meter data. If the body
is valid json and compliant with the spec, then the request will forward
the request to the next in the pipeline, otherwise, it will reject the
request with response code of 400 or 406.
"""
def __init__(self, app, conf):
self.app = app
self.conf = conf
def _is_valid_meter(self, meter):
"""Validate a message
According to the Ceilometer OldSample, the external message format is
{
"counter_name": "instance",
"counter_type": "gauge",
"counter_unit": "instance",
"counter_volume": 1.0,
"message_id": "5460acce-4fd6-480d-ab18-9735ec7b1996",
"project_id": "35b17138-b364-4e6a-a131-8f3099c5be68",
"recorded_at": "2016-04-21T00:07:20.174109",
"resource_id": "bd9431c1-8d69-4ad3-803a-8d4a6b89fd36",
"resource_metadata": {
"name1": "value1",
"name2": "value2"
},
"source": "openstack",
"timestamp": "2016-04-21T00:07:20.174114",
"user_id": "efd87807-12d2-4b38-9c70-5f5c2ac427ff"
}
Once this is validated, the message needs to be transformed into
the following internal format:
The current valid message format is as follows (interna):
{
"meter": {"something": "The meter as a JSON object"},
"meta": {
"tenantId": "the tenant ID acquired",
"region": "the region that the metric was submitted under",
},
"creation_time": "the time when the API received the metric",
}
"""
if (meter.get('counter_name') and meter.get('counter_volume') and
meter.get('message_id') and meter.get('project_id') and
meter.get('source') and meter.get('timestamp') and
meter.get('user_id')):
return True
else:
return False
def __call__(self, env, start_response):
# if request starts with /datapoints/, then let it go on.
# this login middle
if (env.get('PATH_INFO', '').startswith('/v2.0/meters') and
env.get('REQUEST_METHOD', '') == 'POST'):
# We only check the requests which are posting against meters
# endpoint
try:
body = env['wsgi.input'].read()
meters = json.loads(body)
# Do business logic validation here.
is_valid = True
if isinstance(meters, list):
for meter in meters:
if not self._is_valid_meter(meter):
is_valid = False
break
else:
is_valid = self._is_valid_meter(meters)
if is_valid:
# If the message is valid, then wrap it into this internal
# format. The tenantId should be available from the
# request since this should have been authenticated.
# ideally this transformation should be done somewhere
# else. For the sake of simplicity, do the simple one
# here to make the life a bit easier.
# TODO(HP) Add logic to get region id from request header
# HTTP_X_SERVICE_CATALOG, then find endpoints, then region
region_id = None
msg = {'meter': meters,
'meta': {'tenantId': env.get('HTTP_X_PROJECT_ID'),
'region': region_id},
'creation_time': datetime.datetime.now()}
env['wsgi.input'] = StringIO.StringIO(json.dumps(msg))
return self.app(env, start_response)
except Exception:
pass
# It is either invalid or exceptioned out while parsing json
# we will send the request back with 400.
start_response("400 Bad Request", [], '')
return []
else:
# not a metric post request, move on.
return self.app(env, start_response)
def filter_factory(global_conf, **local_conf):
def validator_filter(app):
return MeterValidator(app, local_conf)
return validator_filter

View File

@ -23,7 +23,6 @@ except ImportError:
class MetricValidator(object):
"""middleware that validate the metric input stream.
This middleware checks if the input stream actually follows metric spec
and all the messages in the request has valid metric data. If the body
is valid json and compliant with the spec, then the request will forward
@ -36,7 +35,6 @@ class MetricValidator(object):
def _is_valid_metric(self, metric):
"""Validate a message
The external message format is
{
"name":"name1",
@ -47,10 +45,8 @@ class MetricValidator(object):
"timestamp":1405630174,
"value":1.0
}
Once this is validated, the message needs to be transformed into
the following internal format:
The current valid message format is as follows (interna):
{
"metric": {"something": "The metric as a JSON object"},

View File

@ -0,0 +1,147 @@
# Copyright 2013 IBM Corp
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import falcon
import mock
from oslo_config import fixture as fixture_config
from oslotest import base
import requests
from kiloeyes.common import kafka_conn
from kiloeyes.v2.elasticsearch import meters
try:
import ujson as json
except ImportError:
import json
class TestMeterDispatcher(base.BaseTestCase):
def setUp(self):
super(TestMeterDispatcher, self).setUp()
self.CONF = self.useFixture(fixture_config.Config()).conf
self.CONF.set_override('uri', 'fake_url', group='kafka_opts')
self.CONF.set_override('topic', 'fake', group='meters')
self.CONF.set_override('doc_type', 'fake', group='meters')
self.CONF.set_override('index_prefix', 'also_fake', group='meters')
self.CONF.set_override('index_template', 'etc/metrics.template',
group='meters')
self.CONF.set_override('uri', 'http://fake_es_uri', group='es_conn')
res = mock.Mock()
res.status_code = 200
res.json.return_value = {"data": {"mappings": {"fake": {
"properties": {
"dimensions": {"properties": {
"key1": {"type": "long"}, "key2": {"type": "long"},
"rkey0": {"type": "long"}, "rkey1": {"type": "long"},
"rkey2": {"type": "long"}, "rkey3": {"type": "long"}}},
"name": {"type": "string", "index": "not_analyzed"},
"timestamp": {"type": "string", "index": "not_analyzed"},
"value": {"type": "double"}}}}}}
put_res = mock.Mock()
put_res.status_code = '200'
with mock.patch.object(requests, 'get',
return_value=res):
with mock.patch.object(requests, 'put', return_value=put_res):
self.dispatcher = meters.MeterDispatcher({})
def test_initialization(self):
# test that the kafka connection uri should be 'fake' as it was passed
# in from configuration
self.assertEqual(self.dispatcher._kafka_conn.uri, 'fake_url')
# test that the topic is meters as it was passed into dispatcher
self.assertEqual(self.dispatcher._kafka_conn.topic, 'fake')
# test that the doc type of the es connection is fake
self.assertEqual(self.dispatcher._es_conn.doc_type, 'fake')
self.assertEqual(self.dispatcher._es_conn.uri, 'http://fake_es_uri/')
# test that the query url is correctly formed
self.assertEqual(self.dispatcher._query_url, (
'http://fake_es_uri/also_fake*/fake/_search?search_type=count'))
def test_post_data(self):
with mock.patch.object(kafka_conn.KafkaConnection, 'send_messages',
return_value=204):
res = mock.Mock()
self.dispatcher.post_data(mock.Mock(), res)
# test that the response code is 204
self.assertEqual(getattr(falcon, 'HTTP_204'), res.status)
with mock.patch.object(kafka_conn.KafkaConnection, 'send_messages',
return_value=400):
res = mock.Mock()
self.dispatcher.post_data(mock.Mock(), res)
# test that the response code is 204
self.assertEqual(getattr(falcon, 'HTTP_400'), res.status)
def test_get_meters(self):
res = mock.Mock()
req = mock.Mock()
def _side_effect(arg):
if arg == 'name':
return 'tongli'
elif arg == 'dimensions':
return 'key1:100, key2:200'
req.get_param.side_effect = _side_effect
req_result = mock.Mock()
response_str = """
{"aggregations":{"by_name":{"doc_count_error_upper_bound":0,
"sum_other_doc_count":0,"buckets":[{"key":"BABMGD","doc_count":300,
"by_dim":{"buckets":[{"key": "64e6ce08b3b8547b7c32e5cfa5b7d81f",
"doc_count":300,"meters":{"hits":{"hits":[{ "_type": "metrics",
"_id": "AVOziWmP6-pxt0dRmr7j", "_index": "data_20160401000000",
"_source":{"name":"BABMGD", "value": 4,
"timestamp": 1461337094000,
"dimensions_hash": "0afdb86f508962bb5d8af52df07ef35a",
"project_id": "35b17138-b364-4e6a-a131-8f3099c5be68",
"tenant_id": "bd9431c1-8d69-4ad3-803a-8d4a6b89fd36",
"user_agent": "openstack", "dimensions": null,
"user": "admin", "value_meta": null, "tenant": "admin",
"user_id": "efd87807-12d2-4b38-9c70-5f5c2ac427ff"}}]}}}]}}]}}}
"""
req_result.json.return_value = json.loads(response_str)
req_result.status_code = 200
with mock.patch.object(requests, 'post', return_value=req_result):
self.dispatcher.get_meter(req, res)
# test that the response code is 200
self.assertEqual(res.status, getattr(falcon, 'HTTP_200'))
obj = json.loads(res.body)
self.assertEqual(obj[0]['name'], 'BABMGD')
self.assertEqual(obj[0]['meter_id'], 'AVOziWmP6-pxt0dRmr7j')
self.assertEqual(obj[0]['type'], 'metrics')
self.assertEqual(obj[0]['user_id'],
'efd87807-12d2-4b38-9c70-5f5c2ac427ff')
self.assertEqual(obj[0]['project_id'],
'35b17138-b364-4e6a-a131-8f3099c5be68')
self.assertEqual(len(obj), 1)
def test_post_meters(self):
with mock.patch.object(kafka_conn.KafkaConnection, 'send_messages',
return_value=204):
res = mock.Mock()
self.dispatcher.post_meters(mock.Mock(), res)
self.assertEqual(getattr(falcon, 'HTTP_204'), res.status)

View File

@ -0,0 +1,208 @@
# Copyright 2013 IBM Corp
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import falcon
from oslo_config import cfg
from oslo_log import log
import requests
from stevedore import driver
from kiloeyes.common import es_conn
from kiloeyes.common import kafka_conn
from kiloeyes.common import namespace
from kiloeyes.common import resource_api
from kiloeyes.v2.elasticsearch import metrics
try:
import ujson as json
except ImportError:
import json
METERS_OPTS = [
cfg.StrOpt('topic', default='metrics',
help='The topic that meters will be published to.'),
cfg.StrOpt('doc_type', default='metrics',
help='The doc type that meters will be saved into.'),
cfg.StrOpt('index_strategy', default='fixed',
help='The index strategy used to create index name.'),
cfg.StrOpt('index_prefix', default='data_',
help='The index prefix where meters were saved to.'),
cfg.StrOpt('index_template', default='/etc/kiloeyes/metrics.template',
help='The index template which meters index should use.'),
cfg.IntOpt('size', default=10000,
help=('The query result limit. Any result set more than '
'the limit will be discarded. To see all the matching '
'result, narrow your search by using a small time '
'window or strong matching name')),
]
cfg.CONF.register_opts(METERS_OPTS, group="meters")
LOG = log.getLogger(__name__)
UPDATED = str(datetime.datetime(2014, 1, 1, 0, 0, 0))
class MeterDispatcher(object):
def __init__(self, global_conf):
LOG.debug('initializing V2API!')
super(MeterDispatcher, self).__init__()
self.topic = cfg.CONF.meters.topic
self.doc_type = cfg.CONF.meters.doc_type
self.index_template = cfg.CONF.meters.index_template
self.size = cfg.CONF.meters.size
self._kafka_conn = kafka_conn.KafkaConnection(self.topic)
# load index strategy
if cfg.CONF.meters.index_strategy:
self.index_strategy = driver.DriverManager(
namespace.STRATEGY_NS,
cfg.CONF.meters.index_strategy,
invoke_on_load=True,
invoke_kwds={}).driver
LOG.debug(dir(self.index_strategy))
else:
self.index_strategy = None
self.index_prefix = cfg.CONF.meters.index_prefix
self._es_conn = es_conn.ESConnection(
self.doc_type, self.index_strategy, self.index_prefix)
# Setup the get meters query body pattern
self._query_body = {
"query": {"bool": {"must": []}},
"size": self.size}
self._aggs_body = {}
self._stats_body = {}
self._sort_clause = []
# Setup the get meters query url, the url should be similar to this:
# http://host:port/data_20141201/meters/_search
# the url should be made of es_conn uri, the index prefix, meters
# dispatcher topic, then add the key word _search.
self._query_url = ''.join([self._es_conn.uri,
self._es_conn.index_prefix, '*/',
cfg.CONF.meters.topic,
'/_search?search_type=count'])
# Setup meters query aggregation command. To see the structure of
# the aggregation, copy and paste it to a json formatter.
self._meters_agg = """
{"by_name":{"terms":{"field":"name","size":%(size)d},
"aggs":{"by_dim":{"terms":{"field":"dimensions_hash","size":%(size)d},
"aggs":{"meters":{"top_hits":{"_source":{"exclude":
["dimensions_hash","timestamp","value"]},"size":1}}}}}}}
"""
self.setup_index_template()
def setup_index_template(self):
status = '400'
with open(self.index_template) as template_file:
template_path = ''.join([self._es_conn.uri,
'/_template/metrics'])
es_res = requests.put(template_path, data=template_file.read())
status = getattr(falcon, 'HTTP_%s' % es_res.status_code)
if status == '400':
LOG.error('Metrics template can not be created. Status code %s'
% status)
exit(1)
else:
LOG.debug('Index template set successfully! Status %s' % status)
def post_data(self, req, res):
msg = ""
LOG.debug('@$Post Message is %s' % msg)
LOG.debug('Getting the call.')
msg = req.stream.read()
code = self._kafka_conn.send_messages(msg)
res.status = getattr(falcon, 'HTTP_' + str(code))
def _get_agg_response(self, res):
if res and res.status_code == 200:
obj = res.json()
if obj:
return obj.get('aggregations')
return None
else:
return None
@resource_api.Restify('/v2.0/meters', method='get')
def get_meter(self, req, res):
LOG.debug('The meters GET request is received')
# process query condition
query = []
metrics.ParamUtil.common(req, query)
_meters_ag = self._meters_agg % {"size": self.size}
if query:
body = ('{"query":{"bool":{"must":' + json.dumps(query) + '}},'
'"size":' + str(self.size) + ','
'"aggs":' + _meters_ag + '}')
else:
body = '{"aggs":' + _meters_ag + '}'
LOG.debug('Request body:' + body)
LOG.debug('Request url:' + self._query_url)
es_res = requests.post(self._query_url, data=body)
res.status = getattr(falcon, 'HTTP_%s' % es_res.status_code)
LOG.debug('Query to ElasticSearch returned: %s' % es_res.status_code)
res_data = self._get_agg_response(es_res)
if res_data:
# convert the response into ceilometer meter format
aggs = res_data['by_name']['buckets']
flag = {'is_first': True}
def _render_hits(item):
_id = item['meters']['hits']['hits'][0]['_id']
_type = item['meters']['hits']['hits'][0]['_type']
_source = item['meters']['hits']['hits'][0]['_source']
rslt = ('{"meter_id":' + json.dumps(_id) + ','
'"name":' + json.dumps(_source['name']) + ','
'"project_id":' +
json.dumps(_source['project_id']) + ','
'"resource_id":' +
json.dumps(_source['tenant_id']) + ','
'"source":' + json.dumps(_source['user_agent']) + ','
'"type":' + json.dumps(_type) + ','
'"unit":null,'
'"user_id":' + json.dumps(_source['user_id']) + '}')
if flag['is_first']:
flag['is_first'] = False
return rslt
else:
return ',' + rslt
def _make_body(buckets):
yield '['
for by_name in buckets:
if by_name['by_dim']:
for by_dim in by_name['by_dim']['buckets']:
yield _render_hits(by_dim)
yield ']'
res.body = ''.join(_make_body(aggs))
res.content_type = 'application/json;charset=utf-8'
else:
res.body = ''
@resource_api.Restify('/v2.0/meters/', method='post')
def post_meters(self, req, res):
self.post_data(req, res)

View File

@ -50,6 +50,7 @@ kiloeyes.dispatcher =
alarmdefinitions = kiloeyes.v2.elasticsearch.alarmdefinitions:AlarmDefinitionDispatcher
notificationmethods = kiloeyes.v2.elasticsearch.notificationmethods:NotificationMethodDispatcher
alarms = kiloeyes.v2.elasticsearch.alarms:AlarmDispatcher
meters = kiloeyes.v2.elasticsearch.meters:MeterDispatcher
kiloeyes.index.strategy =
timed = kiloeyes.microservice.timed_strategy:TimedStrategy
@ -64,6 +65,7 @@ paste.filter_factory =
login = kiloeyes.middleware.login:filter_factory
inspector = kiloeyes.middleware.inspector:filter_factory
metric_validator = kiloeyes.middleware.metric_validator:filter_factory
meter_validator = kiloeyes.middleware.meter_validator:filter_factory
[pbr]
warnerrors = True