From 53f783b695affb3c7541b55dc2e1d885b6f7beda Mon Sep 17 00:00:00 2001 From: Adrian Czarnecki Date: Tue, 9 Apr 2019 13:13:45 +0200 Subject: [PATCH] Add monasca-log-api source code It is another step towards merging this two api Story: 2003881 Task: 30532 Change-Id: Icc6460e3a136294462f5ceefdba1751187eb2363 --- monasca_log_api_code/__init__.py | 0 monasca_log_api_code/app/__init__.py | 0 monasca_log_api_code/app/api.py | 126 +++++ monasca_log_api_code/app/base/__init__.py | 0 .../app/base/error_handlers.py | 28 + monasca_log_api_code/app/base/exceptions.py | 38 ++ .../app/base/log_publisher.py | 252 +++++++++ monasca_log_api_code/app/base/model.py | 119 +++++ monasca_log_api_code/app/base/request.py | 109 ++++ .../app/base/request_context.py | 36 ++ monasca_log_api_code/app/base/validation.py | 267 ++++++++++ .../app/controller/__init__.py | 0 .../app/controller/api/__init__.py | 0 .../app/controller/api/headers.py | 24 + .../app/controller/api/healthcheck_api.py | 60 +++ .../app/controller/api/logs_api.py | 88 ++++ .../app/controller/api/versions_api.py | 34 ++ .../app/controller/healthchecks.py | 60 +++ .../app/controller/v2/__init__.py | 0 .../app/controller/v2/aid/__init__.py | 0 .../app/controller/v2/aid/service.py | 157 ++++++ .../app/controller/v2/logs.py | 104 ++++ .../app/controller/v3/__init__.py | 0 .../app/controller/v3/aid/__init__.py | 0 .../app/controller/v3/aid/bulk_processor.py | 157 ++++++ .../app/controller/v3/aid/helpers.py | 62 +++ .../app/controller/v3/logs.py | 110 ++++ .../app/controller/versions.py | 128 +++++ monasca_log_api_code/app/main.py | 46 ++ monasca_log_api_code/app/wsgi.py | 32 ++ monasca_log_api_code/conf/__init__.py | 70 +++ monasca_log_api_code/conf/healthcheck.py | 36 ++ monasca_log_api_code/conf/log_publisher.py | 41 ++ monasca_log_api_code/conf/monitoring.py | 49 ++ monasca_log_api_code/conf/role_middleware.py | 47 ++ monasca_log_api_code/conf/service.py | 37 ++ monasca_log_api_code/config.py | 82 +++ monasca_log_api_code/db/__init__.py | 0 monasca_log_api_code/db/common/__init__.py | 0 monasca_log_api_code/db/common/model.py | 43 ++ monasca_log_api_code/db/repo/__init__.py | 0 .../db/repo/logs_repository.py | 74 +++ monasca_log_api_code/healthcheck/__init__.py | 1 + .../healthcheck/kafka_check.py | 98 ++++ monasca_log_api_code/middleware/__init__.py | 0 .../middleware/role_middleware.py | 152 ++++++ monasca_log_api_code/monitoring/__init__.py | 0 monasca_log_api_code/monitoring/client.py | 78 +++ monasca_log_api_code/monitoring/metrics.py | 47 ++ monasca_log_api_code/policies/__init__.py | 73 +++ monasca_log_api_code/policies/healthchecks.py | 44 ++ monasca_log_api_code/policies/logs.py | 43 ++ monasca_log_api_code/policies/versions.py | 38 ++ monasca_log_api_code/tests/__init__.py | 0 monasca_log_api_code/tests/base.py | 220 ++++++++ monasca_log_api_code/tests/test_config.py | 40 ++ .../tests/test_healthchecks.py | 75 +++ .../tests/test_kafka_check.py | 71 +++ .../tests/test_log_publisher.py | 294 +++++++++++ monasca_log_api_code/tests/test_logs.py | 269 ++++++++++ monasca_log_api_code/tests/test_logs_v3.py | 317 ++++++++++++ monasca_log_api_code/tests/test_monitoring.py | 52 ++ monasca_log_api_code/tests/test_policy.py | 212 ++++++++ monasca_log_api_code/tests/test_request.py | 100 ++++ .../tests/test_role_middleware.py | 226 ++++++++ monasca_log_api_code/tests/test_service.py | 483 ++++++++++++++++++ .../tests/test_v2_v3_compare.py | 119 +++++ monasca_log_api_code/tests/test_version.py | 22 + monasca_log_api_code/tests/test_versions.py | 120 +++++ monasca_log_api_code/version.py | 18 + 70 files changed, 5728 insertions(+) create mode 100644 monasca_log_api_code/__init__.py create mode 100644 monasca_log_api_code/app/__init__.py create mode 100644 monasca_log_api_code/app/api.py create mode 100644 monasca_log_api_code/app/base/__init__.py create mode 100644 monasca_log_api_code/app/base/error_handlers.py create mode 100644 monasca_log_api_code/app/base/exceptions.py create mode 100644 monasca_log_api_code/app/base/log_publisher.py create mode 100644 monasca_log_api_code/app/base/model.py create mode 100644 monasca_log_api_code/app/base/request.py create mode 100644 monasca_log_api_code/app/base/request_context.py create mode 100644 monasca_log_api_code/app/base/validation.py create mode 100644 monasca_log_api_code/app/controller/__init__.py create mode 100644 monasca_log_api_code/app/controller/api/__init__.py create mode 100644 monasca_log_api_code/app/controller/api/headers.py create mode 100644 monasca_log_api_code/app/controller/api/healthcheck_api.py create mode 100644 monasca_log_api_code/app/controller/api/logs_api.py create mode 100644 monasca_log_api_code/app/controller/api/versions_api.py create mode 100644 monasca_log_api_code/app/controller/healthchecks.py create mode 100644 monasca_log_api_code/app/controller/v2/__init__.py create mode 100644 monasca_log_api_code/app/controller/v2/aid/__init__.py create mode 100644 monasca_log_api_code/app/controller/v2/aid/service.py create mode 100644 monasca_log_api_code/app/controller/v2/logs.py create mode 100644 monasca_log_api_code/app/controller/v3/__init__.py create mode 100644 monasca_log_api_code/app/controller/v3/aid/__init__.py create mode 100644 monasca_log_api_code/app/controller/v3/aid/bulk_processor.py create mode 100644 monasca_log_api_code/app/controller/v3/aid/helpers.py create mode 100644 monasca_log_api_code/app/controller/v3/logs.py create mode 100644 monasca_log_api_code/app/controller/versions.py create mode 100644 monasca_log_api_code/app/main.py create mode 100644 monasca_log_api_code/app/wsgi.py create mode 100644 monasca_log_api_code/conf/__init__.py create mode 100644 monasca_log_api_code/conf/healthcheck.py create mode 100644 monasca_log_api_code/conf/log_publisher.py create mode 100644 monasca_log_api_code/conf/monitoring.py create mode 100644 monasca_log_api_code/conf/role_middleware.py create mode 100644 monasca_log_api_code/conf/service.py create mode 100644 monasca_log_api_code/config.py create mode 100644 monasca_log_api_code/db/__init__.py create mode 100644 monasca_log_api_code/db/common/__init__.py create mode 100644 monasca_log_api_code/db/common/model.py create mode 100644 monasca_log_api_code/db/repo/__init__.py create mode 100644 monasca_log_api_code/db/repo/logs_repository.py create mode 100644 monasca_log_api_code/healthcheck/__init__.py create mode 100644 monasca_log_api_code/healthcheck/kafka_check.py create mode 100644 monasca_log_api_code/middleware/__init__.py create mode 100644 monasca_log_api_code/middleware/role_middleware.py create mode 100644 monasca_log_api_code/monitoring/__init__.py create mode 100644 monasca_log_api_code/monitoring/client.py create mode 100644 monasca_log_api_code/monitoring/metrics.py create mode 100644 monasca_log_api_code/policies/__init__.py create mode 100644 monasca_log_api_code/policies/healthchecks.py create mode 100644 monasca_log_api_code/policies/logs.py create mode 100644 monasca_log_api_code/policies/versions.py create mode 100644 monasca_log_api_code/tests/__init__.py create mode 100644 monasca_log_api_code/tests/base.py create mode 100644 monasca_log_api_code/tests/test_config.py create mode 100644 monasca_log_api_code/tests/test_healthchecks.py create mode 100644 monasca_log_api_code/tests/test_kafka_check.py create mode 100644 monasca_log_api_code/tests/test_log_publisher.py create mode 100644 monasca_log_api_code/tests/test_logs.py create mode 100644 monasca_log_api_code/tests/test_logs_v3.py create mode 100644 monasca_log_api_code/tests/test_monitoring.py create mode 100644 monasca_log_api_code/tests/test_policy.py create mode 100644 monasca_log_api_code/tests/test_request.py create mode 100644 monasca_log_api_code/tests/test_role_middleware.py create mode 100644 monasca_log_api_code/tests/test_service.py create mode 100644 monasca_log_api_code/tests/test_v2_v3_compare.py create mode 100644 monasca_log_api_code/tests/test_version.py create mode 100644 monasca_log_api_code/tests/test_versions.py create mode 100644 monasca_log_api_code/version.py diff --git a/monasca_log_api_code/__init__.py b/monasca_log_api_code/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monasca_log_api_code/app/__init__.py b/monasca_log_api_code/app/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monasca_log_api_code/app/api.py b/monasca_log_api_code/app/api.py new file mode 100644 index 000000000..21b33e95d --- /dev/null +++ b/monasca_log_api_code/app/api.py @@ -0,0 +1,126 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Module contains factories to initializes various applications +of monasca-log-api +""" + +import falcon +import six +from monasca_log_api import config +from monasca_log_api.app.base import error_handlers +from monasca_log_api.app.base import request +from monasca_log_api.app.controller import healthchecks +from monasca_log_api.app.controller import versions +from monasca_log_api.app.controller.v2 import logs as v2_logs +from monasca_log_api.app.controller.v3 import logs as v3_logs +from oslo_log import log + + +def error_trap(app_name): + """Decorator trapping any error during application boot time""" + + @six.wraps(error_trap) + def _wrapper(func): + + @six.wraps(_wrapper) + def _inner_wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception: + logger = log.getLogger(__name__) + logger.exception('Failed to load application \'%s\'', app_name) + raise + + return _inner_wrapper + + return _wrapper + + +def singleton_config(func): + """Decorator ensuring that configuration is loaded only once.""" + + @six.wraps(singleton_config) + def _wrapper(global_config, **local_conf): + config.parse_args() + return func(global_config, **local_conf) + + return _wrapper + + +@error_trap('version') +def create_version_app(global_conf, **local_conf): + """Creates Version application""" + + ctrl = versions.Versions() + controllers = { + '/': ctrl, # redirect http://host:port/ down to Version app + # avoid conflicts with actual pipelines and 404 error + '/version': ctrl, # list all the versions + '/version/{version_id}': ctrl # display details of the version + } + + wsgi_app = falcon.API( + request_type=request.Request + ) + for route, ctrl in controllers.items(): + wsgi_app.add_route(route, ctrl) + return wsgi_app + + +@error_trap('healthcheck') +def create_healthcheck_app(global_conf, **local_conf): + """Creates Healthcheck application""" + + ctrl = healthchecks.HealthChecks() + controllers = { + '/': ctrl + } + + wsgi_app = falcon.API( + request_type=request.Request + ) + for route, ctrl in controllers.items(): + wsgi_app.add_route(route, ctrl) + return wsgi_app + + +@error_trap('api') +@singleton_config +def create_api_app(global_conf, **local_conf): + """Creates MainAPI application""" + + controllers = {} + api_version = global_conf.get('api_version') + + if api_version == 'v2.0': + controllers.update({ + '/log/single': v2_logs.Logs() + }) + elif api_version == 'v3.0': + controllers.update({ + '/logs': v3_logs.Logs() + }) + + wsgi_app = falcon.API( + request_type=request.Request + ) + + for route, ctrl in controllers.items(): + wsgi_app.add_route(route, ctrl) + + error_handlers.register_error_handlers(wsgi_app) + + return wsgi_app diff --git a/monasca_log_api_code/app/base/__init__.py b/monasca_log_api_code/app/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monasca_log_api_code/app/base/error_handlers.py b/monasca_log_api_code/app/base/error_handlers.py new file mode 100644 index 000000000..db9865119 --- /dev/null +++ b/monasca_log_api_code/app/base/error_handlers.py @@ -0,0 +1,28 @@ +# Copyright 2016 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import falcon + +from monasca_log_api.app.base import model + + +def log_envelope_exception_handler(ex, req, resp, params): + raise falcon.HTTPUnprocessableEntity( + title='Failed to create Envelope', + description=ex.message) + + +def register_error_handlers(app): + app.add_error_handler(model.LogEnvelopeException, + log_envelope_exception_handler) diff --git a/monasca_log_api_code/app/base/exceptions.py b/monasca_log_api_code/app/base/exceptions.py new file mode 100644 index 000000000..2886c0b67 --- /dev/null +++ b/monasca_log_api_code/app/base/exceptions.py @@ -0,0 +1,38 @@ +# Copyright 2015 kornicameister@gmail.com +# Copyright 2015 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import falcon + + +HTTP_422 = '422 Unprocessable Entity' + + +class HTTPUnprocessableEntity(falcon.OptionalRepresentation, falcon.HTTPError): + """HTTPUnprocessableEntity http error. + + HTTPError that comes with '422 Unprocessable Entity' status + + :argument: message(str) - meaningful description of what caused an error + :argument: kwargs - any other option defined in + :py:class:`falcon.OptionalRepresentation` and + :py:class:`falcon.HTTPError` + """ + def __init__(self, message, **kwargs): + falcon.HTTPError.__init__(self, + HTTP_422, + 'unprocessable_entity', + message, + **kwargs + ) diff --git a/monasca_log_api_code/app/base/log_publisher.py b/monasca_log_api_code/app/base/log_publisher.py new file mode 100644 index 000000000..14ace4a3c --- /dev/null +++ b/monasca_log_api_code/app/base/log_publisher.py @@ -0,0 +1,252 @@ +# Copyright 2015 kornicameister@gmail.com +# Copyright 2016-2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import time +import falcon +from monasca_common.kafka import producer +from monasca_common.rest import utils as rest_utils +from monasca_log_api import conf +from monasca_log_api.app.base import model +from monasca_log_api.monitoring import client +from monasca_log_api.monitoring import metrics +from oslo_log import log +from oslo_utils import encodeutils + + + + + +LOG = log.getLogger(__name__) +CONF = conf.CONF + +_RETRY_AFTER = 60 +_TIMESTAMP_KEY_SIZE = len( + bytearray(str(int(time.time() * 1000)).encode('utf-8'))) +_TRUNCATED_PROPERTY_SIZE = len( + bytearray('"truncated": true'.encode('utf-8'))) +_KAFKA_META_DATA_SIZE = 32 +_TRUNCATION_SAFE_OFFSET = 1 + + +class InvalidMessageException(Exception): + pass + + +class LogPublisher(object): + """Publishes log data to Kafka + + LogPublisher is able to send single message to multiple configured topic. + It uses following configuration written in conf file :: + + [log_publisher] + topics = 'logs' + kafka_url = 'localhost:8900' + + Note: + Uses :py:class:`monasca_common.kafka.producer.KafkaProducer` + to ship logs to kafka. For more details + see `monasca_common`_ github repository. + + .. _monasca_common: https://github.com/openstack/monasca-common + + """ + + def __init__(self): + self._topics = CONF.log_publisher.topics + self.max_message_size = CONF.log_publisher.max_message_size + + self._kafka_publisher = producer.KafkaProducer( + url=CONF.log_publisher.kafka_url + ) + if CONF.monitoring.enable: + self._statsd = client.get_client() + + # setup counter, gauges etc + self._logs_published_counter = self._statsd.get_counter( + metrics.LOGS_PUBLISHED_METRIC + ) + self._publish_time_ms = self._statsd.get_timer( + metrics.LOGS_PUBLISH_TIME_METRIC + ) + self._logs_lost_counter = self._statsd.get_counter( + metrics.LOGS_PUBLISHED_LOST_METRIC + ) + self._logs_truncated_gauge = self._statsd.get_gauge( + metrics.LOGS_TRUNCATED_METRIC + ) + + LOG.info('Initializing LogPublisher <%s>', self) + + def send_message(self, messages): + """Sends message to each configured topic. + + Note: + Falsy messages (i.e. empty) are not shipped to kafka + + See also + * :py:class:`monasca_log_api.common.model.Envelope` + * :py:meth:`._is_message_valid` + + :param dict|list messages: instance (or instances) of log envelope + """ + + if not messages: + return + if not isinstance(messages, list): + messages = [messages] + + sent_counter = 0 + num_of_msgs = len(messages) + + LOG.debug('About to publish %d messages to %s topics', + num_of_msgs, self._topics) + + try: + send_messages = [] + + for message in messages: + msg = self._transform_message(message) + send_messages.append(msg) + if CONF.monitoring.enable: + with self._publish_time_ms.time(name=None): + self._publish(send_messages) + else: + self._publish(send_messages) + + sent_counter = len(send_messages) + except Exception as ex: + LOG.exception('Failure in publishing messages to kafka') + raise ex + finally: + self._after_publish(sent_counter, num_of_msgs) + + def _transform_message(self, message): + """Transforms message into JSON. + + Method executes transformation operation for + single element. Operation is set of following + operations: + + * checking if message is valid + (:py:func:`.LogPublisher._is_message_valid`) + * truncating message if necessary + (:py:func:`.LogPublisher._truncate`) + + :param model.Envelope message: instance of message + :return: serialized message + :rtype: str + """ + if not self._is_message_valid(message): + raise InvalidMessageException() + truncated = self._truncate(message) + return encodeutils.safe_encode(truncated, incoming='utf-8') + + def _truncate(self, envelope): + """Truncates the message if needed. + + Each message send to kafka is verified. + Method checks if message serialized to json + exceeds maximum allowed size that can be posted to kafka + queue. If so, method truncates message property of the log + by difference between message and allowed size. + + :param Envelope envelope: original envelope + :return: serialized message + :rtype: str + """ + + msg_str = model.serialize_envelope(envelope) + envelope_size = ((len(bytearray(msg_str, 'utf-8', 'replace')) + + _TIMESTAMP_KEY_SIZE + + _KAFKA_META_DATA_SIZE) + if msg_str is not None else -1) + + diff_size = ((envelope_size - self.max_message_size) + + _TRUNCATION_SAFE_OFFSET) + + if diff_size > 1: + truncated_by = diff_size + _TRUNCATED_PROPERTY_SIZE + + LOG.warning(('Detected message that exceeds %d bytes,' + 'message will be truncated by %d bytes'), + self.max_message_size, + truncated_by) + + log_msg = envelope['log']['message'] + truncated_log_msg = log_msg[:-truncated_by] + + envelope['log']['truncated'] = True + envelope['log']['message'] = truncated_log_msg + if CONF.monitoring.enable: + self._logs_truncated_gauge.send(name=None, value=truncated_by) + + msg_str = rest_utils.as_json(envelope) + else: + if CONF.monitoring.enable: + self._logs_truncated_gauge.send(name=None, value=0) + + return msg_str + + def _publish(self, messages): + """Publishes messages to kafka. + + :param list messages: list of messages + """ + num_of_msg = len(messages) + + LOG.debug('Publishing %d messages', num_of_msg) + + try: + for topic in self._topics: + self._kafka_publisher.publish( + topic, + messages + ) + LOG.debug('Sent %d messages to topic %s', num_of_msg, topic) + except Exception as ex: + raise falcon.HTTPServiceUnavailable('Service unavailable', + str(ex), 60) + + @staticmethod + def _is_message_valid(message): + """Validates message before sending. + + Methods checks if message is :py:class:`model.Envelope`. + By being instance of this class it is ensured that all required + keys are found and they will have their values. + + """ + return message and isinstance(message, model.Envelope) + + def _after_publish(self, send_count, to_send_count): + """Executed after publishing to sent metrics. + + :param int send_count: how many messages have been sent + :param int to_send_count: how many messages should be sent + + """ + + failed_to_send = to_send_count - send_count + + if failed_to_send == 0: + LOG.debug('Successfully published all [%d] messages', + send_count) + else: + error_str = ('Failed to send all messages, %d ' + 'messages out of %d have not been published') + LOG.error(error_str, failed_to_send, to_send_count) + if CONF.monitoring.enable: + self._logs_published_counter.increment(value=send_count) + self._logs_lost_counter.increment(value=failed_to_send) diff --git a/monasca_log_api_code/app/base/model.py b/monasca_log_api_code/app/base/model.py new file mode 100644 index 000000000..4a24739e4 --- /dev/null +++ b/monasca_log_api_code/app/base/model.py @@ -0,0 +1,119 @@ +# Copyright 2016 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_utils import timeutils +import six + +from monasca_common.rest import utils as rest_utils + + +def serialize_envelope(envelope): + """Returns json representation of an envelope. + + :return: json object of envelope + :rtype: six.text_type + + """ + json = rest_utils.as_json(envelope, ensure_ascii=False) + + if six.PY2: + raw = six.text_type(json.replace(r'\\', r'\\\\'), encoding='utf-8', + errors='replace') + else: + raw = json + + return raw + + +class LogEnvelopeException(Exception): + pass + + +class Envelope(dict): + def __init__(self, log, meta): + if not log: + error_msg = 'Envelope cannot be created without log' + raise LogEnvelopeException(error_msg) + if 'tenantId' not in meta or not meta.get('tenantId'): + error_msg = 'Envelope cannot be created without tenant' + raise LogEnvelopeException(error_msg) + + creation_time = self._get_creation_time() + super(Envelope, self).__init__( + log=log, + creation_time=creation_time, + meta=meta + ) + + @staticmethod + def _get_creation_time(): + return timeutils.utcnow_ts() + + @classmethod + def new_envelope(cls, log, tenant_id, region, dimensions=None): + """Creates new log envelope + + Log envelope is combined ouf of following properties + + * log - dict + * creation_time - timestamp + * meta - meta block + + Example output json would like this: + + .. code-block:: json + + { + "log": { + "message": "Some message", + "dimensions": { + "hostname": "devstack" + } + }, + "creation_time": 1447834886, + "meta": { + "tenantId": "e4bd29509eda473092d32aadfee3e7b1", + "region": "pl" + } + } + + :param dict log: original log element (containing message and other + params + :param str tenant_id: tenant id to be put in meta field + :param str region: region to be put in meta field + :param dict dimensions: additional dimensions to be appended to log + object dimensions + + """ + if dimensions: + log['dimensions'].update(dimensions) + + log_meta = { + 'region': region, + 'tenantId': tenant_id + } + + return cls(log, log_meta) + + @property + def log(self): + return self.get('log', None) + + @property + def creation_time(self): + return self.get('creation_time', None) + + @property + def meta(self): + return self.get('meta', None) diff --git a/monasca_log_api_code/app/base/request.py b/monasca_log_api_code/app/base/request.py new file mode 100644 index 000000000..f9836804f --- /dev/null +++ b/monasca_log_api_code/app/base/request.py @@ -0,0 +1,109 @@ +# Copyright 2016 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import falcon +from monasca_common.policy import policy_engine as policy +from monasca_log_api import policies +from monasca_log_api.app.base import request_context +from monasca_log_api.app.base import validation + +policy.POLICIES = policies + + +_TENANT_ID_PARAM = 'tenant_id' +"""Name of the query-param pointing at project-id (tenant-id)""" + + +class Request(falcon.Request): + """Variation of falcon.Request with context + + Following class enhances :py:class:`falcon.Request` with + :py:class:`context.RequestContext`. + + """ + + def __init__(self, env, options=None): + super(Request, self).__init__(env, options) + self.context = request_context.RequestContext.from_environ(self.env) + + def validate(self, content_types): + """Performs common request validation + + Validation checklist (in that order): + + * :py:func:`validation.validate_content_type` + * :py:func:`validation.validate_payload_size` + * :py:func:`validation.validate_cross_tenant` + + :param content_types: allowed content-types handler supports + :type content_types: list + :raises Exception: if any of the validation fails + + """ + validation.validate_content_type(self, content_types) + validation.validate_payload_size(self) + validation.validate_cross_tenant( + tenant_id=self.project_id, + roles=self.roles, + cross_tenant_id=self.cross_project_id + ) + + @property + def project_id(self): + """Returns project-id (tenant-id) + + :return: project-id + :rtype: str + + """ + return self.context.project_id + + @property + def cross_project_id(self): + """Returns project-id (tenant-id) found in query params. + + This particular project-id is later on identified as + cross-project-id + + :return: project-id + :rtype: str + + """ + return self.get_param(_TENANT_ID_PARAM, required=False) + + @property + def user_id(self): + """Returns user-id + + :return: user-id + :rtype: str + + """ + return self.context.user + + @property + def roles(self): + """Returns roles associated with user + + :return: user's roles + :rtype: list + + """ + return self.context.roles + + def can(self, action, target=None): + return self.context.can(action, target) + + def __repr__(self): + return '%s, context=%s' % (self.path, self.context) diff --git a/monasca_log_api_code/app/base/request_context.py b/monasca_log_api_code/app/base/request_context.py new file mode 100644 index 000000000..d04bd2cb0 --- /dev/null +++ b/monasca_log_api_code/app/base/request_context.py @@ -0,0 +1,36 @@ +# Copyright 2017 FUJITSU LIMITED +# Copyright 2018 OP5 AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from monasca_common.policy import policy_engine as policy +from monasca_log_api import policies +from oslo_context import context + + +policy.POLICIES = policies + + +class RequestContext(context.RequestContext): + """RequestContext. + + RequestContext is customized version of + :py:class:oslo_context.context.RequestContext. + """ + + def can(self, action, target=None): + if target is None: + target = {'project_id': self.project_id, + 'user_id': self.user_id} + + return policy.authorize(self, action=action, target=target) diff --git a/monasca_log_api_code/app/base/validation.py b/monasca_log_api_code/app/base/validation.py new file mode 100644 index 000000000..f5c861b49 --- /dev/null +++ b/monasca_log_api_code/app/base/validation.py @@ -0,0 +1,267 @@ +# Copyright 2016-2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import re +import falcon +import six +from monasca_log_api import conf +from monasca_log_api.app.base import exceptions +from oslo_log import log + + + +LOG = log.getLogger(__name__) +CONF = conf.CONF + +APPLICATION_TYPE_CONSTRAINTS = { + 'MAX_LENGTH': 255, + 'PATTERN': re.compile('^[a-zA-Z0-9_.\\-]+$') +} +"""Application type constraint used in validation. + +See :py:func:`Validations.validate_application_type` +""" +DIMENSION_NAME_CONSTRAINTS = { + 'MAX_LENGTH': 255, + 'PATTERN': re.compile('[^><={}(), \'";&]+$') +} +"""Constraint for name of single dimension. + +See :py:func:`Validations.validate_dimensions` +""" +DIMENSION_VALUE_CONSTRAINTS = { + 'MAX_LENGTH': 255 +} +"""Constraint for value of single dimension. + +See :py:func:`Validations.validate_dimensions` +""" + + +def validate_application_type(application_type=None): + """Validates application type. + + Validation won't take place if application_type is None. + For details see: :py:data:`APPLICATION_TYPE_CONSTRAINTS` + + :param str application_type: application type + """ + + def validate_length(): + if (len(application_type) > + APPLICATION_TYPE_CONSTRAINTS['MAX_LENGTH']): + msg = ('Application type {type} must be ' + '{length} characters or less') + raise exceptions.HTTPUnprocessableEntity( + msg.format( + type=application_type, + length=APPLICATION_TYPE_CONSTRAINTS[ + 'MAX_LENGTH'] + ) + ) + + def validate_match(): + if (not APPLICATION_TYPE_CONSTRAINTS['PATTERN'] + .match(application_type)): + raise exceptions.HTTPUnprocessableEntity( + 'Application type %s may only contain: "a-z A-Z 0-9 _ - ."' + % application_type + ) + + if application_type: + validate_length() + validate_match() + + +def _validate_dimension_name(name): + try: + if len(name) > DIMENSION_NAME_CONSTRAINTS['MAX_LENGTH']: + raise exceptions.HTTPUnprocessableEntity( + 'Dimension name %s must be 255 characters or less' % + name + ) + if name[0] == '_': + raise exceptions.HTTPUnprocessableEntity( + 'Dimension name %s cannot start with underscore (_)' % + name + ) + if not DIMENSION_NAME_CONSTRAINTS['PATTERN'].match(name): + raise exceptions.HTTPUnprocessableEntity( + 'Dimension name %s may not contain: %s' % + (name, '> < = { } ( ) \' " , ; &') + ) + except (TypeError, IndexError): + raise exceptions.HTTPUnprocessableEntity( + 'Dimension name cannot be empty' + ) + + +def _validate_dimension_value(value): + try: + value[0] + if len(value) > DIMENSION_VALUE_CONSTRAINTS['MAX_LENGTH']: + raise exceptions.HTTPUnprocessableEntity( + 'Dimension value %s must be 255 characters or less' % + value + ) + except (TypeError, IndexError): + raise exceptions.HTTPUnprocessableEntity( + 'Dimension value cannot be empty' + ) + + +def validate_dimensions(dimensions): + """Validates dimensions type. + + Empty dimensions are not being validated. + For details see: + + :param dict dimensions: dimensions to validate + + * :py:data:`DIMENSION_NAME_CONSTRAINTS` + * :py:data:`DIMENSION_VALUE_CONSTRAINTS` + """ + try: + for dim_name, dim_value in dimensions.items(): + _validate_dimension_name(dim_name) + _validate_dimension_value(dim_value) + except AttributeError: + raise exceptions.HTTPUnprocessableEntity( + 'Dimensions %s must be a dictionary (map)' % dimensions) + + +def validate_content_type(req, allowed): + """Validates content type. + + Method validates request against correct + content type. + + If content-type cannot be established (i.e. header is missing), + :py:class:`falcon.HTTPMissingHeader` is thrown. + If content-type is not **application/json** or **text/plain**, + :py:class:`falcon.HTTPUnsupportedMediaType` is thrown. + + + :param falcon.Request req: current request + :param iterable allowed: allowed content type + + :exception: :py:class:`falcon.HTTPMissingHeader` + :exception: :py:class:`falcon.HTTPUnsupportedMediaType` + """ + content_type = req.content_type + + LOG.debug('Content-Type is %s', content_type) + + if content_type is None or len(content_type) == 0: + raise falcon.HTTPMissingHeader('Content-Type') + + if content_type not in allowed: + sup_types = ', '.join(allowed) + details = ('Only [%s] are accepted as logs representations' + % str(sup_types)) + raise falcon.HTTPUnsupportedMediaType(description=details) + + +def validate_payload_size(req): + """Validates payload size. + + Method validates sent payload size. + It expects that http header **Content-Length** is present. + If it does not, method raises :py:class:`falcon.HTTPLengthRequired`. + Otherwise values is being compared with :: + + [service] + max_log_size = 1048576 + + **max_log_size** refers to the maximum allowed content length. + If it is exceeded :py:class:`falcon.HTTPRequestEntityTooLarge` is + thrown. + + :param falcon.Request req: current request + + :exception: :py:class:`falcon.HTTPLengthRequired` + :exception: :py:class:`falcon.HTTPRequestEntityTooLarge` + + """ + payload_size = req.content_length + max_size = CONF.service.max_log_size + + LOG.debug('Payload (content-length) is %s', str(payload_size)) + + if payload_size is None: + raise falcon.HTTPLengthRequired( + title='Content length header is missing', + description='Content length is required to estimate if ' + 'payload can be processed' + ) + + if payload_size >= max_size: + raise falcon.HTTPRequestEntityTooLarge( + title='Log payload size exceeded', + description='Maximum allowed size is %d bytes' % max_size + ) + + +def validate_is_delegate(roles): + delegate_roles = CONF.roles_middleware.delegate_roles + if roles and delegate_roles: + roles = roles.split(',') if isinstance(roles, six.string_types) \ + else roles + return any(x in set(delegate_roles) for x in roles) + return False + + +def validate_cross_tenant(tenant_id, cross_tenant_id, roles): + + if not validate_is_delegate(roles): + if cross_tenant_id: + raise falcon.HTTPForbidden( + 'Permission denied', + 'Projects %s cannot POST cross tenant logs' % tenant_id + ) + + +def validate_log_message(log_object): + """Validates log property. + + Log property should have message property. + + Args: + log_object (dict): log property + """ + if 'message' not in log_object: + raise exceptions.HTTPUnprocessableEntity( + 'Log property should have message' + ) + + +def validate_authorization(http_request, authorized_rules_list): + """Validates whether is authorized according to provided policy rules list. + + If authorization fails, 401 is thrown with appropriate description. + Additionally response specifies 'WWW-Authenticate' header with 'Token' + value challenging the client to use different token (the one with + different set of roles which can access the service). + """ + challenge = 'Token' + for rule in authorized_rules_list: + try: + http_request.can(rule) + return + except Exception as ex: + LOG.debug(ex) + + raise falcon.HTTPUnauthorized('Forbidden', + 'The request does not have access to this service', + challenge) diff --git a/monasca_log_api_code/app/controller/__init__.py b/monasca_log_api_code/app/controller/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monasca_log_api_code/app/controller/api/__init__.py b/monasca_log_api_code/app/controller/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monasca_log_api_code/app/controller/api/headers.py b/monasca_log_api_code/app/controller/api/headers.py new file mode 100644 index 000000000..1a6734c8f --- /dev/null +++ b/monasca_log_api_code/app/controller/api/headers.py @@ -0,0 +1,24 @@ +# Copyright 2015 kornicameister@gmail.com +# Copyright 2015 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections + +LogApiHeader = collections.namedtuple('LogApiHeader', ['name', 'is_required']) +"""Tuple describing a header.""" + +X_TENANT_ID = LogApiHeader(name='X-Tenant-Id', is_required=False) +X_ROLES = LogApiHeader(name='X-Roles', is_required=False) +X_APPLICATION_TYPE = LogApiHeader(name='X-Application-Type', is_required=False) +X_DIMENSIONS = LogApiHeader(name='X_Dimensions', is_required=False) diff --git a/monasca_log_api_code/app/controller/api/healthcheck_api.py b/monasca_log_api_code/app/controller/api/healthcheck_api.py new file mode 100644 index 000000000..a1eda8419 --- /dev/null +++ b/monasca_log_api_code/app/controller/api/healthcheck_api.py @@ -0,0 +1,60 @@ +# Copyright 2016 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections + +import falcon +from oslo_log import log + +LOG = log.getLogger(__name__) + +HealthCheckResult = collections.namedtuple('HealthCheckResult', + ['status', 'details']) + + +# TODO(feature) monasca-common candidate +class HealthChecksApi(object): + """HealthChecks Api + + HealthChecksApi server information regarding health of the API. + + """ + + def __init__(self): + super(HealthChecksApi, self).__init__() + LOG.info('Initializing HealthChecksApi!') + + def on_get(self, req, res): + """Complex healthcheck report on GET. + + Returns complex report regarding API well being + and all dependent services. + + :param falcon.Request req: current request + :param falcon.Response res: current response + """ + res.status = falcon.HTTP_501 + + def on_head(self, req, res): + """Simple healthcheck report on HEAD. + + In opposite to :py:meth:`.HealthChecksApi.on_get`, this + method is supposed to execute ASAP to inform user that + API is up and running. + + :param falcon.Request req: current request + :param falcon.Response res: current response + + """ + res.status = falcon.HTTP_501 diff --git a/monasca_log_api_code/app/controller/api/logs_api.py b/monasca_log_api_code/app/controller/api/logs_api.py new file mode 100644 index 000000000..004627665 --- /dev/null +++ b/monasca_log_api_code/app/controller/api/logs_api.py @@ -0,0 +1,88 @@ +# Copyright 2015 kornicameister@gmail.com +# Copyright 2016 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import falcon +from monasca_log_api import conf +from monasca_log_api.monitoring import client +from monasca_log_api.monitoring import metrics +from oslo_log import log + + +CONF = conf.CONF +LOG = log.getLogger(__name__) + + +class LogsApi(object): + """Logs API. + + Logs API acts as RESTful endpoint accepting + messages contains collected log entries from the system. + Works as gateway for any further processing for accepted data. + + """ + def __init__(self): + super(LogsApi, self).__init__() + if CONF.monitoring.enable: + self._statsd = client.get_client() + + # create_common counters, gauges etc. + self._metrics_dimensions = dimensions = {'version': self.version} + + self._logs_in_counter = self._statsd.get_counter( + name=metrics.LOGS_RECEIVED_METRIC, + dimensions=dimensions + ) + self._logs_size_gauge = self._statsd.get_gauge( + name=metrics.LOGS_RECEIVED_BYTE_SIZE_METRICS, + dimensions=dimensions + ) + self._logs_rejected_counter = self._statsd.get_counter( + name=metrics.LOGS_REJECTED_METRIC, + dimensions=dimensions + ) + self._logs_processing_time = self._statsd.get_timer( + name=metrics.LOGS_PROCESSING_TIME_METRIC, + dimensions=dimensions + ) + + LOG.info('Initializing LogsApi %s!' % self.version) + + def on_post(self, req, res): + """Accepts sent logs as text or json. + + Accepts logs sent to resource which should + be sent to kafka queue. + + :param req: current request + :param res: current response + + """ + res.status = falcon.HTTP_501 # pragma: no cover + + def on_get(self, req, res): + """Queries logs matching specified dimension values. + + Performs queries on the underlying log storage + against a time range and set of dimension values. + + :param req: current request + :param res: current response + + """ + res.status = falcon.HTTP_501 # pragma: no cover + + @property + def version(self): + return getattr(self, 'VERSION') diff --git a/monasca_log_api_code/app/controller/api/versions_api.py b/monasca_log_api_code/app/controller/api/versions_api.py new file mode 100644 index 000000000..bdc078f8c --- /dev/null +++ b/monasca_log_api_code/app/controller/api/versions_api.py @@ -0,0 +1,34 @@ +# Copyright 2015 kornicameister@gmail.com +# Copyright 2015 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import falcon +from oslo_log import log + +LOG = log.getLogger(__name__) + + +class VersionsAPI(object): + """Versions API + + VersionsAPI returns information about API itself. + + """ + + def __init__(self): + super(VersionsAPI, self).__init__() + LOG.info('Initializing VersionsAPI!') + + def on_get(self, req, res, version_id): + res.status = falcon.HTTP_501 diff --git a/monasca_log_api_code/app/controller/healthchecks.py b/monasca_log_api_code/app/controller/healthchecks.py new file mode 100644 index 000000000..04cd1b465 --- /dev/null +++ b/monasca_log_api_code/app/controller/healthchecks.py @@ -0,0 +1,60 @@ +# Copyright 2016 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import falcon +from monasca_common.rest import utils as rest_utils + +from monasca_log_api.app.base.validation import validate_authorization +from monasca_log_api.app.controller.api import healthcheck_api +from monasca_log_api.healthcheck import kafka_check + + +class HealthChecks(healthcheck_api.HealthChecksApi): + # response configuration + CACHE_CONTROL = ['must-revalidate', 'no-cache', 'no-store'] + + # response codes + HEALTHY_CODE_GET = falcon.HTTP_OK + HEALTHY_CODE_HEAD = falcon.HTTP_NO_CONTENT + NOT_HEALTHY_CODE = falcon.HTTP_SERVICE_UNAVAILABLE + + def __init__(self): + self._kafka_check = kafka_check.KafkaHealthCheck() + super(HealthChecks, self).__init__() + + def on_head(self, req, res): + validate_authorization(req, ['log_api:healthcheck:head']) + res.status = self.HEALTHY_CODE_HEAD + res.cache_control = self.CACHE_CONTROL + + def on_get(self, req, res): + # at this point we know API is alive, so + # keep up good work and verify kafka status + validate_authorization(req, ['log_api:healthcheck:get']) + kafka_result = self._kafka_check.healthcheck() + + # in case it'd be unhealthy, + # message will contain error string + status_data = { + 'kafka': kafka_result.message + } + + # Really simple approach, ideally that should be + # part of monasca-common with some sort of registration of + # healthchecks concept + + res.status = (self.HEALTHY_CODE_GET + if kafka_result.healthy else self.NOT_HEALTHY_CODE) + res.cache_control = self.CACHE_CONTROL + res.body = rest_utils.as_json(status_data) diff --git a/monasca_log_api_code/app/controller/v2/__init__.py b/monasca_log_api_code/app/controller/v2/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monasca_log_api_code/app/controller/v2/aid/__init__.py b/monasca_log_api_code/app/controller/v2/aid/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monasca_log_api_code/app/controller/v2/aid/service.py b/monasca_log_api_code/app/controller/v2/aid/service.py new file mode 100644 index 000000000..b0a34865e --- /dev/null +++ b/monasca_log_api_code/app/controller/v2/aid/service.py @@ -0,0 +1,157 @@ +# Copyright 2015 kornicameister@gmail.com +# Copyright 2016 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime +from monasca_common.rest import utils as rest_utils +from monasca_log_api import conf +from monasca_log_api.app.base import exceptions +from monasca_log_api.app.base import model +from monasca_log_api.app.base import validation +from oslo_config import cfg +from oslo_log import log + + + + + +LOG = log.getLogger(__name__) +CONF = conf.CONF + +EPOCH_START = datetime.datetime(1970, 1, 1) + + +class LogCreator(object): + """Transforms logs, + + Takes care of transforming information received via + HTTP requests into log and log envelopes objects. + + For more details see following: + + * :py:func:`LogCreator.new_log` + * :py:func:`LogCreator.new_log_envelope` + + """ + + def __init__(self): + self._log = log.getLogger('service.LogCreator') + self._log.info('Initializing LogCreator') + + @staticmethod + def _create_meta_info(tenant_id): + """Creates meta block for log envelope. + + Additionally method accesses oslo configuration, + looking for *service.region* configuration property. + + For more details see :py:data:`service_opts` + + :param tenant_id: ID of the tenant + :type tenant_id: str + :return: meta block + :rtype: dict + + """ + return { + 'tenantId': tenant_id, + 'region': cfg.CONF.service.region + } + + def new_log(self, + application_type, + dimensions, + payload, + content_type='application/json', + validate=True): + """Creates new log object. + + :param str application_type: origin of the log + :param dict dimensions: dictionary of dimensions (any data sent to api) + :param stream payload: stream to read log entry from + :param str content_type: actual content type used to send data to + server + :param bool validate: by default True, marks if log should be validated + :return: log object + :rtype: dict + + :keyword: log_object + """ + + payload = rest_utils.read_body(payload, content_type) + if not payload: + return None + + # normalize_yet_again + application_type = parse_application_type(application_type) + dimensions = parse_dimensions(dimensions) + + if validate: + self._log.debug('Validation enabled, proceeding with validation') + validation.validate_application_type(application_type) + validation.validate_dimensions(dimensions) + + self._log.debug( + 'application_type=%s,dimensions=%s' % ( + application_type, dimensions) + ) + + log_object = {} + if content_type == 'application/json': + log_object.update(payload) + else: + log_object.update({'message': payload}) + + validation.validate_log_message(log_object) + + dimensions['component'] = application_type + log_object.update({'dimensions': dimensions}) + + return log_object + + def new_log_envelope(self, log_object, tenant_id): + return model.Envelope( + log=log_object, + meta=self._create_meta_info(tenant_id) + ) + + +def parse_application_type(app_type): + if app_type: + app_type = app_type.strip() + return app_type if app_type else None + + +def parse_dimensions(dimensions): + if not dimensions: + raise exceptions.HTTPUnprocessableEntity('Dimension are required') + + new_dimensions = {} + dimensions = map(str.strip, dimensions.split(',')) + + for dim in dimensions: + if not dim: + raise exceptions.HTTPUnprocessableEntity( + 'Dimension cannot be empty') + elif ':' not in dim: + raise exceptions.HTTPUnprocessableEntity( + '%s is not a valid dimension' % dim) + + dim = dim.split(':') + name = str(dim[0].strip()) if dim[0] else None + value = str(dim[1].strip()) if dim[1] else None + if name and value: + new_dimensions.update({name: value}) + + return new_dimensions diff --git a/monasca_log_api_code/app/controller/v2/logs.py b/monasca_log_api_code/app/controller/v2/logs.py new file mode 100644 index 000000000..df28fb32b --- /dev/null +++ b/monasca_log_api_code/app/controller/v2/logs.py @@ -0,0 +1,104 @@ +# Copyright 2015 kornicameister@gmail.com +# Copyright 2016-2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import falcon +import six +from monasca_log_api import conf +from monasca_log_api.app.base import log_publisher +from monasca_log_api.app.base.validation import validate_authorization +from monasca_log_api.app.controller.api import headers +from monasca_log_api.app.controller.api import logs_api +from monasca_log_api.app.controller.v2.aid import service + + +CONF = conf.CONF +_DEPRECATED_INFO = ('/v2.0/log/single has been deprecated. ' + 'Please use /v3.0/logs') + + +class Logs(logs_api.LogsApi): + """Logs Api V2.""" + + VERSION = 'v2.0' + SUPPORTED_CONTENT_TYPES = {'application/json', 'text/plain'} + + def __init__(self): + self._log_creator = service.LogCreator() + self._kafka_publisher = log_publisher.LogPublisher() + super(Logs, self).__init__() + + @falcon.deprecated(_DEPRECATED_INFO) + def on_post(self, req, res): + validate_authorization(req, ['log_api:logs:post']) + if CONF.monitoring.enable: + with self._logs_processing_time.time(name=None): + self.process_on_post_request(req, res) + else: + self.process_on_post_request(req, res) + + def process_on_post_request(self, req, res): + try: + req.validate(self.SUPPORTED_CONTENT_TYPES) + tenant_id = (req.project_id if req.project_id + else req.cross_project_id) + + log = self.get_log(request=req) + envelope = self.get_envelope( + log=log, + tenant_id=tenant_id + ) + if CONF.monitoring.enable: + self._logs_size_gauge.send(name=None, + value=int(req.content_length)) + self._logs_in_counter.increment() + except Exception: + # any validation that failed means + # log is invalid and rejected + if CONF.monitoring.enable: + self._logs_rejected_counter.increment() + raise + + self._kafka_publisher.send_message(envelope) + + res.status = falcon.HTTP_204 + res.add_link( + target=str(_get_v3_link(req)), + rel='current', # [RFC5005] + title='V3 Logs', + type_hint='application/json' + ) + res.append_header('DEPRECATED', 'true') + + def get_envelope(self, log, tenant_id): + return self._log_creator.new_log_envelope( + log_object=log, + tenant_id=tenant_id + ) + + def get_log(self, request): + return self._log_creator.new_log( + application_type=request.get_header(*headers.X_APPLICATION_TYPE), + dimensions=request.get_header(*headers.X_DIMENSIONS), + payload=request.stream, + content_type=request.content_type + ) + + +def _get_v3_link(req): + self_uri = req.uri + if six.PY2: + self_uri = self_uri.decode('UTF-8') + base_uri = self_uri.replace(req.relative_uri, '') + return '%s/v3.0/logs' % base_uri diff --git a/monasca_log_api_code/app/controller/v3/__init__.py b/monasca_log_api_code/app/controller/v3/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monasca_log_api_code/app/controller/v3/aid/__init__.py b/monasca_log_api_code/app/controller/v3/aid/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monasca_log_api_code/app/controller/v3/aid/bulk_processor.py b/monasca_log_api_code/app/controller/v3/aid/bulk_processor.py new file mode 100644 index 000000000..11296b7c2 --- /dev/null +++ b/monasca_log_api_code/app/controller/v3/aid/bulk_processor.py @@ -0,0 +1,157 @@ +# Copyright 2016 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from monasca_log_api import conf +from monasca_log_api.app.base import log_publisher +from monasca_log_api.app.base import model +from monasca_log_api.app.base import validation +from oslo_log import log + +LOG = log.getLogger(__name__) +CONF = conf.CONF + + +class BulkProcessor(log_publisher.LogPublisher): + """BulkProcessor for effective log processing and publishing. + + BulkProcessor is customized version of + :py:class:`monasca_log_api.app.base.log_publisher.LogPublisher` + that utilizes processing of bulk request inside single loop. + + """ + + def __init__(self, logs_in_counter=None, logs_rejected_counter=None): + """Initializes BulkProcessor. + + :param logs_in_counter: V3 received logs counter + :param logs_rejected_counter: V3 rejected logs counter + """ + super(BulkProcessor, self).__init__() + + if CONF.monitoring.enable: + assert logs_in_counter is not None + assert logs_rejected_counter is not None + self._logs_in_counter = logs_in_counter + self._logs_rejected_counter = logs_rejected_counter + + self.service_region = CONF.service.region + + def send_message(self, logs, global_dimensions=None, log_tenant_id=None): + """Sends bulk package to kafka + + :param list logs: received logs + :param dict global_dimensions: global dimensions for each log + :param str log_tenant_id: tenant who sent logs + """ + + num_of_msgs = len(logs) if logs else 0 + sent_count = 0 + to_send_msgs = [] + + LOG.debug('Bulk package ', + num_of_msgs, global_dimensions, log_tenant_id) + + try: + for log_el in logs: + t_el = self._transform_message(log_el, + global_dimensions, + log_tenant_id) + if t_el: + to_send_msgs.append(t_el) + if CONF.monitoring.enable: + with self._publish_time_ms.time(name=None): + self._publish(to_send_msgs) + else: + self._publish(to_send_msgs) + + sent_count = len(to_send_msgs) + + except Exception as ex: + LOG.error('Failed to send bulk package ', + num_of_msgs, global_dimensions) + LOG.exception(ex) + raise ex + finally: + if CONF.monitoring.enable: + self._update_counters(len(to_send_msgs), num_of_msgs) + self._after_publish(sent_count, len(to_send_msgs)) + + def _update_counters(self, in_counter, to_send_counter): + rejected_counter = to_send_counter - in_counter + + self._logs_in_counter.increment(value=in_counter) + self._logs_rejected_counter.increment(value=rejected_counter) + + def _transform_message(self, log_element, *args): + try: + validation.validate_log_message(log_element) + + log_envelope = model.Envelope.new_envelope( + log=log_element, + tenant_id=args[1], + region=self.service_region, + dimensions=self._get_dimensions(log_element, + global_dims=args[0]) + ) + + msg_payload = (super(BulkProcessor, self) + ._transform_message(log_envelope)) + + return msg_payload + except Exception as ex: + LOG.error('Log transformation failed, rejecting log') + LOG.exception(ex) + + return None + + def _create_envelope(self, log_element, tenant_id, dimensions=None): + """Create a log envelope. + + :param dict log_element: raw log element + :param str tenant_id: tenant who sent logs + :param dict dimensions: log dimensions + :return: log envelope + :rtype: model.Envelope + + """ + return + + def _get_dimensions(self, log_element, global_dims=None): + """Get the dimensions of log element. + + If global dimensions are specified and passed to this method, + both instances are merged with each other. + + If neither is specified empty dictionary is returned. + + If only local dimensions are specified they are returned without any + additional operations. The last statement applies also + to global dimensions. + + :param dict log_element: raw log instance + :param dict global_dims: global dimensions or None + :return: local dimensions merged with global dimensions + :rtype: dict + """ + local_dims = log_element.get('dimensions', {}) + + if not global_dims: + global_dims = {} + if local_dims: + validation.validate_dimensions(local_dims) + + dimensions = global_dims.copy() + dimensions.update(local_dims) + + return dimensions diff --git a/monasca_log_api_code/app/controller/v3/aid/helpers.py b/monasca_log_api_code/app/controller/v3/aid/helpers.py new file mode 100644 index 000000000..794be4906 --- /dev/null +++ b/monasca_log_api_code/app/controller/v3/aid/helpers.py @@ -0,0 +1,62 @@ +# Copyright 2014 Hewlett-Packard +# Copyright 2015 Cray Inc. All Rights Reserved. +# Copyright 2016 Hewlett Packard Enterprise Development Company LP +# Copyright 2016 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import falcon +from monasca_common.rest import utils as rest_utils +from monasca_log_api.app.base import exceptions +from monasca_log_api.app.base import validation +from oslo_log import log + + +LOG = log.getLogger(__name__) + + +def read_json_msg_body(req): + """Read the json_msg from the http request body and return them as JSON. + + :param req: HTTP request object. + :return: Returns the metrics as a JSON object. + :raises falcon.HTTPBadRequest: + """ + try: + msg = req.stream.read() + json_msg = rest_utils.from_json(msg) + return json_msg + + except rest_utils.exceptions.DataConversionException as ex: + LOG.debug(ex) + raise falcon.HTTPBadRequest('Bad request', + 'Request body is not valid JSON') + except ValueError as ex: + LOG.debug(ex) + raise falcon.HTTPBadRequest('Bad request', + 'Request body is not valid JSON') + + +def get_global_dimensions(request_body): + """Get the top level dimensions in the HTTP request body.""" + global_dims = request_body.get('dimensions', {}) + validation.validate_dimensions(global_dims) + return global_dims + + +def get_logs(request_body): + """Get the logs in the HTTP request body.""" + if 'logs' not in request_body: + raise exceptions.HTTPUnprocessableEntity( + 'Unprocessable Entity Logs not found') + return request_body['logs'] diff --git a/monasca_log_api_code/app/controller/v3/logs.py b/monasca_log_api_code/app/controller/v3/logs.py new file mode 100644 index 000000000..97900849b --- /dev/null +++ b/monasca_log_api_code/app/controller/v3/logs.py @@ -0,0 +1,110 @@ +# Copyright 2016 Hewlett Packard Enterprise Development Company, L.P. +# Copyright 2016-2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import falcon +from monasca_log_api import conf +from monasca_log_api.app.base import exceptions +from monasca_log_api.app.base import validation +from monasca_log_api.app.controller.api import logs_api +from monasca_log_api.app.controller.v3.aid import bulk_processor +from monasca_log_api.app.controller.v3.aid import helpers +from monasca_log_api.monitoring import metrics +from oslo_log import log + + + +CONF = conf.CONF +LOG = log.getLogger(__name__) + + +class Logs(logs_api.LogsApi): + + VERSION = 'v3.0' + SUPPORTED_CONTENT_TYPES = {'application/json'} + + def __init__(self): + super(Logs, self).__init__() + + if CONF.monitoring.enable: + self._processor = bulk_processor.BulkProcessor( + logs_in_counter=self._logs_in_counter, + logs_rejected_counter=self._logs_rejected_counter + ) + self._bulks_rejected_counter = self._statsd.get_counter( + name=metrics.LOGS_BULKS_REJECTED_METRIC, + dimensions=self._metrics_dimensions + ) + else: + self._processor = bulk_processor.BulkProcessor() + + def on_post(self, req, res): + validation.validate_authorization(req, ['log_api:logs:post']) + if CONF.monitoring.enable: + with self._logs_processing_time.time(name=None): + self.process_on_post_request(req, res) + else: + self.process_on_post_request(req, res) + + def process_on_post_request(self, req, res): + try: + req.validate(self.SUPPORTED_CONTENT_TYPES) + + request_body = helpers.read_json_msg_body(req) + + log_list = self._get_logs(request_body) + global_dimensions = self._get_global_dimensions(request_body) + + except Exception as ex: + LOG.error('Entire bulk package has been rejected') + LOG.exception(ex) + if CONF.monitoring.enable: + self._bulks_rejected_counter.increment(value=1) + + raise ex + + if CONF.monitoring.enable: + self._bulks_rejected_counter.increment(value=0) + self._logs_size_gauge.send(name=None, + value=int(req.content_length)) + + tenant_id = (req.cross_project_id if req.cross_project_id + else req.project_id) + + try: + self._processor.send_message( + logs=log_list, + global_dimensions=global_dimensions, + log_tenant_id=tenant_id + ) + except Exception as ex: + res.status = getattr(ex, 'status', falcon.HTTP_500) + return + + res.status = falcon.HTTP_204 + + @staticmethod + def _get_global_dimensions(request_body): + """Get the top level dimensions in the HTTP request body.""" + global_dims = request_body.get('dimensions', {}) + validation.validate_dimensions(global_dims) + return global_dims + + @staticmethod + def _get_logs(request_body): + """Get the logs in the HTTP request body.""" + if 'logs' not in request_body: + raise exceptions.HTTPUnprocessableEntity( + 'Unprocessable Entity Logs not found') + return request_body['logs'] diff --git a/monasca_log_api_code/app/controller/versions.py b/monasca_log_api_code/app/controller/versions.py new file mode 100644 index 000000000..cf3e3e598 --- /dev/null +++ b/monasca_log_api_code/app/controller/versions.py @@ -0,0 +1,128 @@ +# Copyright 2015 kornicameister@gmail.com +# Copyright 2016 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import falcon +import six + +from monasca_common.rest import utils as rest_utils + +from monasca_log_api.app.base.validation import validate_authorization +from monasca_log_api.app.controller.api import versions_api + +_VERSIONS_TPL_DICT = { + 'v2.0': { + 'id': 'v2.0', + 'links': [ + { + 'rel': 'logs', + 'href': '/log/single' + } + ], + 'status': 'DEPRECATED', + 'updated': "2015-09-01T00:00:00Z" + }, + 'v3.0': { + 'id': 'v3.0', + 'links': [ + { + 'rel': 'logs', + 'href': '/logs' + } + ], + 'status': 'CURRENT', + 'updated': "2016-03-01T00:00:00Z" + } +} + + +class Versions(versions_api.VersionsAPI): + """Versions Api""" + + @staticmethod + def handle_none_version_id(req, res, result): + for version in _VERSIONS_TPL_DICT: + selected_version = _parse_version(version, req) + result['elements'].append(selected_version) + res.body = rest_utils.as_json(result, sort_keys=True) + res.status = falcon.HTTP_200 + + @staticmethod + def handle_version_id(req, res, result, version_id): + if version_id in _VERSIONS_TPL_DICT: + result['elements'].append(_parse_version(version_id, req)) + res.body = rest_utils.as_json(result, sort_keys=True) + res.status = falcon.HTTP_200 + else: + error_body = {'message': '%s is not valid version' % version_id} + res.body = rest_utils.as_json(error_body) + res.status = falcon.HTTP_400 + + def on_get(self, req, res, version_id=None): + validate_authorization(req, ['log_api:versions:get']) + result = { + 'links': _get_common_links(req), + 'elements': [] + } + if version_id is None: + self.handle_none_version_id(req, res, result) + else: + self.handle_version_id(req, res, result, version_id) + + +def _get_common_links(req): + self_uri = req.uri + if six.PY2: + self_uri = self_uri.decode(rest_utils.ENCODING) + base_uri = self_uri.replace(req.path, '') + return [ + { + 'rel': 'self', + 'href': self_uri + }, + { + 'rel': 'version', + 'href': '%s/version' % base_uri + }, + { + 'rel': 'healthcheck', + 'href': '%s/healthcheck' % base_uri + } + ] + + +def _parse_version(version_id, req): + self_uri = req.uri + if six.PY2: + self_uri = self_uri.decode(rest_utils.ENCODING) + base_uri = self_uri.replace(req.path, '') + + # need to get template dict, consecutive calls + # needs to operate on unmodified instance + + selected_version = _VERSIONS_TPL_DICT[version_id].copy() + raw_links = selected_version['links'] + links = [] + + for link in raw_links: + raw_link_href = link.get('href') + raw_link_rel = link.get('rel') + link_href = base_uri + '/' + version_id + raw_link_href + links.append({ + 'href': link_href, + 'rel': raw_link_rel + }) + selected_version['links'] = links + + return selected_version diff --git a/monasca_log_api_code/app/main.py b/monasca_log_api_code/app/main.py new file mode 100644 index 000000000..d49bed171 --- /dev/null +++ b/monasca_log_api_code/app/main.py @@ -0,0 +1,46 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Allows to run monasca-log-api from within local [dev] environment. +Primarily used for development. +""" + +import sys +from monasca_log_api import version +from paste import deploy +from paste import httpserver + + +def get_wsgi_app(): + config_dir = 'etc/monasca' + + return deploy.loadapp( + 'config:%s/log-api-paste.ini' % config_dir, + relative_to='./', + name='main' + ) + + +def main(): + wsgi_app = get_wsgi_app() + server_version = 'log-api/%s' % version.version_str + + server = httpserver.serve(application=wsgi_app, host='127.0.0.1', + port=5607, server_version=server_version) + return server + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/monasca_log_api_code/app/wsgi.py b/monasca_log_api_code/app/wsgi.py new file mode 100644 index 000000000..61cd25fcf --- /dev/null +++ b/monasca_log_api_code/app/wsgi.py @@ -0,0 +1,32 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Use this file for deploying the API under mod_wsgi. +""" + +from paste import deploy + +application = None + + +def main(): + base_dir = '/etc/monasca/' + conf = '%slog-api-paste.ini' % base_dir + app = deploy.loadapp('config:%s' % conf) + return app + + +if __name__ == '__main__' or __name__.startswith('_mod_wsgi'): + application = main() diff --git a/monasca_log_api_code/conf/__init__.py b/monasca_log_api_code/conf/__init__.py new file mode 100644 index 000000000..7d4dfe8aa --- /dev/null +++ b/monasca_log_api_code/conf/__init__.py @@ -0,0 +1,70 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import pkgutil + +from oslo_config import cfg +from oslo_log import log +from oslo_utils import importutils + +CONF = cfg.CONF +LOG = log.getLogger(__name__) + + +def load_conf_modules(): + """Loads all modules that contain configuration + + Method iterates over modules of :py:module:`monasca_log_api.conf` + and imports only those that contain following methods: + + - list_opts (required by oslo_config.genconfig) + - register_opts (required by :py:currentmodule:) + + """ + for modname in _list_module_names(): + mod = importutils.import_module('monasca_log_api.conf.' + modname) + required_funcs = ['register_opts', 'list_opts'] + for func in required_funcs: + if hasattr(mod, func): + yield mod + + +def _list_module_names(): + package_path = os.path.dirname(os.path.abspath(__file__)) + for _, modname, ispkg in pkgutil.iter_modules(path=[package_path]): + if not (modname == "opts" and ispkg): + yield modname + + +def register_opts(): + """Registers all conf modules opts + + This method allows different modules to register + opts according to their needs. + + """ + for mod in load_conf_modules(): + mod.register_opts(CONF) + + +def list_opts(): + """Lists all conf modules opts. + + Goes through all conf modules and yields their opts + + """ + for mod in load_conf_modules(): + mod_opts = mod.list_opts() + yield mod_opts[0], mod_opts[1] diff --git a/monasca_log_api_code/conf/healthcheck.py b/monasca_log_api_code/conf/healthcheck.py new file mode 100644 index 000000000..00cf215bd --- /dev/null +++ b/monasca_log_api_code/conf/healthcheck.py @@ -0,0 +1,36 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +kafka_check_opts = [ + cfg.StrOpt('kafka_url', + required=True, + help='Url to kafka server'), + cfg.ListOpt('kafka_topics', + required=True, + default=['logs'], + help='Verify existence of configured topics') +] +kafka_check_group = cfg.OptGroup(name='kafka_healthcheck', + title='kafka_healthcheck') + + +def register_opts(conf): + conf.register_group(kafka_check_group) + conf.register_opts(kafka_check_opts, kafka_check_group) + + +def list_opts(): + return kafka_check_group, kafka_check_opts diff --git a/monasca_log_api_code/conf/log_publisher.py b/monasca_log_api_code/conf/log_publisher.py new file mode 100644 index 000000000..5a7707290 --- /dev/null +++ b/monasca_log_api_code/conf/log_publisher.py @@ -0,0 +1,41 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +_MAX_MESSAGE_SIZE = 1048576 + +log_publisher_opts = [ + cfg.StrOpt('kafka_url', + required=True, + help='Url to kafka server'), + cfg.MultiStrOpt('topics', + default=['logs'], + help='Consumer topics'), + cfg.IntOpt('max_message_size', + default=_MAX_MESSAGE_SIZE, + required=True, + help=('Message max size that can be sent ' + 'to kafka, default to %d bytes' % _MAX_MESSAGE_SIZE)) +] +log_publisher_group = cfg.OptGroup(name='log_publisher', title='log_publisher') + + +def register_opts(conf): + conf.register_group(log_publisher_group) + conf.register_opts(log_publisher_opts, log_publisher_group) + + +def list_opts(): + return log_publisher_group, log_publisher_opts diff --git a/monasca_log_api_code/conf/monitoring.py b/monasca_log_api_code/conf/monitoring.py new file mode 100644 index 000000000..5567a6873 --- /dev/null +++ b/monasca_log_api_code/conf/monitoring.py @@ -0,0 +1,49 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +_DEFAULT_HOST = '127.0.0.1' +_DEFAULT_PORT = 8125 +_DEFAULT_BUFFER_SIZE = 50 + +monitoring_opts = [ + cfg.BoolOpt('enable', + default=True, + help='Determine if self monitoring is enabled'), + cfg.HostAddressOpt('statsd_host', + default=_DEFAULT_HOST, + help=('IP address or host domain name of statsd server, default to %s' + % _DEFAULT_HOST)), + cfg.PortOpt('statsd_port', + default=_DEFAULT_PORT, + help='Port of statsd server, default to %d' % _DEFAULT_PORT), + cfg.IntOpt('statsd_buffer', + default=_DEFAULT_BUFFER_SIZE, + required=True, + help=('Maximum number of metric to buffer before sending, ' + 'default to %d' % _DEFAULT_BUFFER_SIZE)), + cfg.DictOpt('dimensions', default={}, + required=False, help='Additional dimensions that can be set') +] +monitoring_group = cfg.OptGroup(name='monitoring', title='monitoring') + + +def register_opts(conf): + conf.register_group(monitoring_group) + conf.register_opts(monitoring_opts, monitoring_group) + + +def list_opts(): + return monitoring_group, monitoring_opts diff --git a/monasca_log_api_code/conf/role_middleware.py b/monasca_log_api_code/conf/role_middleware.py new file mode 100644 index 000000000..bb460f47a --- /dev/null +++ b/monasca_log_api_code/conf/role_middleware.py @@ -0,0 +1,47 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +role_m_opts = [ + cfg.ListOpt(name='path', + default='/', + help='List of paths where middleware applies to'), + cfg.ListOpt(name='default_roles', + default=['monasca-user'], + help='List of roles allowed to enter api'), + cfg.ListOpt(name='agent_roles', + default=None, + help=('List of roles, that if set, mean that request ' + 'comes from agent, thus is authorized in the same ' + 'time')), + cfg.ListOpt(name='delegate_roles', + default=['admin'], + help=('Roles that are allowed to POST logs on ' + 'behalf of another tenant (project)')), + cfg.ListOpt(name='check_roles', + default=['@'], + help=('Roles that are allowed to do check ' + 'version and health')) +] +role_m_group = cfg.OptGroup(name='roles_middleware', title='roles_middleware') + + +def register_opts(conf): + conf.register_group(role_m_group) + conf.register_opts(role_m_opts, role_m_group) + + +def list_opts(): + return role_m_group, role_m_opts diff --git a/monasca_log_api_code/conf/service.py b/monasca_log_api_code/conf/service.py new file mode 100644 index 000000000..0ce2496d1 --- /dev/null +++ b/monasca_log_api_code/conf/service.py @@ -0,0 +1,37 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +_DEFAULT_MAX_LOG_SIZE = 1024 * 1024 + +service_opts = [ + cfg.StrOpt('region', + default=None, + help='Region'), + cfg.IntOpt('max_log_size', + default=_DEFAULT_MAX_LOG_SIZE, + help=('Refers to payload/envelope size. If either is exceeded' + 'API will throw an error')) +] +service_group = cfg.OptGroup(name='service', title='service') + + +def register_opts(conf): + conf.register_group(service_group) + conf.register_opts(service_opts, service_group) + + +def list_opts(): + return service_group, service_opts diff --git a/monasca_log_api_code/config.py b/monasca_log_api_code/config.py new file mode 100644 index 000000000..2d7871050 --- /dev/null +++ b/monasca_log_api_code/config.py @@ -0,0 +1,82 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +from monasca_log_api import conf +from monasca_log_api import version +from oslo_config import cfg +from oslo_log import log +from oslo_policy import opts as policy_opts + + +CONF = conf.CONF +LOG = log.getLogger(__name__) + +_CONF_LOADED = False +_GUNICORN_MARKER = 'gunicorn' + + +def _is_running_under_gunicorn(): + """Evaluates if api runs under gunicorn""" + content = filter(lambda x: x != sys.executable and _GUNICORN_MARKER in x, + sys.argv or []) + return len(list(content) if not isinstance(content, list) else content) > 0 + + +def get_config_files(): + """Get the possible configuration files accepted by oslo.config + + This also includes the deprecated ones + """ + # default files + conf_files = cfg.find_config_files(project='monasca', + prog='monasca-log-api') + # deprecated config files (only used if standard config files are not there) + if len(conf_files) == 0: + old_conf_files = cfg.find_config_files(project='monasca', + prog='log-api') + if len(old_conf_files) > 0: + LOG.warning('Found deprecated old location "{}" ' + 'of main configuration file'.format(old_conf_files)) + conf_files += old_conf_files + return conf_files + + +def parse_args(argv=None): + global _CONF_LOADED + if _CONF_LOADED: + LOG.debug('Configuration has been already loaded') + return + + log.set_defaults() + log.register_options(CONF) + + argv = (argv if argv is not None else sys.argv[1:]) + args = ([] if _is_running_under_gunicorn() else argv or []) + + CONF(args=args, + prog=sys.argv[1:], + project='monasca', + version=version.version_str, + default_config_files=get_config_files(), + description='RESTful API to collect log files') + + log.setup(CONF, + product_name='monasca-log-api', + version=version.version_str) + + conf.register_opts() + policy_opts.set_defaults(CONF) + + _CONF_LOADED = True diff --git a/monasca_log_api_code/db/__init__.py b/monasca_log_api_code/db/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monasca_log_api_code/db/common/__init__.py b/monasca_log_api_code/db/common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monasca_log_api_code/db/common/model.py b/monasca_log_api_code/db/common/model.py new file mode 100644 index 000000000..60c0cabd1 --- /dev/null +++ b/monasca_log_api_code/db/common/model.py @@ -0,0 +1,43 @@ +# Copyright 2017 StackHPC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections + + +class Dimension(collections.namedtuple('Dimension', 'name values')): + """Representation of dimension names and optional values list. + + Named-tuple type to represent the pairing of a dimension name and an + optional list of values. + + :ivar name: Name of the dimension to reference. + :ivar values: Optional list of values associated with the dimension. + + :vartype name: str + :vartype values: None or list[str] + """ + + +class SortBy(collections.namedtuple('SortBy', 'field direction')): + """Representation of an individual sorting directive. + + Named-tuple type to represent a directive for indicating how a result set + should be sorted. + + :ivar field: Name of the field which is provides the values to sort by. + :ivar direction: Either 'asc' or 'desc' specifying the order of values. + + :vartype name: str + :vartype values: str + """ diff --git a/monasca_log_api_code/db/repo/__init__.py b/monasca_log_api_code/db/repo/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monasca_log_api_code/db/repo/logs_repository.py b/monasca_log_api_code/db/repo/logs_repository.py new file mode 100644 index 000000000..64e9936f7 --- /dev/null +++ b/monasca_log_api_code/db/repo/logs_repository.py @@ -0,0 +1,74 @@ +# Copyright 2017 StackHPC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class LogsRepository(object): + + def __init__(self): + super(LogsRepository, self).__init__() + + @abc.abstractmethod + def list_logs(self, tenant_id, dimensions, start_time, end_time, offset, + limit, sort_by): + """Obtain log listing based on simple criteria of dimension values. + + Performs queries on the underlying log storage against a time range and + set of dimension values. Additionally, it is possible to optionally + sort results by timestamp. + + :param tenant_id: + Tenant/project id for which to obtain logs (required). + :param dimensions: + List of Dimension tuples containing pairs of dimension names and + optional lists of dimension values. These will be used to filter + the logs returned. If no dimensions are specified, then no + filtering is performed. When multiple values are given, the + dimension must match any of the given values. If None is given, + logs with any value for the dimension will be returned. + :param start_time: + Optional starting time in UNIX time (seconds, inclusive). + :param end_time: + Optional ending time in UNIX time (seconds, inclusive). + :param offset: + Number of matching results to skip past, if specified. + :param limit: + Number of matching results to return (required). + :param sort_by: + List of SortBy tuples specifying fields to sort by and the + direction to sort the result set by. e.g. ('timestamp','asc'). The + direction is specified by either the string 'asc' for ascending + direction, or 'desc' for descending. If not specified, no order + must be enforced and the implementation is free to choose the most + efficient method to return the results. + + :type tenant_id: str + :type dimensions: None or list[Dimension[str, list[str] or None]] + :type start_time: None or int + :type end_time: None or int + :type offset: None or int + :type limit: int + :type sort_by: None or list[SortBy[str, str]] + + :return: + Log messages matching the given criteria. The dict representing + each message entry will contain attributes extracted from the + underlying structure; 'message', 'timestamp' and 'dimensions'. + + :rtype: list[dict] + """ + pass diff --git a/monasca_log_api_code/healthcheck/__init__.py b/monasca_log_api_code/healthcheck/__init__.py new file mode 100644 index 000000000..72fb77504 --- /dev/null +++ b/monasca_log_api_code/healthcheck/__init__.py @@ -0,0 +1 @@ +"""Base package for monasca-log-api healthcheck""" diff --git a/monasca_log_api_code/healthcheck/kafka_check.py b/monasca_log_api_code/healthcheck/kafka_check.py new file mode 100644 index 000000000..058c6cb95 --- /dev/null +++ b/monasca_log_api_code/healthcheck/kafka_check.py @@ -0,0 +1,98 @@ +# Copyright 2015-2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +from monasca_common.kafka_lib import client +from monasca_log_api import conf +from oslo_log import log +from six import PY3 + + +LOG = log.getLogger(__name__) +CONF = conf.CONF + + +CheckResult = collections.namedtuple('CheckResult', ['healthy', 'message']) +"""Result from the healthcheck, contains healthy(boolean) and message""" + + +# TODO(feature) monasca-common candidate +class KafkaHealthCheck(object): + """Evaluates kafka health + + Healthcheck verifies if: + + * kafka server is up and running + * there is a configured topic in kafka + + If following conditions are met healthcheck returns healthy status. + Otherwise unhealthy status is returned with explanation. + + Example of middleware configuration: + + .. code-block:: ini + + [kafka_healthcheck] + kafka_url = localhost:8900 + kafka_topics = log + + Note: + It is possible to specify multiple topics if necessary. + Just separate them with , + + """ + + def healthcheck(self): + url = CONF.kafka_healthcheck.kafka_url + + try: + kafka_client = client.KafkaClient(hosts=url) + except client.KafkaUnavailableError as ex: + LOG.error(repr(ex)) + error_str = 'Could not connect to kafka at %s' % url + return CheckResult(healthy=False, message=error_str) + + result = self._verify_topics(kafka_client) + self._disconnect_gracefully(kafka_client) + + return result + + # noinspection PyMethodMayBeStatic + def _verify_topics(self, kafka_client): + topics = CONF.kafka_healthcheck.kafka_topics + + if PY3: + topics = tuple(topic.encode('utf-8') for topic in topics) + + for t in topics: + # kafka client loads metadata for topics as fast + # as possible (happens in __init__), therefore this + # topic_partitions is sure to be filled + for_topic = t in kafka_client.topic_partitions + if not for_topic: + error_str = 'Kafka: Topic %s not found' % t + LOG.error(error_str) + return CheckResult(healthy=False, message=error_str) + + return CheckResult(healthy=True, message='OK') + + # noinspection PyMethodMayBeStatic + def _disconnect_gracefully(self, kafka_client): + # at this point, client is connected so it must be closed + # regardless of topic existence + try: + kafka_client.close() + except Exception as ex: + # log that something went wrong and move on + LOG.error(repr(ex)) diff --git a/monasca_log_api_code/middleware/__init__.py b/monasca_log_api_code/middleware/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monasca_log_api_code/middleware/role_middleware.py b/monasca_log_api_code/middleware/role_middleware.py new file mode 100644 index 000000000..20e27e66e --- /dev/null +++ b/monasca_log_api_code/middleware/role_middleware.py @@ -0,0 +1,152 @@ +# Copyright 2015-2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from monasca_log_api import conf +from oslo_log import log +from oslo_middleware import base as om +from webob import response + + +CONF = conf.CONF +LOG = log.getLogger(__name__) + +_X_IDENTITY_STATUS = 'X-Identity-Status' +_X_ROLES = 'X-Roles' +_X_MONASCA_LOG_AGENT = 'X-MONASCA-LOG-AGENT' +_CONFIRMED_STATUS = 'Confirmed' + + +def _ensure_lower_roles(roles): + if not roles: + return [] + return [role.strip().lower() for role in roles] + + +def _intersect(a, b): + return list(set(a) & set(b)) + + +class RoleMiddleware(om.ConfigurableMiddleware): + """Authorization middleware for X-Roles header. + + RoleMiddleware is responsible for authorizing user's + access against **X-Roles** header. Middleware + expects authentication to be completed (i.e. keystone middleware + has been already called). + + If tenant is authenticated and authorized middleware + exits silently (that is considered a success). Otherwise + middleware produces JSON response according to following schema + + .. code-block:: javascript + + { + 'title': u'Unauthorized', + 'message': explanation (str) + } + + Configuration example + + .. code-block:: cfg + + [roles_middleware] + path = /v2.0/log + default_roles = monasca-user + agent_roles = monasca-log-agent + delegate_roles = admin + + Configuration explained: + + * path (list) - path (or list of paths) middleware should be applied + * agent_roles (list) - list of roles that identifies tenant as an agent + * default_roles (list) - list of roles that should be authorized + * delegate_roles (list) - list of roles that are allowed to POST logs on + behalf of another tenant (project) + + Note: + Being an agent means that tenant is automatically authorized. + Note: + Middleware works only for configured paths and for all + requests apart from HTTP method **OPTIONS**. + + """ + + def __init__(self, application, conf=None): + super(RoleMiddleware, self).__init__(application, conf) + middleware = CONF.roles_middleware + + self._path = middleware.path + self._default_roles = _ensure_lower_roles(middleware.default_roles) + self._agent_roles = _ensure_lower_roles(middleware.agent_roles) + + LOG.debug('RolesMiddleware initialized for paths=%s', self._path) + + def process_request(self, req): + if not self._can_apply_middleware(req): + LOG.debug('%s skipped in role middleware', req.path) + return None + + is_authenticated = self._is_authenticated(req) + is_agent = self._is_agent(req) + tenant_id = req.headers.get('X-Tenant-Id') + + req.environ[_X_MONASCA_LOG_AGENT] = is_agent + + LOG.debug('%s is authenticated=%s, log_agent=%s', + tenant_id, is_authenticated, is_agent) + + if is_authenticated: + LOG.debug('%s has been authenticated', tenant_id) + return # do return nothing to enter API internal + + explanation = u'Failed to authenticate request for %s' % tenant_id + LOG.error(explanation) + json_body = {u'title': u'Unauthorized', u'message': explanation} + return response.Response(status=401, + json_body=json_body, + content_type='application/json') + + def _is_agent(self, req): + headers = req.headers + roles = headers.get(_X_ROLES) + + if not roles: + LOG.warning('Couldn\'t locate %s header,or it was empty', _X_ROLES) + return False + else: + roles = _ensure_lower_roles(roles.split(',')) + + is_agent = len(_intersect(roles, self._agent_roles)) > 0 + + return is_agent + + def _is_authenticated(self, req): + headers = req.headers + if _X_IDENTITY_STATUS in headers: + status = req.headers.get(_X_IDENTITY_STATUS) + return _CONFIRMED_STATUS == status + return False + + def _can_apply_middleware(self, req): + path = req.path + method = req.method + + if method == 'OPTIONS': + return False + + if self._path: + for p in self._path: + if path.startswith(p): + return True + return False # if no configured paths, or nothing matches diff --git a/monasca_log_api_code/monitoring/__init__.py b/monasca_log_api_code/monitoring/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monasca_log_api_code/monitoring/client.py b/monasca_log_api_code/monitoring/client.py new file mode 100644 index 000000000..0afbd8479 --- /dev/null +++ b/monasca_log_api_code/monitoring/client.py @@ -0,0 +1,78 @@ +# Copyright 2016-2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import monascastatsd +from monasca_log_api import conf +from oslo_log import log + +LOG = log.getLogger(__name__) +CONF = conf.CONF + +_CLIENT_NAME = 'monasca' +_DEFAULT_DIMENSIONS = { + 'service': 'monitoring', + 'component': 'monasca-log-api' +} + + +def get_client(dimensions=None): + """Creates statsd client + + Creates monasca-statsd client using configuration from + config file and supplied dimensions. + + Configuration is composed out of :: + + [monitoring] + statsd_host = 192.168.10.4 + statsd_port = 8125 + statsd_buffer = 50 + + Dimensions are appended to following dictionary :: + + { + 'service': 'monitoring', + 'component': 'monasca-log-api' + } + + Note: + Passed dimensions do not override those specified in + dictionary above + + :param dict dimensions: Optional dimensions + :return: statsd client + :rtype: monascastatsd.Client + """ + dims = _DEFAULT_DIMENSIONS.copy() + if dimensions: + for key, val in dimensions.items(): + if key not in _DEFAULT_DIMENSIONS: + dims[key] = val + else: + LOG.warning('Cannot override fixed dimension %s=%s', key, + _DEFAULT_DIMENSIONS[key]) + + connection = monascastatsd.Connection( + host=CONF.monitoring.statsd_host, + port=CONF.monitoring.statsd_port, + max_buffer_size=CONF.monitoring.statsd_buffer + ) + client = monascastatsd.Client(name=_CLIENT_NAME, + connection=connection, + dimensions=dims) + + LOG.debug('Created statsd client %s[%s] = %s:%d', _CLIENT_NAME, dims, + CONF.monitoring.statsd_host, CONF.monitoring.statsd_port) + + return client diff --git a/monasca_log_api_code/monitoring/metrics.py b/monasca_log_api_code/monitoring/metrics.py new file mode 100644 index 000000000..97e3677bc --- /dev/null +++ b/monasca_log_api_code/monitoring/metrics.py @@ -0,0 +1,47 @@ +# Copyright 2016 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +LOGS_RECEIVED_METRIC = 'log.in_logs' +"""Metrics sent with amount of logs (not requests) API receives""" + +LOGS_REJECTED_METRIC = 'log.in_logs_rejected' +"""Metric sent with amount of logs that were rejected +(i.e. invalid dimension)""" + +LOGS_BULKS_REJECTED_METRIC = 'log.in_bulks_rejected' +"""Metric sent with amount of bulk packages that were rejected due +to early stage validation (content-length, content-type). +Only valid for v3.0. +""" + +LOGS_RECEIVED_BYTE_SIZE_METRICS = 'log.in_logs_bytes' +"""Metric sent with size of payloads(a.k.a. Content-Length) + (in bytes) API receives""" + +LOGS_PROCESSING_TIME_METRIC = 'log.processing_time_ms' +"""Metric sent with time that log-api needed to process each received log. +Metric does not include time needed to authorize requests.""" + +LOGS_PUBLISHED_METRIC = 'log.out_logs' +"""Metric sent with amount of logs published to kafka""" + +LOGS_PUBLISHED_LOST_METRIC = 'log.out_logs_lost' +"""Metric sent with amount of logs that were lost due to critical error in +publish phase.""" + +LOGS_PUBLISH_TIME_METRIC = 'log.publish_time_ms' +"""Metric sent with time that publishing took""" + +LOGS_TRUNCATED_METRIC = 'log.out_logs_truncated_bytes' +"""Metric sent with amount of truncated bytes from log message""" diff --git a/monasca_log_api_code/policies/__init__.py b/monasca_log_api_code/policies/__init__.py new file mode 100644 index 000000000..645ccbfea --- /dev/null +++ b/monasca_log_api_code/policies/__init__.py @@ -0,0 +1,73 @@ +# Copyright 2017 FUJITSU LIMITED +# Copyright 2018 OP5 AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import pkgutil +from monasca_log_api.conf import role_middleware +from oslo_config import cfg +from oslo_log import log +from oslo_utils import importutils + + + +LOG = log.getLogger(__name__) +_BASE_MOD_PATH = 'monasca_log_api.policies.' +CONF = cfg.CONF + + +def roles_list_to_check_str(roles_list): + if roles_list: + converted_roles_list = ["role:" + role if role != '@' else role for role in roles_list] + return ' or '.join(converted_roles_list) + else: + return None + + +role_middleware.register_opts(CONF) + + +def load_policy_modules(): + """Load all modules that contain policies. + + Method iterates over modules of :py:mod:`monasca_events_api.policies` + and imports only those that contain following methods: + + - list_rules + + """ + for modname in _list_module_names(): + mod = importutils.import_module(_BASE_MOD_PATH + modname) + if hasattr(mod, 'list_rules'): + yield mod + + +def _list_module_names(): + package_path = os.path.dirname(os.path.abspath(__file__)) + for _, modname, ispkg in pkgutil.iter_modules(path=[package_path]): + if not (modname == "opts" and ispkg): + yield modname + + +def list_rules(): + """List all policy modules rules. + + Goes through all policy modules and yields their rules + + """ + all_rules = [] + for mod in load_policy_modules(): + rules = mod.list_rules() + all_rules.extend(rules) + return all_rules diff --git a/monasca_log_api_code/policies/healthchecks.py b/monasca_log_api_code/policies/healthchecks.py new file mode 100644 index 000000000..adc8e5531 --- /dev/null +++ b/monasca_log_api_code/policies/healthchecks.py @@ -0,0 +1,44 @@ +# Copyright 2018 OP5 AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from monasca_log_api import policies +from oslo_config import cfg +from oslo_policy import policy + + +CHECK_AUTHORIZED_ROLES = policies.roles_list_to_check_str( + cfg.CONF.roles_middleware.check_roles) + +rules = [ + policy.DocumentedRuleDefault( + name='log_api:healthcheck:head', + check_str=CHECK_AUTHORIZED_ROLES, + description='Healthcheck head rule', + operations=[ + {'path': '/healthcheck', 'method': 'HEAD'} + ] + ), + policy.DocumentedRuleDefault( + name='log_api:healthcheck:get', + check_str=CHECK_AUTHORIZED_ROLES, + description='Healthcheck get rule', + operations=[ + {'path': '/healthcheck', 'method': 'GET'} + ] + ), +] + + +def list_rules(): + return rules diff --git a/monasca_log_api_code/policies/logs.py b/monasca_log_api_code/policies/logs.py new file mode 100644 index 000000000..0b8cf3cd7 --- /dev/null +++ b/monasca_log_api_code/policies/logs.py @@ -0,0 +1,43 @@ +# Copyright 2018 OP5 AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from monasca_log_api import policies +from oslo_config import cfg +from oslo_policy import policy + + +DEFAULT_AUTHORIZED_ROLES = policies.roles_list_to_check_str( + cfg.CONF.roles_middleware.default_roles) +AGENT_AUTHORIZED_ROLES = policies.roles_list_to_check_str( + cfg.CONF.roles_middleware.agent_roles) +DELEGATE_AUTHORIZED_ROLES = policies.roles_list_to_check_str( + cfg.CONF.roles_middleware.delegate_roles) + +rules = [ + policy.DocumentedRuleDefault( + name='log_api:logs:post', + check_str=' or '.join(filter(None, [AGENT_AUTHORIZED_ROLES, + DEFAULT_AUTHORIZED_ROLES, + DELEGATE_AUTHORIZED_ROLES])), + description='Logs post rule', + operations=[ + {'path': '/logs', 'method': 'POST'}, + {'path': '/log/single', 'method': 'POST'} + ] + ) +] + + +def list_rules(): + return rules diff --git a/monasca_log_api_code/policies/versions.py b/monasca_log_api_code/policies/versions.py new file mode 100644 index 000000000..fa0757d66 --- /dev/null +++ b/monasca_log_api_code/policies/versions.py @@ -0,0 +1,38 @@ +# Copyright 2018 OP5 AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from monasca_log_api import policies +from oslo_config import cfg +from oslo_policy import policy + + +CHECK_AUTHORIZED_ROLES = policies.roles_list_to_check_str( + cfg.CONF.roles_middleware.check_roles) + +rules = [ + policy.DocumentedRuleDefault( + name='log_api:versions:get', + check_str=CHECK_AUTHORIZED_ROLES, + description='Versions get rule', + operations=[ + {'path': '/', 'method': 'GET'}, + {'path': '/version', 'method': 'GET'}, + {'path': '/version/{version_id}', 'method': 'GET'} + ] + ) +] + + +def list_rules(): + return rules diff --git a/monasca_log_api_code/tests/__init__.py b/monasca_log_api_code/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/monasca_log_api_code/tests/base.py b/monasca_log_api_code/tests/base.py new file mode 100644 index 000000000..ca11e19cb --- /dev/null +++ b/monasca_log_api_code/tests/base.py @@ -0,0 +1,220 @@ +# coding=utf-8 +# Copyright 2015 kornicameister@gmail.com +# Copyright 2015-2017 FUJITSU LIMITED +# Copyright 2018 OP5 AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import codecs +import os +import random +import string +import falcon +import fixtures +import mock +import six +from falcon import testing +from monasca_common.policy import policy_engine as policy +from monasca_log_api import conf +from monasca_log_api import config +from monasca_log_api import policies +from monasca_log_api.app.base import request +from oslo_config import fixture as oo_cfg +from oslo_context import fixture as oo_ctx +from oslo_serialization import jsonutils +from oslotest import base as oslotest_base + + + + + +policy.POLICIES = policies + + +class MockedAPI(falcon.API): + """MockedAPI + + Subclasses :py:class:`falcon.API` in order to overwrite + request_type property with custom :py:class:`request.Request` + + """ + + def __init__(self): + super(MockedAPI, self).__init__( + media_type=falcon.DEFAULT_MEDIA_TYPE, + request_type=request.Request, + response_type=falcon.Response, + middleware=None, + router=None + ) + + +def generate_unique_message(size): + letters = string.ascii_letters + + def rand(amount, space=True): + space = ' ' if space else '' + return ''.join((random.choice(letters + space) for _ in range(amount))) + + return rand(size) + + +def _hex_to_unicode(hex_raw): + hex_raw = six.b(hex_raw.replace(' ', '')) + hex_str_raw = codecs.getdecoder('hex')(hex_raw)[0] + hex_str = hex_str_raw.decode('utf-8', 'replace') + return hex_str + +# NOTE(trebskit) => http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt +UNICODE_MESSAGES = [ + # Unicode is evil... + {'case': 'arabic', 'input': 'يونيكود هو الشر'}, + {'case': 'polish', 'input': 'Unicode to zło'}, + {'case': 'greek', 'input': 'Unicode είναι κακό'}, + {'case': 'portuguese', 'input': 'Unicode é malvado'}, + {'case': 'lao', 'input': 'unicode ເປັນຄວາມຊົ່ວຮ້າຍ'}, + {'case': 'german', 'input': 'Unicode ist böse'}, + {'case': 'japanese', 'input': 'ユニコードは悪です'}, + {'case': 'russian', 'input': 'Unicode - зло'}, + {'case': 'urdu', 'input': 'یونیسیڈ برائی ہے'}, + {'case': 'weird', 'input': '🆄🅽🅸🅲🅾🅳🅴 🅸🆂 🅴🆅🅸🅻...'}, # funky, huh ? + # conditions from link above + # 2.3 Other boundary conditions + {'case': 'stress_2_3_1', 'input': _hex_to_unicode('ed 9f bf')}, + {'case': 'stress_2_3_2', 'input': _hex_to_unicode('ee 80 80')}, + {'case': 'stress_2_3_3', 'input': _hex_to_unicode('ef bf bd')}, + {'case': 'stress_2_3_4', 'input': _hex_to_unicode('f4 8f bf bf')}, + {'case': 'stress_2_3_5', 'input': _hex_to_unicode('f4 90 80 80')}, + # 3.5 Impossible byes + {'case': 'stress_3_5_1', 'input': _hex_to_unicode('fe')}, + {'case': 'stress_3_5_2', 'input': _hex_to_unicode('ff')}, + {'case': 'stress_3_5_3', 'input': _hex_to_unicode('fe fe ff ff')}, + # 4.1 Examples of an overlong ASCII character + {'case': 'stress_4_1_1', 'input': _hex_to_unicode('c0 af')}, + {'case': 'stress_4_1_2', 'input': _hex_to_unicode('e0 80 af')}, + {'case': 'stress_4_1_3', 'input': _hex_to_unicode('f0 80 80 af')}, + {'case': 'stress_4_1_4', 'input': _hex_to_unicode('f8 80 80 80 af')}, + {'case': 'stress_4_1_5', 'input': _hex_to_unicode('fc 80 80 80 80 af')}, + # 4.2 Maximum overlong sequences + {'case': 'stress_4_2_1', 'input': _hex_to_unicode('c1 bf')}, + {'case': 'stress_4_2_2', 'input': _hex_to_unicode('e0 9f bf')}, + {'case': 'stress_4_2_3', 'input': _hex_to_unicode('f0 8f bf bf')}, + {'case': 'stress_4_2_4', 'input': _hex_to_unicode('f8 87 bf bf bf')}, + {'case': 'stress_4_2_5', 'input': _hex_to_unicode('fc 83 bf bf bf bf')}, + # 4.3 Overlong representation of the NUL character + {'case': 'stress_4_3_1', 'input': _hex_to_unicode('c0 80')}, + {'case': 'stress_4_3_2', 'input': _hex_to_unicode('e0 80 80')}, + {'case': 'stress_4_3_3', 'input': _hex_to_unicode('f0 80 80 80')}, + {'case': 'stress_4_3_4', 'input': _hex_to_unicode('f8 80 80 80 80')}, + {'case': 'stress_4_3_5', 'input': _hex_to_unicode('fc 80 80 80 80 80')}, + # and some cheesy example from polish novel 'Pan Tadeusz' + {'case': 'mr_t', 'input': 'Hajże na Soplicę!'}, + # it won't be complete without that one + {'case': 'mr_b', 'input': 'Grzegorz Brzęczyszczykiewicz, ' + 'Chrząszczyżewoszyce, powiat Łękołody'}, + # great success, christmas time + {'case': 'olaf', 'input': '☃'} +] + + +class DisableStatsdFixture(fixtures.Fixture): + + def setUp(self): + super(DisableStatsdFixture, self).setUp() + statsd_patch = mock.patch('monascastatsd.Connection') + statsd_patch.start() + self.addCleanup(statsd_patch.stop) + + +class ConfigFixture(oo_cfg.Config): + """Mocks configuration""" + + def __init__(self): + super(ConfigFixture, self).__init__(config.CONF) + + def setUp(self): + super(ConfigFixture, self).setUp() + self.addCleanup(self._clean_config_loaded_flag) + conf.register_opts() + self._set_defaults() + config.parse_args(argv=[]) # prevent oslo from parsing test args + + @staticmethod + def _clean_config_loaded_flag(): + config._CONF_LOADED = False + + def _set_defaults(self): + self.conf.set_default('kafka_url', '127.0.0.1', 'kafka_healthcheck') + self.conf.set_default('kafka_url', '127.0.0.1', 'log_publisher') + + +class PolicyFixture(fixtures.Fixture): + + """Override the policy with a completely new policy file. + + This overrides the policy with a completely fake and synthetic + policy file. + + """ + + def setUp(self): + super(PolicyFixture, self).setUp() + self._prepare_policy() + policy.reset() + policy.init() + + def _prepare_policy(self): + policy_dir = self.useFixture(fixtures.TempDir()) + policy_file = os.path.join(policy_dir.path, 'policy.yaml') + # load the fake_policy data and add the missing default rules. + policy_rules = jsonutils.loads('{}') + self.add_missing_default_rules(policy_rules) + with open(policy_file, 'w') as f: + jsonutils.dump(policy_rules, f) + + BaseTestCase.conf_override(policy_file=policy_file, group='oslo_policy') + BaseTestCase.conf_override(policy_dirs=[], group='oslo_policy') + + @staticmethod + def add_missing_default_rules(rules): + for rule in policies.list_rules(): + if rule.name not in rules: + rules[rule.name] = rule.check_str + + +class BaseTestCase(oslotest_base.BaseTestCase): + + def setUp(self): + super(BaseTestCase, self).setUp() + self.useFixture(ConfigFixture()) + self.useFixture(DisableStatsdFixture()) + self.useFixture(oo_ctx.ClearRequestContext()) + self.useFixture(PolicyFixture()) + + @staticmethod + def conf_override(**kw): + """Override flag variables for a test.""" + group = kw.pop('group', None) + for k, v in kw.items(): + config.CONF.set_override(k, v, group) + + @staticmethod + def conf_default(**kw): + """Override flag variables for a test.""" + group = kw.pop('group', None) + for k, v in kw.items(): + config.CONF.set_default(k, v, group) + + +class BaseApiTestCase(BaseTestCase, testing.TestBase): + api_class = MockedAPI diff --git a/monasca_log_api_code/tests/test_config.py b/monasca_log_api_code/tests/test_config.py new file mode 100644 index 000000000..20f16fe6a --- /dev/null +++ b/monasca_log_api_code/tests/test_config.py @@ -0,0 +1,40 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from monasca_log_api import config +from monasca_log_api.tests import base + + +class TestConfig(base.BaseTestCase): + + @mock.patch('monasca_log_api.config.sys') + def test_should_return_true_if_runs_under_gunicorn(self, sys_patch): + sys_patch.argv = [ + '/bin/gunicorn', + '--capture-output', + '--paste', + 'etc/monasca/log-api-paste.ini', + '--workers', + '1' + ] + sys_patch.executable = '/bin/python' + self.assertTrue(config._is_running_under_gunicorn()) + + @mock.patch('monasca_log_api.config.sys') + def test_should_return_false_if_runs_without_gunicorn(self, sys_patch): + sys_patch.argv = ['/bin/monasca-log-api'] + sys_patch.executable = '/bin/python' + self.assertFalse(config._is_running_under_gunicorn()) diff --git a/monasca_log_api_code/tests/test_healthchecks.py b/monasca_log_api_code/tests/test_healthchecks.py new file mode 100644 index 000000000..a982be188 --- /dev/null +++ b/monasca_log_api_code/tests/test_healthchecks.py @@ -0,0 +1,75 @@ +# Copyright 2016 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import falcon +import mock +import simplejson as json + +from monasca_log_api.app.controller import healthchecks +from monasca_log_api.healthcheck import kafka_check as healthcheck +from monasca_log_api.tests import base + +ENDPOINT = '/healthcheck' + + +class TestApiHealthChecks(base.BaseApiTestCase): + + def before(self): + self.resource = healthchecks.HealthChecks() + self.api.add_route( + ENDPOINT, + self.resource + ) + + def test_should_return_200_for_head(self): + self.simulate_request(ENDPOINT, method='HEAD') + self.assertEqual(falcon.HTTP_NO_CONTENT, self.srmock.status) + + @mock.patch('monasca_log_api.healthcheck.kafka_check.KafkaHealthCheck') + def test_should_report_healthy_if_kafka_healthy(self, kafka_check): + kafka_check.healthcheck.return_value = healthcheck.CheckResult(True, + 'OK') + self.resource._kafka_check = kafka_check + + ret = self.simulate_request(ENDPOINT, + headers={ + 'Content-Type': 'application/json' + }, + decode='utf8', + method='GET') + self.assertEqual(falcon.HTTP_OK, self.srmock.status) + + ret = json.loads(ret) + self.assertIn('kafka', ret) + self.assertEqual('OK', ret.get('kafka')) + + @mock.patch('monasca_log_api.healthcheck.kafka_check.KafkaHealthCheck') + def test_should_report_unhealthy_if_kafka_healthy(self, kafka_check): + url = 'localhost:8200' + err_str = 'Could not connect to kafka at %s' % url + kafka_check.healthcheck.return_value = healthcheck.CheckResult(False, + err_str) + self.resource._kafka_check = kafka_check + + ret = self.simulate_request(ENDPOINT, + headers={ + 'Content-Type': 'application/json' + }, + decode='utf8', + method='GET') + self.assertEqual(falcon.HTTP_SERVICE_UNAVAILABLE, self.srmock.status) + + ret = json.loads(ret) + self.assertIn('kafka', ret) + self.assertEqual(err_str, ret.get('kafka')) diff --git a/monasca_log_api_code/tests/test_kafka_check.py b/monasca_log_api_code/tests/test_kafka_check.py new file mode 100644 index 000000000..21f7dbe1d --- /dev/null +++ b/monasca_log_api_code/tests/test_kafka_check.py @@ -0,0 +1,71 @@ +# Copyright 2015-2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from monasca_common.kafka_lib import client + +from monasca_log_api.healthcheck import kafka_check as kc +from monasca_log_api.tests import base + + +class KafkaCheckLogicTest(base.BaseTestCase): + + mock_kafka_url = 'localhost:1234' + mocked_topics = ['test_1', 'test_2'] + mock_config = { + 'kafka_url': mock_kafka_url, + 'kafka_topics': mocked_topics + } + + def setUp(self): + super(KafkaCheckLogicTest, self).setUp() + self.conf_default(group='kafka_healthcheck', **self.mock_config) + + @mock.patch('monasca_log_api.healthcheck.kafka_check.client.KafkaClient') + def test_should_fail_kafka_unavailable(self, kafka_client): + kafka_client.side_effect = client.KafkaUnavailableError() + kafka_health = kc.KafkaHealthCheck() + result = kafka_health.healthcheck() + + self.assertFalse(result.healthy) + + @mock.patch('monasca_log_api.healthcheck.kafka_check.client.KafkaClient') + def test_should_fail_topic_missing(self, kafka_client): + kafka = mock.Mock() + kafka.topic_partitions = [self.mocked_topics[0]] + kafka_client.return_value = kafka + + kafka_health = kc.KafkaHealthCheck() + result = kafka_health.healthcheck() + + # verify result + self.assertFalse(result.healthy) + + # ensure client was closed + self.assertTrue(kafka.close.called) + + @mock.patch('monasca_log_api.healthcheck.kafka_check.client.KafkaClient') + def test_should_pass(self, kafka_client): + kafka = mock.Mock() + kafka.topic_partitions = self.mocked_topics + kafka_client.return_value = kafka + + kafka_health = kc.KafkaHealthCheck() + result = kafka_health.healthcheck() + + self.assertTrue(result) + + # ensure client was closed + self.assertTrue(kafka.close.called) diff --git a/monasca_log_api_code/tests/test_log_publisher.py b/monasca_log_api_code/tests/test_log_publisher.py new file mode 100644 index 000000000..299dc6cbd --- /dev/null +++ b/monasca_log_api_code/tests/test_log_publisher.py @@ -0,0 +1,294 @@ +# Copyright 2015 kornicameister@gmail.com +# Copyright 2016-2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import datetime +import random +import ujson +import unittest +import mock +import six +from monasca_log_api.app.base import log_publisher +from monasca_log_api.app.base import model +from monasca_log_api.tests import base +from oslo_config import cfg +from oslo_log import log + + + + +LOG = log.getLogger(__name__) +EPOCH_START = datetime.datetime(1970, 1, 1) + + +class TestSendMessage(base.BaseTestCase): + + @mock.patch('monasca_log_api.app.base.log_publisher.producer' + '.KafkaProducer') + def test_should_not_send_empty_message(self, _): + instance = log_publisher.LogPublisher() + + instance._kafka_publisher = mock.Mock() + instance.send_message({}) + + self.assertFalse(instance._kafka_publisher.publish.called) + + @unittest.expectedFailure + def test_should_not_send_message_not_dict(self): + instance = log_publisher.LogPublisher() + not_dict_value = 123 + instance.send_message(not_dict_value) + + @mock.patch('monasca_log_api.app.base.log_publisher.producer' + '.KafkaProducer') + def test_should_not_send_message_missing_keys(self, _): + # checks every combination of missing keys + # test does not rely on those keys having a value or not, + # it simply assumes that values are set but important + # message (i.e. envelope) properties are missing entirely + # that's why there are two loops instead of three + + instance = log_publisher.LogPublisher() + keys = ['log', 'creation_time', 'meta'] + + for key_1 in keys: + diff = keys[:] + diff.remove(key_1) + for key_2 in diff: + message = { + key_1: random.randint(10, 20), + key_2: random.randint(30, 50) + } + self.assertRaises(log_publisher.InvalidMessageException, + instance.send_message, + message) + + @mock.patch('monasca_log_api.app.base.log_publisher.producer' + '.KafkaProducer') + def test_should_not_send_message_missing_values(self, _): + # original message assumes that every property has value + # test modify each property one by one by removing that value + # (i.e. creating false-like value) + instance = log_publisher.LogPublisher() + message = { + 'log': { + 'message': '11' + }, + 'creation_time': 123456, + 'meta': { + 'region': 'pl' + } + } + + for key in message: + tmp_message = message + tmp_message[key] = None + self.assertRaises(log_publisher.InvalidMessageException, + instance.send_message, + tmp_message) + + @mock.patch('monasca_log_api.app.base.log_publisher.producer' + '.KafkaProducer') + def test_should_send_message(self, kafka_producer): + instance = log_publisher.LogPublisher() + instance._kafka_publisher = kafka_producer + instance.send_message({}) + + creation_time = ((datetime.datetime.utcnow() - EPOCH_START) + .total_seconds()) + application_type = 'monasca-log-api' + dimension_1_name = 'disk_usage' + dimension_1_value = '50' + dimension_2_name = 'cpu_time' + dimension_2_value = '60' + + msg = model.Envelope( + log={ + 'message': '1', + 'application_type': application_type, + 'dimensions': { + dimension_1_name: dimension_1_value, + dimension_2_name: dimension_2_value + } + }, + meta={ + 'tenantId': '1' + } + ) + msg['creation_time'] = creation_time + instance.send_message(msg) + + instance._kafka_publisher.publish.assert_called_once_with( + cfg.CONF.log_publisher.topics[0], + [ujson.dumps(msg, ensure_ascii=False).encode('utf-8')]) + + @mock.patch('monasca_log_api.app.base.log_publisher.producer' + '.KafkaProducer') + def test_should_send_message_multiple_topics(self, _): + topics = ['logs', 'analyzer', 'tester'] + self.conf_override(topics=topics, + max_message_size=5000, + group='log_publisher') + + instance = log_publisher.LogPublisher() + instance._kafka_publisher = mock.Mock() + instance.send_message({}) + + creation_time = ((datetime.datetime.utcnow() - EPOCH_START) + .total_seconds()) + dimension_1_name = 'disk_usage' + dimension_1_value = '50' + dimension_2_name = 'cpu_time' + dimension_2_value = '60' + application_type = 'monasca-log-api' + msg = model.Envelope( + log={ + 'message': '1', + 'application_type': application_type, + 'dimensions': { + dimension_1_name: dimension_1_value, + dimension_2_name: dimension_2_value + } + }, + meta={ + 'tenantId': '1' + } + ) + msg['creation_time'] = creation_time + json_msg = ujson.dumps(msg, ensure_ascii=False) + + instance.send_message(msg) + + self.assertEqual(len(topics), + instance._kafka_publisher.publish.call_count) + for topic in topics: + instance._kafka_publisher.publish.assert_any_call( + topic, + [json_msg.encode('utf-8')]) + + @mock.patch('monasca_log_api.app.base.log_publisher.producer' + '.KafkaProducer') + def test_should_send_unicode_message(self, kp): + instance = log_publisher.LogPublisher() + instance._kafka_publisher = kp + + for um in base.UNICODE_MESSAGES: + case, msg = um.values() + try: + envelope = model.Envelope( + log={ + 'message': msg, + 'application_type': 'test', + 'dimensions': { + 'test': 'test_log_publisher', + 'case': 'test_should_send_unicode_message' + } + }, + meta={ + 'tenantId': 1 + } + ) + instance.send_message(envelope) + + expected_message = ujson.dumps(envelope, ensure_ascii=False) + + if six.PY3: + expected_message = expected_message.encode('utf-8') + + instance._kafka_publisher.publish.assert_called_with( + cfg.CONF.log_publisher.topics[0], + [expected_message] + ) + except Exception: + LOG.exception('Failed to evaluate unicode case %s', case) + raise + + +@mock.patch( + 'monasca_log_api.app.base.log_publisher.producer' + '.KafkaProducer') +class TestTruncation(base.BaseTestCase): + EXTRA_CHARS_SIZE = len(bytearray(ujson.dumps({ + 'log': { + 'message': None + } + }), 'utf8')) - 2 + + def test_should_not_truncate_message_if_size_is_smaller(self, _): + diff_size = random.randint(1, 100) + self._run_truncate_test(log_size_factor=-diff_size, + truncate_by=0) + + def test_should_not_truncate_message_if_size_equal_to_max(self, _): + self._run_truncate_test(log_size_factor=0, + truncate_by=0) + + def test_should_truncate_too_big_message(self, _): + diff_size = random.randint(1, 100) + max_size = 1000 + truncate_by = ((max_size - + (max_size - log_publisher._TRUNCATED_PROPERTY_SIZE)) + + log_publisher._TRUNCATION_SAFE_OFFSET + diff_size) + self._run_truncate_test(max_message_size=1000, + log_size_factor=diff_size, + truncate_by=truncate_by) + + def _run_truncate_test(self, + max_message_size=1000, + log_size_factor=0, + truncate_by=0, + gen_fn=base.generate_unique_message): + + log_size = (max_message_size - + TestTruncation.EXTRA_CHARS_SIZE - + log_publisher._KAFKA_META_DATA_SIZE - + log_publisher._TIMESTAMP_KEY_SIZE + + log_size_factor) + + expected_log_message_size = log_size - truncate_by + + self.conf_override( + group='log_publisher', + max_message_size=max_message_size + ) + + log_msg = gen_fn(log_size) + envelope = { + 'log': { + 'message': log_msg + } + } + + instance = log_publisher.LogPublisher() + instance._logs_truncated_gauge.send = meter = mock.Mock() + + envelope_copy = copy.deepcopy(envelope) + json_envelope = instance._truncate(envelope_copy) + + parsed_envelope = ujson.loads(json_envelope) + + parsed_log_message = parsed_envelope['log']['message'] + parsed_log_message_len = len(parsed_log_message) + + if truncate_by > 0: + self.assertNotEqual(envelope['log']['message'], + parsed_log_message) + else: + self.assertEqual(envelope['log']['message'], + parsed_log_message) + + self.assertEqual(expected_log_message_size, parsed_log_message_len) + self.assertEqual(1, meter.call_count) + self.assertEqual(truncate_by, meter.mock_calls[0][2]['value']) diff --git a/monasca_log_api_code/tests/test_logs.py b/monasca_log_api_code/tests/test_logs.py new file mode 100644 index 000000000..5114f4b25 --- /dev/null +++ b/monasca_log_api_code/tests/test_logs.py @@ -0,0 +1,269 @@ +# Copyright 2015 kornicameister@gmail.com +# Copyright 2016 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import falcon +import mock + +from monasca_log_api.app.base import exceptions as log_api_exceptions +from monasca_log_api.app.controller.api import headers +from monasca_log_api.app.controller.v2 import logs +from monasca_log_api.tests import base + +ROLES = 'admin' + + +def _init_resource(test): + resource = logs.Logs() + test.api.add_route('/log/single', resource) + return resource + + +class TestApiLogsVersion(base.BaseApiTestCase): + @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') + @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') + def test_should_return_v2_as_version(self, _, __): + logs_resource = logs.Logs() + self.assertEqual('v2.0', logs_resource.version) + + +class TestApiLogs(base.BaseApiTestCase): + + @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') + @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') + def test_should_contain_deprecated_details_in_successful_response(self, + _, + __): + _init_resource(self) + + self.simulate_request( + '/log/single', + method='POST', + headers={ + headers.X_ROLES.name: ROLES, + headers.X_DIMENSIONS.name: 'a:1', + 'Content-Type': 'application/json', + 'Content-Length': '0' + } + ) + + self.assertEqual(falcon.HTTP_204, self.srmock.status) + self.assertIn('deprecated', self.srmock.headers_dict) + self.assertIn('link', self.srmock.headers_dict) + + @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') + @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') + def test_should_fail_not_delegate_ok_cross_tenant_id(self, _, __): + _init_resource(self) + self.simulate_request( + '/log/single', + method='POST', + query_string='tenant_id=1', + headers={ + 'Content-Type': 'application/json', + 'Content-Length': '0' + } + ) + self.assertEqual(falcon.HTTP_401, self.srmock.status) + + @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') + @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') + def test_should_pass_empty_cross_tenant_id_wrong_role(self, + log_creator, + kafka_publisher): + logs_resource = _init_resource(self) + logs_resource._log_creator = log_creator + logs_resource._kafka_publisher = kafka_publisher + + self.simulate_request( + '/log/single', + method='POST', + headers={ + headers.X_ROLES.name: ROLES, + headers.X_DIMENSIONS.name: 'a:1', + 'Content-Type': 'application/json', + 'Content-Length': '0' + } + ) + self.assertEqual(falcon.HTTP_204, self.srmock.status) + + self.assertEqual(1, kafka_publisher.send_message.call_count) + self.assertEqual(1, log_creator.new_log.call_count) + self.assertEqual(1, log_creator.new_log_envelope.call_count) + + @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') + @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') + def test_should_pass_empty_cross_tenant_id_ok_role(self, + log_creator, + kafka_publisher): + logs_resource = _init_resource(self) + logs_resource._log_creator = log_creator + logs_resource._kafka_publisher = kafka_publisher + + self.simulate_request( + '/log/single', + method='POST', + headers={ + headers.X_ROLES.name: ROLES, + headers.X_DIMENSIONS.name: 'a:1', + 'Content-Type': 'application/json', + 'Content-Length': '0' + } + ) + self.assertEqual(falcon.HTTP_204, self.srmock.status) + + self.assertEqual(1, kafka_publisher.send_message.call_count) + self.assertEqual(1, log_creator.new_log.call_count) + self.assertEqual(1, log_creator.new_log_envelope.call_count) + + @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') + @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') + def test_should_pass_delegate_cross_tenant_id_ok_role(self, + log_creator, + log_publisher): + resource = _init_resource(self) + resource._log_creator = log_creator + resource._kafka_publisher = log_publisher + + self.simulate_request( + '/log/single', + method='POST', + query_string='tenant_id=1', + headers={ + headers.X_ROLES.name: ROLES, + headers.X_DIMENSIONS.name: 'a:1', + 'Content-Type': 'application/json', + 'Content-Length': '0' + } + ) + self.assertEqual(falcon.HTTP_204, self.srmock.status) + + self.assertEqual(1, log_publisher.send_message.call_count) + self.assertEqual(1, log_creator.new_log.call_count) + self.assertEqual(1, log_creator.new_log_envelope.call_count) + + @mock.patch('monasca_common.rest.utils') + @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') + def test_should_fail_empty_dimensions_delegate(self, _, rest_utils): + _init_resource(self) + rest_utils.read_body.return_value = True + + self.simulate_request( + '/log/single', + method='POST', + headers={ + headers.X_ROLES.name: ROLES, + headers.X_DIMENSIONS.name: '', + 'Content-Type': 'application/json', + 'Content-Length': '0' + }, + body='{"message":"test"}' + ) + self.assertEqual(log_api_exceptions.HTTP_422, self.srmock.status) + + @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') + @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') + def test_should_fail_for_invalid_content_type(self, _, __): + _init_resource(self) + + self.simulate_request( + '/log/single', + method='POST', + headers={ + headers.X_ROLES.name: ROLES, + headers.X_DIMENSIONS.name: '', + 'Content-Type': 'video/3gpp', + 'Content-Length': '0' + } + ) + self.assertEqual(falcon.HTTP_415, self.srmock.status) + + @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') + @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') + def test_should_pass_payload_size_not_exceeded(self, _, __): + _init_resource(self) + + max_log_size = 1000 + content_length = max_log_size - 100 + self.conf_override(max_log_size=max_log_size, group='service') + + self.simulate_request( + '/log/single', + method='POST', + headers={ + headers.X_ROLES.name: ROLES, + headers.X_DIMENSIONS.name: '', + 'Content-Type': 'application/json', + 'Content-Length': str(content_length) + } + ) + self.assertEqual(falcon.HTTP_204, self.srmock.status) + + @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') + @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') + def test_should_fail_payload_size_exceeded(self, _, __): + _init_resource(self) + + max_log_size = 1000 + content_length = max_log_size + 100 + self.conf_override(max_log_size=max_log_size, group='service') + + self.simulate_request( + '/log/single', + method='POST', + headers={ + headers.X_ROLES.name: ROLES, + headers.X_DIMENSIONS.name: '', + 'Content-Type': 'application/json', + 'Content-Length': str(content_length) + } + ) + self.assertEqual(falcon.HTTP_413, self.srmock.status) + + @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') + @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') + def test_should_fail_payload_size_equal(self, _, __): + _init_resource(self) + + max_log_size = 1000 + content_length = max_log_size + self.conf_override(max_log_size=max_log_size, group='service') + + self.simulate_request( + '/log/single', + method='POST', + headers={ + headers.X_ROLES.name: ROLES, + headers.X_DIMENSIONS.name: '', + 'Content-Type': 'application/json', + 'Content-Length': str(content_length) + } + ) + self.assertEqual(falcon.HTTP_413, self.srmock.status) + + @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') + @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') + def test_should_fail_content_length(self, _, __): + _init_resource(self) + + self.simulate_request( + '/log/single', + method='POST', + headers={ + headers.X_ROLES.name: ROLES, + headers.X_DIMENSIONS.name: '', + 'Content-Type': 'application/json' + } + ) + self.assertEqual(falcon.HTTP_411, self.srmock.status) diff --git a/monasca_log_api_code/tests/test_logs_v3.py b/monasca_log_api_code/tests/test_logs_v3.py new file mode 100644 index 000000000..15c6808d7 --- /dev/null +++ b/monasca_log_api_code/tests/test_logs_v3.py @@ -0,0 +1,317 @@ +# Copyright 2016-2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import ujson as json +import falcon +import mock +from monasca_log_api.app.base import exceptions as log_api_exceptions +from monasca_log_api.app.controller.api import headers +from monasca_log_api.app.controller.v3 import logs +from monasca_log_api.tests import base + + + + +ENDPOINT = '/logs' +TENANT_ID = 'bob' +ROLES = 'admin' + + +def _init_resource(test): + resource = logs.Logs() + test.api.add_route(ENDPOINT, resource) + return resource + + +def _generate_v3_payload(log_count=None, messages=None): + if not log_count and messages: + log_count = len(messages) + v3_logs = [{ + 'message': messages[it], + 'dimensions': { + 'hostname': 'host_%d' % it, + 'component': 'component_%d' % it, + 'service': 'service_%d' % it + } + } for it in range(log_count)] + else: + v3_logs = [{ + 'message': base.generate_unique_message(100), + 'dimensions': { + 'hostname': 'host_%d' % it, + 'component': 'component_%d' % it, + 'service': 'service_%d' % it + } + } for it in range(log_count)] + v3_body = { + 'dimensions': { + 'origin': __name__ + }, + 'logs': v3_logs + } + + return v3_body, v3_logs + + +class TestApiLogsVersion(base.BaseApiTestCase): + + @mock.patch('monasca_log_api.app.controller.v3.aid' + '.bulk_processor.BulkProcessor') + def test_should_return_v3_as_version(self, _): + logs_resource = logs.Logs() + self.assertEqual('v3.0', logs_resource.version) + + +@mock.patch('monasca_log_api.app.base.log_publisher.producer.KafkaProducer') +@mock.patch('monasca_log_api.monitoring.client.monascastatsd.Connection') +class TestApiLogsMonitoring(base.BaseApiTestCase): + + def test_monitor_bulk_rejected(self, __, _): + res = _init_resource(self) + + in_counter = res._logs_in_counter.increment = mock.Mock() + bulk_counter = res._bulks_rejected_counter.increment = mock.Mock() + rejected_counter = res._logs_rejected_counter.increment = mock.Mock() + size_gauge = res._logs_size_gauge.send = mock.Mock() + + res._get_logs = mock.Mock( + side_effect=log_api_exceptions.HTTPUnprocessableEntity('')) + + log_count = 1 + v3_body, _ = _generate_v3_payload(log_count) + payload = json.dumps(v3_body) + content_length = len(payload) + + self.simulate_request( + ENDPOINT, + method='POST', + headers={ + headers.X_ROLES.name: ROLES, + headers.X_TENANT_ID.name: TENANT_ID, + 'Content-Type': 'application/json', + 'Content-Length': str(content_length) + }, + body=payload + ) + + self.assertEqual(1, bulk_counter.call_count) + self.assertEqual(0, in_counter.call_count) + self.assertEqual(0, rejected_counter.call_count) + self.assertEqual(0, size_gauge.call_count) + + def test_monitor_not_all_logs_ok(self, __, _): + res = _init_resource(self) + + in_counter = res._logs_in_counter.increment = mock.Mock() + bulk_counter = res._bulks_rejected_counter.increment = mock.Mock() + rejected_counter = res._logs_rejected_counter.increment = mock.Mock() + size_gauge = res._logs_size_gauge.send = mock.Mock() + + log_count = 5 + reject_logs = 1 + v3_body, _ = _generate_v3_payload(log_count) + payload = json.dumps(v3_body) + content_length = len(payload) + + side_effects = [{} for ___ in range(log_count - reject_logs)] + side_effects.append(log_api_exceptions.HTTPUnprocessableEntity('')) + + res._processor._get_dimensions = mock.Mock(side_effect=side_effects) + + self.simulate_request( + ENDPOINT, + method='POST', + headers={ + headers.X_ROLES.name: ROLES, + headers.X_TENANT_ID.name: TENANT_ID, + 'Content-Type': 'application/json', + 'Content-Length': str(content_length) + }, + body=payload + ) + + self.assertEqual(1, bulk_counter.call_count) + self.assertEqual(0, + bulk_counter.mock_calls[0][2]['value']) + + self.assertEqual(1, in_counter.call_count) + self.assertEqual(log_count - reject_logs, + in_counter.mock_calls[0][2]['value']) + + self.assertEqual(1, rejected_counter.call_count) + self.assertEqual(reject_logs, + rejected_counter.mock_calls[0][2]['value']) + + self.assertEqual(1, size_gauge.call_count) + self.assertEqual(content_length, + size_gauge.mock_calls[0][2]['value']) + + def test_monitor_all_logs_ok(self, __, _): + res = _init_resource(self) + + in_counter = res._logs_in_counter.increment = mock.Mock() + bulk_counter = res._bulks_rejected_counter.increment = mock.Mock() + rejected_counter = res._logs_rejected_counter.increment = mock.Mock() + size_gauge = res._logs_size_gauge.send = mock.Mock() + + res._send_logs = mock.Mock() + + log_count = 10 + + v3_body, _ = _generate_v3_payload(log_count) + + payload = json.dumps(v3_body) + content_length = len(payload) + self.simulate_request( + ENDPOINT, + method='POST', + headers={ + headers.X_ROLES.name: ROLES, + headers.X_TENANT_ID.name: TENANT_ID, + 'Content-Type': 'application/json', + 'Content-Length': str(content_length) + }, + body=payload + ) + + self.assertEqual(1, bulk_counter.call_count) + self.assertEqual(0, + bulk_counter.mock_calls[0][2]['value']) + + self.assertEqual(1, in_counter.call_count) + self.assertEqual(log_count, + in_counter.mock_calls[0][2]['value']) + + self.assertEqual(1, rejected_counter.call_count) + self.assertEqual(0, + rejected_counter.mock_calls[0][2]['value']) + + self.assertEqual(1, size_gauge.call_count) + self.assertEqual(content_length, + size_gauge.mock_calls[0][2]['value']) + + +class TestApiLogs(base.BaseApiTestCase): + + @mock.patch('monasca_log_api.app.controller.v3.aid.bulk_processor.' + 'BulkProcessor') + def test_should_pass_cross_tenant_id(self, bulk_processor): + logs_resource = _init_resource(self) + logs_resource._processor = bulk_processor + + v3_body, v3_logs = _generate_v3_payload(1) + payload = json.dumps(v3_body) + content_length = len(payload) + self.simulate_request( + '/logs', + method='POST', + query_string='tenant_id=1', + headers={ + headers.X_ROLES.name: ROLES, + 'Content-Type': 'application/json', + 'Content-Length': str(content_length) + }, + body=payload + ) + self.assertEqual(falcon.HTTP_204, self.srmock.status) + logs_resource._processor.send_message.assert_called_with( + logs=v3_logs, + global_dimensions=v3_body['dimensions'], + log_tenant_id='1') + + @mock.patch('monasca_log_api.app.controller.v3.aid.bulk_processor.' + 'BulkProcessor') + def test_should_fail_not_delegate_ok_cross_tenant_id(self, _): + _init_resource(self) + self.simulate_request( + '/logs', + method='POST', + query_string='tenant_id=1', + headers={ + headers.X_ROLES.name: ROLES, + 'Content-Type': 'application/json', + 'Content-Length': '0' + } + ) + self.assertEqual(falcon.HTTP_400, self.srmock.status) + + @mock.patch('monasca_log_api.app.controller.v3.aid.bulk_processor.' + 'BulkProcessor') + def test_should_pass_empty_cross_tenant_id_wrong_role(self, + bulk_processor): + logs_resource = _init_resource(self) + logs_resource._processor = bulk_processor + + v3_body, _ = _generate_v3_payload(1) + payload = json.dumps(v3_body) + content_length = len(payload) + self.simulate_request( + '/logs', + method='POST', + headers={ + headers.X_ROLES.name: ROLES, + 'Content-Type': 'application/json', + 'Content-Length': str(content_length) + }, + body=payload + ) + self.assertEqual(falcon.HTTP_204, self.srmock.status) + self.assertEqual(1, bulk_processor.send_message.call_count) + + @mock.patch('monasca_log_api.app.controller.v3.aid.bulk_processor.' + 'BulkProcessor') + def test_should_pass_empty_cross_tenant_id_ok_role(self, + bulk_processor): + logs_resource = _init_resource(self) + logs_resource._processor = bulk_processor + + v3_body, _ = _generate_v3_payload(1) + payload = json.dumps(v3_body) + content_length = len(payload) + self.simulate_request( + '/logs', + method='POST', + headers={ + headers.X_ROLES.name: ROLES, + 'Content-Type': 'application/json', + 'Content-Length': str(content_length) + }, + body=payload + ) + self.assertEqual(falcon.HTTP_204, self.srmock.status) + self.assertEqual(1, bulk_processor.send_message.call_count) + + +class TestUnicodeLogs(base.BaseApiTestCase): + + @mock.patch('monasca_log_api.app.base.log_publisher.producer.' + 'KafkaProducer') + def test_should_send_unicode_messages(self, _): + _init_resource(self) + + messages = [m['input'] for m in base.UNICODE_MESSAGES] + v3_body, _ = _generate_v3_payload(messages=messages) + payload = json.dumps(v3_body, ensure_ascii=False) + content_length = len(payload) + self.simulate_request( + '/logs', + method='POST', + headers={ + headers.X_ROLES.name: ROLES, + 'Content-Type': 'application/json', + 'Content-Length': str(content_length) + }, + body=payload + ) + self.assertEqual(falcon.HTTP_204, self.srmock.status) diff --git a/monasca_log_api_code/tests/test_monitoring.py b/monasca_log_api_code/tests/test_monitoring.py new file mode 100644 index 000000000..1048216c7 --- /dev/null +++ b/monasca_log_api_code/tests/test_monitoring.py @@ -0,0 +1,52 @@ +# Copyright 2016-2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from monasca_log_api.monitoring import client +from monasca_log_api.tests import base + + +class TestMonitoring(base.BaseTestCase): + + @mock.patch('monasca_log_api.monitoring.client.monascastatsd') + def test_should_use_default_dimensions_if_none_specified(self, + monascastatsd): + client.get_client() + + statsd_client = monascastatsd.Client + + expected_dimensions = client._DEFAULT_DIMENSIONS + actual_dimensions = statsd_client.call_args[1]['dimensions'] + + self.assertEqual(1, statsd_client.call_count) + self.assertEqual(expected_dimensions, actual_dimensions) + + @mock.patch('monasca_log_api.monitoring.client.monascastatsd') + def test_should_not_override_fixed_dimensions(self, + monascastatsd): + dims = { + 'service': 'foo', + 'component': 'bar' + } + + client.get_client(dims) + + statsd_client = monascastatsd.Client + + expected_dimensions = client._DEFAULT_DIMENSIONS + actual_dimensions = statsd_client.call_args[1]['dimensions'] + + self.assertEqual(1, statsd_client.call_count) + self.assertEqual(expected_dimensions, actual_dimensions) diff --git a/monasca_log_api_code/tests/test_policy.py b/monasca_log_api_code/tests/test_policy.py new file mode 100644 index 000000000..fbb595005 --- /dev/null +++ b/monasca_log_api_code/tests/test_policy.py @@ -0,0 +1,212 @@ +# Copyright 2016-2017 FUJITSU LIMITED +# Copyright 2018 OP5 AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from falcon import testing +from monasca_common.policy import policy_engine as policy +from monasca_log_api.app.base import request +from monasca_log_api.policies import roles_list_to_check_str +from monasca_log_api.tests import base +from oslo_context import context +from oslo_policy import policy as os_policy + + +class TestPolicyFileCase(base.BaseTestCase): + def setUp(self): + super(TestPolicyFileCase, self).setUp() + self.context = context.RequestContext(user='fake', + tenant='fake', + roles=['fake']) + self.target = {'tenant_id': 'fake'} + + def test_modified_policy_reloads(self): + tmp_file = \ + self.create_tempfiles(files=[('policies', '{}')], ext='.yaml')[0] + base.BaseTestCase.conf_override(policy_file=tmp_file, + group='oslo_policy') + + policy.reset() + policy.init() + action = 'example:test' + rule = os_policy.RuleDefault(action, '') + policy._ENFORCER.register_defaults([rule]) + + with open(tmp_file, 'w') as policy_file: + policy_file.write('{"example:test": ""}') + policy.authorize(self.context, action, self.target) + + with open(tmp_file, 'w') as policy_file: + policy_file.write('{"example:test": "!"}') + policy._ENFORCER.load_rules(True) + self.assertRaises(os_policy.PolicyNotAuthorized, policy.authorize, + self.context, action, self.target) + + +class TestPolicyCase(base.BaseTestCase): + def setUp(self): + super(TestPolicyCase, self).setUp() + rules = [ + os_policy.RuleDefault("true", "@"), + os_policy.RuleDefault("example:allowed", "@"), + os_policy.RuleDefault("example:denied", "!"), + os_policy.RuleDefault("example:lowercase_monasca_user", + "role:monasca_user or role:sysadmin"), + os_policy.RuleDefault("example:uppercase_monasca_user", + "role:MONASCA_USER or role:sysadmin"), + ] + policy.reset() + policy.init() + policy._ENFORCER.register_defaults(rules) + + def test_authorize_nonexist_action_throws(self): + action = "example:noexist" + ctx = request.Request( + testing.create_environ( + path="/", + headers={ + "X_USER_ID": "fake", + "X_PROJECT_ID": "fake", + "X_ROLES": "member" + } + ) + ) + self.assertRaises(os_policy.PolicyNotRegistered, policy.authorize, + ctx.context, action, {}) + + def test_authorize_bad_action_throws(self): + action = "example:denied" + ctx = request.Request( + testing.create_environ( + path="/", + headers={ + "X_USER_ID": "fake", + "X_PROJECT_ID": "fake", + "X_ROLES": "member" + } + ) + ) + self.assertRaises(os_policy.PolicyNotAuthorized, policy.authorize, + ctx.context, action, {}) + + def test_authorize_bad_action_no_exception(self): + action = "example:denied" + ctx = request.Request( + testing.create_environ( + path="/", + headers={ + "X_USER_ID": "fake", + "X_PROJECT_ID": "fake", + "X_ROLES": "member" + } + ) + ) + result = policy.authorize(ctx.context, action, {}, False) + self.assertFalse(result) + + def test_authorize_good_action(self): + action = "example:allowed" + ctx = request.Request( + testing.create_environ( + path="/", + headers={ + "X_USER_ID": "fake", + "X_PROJECT_ID": "fake", + "X_ROLES": "member" + } + ) + ) + result = policy.authorize(ctx.context, action, {}, False) + self.assertTrue(result) + + def test_ignore_case_role_check(self): + lowercase_action = "example:lowercase_monasca_user" + uppercase_action = "example:uppercase_monasca_user" + + monasca_user_context = request.Request( + testing.create_environ( + path="/", + headers={ + "X_USER_ID": "monasca_user", + "X_PROJECT_ID": "fake", + "X_ROLES": "MONASCA_user" + } + ) + ) + self.assertTrue(policy.authorize(monasca_user_context.context, + lowercase_action, + {})) + self.assertTrue(policy.authorize(monasca_user_context.context, + uppercase_action, + {})) + + +class RegisteredPoliciesTestCase(base.BaseTestCase): + def __init__(self, *args, **kwds): + super(RegisteredPoliciesTestCase, self).__init__(*args, **kwds) + self.default_roles = ['monasca-user', 'admin'] + + def test_healthchecks_policies_roles(self): + healthcheck_policies = { + 'log_api:healthcheck:head': ['any_role'], + 'log_api:healthcheck:get': ['any_role'] + } + + self._assert_rules(healthcheck_policies) + + def test_versions_policies_roles(self): + versions_policies = { + 'log_api:versions:get': ['any_role'] + } + + self._assert_rules(versions_policies) + + def test_logs_policies_roles(self): + + logs_policies = { + 'log_api:logs:post': self.default_roles + } + + self._assert_rules(logs_policies) + + def _assert_rules(self, policies_list): + for policy_name in policies_list: + registered_rule = policy.get_rules()[policy_name] + if hasattr(registered_rule, 'rules'): + self.assertEqual(len(registered_rule.rules), + len(policies_list[policy_name])) + for role in policies_list[policy_name]: + ctx = self._get_request_context(role) + self.assertTrue(policy.authorize(ctx.context, + policy_name, + {}) + ) + + @staticmethod + def _get_request_context(role): + return request.Request( + testing.create_environ( + path='/', + headers={'X_ROLES': role} + ) + ) + + +class PolicyUtilsTestCase(base.BaseTestCase): + def test_roles_list_to_check_str(self): + self.assertEqual(roles_list_to_check_str(['test_role']), 'role:test_role') + self.assertEqual(roles_list_to_check_str(['role1', 'role2', 'role3']), + 'role:role1 or role:role2 or role:role3') + self.assertEqual(roles_list_to_check_str(['@']), '@') + self.assertEqual(roles_list_to_check_str(['role1', '@', 'role2']), + 'role:role1 or @ or role:role2') + self.assertIsNone(roles_list_to_check_str(None)) diff --git a/monasca_log_api_code/tests/test_request.py b/monasca_log_api_code/tests/test_request.py new file mode 100644 index 000000000..84aa255c0 --- /dev/null +++ b/monasca_log_api_code/tests/test_request.py @@ -0,0 +1,100 @@ +# Copyright 2016-2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from falcon import testing +from mock import mock + +from monasca_log_api.app.base import request +from monasca_log_api.app.base import validation +from monasca_log_api.tests import base + + +class TestRequest(base.BaseTestCase): + + def test_use_context_from_request(self): + req = request.Request( + testing.create_environ( + path='/', + headers={ + 'X_AUTH_TOKEN': '111', + 'X_USER_ID': '222', + 'X_PROJECT_ID': '333', + 'X_ROLES': 'terminator,predator' + } + ) + ) + + self.assertEqual('111', req.context.auth_token) + self.assertEqual('222', req.user_id) + self.assertEqual('333', req.project_id) + self.assertEqual(['terminator', 'predator'], req.roles) + + def test_validate_context_type(self): + with mock.patch.object(validation, + 'validate_content_type') as vc_type, \ + mock.patch.object(validation, + 'validate_payload_size') as vp_size, \ + mock.patch.object(validation, + 'validate_cross_tenant') as vc_tenant: + req = request.Request(testing.create_environ()) + vc_type.side_effect = Exception() + + try: + req.validate(['test']) + except Exception as ex: + self.assertEqual(1, vc_type.call_count) + self.assertEqual(0, vp_size.call_count) + self.assertEqual(0, vc_tenant.call_count) + + self.assertIsInstance(ex, Exception) + + def test_validate_payload_size(self): + with mock.patch.object(validation, + 'validate_content_type') as vc_type, \ + mock.patch.object(validation, + 'validate_payload_size') as vp_size, \ + mock.patch.object(validation, + 'validate_cross_tenant') as vc_tenant: + + req = request.Request(testing.create_environ()) + vp_size.side_effect = Exception() + + try: + req.validate(['test']) + except Exception as ex: + self.assertEqual(1, vc_type.call_count) + self.assertEqual(1, vp_size.call_count) + self.assertEqual(0, vc_tenant.call_count) + + self.assertIsInstance(ex, Exception) + + def test_validate_cross_tenant(self): + with mock.patch.object(validation, + 'validate_content_type') as vc_type, \ + mock.patch.object(validation, + 'validate_payload_size') as vp_size, \ + mock.patch.object(validation, + 'validate_cross_tenant') as vc_tenant: + + req = request.Request(testing.create_environ()) + vc_tenant.side_effect = Exception() + + try: + req.validate(['test']) + except Exception as ex: + self.assertEqual(1, vc_type.call_count) + self.assertEqual(1, vp_size.call_count) + self.assertEqual(1, vc_tenant.call_count) + + self.assertIsInstance(ex, Exception) diff --git a/monasca_log_api_code/tests/test_role_middleware.py b/monasca_log_api_code/tests/test_role_middleware.py new file mode 100644 index 000000000..047e33011 --- /dev/null +++ b/monasca_log_api_code/tests/test_role_middleware.py @@ -0,0 +1,226 @@ +# Copyright 2015-2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +from monasca_log_api.middleware import role_middleware as rm +from monasca_log_api.tests import base +from webob import response + + + +class SideLogicTestEnsureLowerRoles(base.BaseTestCase): + + def test_should_ensure_lower_roles(self): + roles = ['CMM-Admin', ' CmM-User '] + expected = ['cmm-admin', 'cmm-user'] + self.assertItemsEqual(expected, rm._ensure_lower_roles(roles)) + + def test_should_return_empty_array_for_falsy_input_1(self): + roles = [] + expected = [] + self.assertItemsEqual(expected, rm._ensure_lower_roles(roles)) + + def test_should_return_empty_array_for_falsy_input_2(self): + roles = None + expected = [] + self.assertItemsEqual(expected, rm._ensure_lower_roles(roles)) + + +class SideLogicTestIntersect(base.BaseTestCase): + + def test_should_intersect_seqs(self): + seq_1 = [1, 2, 3] + seq_2 = [2] + + expected = [2] + + self.assertItemsEqual(expected, rm._intersect(seq_1, seq_2)) + self.assertItemsEqual(expected, rm._intersect(seq_2, seq_1)) + + def test_should_intersect_empty(self): + seq_1 = [] + seq_2 = [] + + expected = [] + + self.assertItemsEqual(expected, rm._intersect(seq_1, seq_2)) + self.assertItemsEqual(expected, rm._intersect(seq_2, seq_1)) + + def test_should_not_intersect_without_common_elements(self): + seq_1 = [1, 2, 3] + seq_2 = [4, 5, 6] + + expected = [] + + self.assertItemsEqual(expected, rm._intersect(seq_1, seq_2)) + self.assertItemsEqual(expected, rm._intersect(seq_2, seq_1)) + + +class RolesMiddlewareSideLogicTest(base.BaseTestCase): + + def test_should_apply_middleware_for_valid_path(self): + paths = ['/', '/v2.0/', '/v2.0/log/'] + + instance = rm.RoleMiddleware(None) + instance._path = paths + + for p in paths: + req = mock.Mock() + req.method = 'GET' + req.path = p + self.assertTrue(instance._can_apply_middleware(req)) + + def test_should_apply_middleware_for_invalid_path(self): + paths = ['/v2.0/', '/v2.0/log/'] + + instance = rm.RoleMiddleware(None) + instance._path = paths + + for p in paths: + pp = 'test/%s' % p + req = mock.Mock() + req.method = 'GET' + req.path = pp + self.assertFalse(instance._can_apply_middleware(req)) + + def test_should_reject_OPTIONS_request(self): + instance = rm.RoleMiddleware(None) + req = mock.Mock() + req.method = 'OPTIONS' + req.path = '/' + self.assertFalse(instance._can_apply_middleware(req)) + + def test_should_return_true_if_authenticated(self): + instance = rm.RoleMiddleware(None) + + req = mock.Mock() + req.headers = {rm._X_IDENTITY_STATUS: rm._CONFIRMED_STATUS} + + self.assertTrue(instance._is_authenticated(req)) + + def test_should_return_false_if_not_authenticated(self): + instance = rm.RoleMiddleware(None) + + req = mock.Mock() + req.headers = {rm._X_IDENTITY_STATUS: 'Some_Other_Status'} + + self.assertFalse(instance._is_authenticated(req)) + + def test_should_return_false_if_identity_status_not_found(self): + instance = rm.RoleMiddleware(None) + + req = mock.Mock() + req.headers = {} + + self.assertFalse(instance._is_authenticated(req)) + + def test_should_return_true_if_is_agent(self): + roles = 'cmm-admin,cmm-user' + roles_array = roles.split(',') + + default_roles = [roles_array[0]] + admin_roles = [roles_array[1]] + + instance = rm.RoleMiddleware(None) + instance._default_roles = default_roles + instance._agent_roles = admin_roles + + req = mock.Mock() + req.headers = {rm._X_ROLES: roles} + + is_agent = instance._is_agent(req) + + self.assertTrue(is_agent) + + +class RolesMiddlewareLogicTest(base.BaseTestCase): + + def test_not_process_further_if_cannot_apply_path(self): + roles = 'cmm-admin,cmm-user' + roles_array = roles.split(',') + + default_roles = [roles_array[0]] + admin_roles = [roles_array[1]] + + instance = rm.RoleMiddleware(None) + instance._default_roles = default_roles + instance._agent_roles = admin_roles + instance._path = ['/test'] + + # spying + instance._is_authenticated = mock.Mock() + instance._is_agent = mock.Mock() + + req = mock.Mock() + req.headers = {rm._X_ROLES: roles} + req.path = '/different/test' + + instance.process_request(req=req) + + self.assertFalse(instance._is_authenticated.called) + self.assertFalse(instance._is_agent.called) + + def test_not_process_further_if_cannot_apply_method(self): + roles = 'cmm-admin,cmm-user' + roles_array = roles.split(',') + + default_roles = [roles_array[0]] + admin_roles = [roles_array[1]] + + instance = rm.RoleMiddleware(None) + instance._default_roles = default_roles + instance._agent_roles = admin_roles + instance._path = ['/test'] + + # spying + instance._is_authenticated = mock.Mock() + instance._is_agent = mock.Mock() + + req = mock.Mock() + req.headers = {rm._X_ROLES: roles} + req.path = '/test' + req.method = 'OPTIONS' + + instance.process_request(req=req) + + self.assertFalse(instance._is_authenticated.called) + self.assertFalse(instance._is_agent.called) + + def test_should_produce_json_response_if_not_authenticated( + self): + instance = rm.RoleMiddleware(None) + is_agent = True + is_authenticated = False + + instance._can_apply_middleware = mock.Mock(return_value=True) + instance._is_agent = mock.Mock(return_value=is_agent) + instance._is_authenticated = mock.Mock(return_value=is_authenticated) + + req = mock.Mock() + req.environ = {} + req.headers = { + 'X-Tenant-Id': '11111111' + } + + result = instance.process_request(req=req) + + self.assertIsNotNone(result) + self.assertIsInstance(result, response.Response) + + status = result.status_code + json_body = result.json_body + message = json_body.get('message') + + self.assertIn('Failed to authenticate request for', message) + self.assertEqual(401, status) diff --git a/monasca_log_api_code/tests/test_service.py b/monasca_log_api_code/tests/test_service.py new file mode 100644 index 000000000..72e35e427 --- /dev/null +++ b/monasca_log_api_code/tests/test_service.py @@ -0,0 +1,483 @@ +# Copyright 2015 kornicameister@gmail.com +# Copyright 2016-2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime +import unittest +import mock +from falcon import errors +from falcon import testing +from monasca_log_api.app.base import exceptions +from monasca_log_api.app.base import validation +from monasca_log_api.app.controller.v2.aid import service as aid_service +from monasca_log_api.tests import base + + + + + +class IsDelegate(base.BaseTestCase): + + def __init__(self, *args, **kwargs): + super(IsDelegate, self).__init__(*args, **kwargs) + self._roles = ['admin'] + + def test_is_delegate_ok_role(self): + self.assertTrue(validation.validate_is_delegate(self._roles)) + + def test_is_delegate_ok_role_in_roles(self): + self._roles.extend(['a_role', 'b_role']) + self.assertTrue(validation.validate_is_delegate(self._roles)) + + def test_is_delegate_not_ok_role(self): + roles = ['a_role', 'b_role'] + self.assertFalse(validation.validate_is_delegate(roles)) + + +class ParseDimensions(base.BaseTestCase): + def test_should_fail_for_empty_dimensions(self): + self.assertRaises(exceptions.HTTPUnprocessableEntity, + aid_service.parse_dimensions, + '') + self.assertRaises(exceptions.HTTPUnprocessableEntity, + aid_service.parse_dimensions, + None) + + def test_should_fail_for_empty_dim_in_dimensions(self): + err = self.assertRaises(exceptions.HTTPUnprocessableEntity, + aid_service.parse_dimensions, + ',') + self.assertEqual(err.description, 'Dimension cannot be empty') + + def test_should_fail_for_invalid_dim_in_dimensions(self): + invalid_dim = 'a' + err = self.assertRaises(exceptions.HTTPUnprocessableEntity, + aid_service.parse_dimensions, + invalid_dim) + self.assertEqual(err.description, '%s is not a valid dimension' + % invalid_dim) + + def test_should_pass_for_valid_dimensions(self): + dimensions = 'a:1,b:2' + expected = { + 'a': '1', + 'b': '2' + } + + self.assertDictEqual(expected, + aid_service.parse_dimensions(dimensions)) + + +class ParseApplicationType(base.BaseTestCase): + def test_should_return_none_for_none(self): + self.assertIsNone(aid_service.parse_application_type(None)) + + def test_should_return_none_for_empty(self): + self.assertIsNone(aid_service.parse_application_type('')) + + def test_should_return_none_for_whitespace_filled(self): + self.assertIsNone(aid_service.parse_application_type(' ')) + + def test_should_return_value_for_ok_value(self): + app_type = 'monasca' + self.assertEqual(app_type, + aid_service.parse_application_type(app_type)) + + def test_should_return_value_for_ok_value_with_spaces(self): + app_type = ' monasca ' + expected = 'monasca' + self.assertEqual(expected, + aid_service.parse_application_type(app_type)) + + +class ApplicationTypeValidations(base.BaseTestCase): + def test_should_pass_for_empty_app_type(self): + validation.validate_application_type() + validation.validate_application_type('') + + def test_should_fail_for_invalid_length(self): + r_app_type = testing.rand_string(300, 600) + err = self.assertRaises(exceptions.HTTPUnprocessableEntity, + validation.validate_application_type, + r_app_type) + + length = validation.APPLICATION_TYPE_CONSTRAINTS['MAX_LENGTH'] + msg = ('Application type {type} must be ' + '{length} characters or less'.format(type=r_app_type, + length=length)) + + self.assertEqual(err.description, msg) + + def test_should_fail_for_invalid_content(self): + r_app_type = '%#$@!' + + err = self.assertRaises(exceptions.HTTPUnprocessableEntity, + validation.validate_application_type, + r_app_type) + msg = ('Application type %s may only contain: "a-z A-Z 0-9 _ - ."' % + r_app_type) + self.assertEqual(err.description, msg) + + def test_should_pass_for_ok_app_type(self): + r_app_type = 'monasca' + validation.validate_application_type(r_app_type) + + +class DimensionsValidations(base.BaseTestCase): + @unittest.expectedFailure + def test_should_fail_for_none_dimensions(self): + validation.validate_dimensions(None) + + @unittest.expectedFailure + def test_should_fail_pass_for_non_iterable_dimensions_str(self): + validation.validate_dimensions('') + + @unittest.expectedFailure + def test_should_fail_pass_for_non_iterable_dimensions_number(self): + validation.validate_dimensions(1) + + def test_should_pass_for_empty_dimensions_array(self): + validation.validate_dimensions({}) + + def test_should_fail_too_empty_name(self): + dimensions = {'': 1} + + err = self.assertRaises(exceptions.HTTPUnprocessableEntity, + validation.validate_dimensions, + dimensions) + msg = 'Dimension name cannot be empty' + self.assertEqual(err.description, msg) + + def test_should_fail_too_long_name(self): + name = testing.rand_string(256, 260) + dimensions = {name: 1} + + err = self.assertRaises(exceptions.HTTPUnprocessableEntity, + validation.validate_dimensions, + dimensions) + msg = 'Dimension name %s must be 255 characters or less' % name + self.assertEqual(err.description, msg) + + def test_should_fail_underscore_at_begin(self): + name = '_aDim' + dimensions = {name: 1} + + err = self.assertRaises(exceptions.HTTPUnprocessableEntity, + validation.validate_dimensions, + dimensions) + msg = 'Dimension name %s cannot start with underscore (_)' % name + self.assertEqual(err.description, msg) + + def test_should_fail_invalid_chars(self): + name = '<>' + dimensions = {name: 1} + + err = self.assertRaises(exceptions.HTTPUnprocessableEntity, + validation.validate_dimensions, + dimensions) + invalid_chars = '> < = { } ( ) \' " , ; &' + msg = 'Dimension name %s may not contain: %s' % (name, invalid_chars) + self.assertEqual(err.description, msg) + + def test_should_fail_ok_name_empty_value(self): + name = 'monasca' + dimensions = {name: ''} + + err = self.assertRaises(exceptions.HTTPUnprocessableEntity, + validation.validate_dimensions, + dimensions) + msg = 'Dimension value cannot be empty' + self.assertEqual(err.description, msg) + + def test_should_fail_ok_name_too_long_value(self): + name = 'monasca' + value = testing.rand_string(256, 300) + dimensions = {name: value} + + err = self.assertRaises(exceptions.HTTPUnprocessableEntity, + validation.validate_dimensions, + dimensions) + msg = 'Dimension value %s must be 255 characters or less' % value + self.assertEqual(err.description, msg) + + def test_should_pass_ok_name_ok_value_empty_service(self): + name = 'monasca' + value = '1' + dimensions = {name: value} + validation.validate_dimensions(dimensions) + + def test_should_pass_ok_name_ok_value_service_SERVICE_DIMENSIONS_as_name( + self): + name = 'some_name' + value = '1' + dimensions = {name: value} + validation.validate_dimensions(dimensions) + + +class ContentTypeValidations(base.BaseTestCase): + def test_should_pass_text_plain(self): + content_type = 'text/plain' + allowed_types = ['text/plain'] + + req = mock.Mock() + req.content_type = content_type + validation.validate_content_type(req, allowed_types) + + def test_should_pass_application_json(self): + content_type = 'application/json' + allowed_types = ['application/json'] + + req = mock.Mock() + req.content_type = content_type + + validation.validate_content_type(req, allowed_types) + + def test_should_fail_invalid_content_type(self): + content_type = 'no/such/type' + allowed_types = ['application/json'] + + req = mock.Mock() + req.content_type = content_type + + self.assertRaises( + errors.HTTPUnsupportedMediaType, + validation.validate_content_type, + req, + allowed_types + ) + + def test_should_fail_missing_header(self): + content_type = None + allowed_types = ['application/json'] + + req = mock.Mock() + req.content_type = content_type + + self.assertRaises( + errors.HTTPMissingHeader, + validation.validate_content_type, + req, + allowed_types + ) + + +class PayloadSizeValidations(base.BaseTestCase): + + def test_should_fail_missing_header(self): + content_length = None + req = mock.Mock() + req.content_length = content_length + self.assertRaises( + errors.HTTPLengthRequired, + validation.validate_payload_size, + req + ) + + def test_should_pass_limit_not_exceeded(self): + content_length = 120 + max_log_size = 240 + self.conf_override(max_log_size=max_log_size, + group='service') + + req = mock.Mock() + req.content_length = content_length + + validation.validate_payload_size(req) + + def test_should_fail_limit_exceeded(self): + content_length = 120 + max_log_size = 60 + self.conf_override(max_log_size=max_log_size, + group='service') + + req = mock.Mock() + req.content_length = content_length + + self.assertRaises( + errors.HTTPRequestEntityTooLarge, + validation.validate_payload_size, + req + ) + + def test_should_fail_limit_equal(self): + content_length = 120 + max_log_size = 120 + self.conf_override(max_log_size=max_log_size, + group='service') + + req = mock.Mock() + req.content_length = content_length + + self.assertRaises( + errors.HTTPRequestEntityTooLarge, + validation.validate_payload_size, + req + ) + + +class LogMessageValidations(base.BaseTestCase): + def test_should_pass_message_in_log_property(self): + log_object = { + 'message': 'some messages', + 'application_type': 'monasca-log-api', + 'dimensions': { + 'hostname': 'devstack' + } + } + validation.validate_log_message(log_object) + + @unittest.expectedFailure + def test_should_fail_pass_for_non_message_in_log_property(self): + log_object = { + 'massage': 'some messages', + 'application_type': 'monasca-log-api', + 'dimensions': { + 'hostname': 'devstack' + } + } + validation.validate_log_message(log_object) + + def test_should_fail_with_empty_message(self): + self.assertRaises(exceptions.HTTPUnprocessableEntity, + validation.validate_log_message, {}) + + +class LogsCreatorNewLog(base.BaseTestCase): + def setUp(self): + super(LogsCreatorNewLog, self).setUp() + self.instance = aid_service.LogCreator() + + @mock.patch('io.IOBase') + def test_should_create_log_from_json(self, payload): + msg = u'Hello World' + path = u'/var/log/messages' + json_msg = u'{"path":"%s","message":"%s"}' % (path, msg) + app_type = 'monasca' + dimensions = 'cpu_time:30' + payload.read.return_value = json_msg + + expected_log = { + 'message': msg, + 'dimensions': { + 'component': app_type, + 'cpu_time': '30' + }, + 'path': path + } + + self.assertEqual(expected_log, self.instance.new_log( + application_type=app_type, + dimensions=dimensions, + payload=payload + )) + + @mock.patch('io.IOBase') + def test_should_create_log_from_text(self, payload): + msg = u'Hello World' + app_type = 'monasca' + dimension_name = 'cpu_time' + dimension_value = 30 + dimensions = '%s:%s' % (dimension_name, str(dimension_value)) + payload.read.return_value = msg + + expected_log = { + 'message': msg, + 'dimensions': { + 'component': app_type, + dimension_name: str(dimension_value) + } + } + + self.assertEqual(expected_log, self.instance.new_log( + application_type=app_type, + dimensions=dimensions, + payload=payload, + content_type='text/plain' + )) + + +class LogCreatorNewEnvelope(base.BaseTestCase): + def setUp(self): + super(LogCreatorNewEnvelope, self).setUp() + self.instance = aid_service.LogCreator() + + def test_should_create_envelope(self): + msg = u'Hello World' + path = u'/var/log/messages' + app_type = 'monasca' + dimension_name = 'cpu_time' + dimension_value = 30 + expected_log = { + 'message': msg, + 'application_type': app_type, + 'dimensions': { + dimension_name: str(dimension_value) + }, + 'path': path + } + tenant_id = 'a_tenant' + none = None + meta = {'tenantId': tenant_id, 'region': none} + timestamp = (datetime.datetime.utcnow() - + datetime.datetime(1970, 1, 1)).total_seconds() + expected_envelope = { + 'log': expected_log, + 'creation_time': timestamp, + 'meta': meta + } + + with mock.patch.object(self.instance, '_create_meta_info', + return_value=meta): + actual_envelope = self.instance.new_log_envelope(expected_log, + tenant_id) + + self.assertEqual(expected_envelope.get('log'), + actual_envelope.get('log')) + self.assertEqual(expected_envelope.get('meta'), + actual_envelope.get('meta')) + self.assertDictEqual( + expected_envelope.get('log').get('dimensions'), + actual_envelope.get('log').get('dimensions')) + + @unittest.expectedFailure + def test_should_not_create_log_none(self): + log_object = None + tenant_id = 'a_tenant' + + self.instance.new_log_envelope(log_object, tenant_id) + + @unittest.expectedFailure + def test_should_not_create_log_empty(self): + log_object = {} + tenant_id = 'a_tenant' + + self.instance.new_log_envelope(log_object, tenant_id) + + @unittest.expectedFailure + def test_should_not_create_tenant_none(self): + log_object = { + 'message': '' + } + tenant_id = None + + self.instance.new_log_envelope(log_object, tenant_id) + + @unittest.expectedFailure + def test_should_not_create_tenant_empty(self): + log_object = { + 'message': '' + } + tenant_id = '' + + self.instance.new_log_envelope(log_object, tenant_id) diff --git a/monasca_log_api_code/tests/test_v2_v3_compare.py b/monasca_log_api_code/tests/test_v2_v3_compare.py new file mode 100644 index 000000000..05ce23d56 --- /dev/null +++ b/monasca_log_api_code/tests/test_v2_v3_compare.py @@ -0,0 +1,119 @@ +# Copyright 2016 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import ujson as json +import mock +from monasca_log_api.app.controller.api import headers +from monasca_log_api.app.controller.v2 import logs as v2_logs +from monasca_log_api.app.controller.v3 import logs as v3_logs +from monasca_log_api.tests import base + + + + +class TestApiSameV2V3Output(base.BaseApiTestCase): + + # noinspection PyProtectedMember + @mock.patch('monasca_log_api.app.base.log_publisher.' + 'producer.KafkaProducer') + def test_send_identical_messages(self, _): + # mocks only log publisher, so the last component that actually + # sends data to kafka + # case is to verify if publisher was called with same arguments + # for both cases + + v2 = v2_logs.Logs() + v3 = v3_logs.Logs() + + publish_mock = mock.Mock() + + v2._kafka_publisher._kafka_publisher.publish = publish_mock + v3._processor._kafka_publisher.publish = publish_mock + + component = 'monasca-log-api' + service = 'laas' + hostname = 'kornik' + tenant_id = 'ironMan' + roles = 'admin' + + v2_dimensions = 'hostname:%s,service:%s' % (hostname, service) + v3_dimensions = { + 'hostname': hostname, + 'component': component, + 'service': service + } + + v2_body = { + 'message': 'test' + } + + v3_body = { + 'logs': [ + { + 'message': 'test', + 'dimensions': v3_dimensions + } + ] + } + + self.api.add_route('/v2.0', v2) + self.api.add_route('/v3.0', v3) + + self.simulate_request( + '/v2.0', + method='POST', + headers={ + headers.X_ROLES.name: roles, + headers.X_DIMENSIONS.name: v2_dimensions, + headers.X_APPLICATION_TYPE.name: component, + headers.X_TENANT_ID.name: tenant_id, + 'Content-Type': 'application/json', + 'Content-Length': '100' + }, + body=json.dumps(v2_body) + ) + + self.simulate_request( + '/v3.0', + method='POST', + headers={ + headers.X_ROLES.name: roles, + headers.X_TENANT_ID.name: tenant_id, + 'Content-Type': 'application/json', + 'Content-Length': '100' + }, + body=json.dumps(v3_body) + ) + + self.assertEqual(2, publish_mock.call_count) + + # in v2 send_messages is called with single envelope + v2_send_msg_arg = publish_mock.mock_calls[0][1][1] + + # in v3 it is always called with list of envelopes + v3_send_msg_arg = publish_mock.mock_calls[1][1][1] + + self.maxDiff = None + + # at this point we know that both args should be identical + self.assertEqual(type(v2_send_msg_arg), type(v3_send_msg_arg)) + self.assertIsInstance(v3_send_msg_arg, list) + + self.assertEqual(len(v2_send_msg_arg), len(v3_send_msg_arg)) + self.assertEqual(1, len(v2_send_msg_arg)) + + v2_msg_as_dict = json.loads(v2_send_msg_arg[0]) + v3_msg_as_dict = json.loads(v3_send_msg_arg[0]) + + self.assertDictEqual(v2_msg_as_dict, v3_msg_as_dict) diff --git a/monasca_log_api_code/tests/test_version.py b/monasca_log_api_code/tests/test_version.py new file mode 100644 index 000000000..9e6e8a8cd --- /dev/null +++ b/monasca_log_api_code/tests/test_version.py @@ -0,0 +1,22 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from monasca_log_api import version +from monasca_log_api.tests import base + + + +class TestAppVersion(base.BaseTestCase): + def test_should_report_version(self): + self.assertIsNotNone(version.version_str) diff --git a/monasca_log_api_code/tests/test_versions.py b/monasca_log_api_code/tests/test_versions.py new file mode 100644 index 000000000..8bf37f318 --- /dev/null +++ b/monasca_log_api_code/tests/test_versions.py @@ -0,0 +1,120 @@ +# Copyright 2016 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import ujson as json +import falcon +from monasca_log_api.app.controller import versions +from monasca_log_api.tests import base + + + + +def _get_versioned_url(version_id): + return '/version/%s' % version_id + + +class TestApiVersions(base.BaseApiTestCase): + + def before(self): + self.versions = versions.Versions() + self.api.add_route("/version/", self.versions) + self.api.add_route("/version/{version_id}", self.versions) + + def test_should_fail_for_unsupported_version(self): + unsupported_version = 'v5.0' + uri = _get_versioned_url(unsupported_version) + + self.simulate_request( + uri, + method='GET', + headers={ + 'Content-Type': 'application/json' + } + ) + + self.assertEqual(falcon.HTTP_400, self.srmock.status) + + def test_should_return_all_supported_versions(self): + + def _check_elements(): + self.assertIn('elements', response) + elements = response.get('elements') + self.assertIsInstance(elements, list) + + for el in elements: + # do checkup by expected keys + self.assertIn('id', el) + self.assertItemsEqual([ + u'id', + u'links', + u'status', + u'updated' + ], el.keys()) + + ver = el.get('id') + self.assertIn(ver, expected_versions) + + def _check_global_links(): + self.assertIn('links', response) + links = response.get('links') + self.assertIsInstance(links, list) + + for link in links: + self.assertIn('rel', link) + key = link.get('rel') + self.assertIn(key, expected_links_keys) + + expected_versions = 'v2.0', 'v3.0' + expected_links_keys = 'self', 'version', 'healthcheck' + + res = self.simulate_request( + '/version', + method='GET', + headers={ + 'Content-Type': 'application/json' + }, + decode='utf-8' + ) + self.assertEqual(falcon.HTTP_200, self.srmock.status) + + response = json.loads(res) + + _check_elements() + _check_global_links() + + def test_should_return_expected_version_id(self): + expected_versions = 'v2.0', 'v3.0' + for expected_version in expected_versions: + uri = _get_versioned_url(expected_version) + res = self.simulate_request( + uri, + method='GET', + headers={ + 'Content-Type': 'application/json' + }, + decode='utf-8' + ) + self.assertEqual(falcon.HTTP_200, self.srmock.status) + + response = json.loads(res) + self.assertIn('elements', response) + self.assertIn('links', response) + + elements = response.get('elements') + self.assertIsInstance(elements, list) + self.assertEqual(1, len(elements)) + + el = elements[0] + ver = el.get('id') + self.assertEqual(expected_version, ver) diff --git a/monasca_log_api_code/version.py b/monasca_log_api_code/version.py new file mode 100644 index 000000000..7b20ca390 --- /dev/null +++ b/monasca_log_api_code/version.py @@ -0,0 +1,18 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pbr.version + +version_info = pbr.version.VersionInfo('monasca-log-api') +version_str = version_info.version_string()