From e1a9b9a96a56ec5476a21556e5ace61996afd74c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Tr=C4=99bski?= Date: Tue, 16 May 2017 00:14:56 +0200 Subject: [PATCH] Integrate with oslo.conf and oslo.log Change upgrades the monasca-notification to leverage the capabilities of both oslo.log and oslo.conf: - configuration of logging separated from application settings - ability to enforce data types for application settings - ability to use oslo.config-generator capabilities - automatic configuration parsing done by oslo.cfg That change will bring it closer to the rest of monasca components where such transition has happened already. However, in the rest of monasca, oslo.cfg was partially or fully implemented whereas monasca-notification has been relying on YAML based configuration file. Therefore backward compatybility for such format will be kept for now. Story: 2000959 Task: 4093 Task: 4092 Change-Id: Ia75c3b60d0fada854178f21ca5ccb9e6a880f37f --- .gitignore | 2 + config-generator/notification.conf | 5 + etc/monasca/notification-logging.conf | 46 ++++ .../common/repositories/mysql/mysql_repo.py | 18 +- .../common/repositories/orm/orm_repo.py | 32 +-- .../repositories/postgres/pgsql_repo.py | 4 +- monasca_notification/common/utils.py | 39 ++- monasca_notification/conf/__init__.py | 162 +++++++++++++ monasca_notification/conf/cli.py | 40 ++++ monasca_notification/conf/database.py | 129 ++++++++++ monasca_notification/conf/kafka.py | 77 ++++++ monasca_notification/conf/notifiers.py | 53 ++++ monasca_notification/conf/processors.py | 55 +++++ monasca_notification/conf/queues.py | 45 ++++ monasca_notification/conf/retry.py | 41 ++++ monasca_notification/conf/statsd.py | 41 ++++ monasca_notification/conf/types.py | 102 ++++++++ monasca_notification/conf/zookeeper.py | 61 +++++ monasca_notification/config.py | 64 +++++ monasca_notification/main.py | 77 +++--- monasca_notification/notification_engine.py | 45 ++-- monasca_notification/periodic_engine.py | 26 +- .../plugins/abstract_notifier.py | 5 +- .../plugins/email_notifier.py | 58 +++-- .../plugins/hipchat_notifier.py | 50 +++- monasca_notification/plugins/jira_notifier.py | 69 ++++-- .../plugins/pagerduty_notifier.py | 40 +++- .../plugins/slack_notifier.py | 52 ++-- .../plugins/webhook_notifier.py | 34 ++- .../processors/alarm_processor.py | 16 +- .../processors/notification_processor.py | 14 +- monasca_notification/retry_engine.py | 45 ++-- monasca_notification/types/notifiers.py | 50 ++-- monasca_notification/version.py | 23 ++ requirements.txt | 7 +- setup.cfg | 4 +- test-requirements.txt | 14 +- tests/base.py | 83 +++++++ tests/resources/notification.yaml | 125 ++++++++++ tests/test_alarm_processor.py | 24 +- tests/test_config.py | 226 ++++++++++++++++++ tests/test_email_notification.py | 38 ++- tests/test_hipchat_notification.py | 11 +- tests/test_jira_notification.py | 66 +++-- tests/test_mysql_repo.py | 19 +- tests/test_notification_processor.py | 46 ++-- tests/test_notifiers.py | 173 +++----------- tests/test_orm_repo.py | 11 +- tests/test_pagerduty_notification.py | 17 +- tests/test_slack_notification.py | 78 +++--- tests/test_utils.py | 26 +- tests/test_webhook_notification.py | 13 +- tox.ini | 8 + 53 files changed, 2004 insertions(+), 605 deletions(-) create mode 100644 config-generator/notification.conf create mode 100644 etc/monasca/notification-logging.conf create mode 100644 monasca_notification/conf/__init__.py create mode 100644 monasca_notification/conf/cli.py create mode 100644 monasca_notification/conf/database.py create mode 100644 monasca_notification/conf/kafka.py create mode 100644 monasca_notification/conf/notifiers.py create mode 100644 monasca_notification/conf/processors.py create mode 100644 monasca_notification/conf/queues.py create mode 100644 monasca_notification/conf/retry.py create mode 100644 monasca_notification/conf/statsd.py create mode 100644 monasca_notification/conf/types.py create mode 100644 monasca_notification/conf/zookeeper.py create mode 100644 monasca_notification/config.py create mode 100644 monasca_notification/version.py create mode 100644 tests/base.py create mode 100644 tests/resources/notification.yaml create mode 100644 tests/test_config.py diff --git a/.gitignore b/.gitignore index b62244d..7475aca 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ dist .stestr/ .coverage.* cover/ + +*.sample diff --git a/config-generator/notification.conf b/config-generator/notification.conf new file mode 100644 index 0000000..14fca52 --- /dev/null +++ b/config-generator/notification.conf @@ -0,0 +1,5 @@ +[DEFAULT] +output_file = etc/monasca/notification.conf.sample +wrap_width = 80 +namespace = monasca_notification +namespace = oslo.log diff --git a/etc/monasca/notification-logging.conf b/etc/monasca/notification-logging.conf new file mode 100644 index 0000000..5b8814c --- /dev/null +++ b/etc/monasca/notification-logging.conf @@ -0,0 +1,46 @@ +[loggers] +keys = root, kafka, zookeeper, statsd + +[handlers] +keys = console, file + +[formatters] +keys = context + +[logger_root] +level = DEBUG +handlers = console, file + +[logger_kafka] +qualname = kafka +level = DEBUG +handlers = console, file +propagate = 0 + +[logger_zookeeper] +qualname = zookeeper +level = DEBUG +handlers = console, file +propagate = 0 + +[logger_statsd] +qualname = statsd +level = DEBUG +handlers = console, file +propagate = 0 + +[handler_console] +class = logging.StreamHandler +args = (sys.stderr,) +level = DEBUG +formatter = context + +[handler_file] +class = logging.handlers.RotatingFileHandler +level = DEBUG +formatter = context +# store up to 5*100MB of logs +args = ('monasca-notification.log', 'a', 104857600, 5) + +[formatter_context] +class = oslo_log.formatters.ContextFormatter diff --git a/monasca_notification/common/repositories/mysql/mysql_repo.py b/monasca_notification/common/repositories/mysql/mysql_repo.py index 2209ee0..eb4a926 100644 --- a/monasca_notification/common/repositories/mysql/mysql_repo.py +++ b/monasca_notification/common/repositories/mysql/mysql_repo.py @@ -11,7 +11,7 @@ # or implied. See the License for the specific language governing permissions and limitations under # the License. -import logging +from oslo_log import log as logging import pymysql from monasca_notification.common.repositories.base import base_repo @@ -24,21 +24,9 @@ log = logging.getLogger(__name__) class MysqlRepo(base_repo.BaseRepo): def __init__(self, config): super(MysqlRepo, self).__init__(config) - if 'ssl' in config['mysql']: - self._mysql_ssl = config['mysql']['ssl'] - else: - self._mysql_ssl = None - - if 'port' in config['mysql']: - self._mysql_port = config['mysql']['port'] - else: - # - # If port isn't specified in the config file, - # set it to the default value. - # - self._mysql_port = 3306 - + self._mysql_ssl = config['mysql']['ssl'] or None self._mysql_host = config['mysql']['host'] + self._mysql_port = config['mysql']['port'] self._mysql_user = config['mysql']['user'] self._mysql_passwd = config['mysql']['passwd'] self._mysql_dbname = config['mysql']['db'] diff --git a/monasca_notification/common/repositories/orm/orm_repo.py b/monasca_notification/common/repositories/orm/orm_repo.py index 508e7f1..cd76d21 100644 --- a/monasca_notification/common/repositories/orm/orm_repo.py +++ b/monasca_notification/common/repositories/orm/orm_repo.py @@ -1,4 +1,4 @@ -# Copyright 2015 FUJITSU LIMITED +# Copyright 2015-2017 FUJITSU LIMITED # (C) Copyright 2015,2016 Hewlett Packard Enterprise Development LP # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except @@ -11,7 +11,8 @@ # or implied. See the License for the specific language governing permissions and limitations under # the License. -import logging +from oslo_config import cfg +from oslo_log import log as logging from sqlalchemy import engine_from_config, MetaData from sqlalchemy.sql import select, bindparam, and_, insert from sqlalchemy.exc import DatabaseError @@ -19,12 +20,15 @@ from sqlalchemy.exc import DatabaseError from monasca_notification.common.repositories import exceptions as exc from monasca_notification.common.repositories.orm import models -log = logging.getLogger(__name__) +LOG = logging.getLogger(__name__) +CONF = cfg.CONF class OrmRepo(object): - def __init__(self, config): - self._orm_engine = engine_from_config(config['database']['orm'], prefix='') + def __init__(self): + self._orm_engine = engine_from_config({ + 'url': CONF.orm.url + }, prefix='') metadata = MetaData() @@ -54,38 +58,38 @@ class OrmRepo(object): def fetch_notifications(self, alarm): try: with self._orm_engine.connect() as conn: - log.debug('Orm query {%s}', str(self._orm_query)) + LOG.debug('Orm query {%s}', str(self._orm_query)) notifications = conn.execute(self._orm_query, alarm_definition_id=alarm['alarmDefinitionId'], alarm_state=alarm['newState']) return [(row[0], row[1].lower(), row[2], row[3], row[4]) for row in notifications] except DatabaseError as e: - log.exception("Couldn't fetch alarms actions %s", e) + LOG.exception("Couldn't fetch alarms actions %s", e) raise exc.DatabaseException(e) def get_alarm_current_state(self, alarm_id): try: with self._orm_engine.connect() as conn: - log.debug('Orm query {%s}', str(self._orm_get_alarm_state)) + LOG.debug('Orm query {%s}', str(self._orm_get_alarm_state)) result = conn.execute(self._orm_get_alarm_state, alarm_id=alarm_id) row = result.fetchone() state = row[0] if row is not None else None return state except DatabaseError as e: - log.exception("Couldn't fetch the current alarm state %s", e) + LOG.exception("Couldn't fetch the current alarm state %s", e) raise exc.DatabaseException(e) def fetch_notification_method_types(self): try: with self._orm_engine.connect() as conn: - log.debug('Orm query {%s}', str(self._orm_nmt_query)) + LOG.debug('Orm query {%s}', str(self._orm_nmt_query)) notification_method_types = conn.execute(self._orm_nmt_query).fetchall() return [row[0] for row in notification_method_types] except DatabaseError as e: - log.exception("Couldn't fetch notification method types %s", e) + LOG.exception("Couldn't fetch notification method types %s", e) raise exc.DatabaseException(e) def insert_notification_method_types(self, notification_types): @@ -98,13 +102,13 @@ class OrmRepo(object): conn.execute(self._orm_add_notification_type, b_name=notification_type) except DatabaseError as e: - log.debug("Failed to insert notification types %s", notification_types) + LOG.debug("Failed to insert notification types %s", notification_types) raise exc.DatabaseException(e) def get_notification(self, notification_id): try: with self._orm_engine.connect() as conn: - log.debug('Orm query {%s}', str(self._orm_get_notification)) + LOG.debug('Orm query {%s}', str(self._orm_get_notification)) result = conn.execute(self._orm_get_notification, notification_id=notification_id) notification = result.fetchone() @@ -113,5 +117,5 @@ class OrmRepo(object): else: return [notification[0], notification[1].lower(), notification[2], notification[3]] except DatabaseError as e: - log.exception("Couldn't fetch the notification method %s", e) + LOG.exception("Couldn't fetch the notification method %s", e) raise exc.DatabaseException(e) diff --git a/monasca_notification/common/repositories/postgres/pgsql_repo.py b/monasca_notification/common/repositories/postgres/pgsql_repo.py index 95235a6..1ba1e7c 100644 --- a/monasca_notification/common/repositories/postgres/pgsql_repo.py +++ b/monasca_notification/common/repositories/postgres/pgsql_repo.py @@ -1,4 +1,4 @@ -# Copyright 2015 FUJITSU LIMITED +# Copyright 2015-2017 FUJITSU LIMITED # (C) Copyright 2016 Hewlett Packard Enterprise Development LP # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except @@ -11,7 +11,7 @@ # or implied. See the License for the specific language governing permissions and limitations under # the License. -import logging +from oslo_log import log as logging import psycopg2 from monasca_notification.common.repositories.base import base_repo diff --git a/monasca_notification/common/utils.py b/monasca_notification/common/utils.py index 712e265..58c1984 100644 --- a/monasca_notification/common/utils.py +++ b/monasca_notification/common/utils.py @@ -13,25 +13,25 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging import monascastatsd -from monasca_common.simport import simport +from oslo_config import cfg +from oslo_log import log from monasca_notification.common.repositories import exceptions from monasca_notification.notification import Notification -log = logging.getLogger(__name__) +LOG = log.getLogger(__name__) +CONF = cfg.CONF NOTIFICATION_DIMENSIONS = {'service': 'monitoring', 'component': 'monasca-notification'} -def get_db_repo(config): - if 'database' in config and 'repo_driver' in config['database']: - return simport.load(config['database']['repo_driver'])(config) - else: - return simport.load('monasca_notification.common.repositories.mysql.mysql_repo:MysqlRepo')(config) +def get_db_repo(): + repo_driver = CONF.database.repo_driver + LOG.debug('Enabling the %s RDB repository', repo_driver) + return repo_driver(CONF) def construct_notification_object(db_repo, notification_json): @@ -47,7 +47,7 @@ def construct_notification_object(db_repo, notification_json): stored_notification = grab_stored_notification_method(db_repo, notification.id) # Notification method was deleted if stored_notification is None: - log.debug("Notification method {0} was deleted from database. " + LOG.debug("Notification method {0} was deleted from database. " "Will stop sending.".format(notification.id)) return None # Update notification method with most up to date values @@ -58,11 +58,11 @@ def construct_notification_object(db_repo, notification_json): notification.period = stored_notification[3] return notification except exceptions.DatabaseException: - log.warn("Error querying mysql for notification method. " + LOG.warn("Error querying mysql for notification method. " "Using currently cached method.") return notification except Exception as e: - log.warn("Error when attempting to construct notification {0}".format(e)) + LOG.warn("Error when attempting to construct notification {0}".format(e)) return None @@ -70,21 +70,18 @@ def grab_stored_notification_method(db_repo, notification_id): try: stored_notification = db_repo.get_notification(notification_id) except exceptions.DatabaseException: - log.debug('Database Error. Attempting reconnect') + LOG.debug('Database Error. Attempting reconnect') stored_notification = db_repo.get_notification(notification_id) return stored_notification -def get_statsd_client(config, dimensions=None): +def get_statsd_client(dimensions=None): local_dims = dimensions.copy() if dimensions else {} local_dims.update(NOTIFICATION_DIMENSIONS) - if 'statsd' in config: - client = monascastatsd.Client(name='monasca', - host=config['statsd'].get('host', 'localhost'), - port=config['statsd'].get('port', 8125), - dimensions=local_dims) - else: - client = monascastatsd.Client(name='monasca', - dimensions=local_dims) + client = monascastatsd.Client(name='monasca', + host=CONF.statsd.host, + port=CONF.statsd.port, + dimensions=local_dims) + return client diff --git a/monasca_notification/conf/__init__.py b/monasca_notification/conf/__init__.py new file mode 100644 index 0000000..b20e5b2 --- /dev/null +++ b/monasca_notification/conf/__init__.py @@ -0,0 +1,162 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +import copy + +from oslo_config import cfg +from oslo_log import log +from oslo_utils import importutils + +from monasca_notification.conf import cli +from monasca_notification.conf import database +from monasca_notification.conf import kafka +from monasca_notification.conf import notifiers +from monasca_notification.conf import processors +from monasca_notification.conf import queues +from monasca_notification.conf import retry +from monasca_notification.conf import statsd +from monasca_notification.conf import zookeeper + +LOG = log.getLogger(__name__) +CONF = cfg.CONF + +CONF_OPTS = [ + cli, + database, + kafka, + notifiers, + processors, + queues, + retry, + statsd, + zookeeper +] + + +def register_opts(conf=None): + if conf is None: + conf = CONF + for m in CONF_OPTS: + m.register_opts(conf) + + +def list_opts(): + opts = collections.defaultdict(list) + for m in CONF_OPTS: + configs = copy.deepcopy(m.list_opts()) + for key, val in configs.items(): + opts[key].extend(val) + return _tupleize(opts) + + +def load_from_yaml(yaml_config, conf=None): + # build named BACKWARD_MAP to modules set_defaults + + if conf is None: + conf = CONF + + def _noop(*arg, **kwargs): + pass + + def _plain_override(g=None, **opts): + for k, v in opts.items(): + conf.set_override(group=g, name=k, override=v) + + def _load_plugin_settings(**notifiers_cfg): + notifiers_cfg = {t.lower(): v for t, v in notifiers_cfg.items()} + enabled_plugins = notifiers_cfg.pop('plugins', []) + + # We still can have these 3 (email, pagerduty, webhook) + # that are considered as builtin, hence should be always enabled + conf_to_plugin = { + 'email': 'monasca_notification.plugins.' + 'email_notifier:EmailNotifier', + 'pagerduty': 'monasca_notification.plugins.' + 'pagerduty_notifier:PagerdutyNotifier', + 'webhook': 'monasca_notification.plugins.' + 'webhook_notifier:WebhookNotifier' + } + for ctp_key, ctp_clazz in conf_to_plugin.items(): + if ctp_key in notifiers_cfg and ctp_key not in enabled_plugins: + LOG.debug('%s enabled as builtin plugin', ctp_key) + enabled_plugins.append(ctp_clazz) + + _plain_override(g='notification_types', enabled=enabled_plugins) + if not enabled_plugins: + return + + for ep in enabled_plugins: + ep_module = importutils.import_module(ep.split(':')[0]) + ep_clazz = importutils.import_class(ep.replace(':', '.')) + + if not hasattr(ep_module, 'register_opts'): + LOG.debug('%s does not have \'register_opts\' method') + continue + if not hasattr(ep_clazz, 'type'): + LOG.debug('%s does not have \'type\' class variable') + continue + + ep_r_opt = getattr(ep_module, 'register_opts') + ep_type = getattr(ep_clazz, 'type') + + ep_r_opt(conf) # register options + _plain_override(g='%s_notifier' % ep_type, + **notifiers_cfg.get(ep_type)) + + LOG.debug('Registered options and values of the %s notifier', + ep_type) + + def _configure_and_warn_the_logging(logging_config): + LOG.warning('Configuration of the logging system from ' + '\'notification.yml\' has been deprecated and ' + 'Please check how to configure logging with ' + 'oslo.log library.') + import logging.config + logging.config.dictConfig(logging_config) + + mappper = { + 'statsd': [lambda d: _plain_override(g='statsd', **d)], + 'retry': [lambda d: _plain_override(g='retry_engine', **d)], + 'database': [ + lambda d: _plain_override(g='database', repo_driver=d['repo_driver']), + lambda d: _plain_override(g='orm', url=d['orm']['url']) + ], + 'postgresql': [lambda d: _plain_override(g='postgresql', **d)], + 'mysql': [lambda d: _plain_override(g='mysql', **d)], + 'processors': [ + lambda d: _plain_override(g='alarm_processor', + number=d['alarm']['number'], + ttl=d['alarm']['ttl']), + lambda d: _plain_override(g='notification_processor', + number=d['notification']['number']) + ], + 'queues': [lambda d: _plain_override(g='queues', **d)], + 'kafka': [lambda d: _plain_override(g='kafka', **d)], + 'zookeeper': [lambda d: _plain_override(g='zookeeper', **d)], + 'notification_types': [lambda d: _load_plugin_settings(**d)], + 'logging': [_configure_and_warn_the_logging] + } + + for key, opts in yaml_config.items(): + LOG.debug('Loading group %s from deprecated yaml configuration', key) + handlers = mappper.get(key, [_noop]) + if len(handlers) == 1 and handlers[0] == _noop: + LOG.warning('Unmapped configuration group %s from YAML file', key) + [handler(opts) for handler in handlers] + + +def _tupleize(d): + """Convert a dict of options to the 2-tuple format.""" + return [(key, value) for key, value in d.items()] diff --git a/monasca_notification/conf/cli.py b/monasca_notification/conf/cli.py new file mode 100644 index 0000000..572dfca --- /dev/null +++ b/monasca_notification/conf/cli.py @@ -0,0 +1,40 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +cli_opts = [ + cfg.StrOpt(name='yaml_config', default=None, + positional=True, + help='Backward compatible option that allows to pass path ' + 'to YAML file containing configuration ' + 'of monasca-notitifcation.', + deprecated_for_removal=True, + deprecated_since='1.9.0', + deprecated_reason='monasca-notification has moved to ' + 'oslo.conf henceusing YAML based ' + 'configuration will be removed ' + 'after PIKE release.') +] + + +def register_opts(conf): + for opt in cli_opts: + conf.register_cli_opt(opt=opt) + + +def list_opts(): + return { + 'default': cli_opts + } diff --git a/monasca_notification/conf/database.py b/monasca_notification/conf/database.py new file mode 100644 index 0000000..2a654f2 --- /dev/null +++ b/monasca_notification/conf/database.py @@ -0,0 +1,129 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from monasca_notification.conf import types + +_POSTGRESQL = 'postgresql' +_MYSQL = 'mysql' +_ORM = 'orm' +_REPO_DRIVERS_MAP = { + _POSTGRESQL: 'monasca_notification.common.repositories.' + 'postgres.pgsql_repo:PostgresqlRepo', + _MYSQL: 'monasca_notification.common.repositories.' + 'mysql.mysql_repo:MysqlRepo', + _ORM: 'monasca_notification.common.repositories.' + 'orm.orm_repo:OrmRepo' +} +_ACCEPTABLE_DRIVER_KEYS = set(list(_REPO_DRIVERS_MAP.keys()) + + list(_REPO_DRIVERS_MAP.values())) + +_DEFAULT_DB_HOST = '127.0.0.1' +_DEFAULT_DB_USER = 'notification' +_DEFAULT_DB_PASSWORD = 'password' +_DEFAULT_DB_NAME = 'mon' +_DEFAULT_POSTGRESQL_PORT = 5432 +_DEFAULT_MYSQL_PORT = 3306 + +db_group = cfg.OptGroup('database', + title='Database Options', + help='Driver configuration for database connectivity.') + +db_opts = [ + types.PluginOpt(name='repo_driver', choices=_ACCEPTABLE_DRIVER_KEYS, + default=_MYSQL, plugin_map=_REPO_DRIVERS_MAP, + required=True, advanced=True, + help='Driver name (or full class path) that should be ' + 'used to handle RDB connections. Accepts either ' + 'short labels {0} or full class names {1}. ' + 'Configuring either of them will require presence of ' + 'one of following sections {0} inside configuration ' + 'file.'.format(_REPO_DRIVERS_MAP.keys(), + _REPO_DRIVERS_MAP.values()) + ) +] + +orm_group = cfg.OptGroup('orm', + title='ORM Options', + help='Configuration options to configure ' + 'ORM RBD driver.') +orm_opts = [ + cfg.StrOpt(name='url', default=None, + help='Connection string for sqlalchemy.') +] + +mysql_group = cfg.OptGroup('mysql', + title='MySQL Options', + help='Configuration options to configure ' + 'plain MySQL RBD driver.') +mysql_opts = [ + cfg.HostAddressOpt(name='host', default=_DEFAULT_DB_HOST, + help='IP address of MySQL instance.'), + cfg.PortOpt(name='port', default=_DEFAULT_MYSQL_PORT, + help='Port number of MySQL instance.'), + cfg.StrOpt(name='user', default=_DEFAULT_DB_USER, + help='Username to connect to MySQL ' + 'instance and given database.'), + cfg.StrOpt(name='passwd', default=_DEFAULT_DB_PASSWORD, + ignore_case=True, secret=True, + help='Password to connect to MySQL instance ' + 'and given database.'), + cfg.DictOpt(name='ssl', default={}, + help='A dict of arguments similar ' + 'to mysql_ssl_set parameters.'), + cfg.StrOpt(name='db', default=_DEFAULT_DB_NAME, + help='Database name available in given MySQL instance.') +] + +postgresql_group = cfg.OptGroup('postgresql', + title='PostgreSQL Options', + help='Configuration options to configure ' + 'plain PostgreSQL RBD driver.') +postgresql_opts = [ + cfg.HostAddressOpt(name='host', default=_DEFAULT_DB_HOST, + help='IP address of PostgreSQL instance.'), + cfg.PortOpt(name='port', default=_DEFAULT_POSTGRESQL_PORT, + help='Port number of PostgreSQL instance.'), + cfg.StrOpt(name='user', default=_DEFAULT_DB_USER, + help='Username to connect to PostgreSQL ' + 'instance and given database.'), + cfg.StrOpt(name='password', default=_DEFAULT_DB_PASSWORD, + secret=True, help='Password to connect to PostgreSQL ' + 'instance and given database'), + cfg.StrOpt(name='database', default=_DEFAULT_DB_NAME, + help='Database name available in ' + 'given PostgreSQL instance.') +] + + +def register_opts(conf): + conf.register_group(db_group) + conf.register_group(orm_group) + conf.register_group(mysql_group) + conf.register_group(postgresql_group) + + conf.register_opts(db_opts, group=db_group) + conf.register_opts(orm_opts, group=orm_group) + conf.register_opts(mysql_opts, group=mysql_group) + conf.register_opts(postgresql_opts, group=postgresql_group) + + +def list_opts(): + return { + db_group: db_opts, + orm_group: orm_opts, + mysql_group: mysql_opts, + postgresql_group: postgresql_opts, + } diff --git a/monasca_notification/conf/kafka.py b/monasca_notification/conf/kafka.py new file mode 100644 index 0000000..fdee2ac --- /dev/null +++ b/monasca_notification/conf/kafka.py @@ -0,0 +1,77 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from monasca_notification.conf import types + +_DEFAULT_URL = '127.0.0.1:9092' +_DEFAULT_GROUP = 'monasca-notification' +_DEFAULT_ALARM_TOPIC = 'alarm-state-transitions' +_DEFAULT_NOTIFICATION_TOPIC = 'alarm-notifications' +_DEFAULT_RETRY_TOPIC = 'retry-notifications' +_DEFAULT_PERIODIC_TOPICS = { + 60: '60-seconds-notifications' +} +_DEFAULT_MAX_OFFSET_LAG = 600 + +kafka_group = cfg.OptGroup('kafka', + title='Kafka Options', + help='Options under this group allow to configure ' + 'valid connection or Kafka queue.') + +kafka_opts = [ + cfg.ListOpt(name='url', item_type=types.HostAddressPortType(), + bounds=False, + default=_DEFAULT_URL, required=True, + help='List of addresses (with ports) pointing ' + 'at zookeeper cluster.'), + cfg.StrOpt(name='group', default=_DEFAULT_GROUP, + required=True, advanced=True, + help='Consumer\'s group for monasca-notification client.'), + cfg.StrOpt(name='alarm_topic', default=_DEFAULT_ALARM_TOPIC, + required=True, advanced=True, + help='Topic name in kafka where alarm ' + 'transitions are stored.'), + cfg.StrOpt(name='notification_topic', default=_DEFAULT_NOTIFICATION_TOPIC, + required=True, advanced=True, + help='Topic name in kafka where alarm ' + 'notifications are stored.'), + cfg.StrOpt(name='notification_retry_topic', default=_DEFAULT_RETRY_TOPIC, + required=True, advanced=True, + help='Topic name in kafka where notifications, that have ' + 'failed to be sent and are waiting for retry operations, ' + 'are stored.'), + cfg.DictOpt(name='periodic', default=_DEFAULT_PERIODIC_TOPICS, + required=True, advanced=True, + help='Dict of periodic topics. Keys are the period and ' + 'values the actual topic names in kafka where ' + 'notifications are stored.'), + cfg.IntOpt(name='max_offset_lag', default=_DEFAULT_MAX_OFFSET_LAG, + required=True, advanced=True, + help='Maximum lag for topic that is acceptable by ' + 'the monasca-notification. Notifications that are older ' + 'than this offset are skipped.') +] + + +def register_opts(conf): + conf.register_group(kafka_group) + conf.register_opts(kafka_opts, group=kafka_group) + + +def list_opts(): + return { + kafka_group: kafka_opts + } diff --git a/monasca_notification/conf/notifiers.py b/monasca_notification/conf/notifiers.py new file mode 100644 index 0000000..6f2787a --- /dev/null +++ b/monasca_notification/conf/notifiers.py @@ -0,0 +1,53 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +# NOTE(kornicameister) Until https://review.openstack.org/#/c/435136/ +# is merged we only treat these below as plugins. +# WEBHOOK, EMAIL, PAGERDUTY are now treated as built-in & hardcoded +# user has no possibility of enabling/disabling them + +_KEY_MAP = { + 'hipchat': 'monasca_notification.plugins.hipchat_notifier.HipChatNotifier', + 'slack': 'monasca_notification.plugins.slack_notifier.SlackNotifier', + 'jira': 'monasca_notification.plugins.jira_notifier.JiraNotifier' +} + +notifier_group = cfg.OptGroup('notification_types', + title='Notification types', + help='Group allows to configure available ' + 'notifiers inside notification engine.') + +notifier_opts = [ + cfg.ListOpt(name='enabled', default=[], + item_type=lambda x: _KEY_MAP.get(x, x), bounds=False, + advanced=True, sample_default=','.join(_KEY_MAP.keys()), + help='List of enabled notification types. You may specify ' + 'full class name {} ' + 'or shorter label {}.'.format(_KEY_MAP.get('hipchat'), + 'hipchat') + ) +] + + +def register_opts(conf): + conf.register_group(notifier_group) + conf.register_opts(notifier_opts, group=notifier_group) + + +def list_opts(): + return { + notifier_group: notifier_opts, + } diff --git a/monasca_notification/conf/processors.py b/monasca_notification/conf/processors.py new file mode 100644 index 0000000..ece177d --- /dev/null +++ b/monasca_notification/conf/processors.py @@ -0,0 +1,55 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +ap_group = cfg.OptGroup('alarm_processor', + title='Alarm processor group', + help='Options to configure alarm processor.') + +ap_opts = [ + cfg.IntOpt(name='number', min=1, default=2, + help='Number of alarm processors to spawn', + deprecated_for_removal=True, + deprecated_since='1.8.0', + deprecated_reason='Options is not used in the current code ' + 'and will be removed in future releases.'), + cfg.IntOpt(name='ttl', default=14400, + advanced=True, + help='Alarms older than TTL are not processed ' + 'by notification engine.') +] + +np_group = cfg.OptGroup('notification_processor', + title='Notification processor group', + help='Options to configure notification processor.') +np_opts = [ + cfg.IntOpt(name='number', min=1, + default=4, help='Number of notification processors to spawn.') +] + + +def register_opts(conf): + conf.register_group(ap_group) + conf.register_group(np_group) + + conf.register_opts(ap_opts, group=ap_group) + conf.register_opts(np_opts, group=np_group) + + +def list_opts(): + return { + ap_group: ap_opts, + np_group: np_opts + } diff --git a/monasca_notification/conf/queues.py b/monasca_notification/conf/queues.py new file mode 100644 index 0000000..96c59e7 --- /dev/null +++ b/monasca_notification/conf/queues.py @@ -0,0 +1,45 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +queues_group = cfg.OptGroup('queues', + title='Queues Options', + help=('Options under this group allow to ' + 'configure valid connection sizes of ' + 'all queues.')) + +queues_opts = [ + cfg.IntOpt(name='alarms_size', min=1, default=256, + help='Size of the alarms queue.'), + cfg.IntOpt(name='finished_size', min=1, default=256, + help='Size of the finished alarms queue.'), + cfg.IntOpt(name='notifications_size', min=1, default=256, + help='Size of notifications queue.'), + cfg.IntOpt(name='sent_notifications_size', min=1, default=50, + help='Size of sent notifications queue. ' + 'Limiting this size reduces potential or ' + 're-sent notifications after a failure.') +] + + +def register_opts(conf): + conf.register_group(queues_group) + conf.register_opts(queues_opts, group=queues_group) + + +def list_opts(): + return { + queues_group: queues_opts + } diff --git a/monasca_notification/conf/retry.py b/monasca_notification/conf/retry.py new file mode 100644 index 0000000..83b3ff9 --- /dev/null +++ b/monasca_notification/conf/retry.py @@ -0,0 +1,41 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +retry_engine_group = cfg.OptGroup('retry_engine', + title='Retry Engine Options', + help='Options under this group allow to ' + 'configure valid connection ' + 'for retry engine.') + +retry_opts = [ + cfg.IntOpt(name='interval', min=30, default=30, + advanced=True, + help='How often should retry happen.'), + cfg.IntOpt(name='max_attempts', default=5, + advanced=True, + help='How many times should retrying be tried.') +] + + +def register_opts(conf): + conf.register_group(retry_engine_group) + conf.register_opts(retry_opts, group=retry_engine_group) + + +def list_opts(): + return { + retry_engine_group: retry_opts + } diff --git a/monasca_notification/conf/statsd.py b/monasca_notification/conf/statsd.py new file mode 100644 index 0000000..5d5275c --- /dev/null +++ b/monasca_notification/conf/statsd.py @@ -0,0 +1,41 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +_DEFAULT_HOST = '127.0.0.1' +_DEFAULT_PORT = 8125 + +statsd_group = cfg.OptGroup('statsd', + title='statsd Options', + help='Options under this group allow ' + 'to configure valid connection ' + 'to statsd server launched by monasca-agent.') + +statsd_opts = [ + cfg.HostAddressOpt('host', default=_DEFAULT_HOST, + help='IP address of statsd server.'), + cfg.PortOpt('port', default=_DEFAULT_PORT, help='Port of statsd server.'), +] + + +def register_opts(conf): + conf.register_group(statsd_group) + conf.register_opts(statsd_opts, group=statsd_group) + + +def list_opts(): + return { + statsd_group: statsd_opts + } diff --git a/monasca_notification/conf/types.py b/monasca_notification/conf/types.py new file mode 100644 index 0000000..44b5b44 --- /dev/null +++ b/monasca_notification/conf/types.py @@ -0,0 +1,102 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from oslo_config import types +from oslo_log import log +from oslo_utils import importutils + +LOG = log.getLogger(__name__) + + +class Plugin(types.String): + def __init__(self, ignore_missing=False, choices=None, plugin_map=None): + + if not plugin_map: + # since simport is used, we cannot tell where plugin is located + # thus plugin map wouldn't contain it + plugin_map = {} + + super(Plugin, self).__init__(choices, quotes=False, ignore_case=True, + type_name='plugin value') + self._plugin_map = plugin_map + self._ingore_mission = ignore_missing + + def __call__(self, value): + value = super(Plugin, self).__call__(value) + value = self._get_actual_target(value) + cls = None + + try: + value = value.replace(':', '.') + cls = importutils.import_class(value) + except ImportError: + if not self._ingore_mission: + raise ValueError('%s cannot be imported' % value) + else: + LOG.exception('%s cannot be imported', value) + + return cls + + def _get_actual_target(self, value): + + # NOTE(trebskit) missing values will be handled + # by choices from StringType + + if value in self._plugin_map.keys(): + return self._plugin_map[value] + + return value + + +class PluginOpt(cfg.Opt): + def __init__(self, name, choices=None, plugin_map=None, **kwargs): + plugin = Plugin(choices=choices, plugin_map=plugin_map) + super(PluginOpt, self).__init__(name, + type=plugin, + **kwargs) + + +class HostAddressPortType(types.HostAddress): + """HostAddress with additional port""" + + def __init__(self, version=None): + type_name = 'host address port value' + super(HostAddressPortType, self).__init__(version, + type_name=type_name) + + def __call__(self, value): + addr, port = value.split(':') + + addr = self._validate_addr(addr) + port = self._validate_port(port) + + if addr and port: + return '%s:%d' % (addr, port) + raise ValueError('%s is not valid host address with optional port') + + @staticmethod + def _validate_port(port=80): + port = types.Port()(port) + return port + + def _validate_addr(self, addr): + try: + addr = self.ip_address(addr) + except ValueError: + try: + addr = self.hostname(addr) + except ValueError: + raise ValueError("%s is not a valid host address", addr) + return addr diff --git a/monasca_notification/conf/zookeeper.py b/monasca_notification/conf/zookeeper.py new file mode 100644 index 0000000..60c4a9a --- /dev/null +++ b/monasca_notification/conf/zookeeper.py @@ -0,0 +1,61 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from monasca_notification.conf import types + +_DEFAULT_URL = '127.0.0.1:2181' +_DEFAULT_NOTIFICATION_PATH = '/notification/alarms' +_DEFAULT_RETRY_PATH = '/notification/retry' +_DEFAULT_PERIODIC_PATH = { + 60: '/notification/60_seconds' +} + +zookeeper_group = cfg.OptGroup('zookeeper', + title='Zookeeper Options', + help='Options under this group allow to ' + 'configure settings for zookeeper ' + 'handling.') + +zookeeper_opts = [ + cfg.ListOpt(name='url', item_type=types.HostAddressPortType(), + default=_DEFAULT_URL, required=True, + help='List of addresses (with ports) pointing ' + 'at zookeeper cluster.'), + cfg.StrOpt(name='notification_path', default=_DEFAULT_NOTIFICATION_PATH, + required=True, advanced=True, + help='Path in zookeeper tree to track notification offsets.'), + cfg.StrOpt(name='notification_retry_path', default=_DEFAULT_RETRY_PATH, + required=True, advanced=True, + help='Path in zookeeper tree to track notification ' + 'retries offsets.'), + cfg.DictOpt(name='periodic_path', default=_DEFAULT_PERIODIC_PATH, + required=True, advanced=True, + help='Paths in zookeeper tree to track periodic offsets. ' + 'Keys must be integers describing the interval ' + 'of periodic notification. Values are actual ' + 'paths inside zookeeper tree.') +] + + +def register_opts(conf): + conf.register_group(zookeeper_group) + conf.register_opts(zookeeper_opts, group=zookeeper_group) + + +def list_opts(): + return { + zookeeper_group: zookeeper_opts + } diff --git a/monasca_notification/config.py b/monasca_notification/config.py new file mode 100644 index 0000000..9adee1b --- /dev/null +++ b/monasca_notification/config.py @@ -0,0 +1,64 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log +import yaml + +from monasca_notification import conf +from monasca_notification import version + +LOG = log.getLogger(__name__) +CONF = conf.CONF +_CONF_LOADED = False + + +def parse_args(argv, no_yaml=False): + """Sets up configuration of monasca-notification.""" + + global _CONF_LOADED + if _CONF_LOADED: + LOG.debug('Configuration has been already loaded') + return + + conf.register_opts(CONF) + log.register_options(CONF) + + default_log_levels = (log.get_default_log_levels()) + log.set_defaults(default_log_levels=default_log_levels) + + CONF(args=argv, + project='monasca', + prog='notification', + version=version.version_string, + description=''' + monasca-notification is an engine responsible for + transforming alarm transitions into proper notifications + ''') + log.setup(CONF, + product_name='monasca-notification', + version=version.version_string) + + if not no_yaml: + # note(trebskit) used only in test cases as the notification.yml + # will be dropped eventually + set_from_yaml() + + _CONF_LOADED = True + + +def set_from_yaml(): + if CONF.yaml_config: + LOG.info('Detected usage of deprecated YAML configuration') + yaml_cfg = yaml.safe_load(open(CONF.yaml_config, 'rb')) + conf.load_from_yaml(yaml_cfg) diff --git a/monasca_notification/main.py b/monasca_notification/main.py index 8023ff4..8f091a1 100644 --- a/monasca_notification/main.py +++ b/monasca_notification/main.py @@ -18,20 +18,23 @@ This engine reads alarms from Kafka and then notifies the customer using their configured notification method. """ -import logging -import logging.config import multiprocessing import os import signal import sys import time -import yaml +import warnings -from monasca_notification.notification_engine import NotificationEngine -from monasca_notification.periodic_engine import PeriodicEngine -from monasca_notification.retry_engine import RetryEngine +from oslo_log import log + +from monasca_notification import config +from monasca_notification import notification_engine +from monasca_notification import periodic_engine +from monasca_notification import retry_engine + +LOG = log.getLogger(__name__) +CONF = config.CONF -log = logging.getLogger(__name__) processors = [] # global list to facilitate clean signal handling exiting = False @@ -45,10 +48,10 @@ def clean_exit(signum, frame=None): # Since this is set up as a handler for SIGCHLD when this kills one # child it gets another signal, the global exiting avoids this running # multiple times. - log.debug('Exit in progress clean_exit received additional signal %s' % signum) + LOG.debug('Exit in progress clean_exit received additional signal %s' % signum) return - log.info('Received signal %s, beginning graceful shutdown.' % signum) + LOG.info('Received signal %s, beginning graceful shutdown.' % signum) exiting = True wait_for_exit = False @@ -68,7 +71,7 @@ def clean_exit(signum, frame=None): # Kill everything, that didn't already die for child in multiprocessing.active_children(): - log.debug('Killing pid %s' % child.pid) + LOG.debug('Killing pid %s' % child.pid) try: os.kill(child.pid, signal.SIGKILL) except Exception: # nosec @@ -82,50 +85,35 @@ def clean_exit(signum, frame=None): sys.exit(signum) -def start_process(process_type, config, *args): - log.info("start process: {}".format(process_type)) - p = process_type(config, *args) +def start_process(process_type, *args): + LOG.info("start process: {}".format(process_type)) + p = process_type(*args) p.run() def main(argv=None): - if argv is None: - argv = sys.argv - if len(argv) == 2: - config_file = argv[1] - elif len(argv) > 2: - print("Usage: " + argv[0] + " ") - print("Config file defaults to /etc/monasca/notification.yaml") - return 1 - else: - config_file = '/etc/monasca/notification.yaml' + warnings.simplefilter('always') + config.parse_args(argv=argv) - config = yaml.safe_load(open(config_file, 'rb')) - - # Setup logging - try: - if config['logging']['raise_exceptions'] is True: - logging.raiseExceptions = True - else: - logging.raiseExceptions = False - except KeyError: - logging.raiseExceptions = False - pass - logging.config.dictConfig(config['logging']) - - for proc in range(0, config['processors']['notification']['number']): + for proc in range(0, CONF.notification_processor.number): processors.append(multiprocessing.Process( - target=start_process, args=(NotificationEngine, config))) + target=start_process, + args=(notification_engine.NotificationEngine,)) + ) processors.append(multiprocessing.Process( - target=start_process, args=(RetryEngine, config))) + target=start_process, + args=(retry_engine.RetryEngine,)) + ) - if 60 in config['kafka']['periodic']: + if 60 in CONF.kafka.periodic: processors.append(multiprocessing.Process( - target=start_process, args=(PeriodicEngine, config, 60))) + target=start_process, + args=(periodic_engine.PeriodicEngine, 60)) + ) try: - log.info('Starting processes') + LOG.info('Starting processes') for process in processors: process.start() @@ -139,8 +127,9 @@ def main(argv=None): time.sleep(10) except Exception: - log.exception('Error! Exiting.') + LOG.exception('Error! Exiting.') clean_exit(signal.SIGKILL) + if __name__ == "__main__": - sys.exit(main()) + sys.exit(main(sys.argv[1:])) diff --git a/monasca_notification/notification_engine.py b/monasca_notification/notification_engine.py index 79b952a..3673560 100644 --- a/monasca_notification/notification_engine.py +++ b/monasca_notification/notification_engine.py @@ -1,4 +1,5 @@ # (C) Copyright 2015-2016 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,44 +14,40 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import time +from oslo_config import cfg +from oslo_log import log as logging + from monasca_common.kafka import consumer from monasca_common.kafka import producer from monasca_notification.common.utils import get_statsd_client -from monasca_notification.processors.alarm_processor import AlarmProcessor -from monasca_notification.processors.notification_processor import NotificationProcessor +from monasca_notification.processors import alarm_processor as ap +from monasca_notification.processors import notification_processor as np log = logging.getLogger(__name__) +CONF = cfg.CONF class NotificationEngine(object): - def __init__(self, config): - self._topics = {} - self._topics['notification_topic'] = config['kafka']['notification_topic'] - self._topics['retry_topic'] = config['kafka']['notification_retry_topic'] - - self._statsd = get_statsd_client(config) + def __init__(self): + self._statsd = get_statsd_client() self._consumer = consumer.KafkaConsumer( - config['kafka']['url'], - config['zookeeper']['url'], - config['zookeeper']['notification_path'], - config['kafka']['group'], - config['kafka']['alarm_topic']) - self._producer = producer.KafkaProducer(config['kafka']['url']) - self._alarm_ttl = config['processors']['alarm']['ttl'] - self._alarms = AlarmProcessor(self._alarm_ttl, config) - self._notifier = NotificationProcessor(config) - - self._config = config + CONF.kafka.url, + ','.join(CONF.zookeeper.url), + CONF.zookeeper.notification_path, + CONF.kafka.group, + CONF.kafka.alarm_topic) + self._producer = producer.KafkaProducer(CONF.kafka.url) + self._alarms = ap.AlarmProcessor() + self._notifier = np.NotificationProcessor() def _add_periodic_notifications(self, notifications): for notification in notifications: topic = notification.periodic_topic - if topic in self._config['kafka']['periodic'] and notification.type == "webhook": + if topic in CONF.kafka.periodic and notification.type == "webhook": notification.notification_timestamp = time.time() - self._producer.publish(self._config['kafka']['periodic'][topic], + self._producer.publish(CONF.kafka.periodic[topic], [notification.to_json()]) def run(self): @@ -62,9 +59,9 @@ class NotificationEngine(object): self._add_periodic_notifications(notifications) sent, failed = self._notifier.send(notifications) - self._producer.publish(self._topics['notification_topic'], + self._producer.publish(CONF.kafka.notification_topic, [i.to_json() for i in sent]) - self._producer.publish(self._topics['retry_topic'], + self._producer.publish(CONF.kafka.notification_retry_topic, [i.to_json() for i in failed]) self._consumer.commit() diff --git a/monasca_notification/periodic_engine.py b/monasca_notification/periodic_engine.py index 4844848..1c86291 100644 --- a/monasca_notification/periodic_engine.py +++ b/monasca_notification/periodic_engine.py @@ -1,4 +1,5 @@ # (C) Copyright 2016 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +15,11 @@ # limitations under the License. import json -import logging import time +from oslo_config import cfg +from oslo_log import log as logging + from monasca_common.kafka import consumer from monasca_common.kafka import producer from monasca_notification.common.repositories import exceptions @@ -26,25 +29,26 @@ from monasca_notification.common.utils import get_statsd_client from monasca_notification.processors import notification_processor log = logging.getLogger(__name__) +CONF = cfg.CONF class PeriodicEngine(object): - def __init__(self, config, period): - self._topic_name = config['kafka']['periodic'][period] + def __init__(self, period): + self._topic_name = CONF.kafka.periodic[period] - self._statsd = get_statsd_client(config) + self._statsd = get_statsd_client() - zookeeper_path = config['zookeeper']['periodic_path'][period] - self._consumer = consumer.KafkaConsumer(config['kafka']['url'], - config['zookeeper']['url'], + zookeeper_path = CONF.zookeeper.periodic_path[period] + self._consumer = consumer.KafkaConsumer(CONF.kafka.url, + ','.join(CONF.zookeeper.url), zookeeper_path, - config['kafka']['group'], + CONF.kafka.group, self._topic_name) - self._producer = producer.KafkaProducer(config['kafka']['url']) + self._producer = producer.KafkaProducer(CONF.kafka.url) - self._notifier = notification_processor.NotificationProcessor(config) - self._db_repo = get_db_repo(config) + self._notifier = notification_processor.NotificationProcessor() + self._db_repo = get_db_repo() self._period = period def _keep_sending(self, alarm_id, original_state, type, period): diff --git a/monasca_notification/plugins/abstract_notifier.py b/monasca_notification/plugins/abstract_notifier.py index 5426725..dfc9a0e 100644 --- a/monasca_notification/plugins/abstract_notifier.py +++ b/monasca_notification/plugins/abstract_notifier.py @@ -1,4 +1,5 @@ # (C) Copyright 2015 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,10 +24,6 @@ class AbstractNotifier(object): def __init__(self): pass - @abc.abstractproperty - def type(self): - pass - @abc.abstractproperty def statsd_name(self): pass diff --git a/monasca_notification/plugins/email_notifier.py b/monasca_notification/plugins/email_notifier.py index f99e872..571716f 100644 --- a/monasca_notification/plugins/email_notifier.py +++ b/monasca_notification/plugins/email_notifier.py @@ -1,4 +1,5 @@ # (C) Copyright 2015-2016 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,9 +21,13 @@ import six import smtplib import time +from debtcollector import removals +from oslo_config import cfg from monasca_notification.plugins import abstract_notifier +CONF = cfg.CONF + EMAIL_SINGLE_HOST_BASE = u'''On host "{hostname}" for target "{target_host}" {message} Alarm "{alarm_name}" transitioned to the {state} state at {timestamp} UTC @@ -60,22 +65,39 @@ With dimensions {metric_dimensions}''' +def register_opts(conf): + gr = cfg.OptGroup(name='%s_notifier' % EmailNotifier.type) + opts = [ + cfg.StrOpt(name='from_addr'), + cfg.HostAddressOpt(name='server'), + cfg.PortOpt(name='port', default=25), + cfg.IntOpt(name='timeout', default=5, min=1), + cfg.StrOpt(name='user', default=None), + cfg.StrOpt(name='password', default=None, secret=True), + cfg.StrOpt(name='grafana_url', default=None) + ] + + conf.register_group(gr) + conf.register_opts(opts, group=gr) + + class EmailNotifier(abstract_notifier.AbstractNotifier): + type = 'email' + def __init__(self, log): super(EmailNotifier, self).__init__() self._log = log self._smtp = None - self._config = None - def config(self, config): - self._config = config + @removals.remove( + message='Configuration of notifier is available through oslo.cfg', + version='1.9.0', + removal_version='3.0.0' + ) + def config(self, config=None): self._smtp_connect() - @property - def type(self): - return "email" - @property def statsd_name(self): return "sent_smtp_count" @@ -122,7 +144,7 @@ class EmailNotifier(abstract_notifier.AbstractNotifier): return False def _sendmail(self, notification, msg): - self._smtp.sendmail(self._config['from_addr'], + self._smtp.sendmail(CONF.email_notifier.from_addr, notification.address, msg.as_string()) self._log.debug("Sent email to {}, notification {}".format(notification.address, @@ -135,15 +157,19 @@ class EmailNotifier(abstract_notifier.AbstractNotifier): def _smtp_connect(self): """Connect to the smtp server """ - self._log.info("Connecting to Email Server {}".format(self._config['server'])) + self._log.info("Connecting to Email Server {}".format( + CONF.email_notifier.server)) try: - smtp = smtplib.SMTP(self._config['server'], - self._config['port'], - timeout=self._config['timeout']) + smtp = smtplib.SMTP(CONF.email_notifier.server, + CONF.email_notifier.port, + timeout=CONF.email_notifier.timeout) - if ('user', 'password') in self._config.keys(): - smtp.login(self._config['user'], self._config['password']) + email_notifier_user = CONF.email_notifier.user + email_notifier_password = CONF.email_notifier.password + if email_notifier_user and email_notifier_password: + smtp.login(email_notifier_user, + email_notifier_password) self._smtp = smtp return True @@ -222,7 +248,7 @@ class EmailNotifier(abstract_notifier.AbstractNotifier): msg = email.mime.text.MIMEText(text, 'plain', 'utf-8') msg['Subject'] = email.header.Header(subject, 'utf-8') - msg['From'] = self._config['from_addr'] + msg['From'] = CONF.email_notifier.from_addr msg['To'] = notification.address msg['Date'] = email.utils.formatdate(localtime=True, usegmt=True) @@ -237,7 +263,7 @@ class EmailNotifier(abstract_notifier.AbstractNotifier): has been defined. """ - grafana_url = self._config.get('grafana_url', None) + grafana_url = CONF.email_notifier.grafana_url if grafana_url is None: return None diff --git a/monasca_notification/plugins/hipchat_notifier.py b/monasca_notification/plugins/hipchat_notifier.py index fbd3e42..d9a2ea4 100644 --- a/monasca_notification/plugins/hipchat_notifier.py +++ b/monasca_notification/plugins/hipchat_notifier.py @@ -1,4 +1,5 @@ # (C) Copyright 2016-2017 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,10 +17,13 @@ import requests import ujson as json +from debtcollector import removals +from oslo_config import cfg from six.moves import urllib from monasca_notification.plugins import abstract_notifier +CONF = cfg.CONF """ notification.address = https://hipchat.hpcloud.net/v2/room//notification?auth_token=432432 @@ -44,17 +48,34 @@ SEVERITY_COLORS = {"low": 'green', 'critical': 'red'} +def register_opts(conf): + gr = cfg.OptGroup(name='%s_notifier' % HipChatNotifier.type) + opts = [ + cfg.IntOpt(name='timeout', default=5, min=1), + cfg.BoolOpt(name='insecure', default=True), + cfg.StrOpt(name='ca_certs', default=None), + cfg.StrOpt(name='proxy', default=None) + ] + + conf.register_group(gr) + conf.register_opts(opts, group=gr) + + class HipChatNotifier(abstract_notifier.AbstractNotifier): + + type = 'hipchat' + def __init__(self, log): + super(HipChatNotifier, self).__init__() self._log = log - def config(self, config_dict): - self._config = {'timeout': 5} - self._config.update(config_dict) - - @property - def type(self): - return "hipchat" + @removals.remove( + message='Configuration of notifier is available through oslo.cfg', + version='1.9.0', + removal_version='3.0.0' + ) + def config(self, config_dict=None): + pass @property def statsd_name(self): @@ -97,14 +118,17 @@ class HipChatNotifier(abstract_notifier.AbstractNotifier): url = urllib.parse.urljoin(notification.address, urllib.parse.urlparse(notification.address).path) # Default option is to do cert verification - verify = not self._config.get('insecure', True) + verify = not CONF.hipchat_notifier.insecure + ca_certs = CONF.hipchat_notifier.ca_certs + proxy = CONF.hipchat_notifier.proxy + # If ca_certs is specified, do cert validation and ignore insecure flag - if (self._config.get("ca_certs")): - verify = self._config.get("ca_certs") + if ca_certs is not None: + verify = ca_certs proxyDict = None - if (self._config.get("proxy")): - proxyDict = {"https": self._config.get("proxy")} + if proxy is not None: + proxyDict = {'https': proxy} try: # Posting on the given URL @@ -113,7 +137,7 @@ class HipChatNotifier(abstract_notifier.AbstractNotifier): verify=verify, params=query_params, proxies=proxyDict, - timeout=self._config['timeout']) + timeout=CONF.hipchat_notifier.timeout) if result.status_code in range(200, 300): self._log.info("Notification successfully posted.") diff --git a/monasca_notification/plugins/jira_notifier.py b/monasca_notification/plugins/jira_notifier.py index 6bfcab3..86c2341 100644 --- a/monasca_notification/plugins/jira_notifier.py +++ b/monasca_notification/plugins/jira_notifier.py @@ -1,4 +1,5 @@ # (C) Copyright 2016 Hewlett Packard Enterprise Development Company LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,6 +20,9 @@ from six.moves import urllib import ujson as json import yaml +from debtcollector import removals +from oslo_config import cfg + from monasca_notification.plugins.abstract_notifier import AbstractNotifier """ @@ -51,27 +55,40 @@ from monasca_notification.plugins.abstract_notifier import AbstractNotifier """ +CONF = cfg.CONF + + +def register_opts(conf): + gr = cfg.OptGroup(name='%s_notifier' % JiraNotifier.type) + opts = [ + cfg.IntOpt(name='timeout', default=5, min=1), + cfg.StrOpt(name='user', required=False), + cfg.StrOpt(name='password', required=False, secret=True), + cfg.StrOpt(name='custom_formatter', default=None), + cfg.StrOpt(name='proxy', default=None) + ] + + conf.register_group(gr) + conf.register_opts(opts, group=gr) + + class JiraNotifier(AbstractNotifier): + type = 'jira' _search_query = search_query = "project={} and reporter='{}' and summary ~ '{}'" def __init__(self, log): + super(JiraNotifier, self).__init__() self._log = log self.jira_fields_format = None + @removals.remove( + message='Configuration of notifier is available through oslo.cfg', + version='1.9.0', + removal_version='3.0.0' + ) def config(self, config_dict): - self._config = {'timeout': 5} - if not config_dict.get("user") and not config_dict.get("password"): - message = "Missing user and password settings in JIRA plugin configuration" - self._log.exception(message) - raise Exception(message) - - self._config.update(config_dict) - self.jira_fields_format = self._get_jira_custom_format_fields() - - @property - def type(self): - return "jira" + pass @property def statsd_name(self): @@ -80,9 +97,10 @@ class JiraNotifier(AbstractNotifier): def _get_jira_custom_format_fields(self): jira_fields_format = None - if (not self.jira_fields_format and self._config.get("custom_formatter")): + formatter = CONF.jira_notifier.custom_formatter + if not self.jira_fields_format and formatter: try: - with open(self._config.get("custom_formatter")) as f: + with open(formatter, 'r') as f: jira_fields_format = yaml.safe_load(f) except Exception: self._log.exception("Unable to read custom_formatter file. Check file location") @@ -139,8 +157,10 @@ class JiraNotifier(AbstractNotifier): return jira_fields def _build_jira_message(self, notification): - if self._config.get("custom_formatter"): - return self._build_custom_jira_message(notification, self.jira_fields_format) + formatter = CONF.jira_notifier.custom_formatter + if formatter: + return self._build_custom_jira_message(notification, + self._get_jira_custom_format_fields()) return self._build_default_jira_message(notification) @@ -159,13 +179,17 @@ class JiraNotifier(AbstractNotifier): if query_params.get("component"): jira_fields["component"] = query_params["component"][0] - auth = (self._config["user"], self._config["password"]) - proxyDict = None - if (self._config.get("proxy")): - proxyDict = {"https": self._config.get("proxy")} + auth = ( + CONF.jira_notifier.user, + CONF.jira_notifier.password + ) + proxy = CONF.jira_notifier.proxy + proxy_dict = None + if proxy is not None: + proxy_dict = {"https": proxy} try: - jira_obj = jira.JIRA(url, basic_auth=auth, proxies=proxyDict) + jira_obj = jira.JIRA(url, basic_auth=auth, proxies=proxy_dict) self.jira_workflow(jira_fields, jira_obj, notification) except Exception: @@ -192,7 +216,8 @@ class JiraNotifier(AbstractNotifier): issue_dict["components"] = [{"name": jira_fields.get("component")}] search_term = self._search_query.format(issue_dict["project"]["key"], - self._config["user"], notification.alarm_id) + CONF.jira_notifier.user, + notification.alarm_id) issue_list = jira_obj.search_issues(search_term) if not issue_list: self._log.debug("Creating an issue with the data {}".format(issue_dict)) diff --git a/monasca_notification/plugins/pagerduty_notifier.py b/monasca_notification/plugins/pagerduty_notifier.py index 306a2ad..51fa738 100644 --- a/monasca_notification/plugins/pagerduty_notifier.py +++ b/monasca_notification/plugins/pagerduty_notifier.py @@ -1,4 +1,5 @@ # (C) Copyright 2015,2016 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,25 +17,44 @@ import requests import ujson as json +from debtcollector import removals +from oslo_config import cfg + from monasca_notification.plugins import abstract_notifier +CONF = cfg.CONF VALID_HTTP_CODES = [200, 201, 204] +def register_opts(conf): + gr = cfg.OptGroup(name='%s_notifier' % PagerdutyNotifier.type) + opts = [ + cfg.IntOpt(name='timeout', default=5, min=1), + cfg.StrOpt(name='url', + default='https://events.pagerduty.com/' + 'generic/2010-04-15/create_event.json') + ] + + conf.register_group(gr) + conf.register_opts(opts, group=gr) + + class PagerdutyNotifier(abstract_notifier.AbstractNotifier): + + type = 'pagerduty' + def __init__(self, log): + super(PagerdutyNotifier, self).__init__() self._log = log + @removals.remove( + message='Configuration of notifier is available through oslo.cfg', + version='1.9.0', + removal_version='3.0.0' + ) def config(self, config): - self._config = { - 'timeout': 5, - 'url': 'https://events.pagerduty.com/generic/2010-04-15/create_event.json'} - self._config.update(config) - - @property - def type(self): - return "pagerduty" + pass @property def statsd_name(self): @@ -44,7 +64,7 @@ class PagerdutyNotifier(abstract_notifier.AbstractNotifier): """Send pagerduty notification """ - url = self._config['url'] + url = CONF.pagerduty_notifier.url headers = {"content-type": "application/json"} body = {"service_key": notification.address, "event_type": "trigger", @@ -60,7 +80,7 @@ class PagerdutyNotifier(abstract_notifier.AbstractNotifier): result = requests.post(url=url, data=json.dumps(body), headers=headers, - timeout=self._config['timeout']) + timeout=CONF.pagerduty_notifier.timeout) if result.status_code in VALID_HTTP_CODES: return True diff --git a/monasca_notification/plugins/slack_notifier.py b/monasca_notification/plugins/slack_notifier.py index 4b281f4..8173121 100644 --- a/monasca_notification/plugins/slack_notifier.py +++ b/monasca_notification/plugins/slack_notifier.py @@ -1,4 +1,5 @@ # (C) Copyright 2016-2017 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,8 +18,26 @@ import requests from six.moves import urllib import ujson as json +from debtcollector import removals +from oslo_config import cfg + from monasca_notification.plugins import abstract_notifier +CONF = cfg.CONF + + +def register_opts(conf): + gr = cfg.OptGroup(name='%s_notifier' % SlackNotifier.type) + opts = [ + cfg.IntOpt(name='timeout', default=5, min=1), + cfg.BoolOpt(name='insecure', default=True), + cfg.StrOpt(name='ca_certs', default=None), + cfg.StrOpt(name='proxy', default=None) + ] + + conf.register_group(gr) + conf.register_opts(opts, group=gr) + class SlackNotifier(abstract_notifier.AbstractNotifier): """This module is a notification plugin to integrate with Slack. @@ -42,25 +61,24 @@ class SlackNotifier(abstract_notifier.AbstractNotifier): https://api.slack.com/incoming-webhooks """ - CONFIG_CA_CERTS = 'ca_certs' - CONFIG_INSECURE = 'insecure' - CONFIG_PROXY = 'proxy' - CONFIG_TIMEOUT = 'timeout' + type = 'slack' + MAX_CACHE_SIZE = 100 RESPONSE_OK = 'ok' _raw_data_url_caches = [] def __init__(self, log): + super(SlackNotifier, self).__init__() self._log = log + @removals.remove( + message='Configuration of notifier is available through oslo.cfg', + version='1.9.0', + removal_version='3.0.0' + ) def config(self, config_dict): - self._config = {'timeout': 5} - self._config.update(config_dict) - - @property - def type(self): - return "slack" + pass @property def statsd_name(self): @@ -141,12 +159,12 @@ class SlackNotifier(abstract_notifier.AbstractNotifier): # Default option is to do cert verification # If ca_certs is specified, do cert validation and ignore insecure flag - verify = self._config.get(self.CONFIG_CA_CERTS, - (not self._config.get(self.CONFIG_INSECURE, True))) + verify = CONF.slack_notifier.ca_certs or not CONF.slack_notifier.insecure - proxyDict = None - if (self.CONFIG_PROXY in self._config): - proxyDict = {'https': self._config.get(self.CONFIG_PROXY)} + proxy = CONF.slack_notifier.proxy + proxy_dict = None + if proxy is not None: + proxy_dict = {'https': proxy} data_format_list = ['json', 'data'] if url in SlackNotifier._raw_data_url_caches: @@ -159,8 +177,8 @@ class SlackNotifier(abstract_notifier.AbstractNotifier): 'url': url, 'verify': verify, 'params': query_params, - 'proxies': proxyDict, - 'timeout': self._config[self.CONFIG_TIMEOUT], + 'proxies': proxy_dict, + 'timeout': CONF.slack_notifier.timeout, data_format: slack_message } if self._send_message(request_options): diff --git a/monasca_notification/plugins/webhook_notifier.py b/monasca_notification/plugins/webhook_notifier.py index fcf63c7..ac21040 100644 --- a/monasca_notification/plugins/webhook_notifier.py +++ b/monasca_notification/plugins/webhook_notifier.py @@ -1,4 +1,5 @@ # (C) Copyright 2015,2016 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,20 +17,39 @@ import requests import ujson as json +from debtcollector import removals +from oslo_config import cfg + from monasca_notification.plugins import abstract_notifier +CONF = cfg.CONF + + +def register_opts(conf): + gr = cfg.OptGroup(name='%s_notifier' % WebhookNotifier.type) + opts = [ + cfg.IntOpt(name='timeout', default=5, min=1) + ] + + conf.register_group(gr) + conf.register_opts(opts, group=gr) + class WebhookNotifier(abstract_notifier.AbstractNotifier): + + type = 'webhook' + def __init__(self, log): + super(WebhookNotifier, self).__init__() self._log = log + @removals.remove( + message='Configuration of notifier is available through oslo.cfg', + version='1.9.0', + removal_version='3.0.0' + ) def config(self, config_dict): - self._config = {'timeout': 5} - self._config.update(config_dict) - - @property - def type(self): - return "webhook" + pass @property def statsd_name(self): @@ -60,7 +80,7 @@ class WebhookNotifier(abstract_notifier.AbstractNotifier): result = requests.post(url=url, data=json.dumps(body), headers=headers, - timeout=self._config['timeout']) + timeout=CONF.webhook_notifier.timeout) if result.status_code in range(200, 300): self._log.info("Notification successfully posted.") diff --git a/monasca_notification/processors/alarm_processor.py b/monasca_notification/processors/alarm_processor.py index ae2b754..20ac84a 100644 --- a/monasca_notification/processors/alarm_processor.py +++ b/monasca_notification/processors/alarm_processor.py @@ -13,10 +13,11 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. - -import logging -import six import time + +from oslo_config import cfg +from oslo_log import log as logging +import six import ujson as json from monasca_notification.common.repositories import exceptions as exc @@ -27,13 +28,14 @@ from monasca_notification import notification_exceptions log = logging.getLogger(__name__) +CONF = cfg.CONF class AlarmProcessor(object): - def __init__(self, alarm_ttl, config): - self._alarm_ttl = alarm_ttl - self._statsd = get_statsd_client(config) - self._db_repo = get_db_repo(config) + def __init__(self): + self._alarm_ttl = CONF.alarm_processor.ttl + self._statsd = get_statsd_client() + self._db_repo = get_db_repo() @staticmethod def _parse_alarm(alarm_data): diff --git a/monasca_notification/processors/notification_processor.py b/monasca_notification/processors/notification_processor.py index e757e4f..5cab47b 100644 --- a/monasca_notification/processors/notification_processor.py +++ b/monasca_notification/processors/notification_processor.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +from oslo_log import log as logging from monasca_notification.common.repositories import exceptions as exc from monasca_notification.common.utils import get_db_repo @@ -25,12 +25,14 @@ log = logging.getLogger(__name__) class NotificationProcessor(object): - def __init__(self, config): - self.statsd = get_statsd_client(config) + def __init__(self): + self.statsd = get_statsd_client() notifiers.init(self.statsd) - notifiers.load_plugins(config['notification_types']) - notifiers.config(config['notification_types']) - self._db_repo = get_db_repo(config) + + notifiers.load_plugins() + notifiers.config() + + self._db_repo = get_db_repo() self.insert_configured_plugins() def _remaining_plugin_types(self): diff --git a/monasca_notification/retry_engine.py b/monasca_notification/retry_engine.py index 8c4aaca..38a4fc1 100644 --- a/monasca_notification/retry_engine.py +++ b/monasca_notification/retry_engine.py @@ -1,4 +1,5 @@ # (C) Copyright 2015-2016 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +15,11 @@ # limitations under the License. import json -import logging import time +from oslo_config import cfg +from oslo_log import log as logging + from monasca_common.kafka import consumer from monasca_common.kafka import producer from monasca_notification.common.utils import construct_notification_object @@ -25,30 +28,24 @@ from monasca_notification.common.utils import get_statsd_client from monasca_notification.processors import notification_processor log = logging.getLogger(__name__) +CONF = cfg.CONF class RetryEngine(object): - def __init__(self, config): - self._retry_interval = config['retry']['interval'] - self._retry_max = config['retry']['max_attempts'] - - self._topics = {} - self._topics['notification_topic'] = config['kafka']['notification_topic'] - self._topics['retry_topic'] = config['kafka']['notification_retry_topic'] - - self._statsd = get_statsd_client(config) + def __init__(self): + self._statsd = get_statsd_client() self._consumer = consumer.KafkaConsumer( - config['kafka']['url'], - config['zookeeper']['url'], - config['zookeeper']['notification_retry_path'], - config['kafka']['group'], - config['kafka']['notification_retry_topic']) + CONF.kafka.url, + ','.join(CONF.zookeeper.url), + CONF.zookeeper.notification_retry_path, + CONF.kafka.group, + CONF.kafka.notification_retry_topic + ) + self._producer = producer.KafkaProducer(CONF.kafka.url) - self._producer = producer.KafkaProducer(config['kafka']['url']) - - self._notifier = notification_processor.NotificationProcessor(config) - self._db_repo = get_db_repo(config) + self._notifier = notification_processor.NotificationProcessor() + self._db_repo = get_db_repo() def run(self): for raw_notification in self._consumer: @@ -62,7 +59,7 @@ class RetryEngine(object): self._consumer.commit() continue - wait_duration = self._retry_interval - ( + wait_duration = CONF.retry_engine.interval - ( time.time() - notification_data['notification_timestamp']) if wait_duration > 0: @@ -71,19 +68,19 @@ class RetryEngine(object): sent, failed = self._notifier.send([notification]) if sent: - self._producer.publish(self._topics['notification_topic'], + self._producer.publish(CONF.kafka.notification_topic, [notification.to_json()]) if failed: notification.retry_count += 1 notification.notification_timestamp = time.time() - if notification.retry_count < self._retry_max: + if notification.retry_count < CONF.retry_engine.max_attempts: log.error(u"retry failed for {} with name {} " u"at {}. " u"Saving for later retry.".format(notification.type, notification.name, notification.address)) - self._producer.publish(self._topics['retry_topic'], + self._producer.publish(CONF.kafka.notification_retry_topic, [notification.to_json()]) else: log.error(u"retry failed for {} with name {} " @@ -92,6 +89,6 @@ class RetryEngine(object): .format(notification.type, notification.name, notification.address, - self._retry_max)) + CONF.retry_engine.max_attempts)) self._consumer.commit() diff --git a/monasca_notification/types/notifiers.py b/monasca_notification/types/notifiers.py index a1040a1..89cebff 100644 --- a/monasca_notification/types/notifiers.py +++ b/monasca_notification/types/notifiers.py @@ -1,4 +1,5 @@ # (C) Copyright 2015,2016 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,15 +14,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import time -from monasca_common.simport import simport +from debtcollector import removals +from oslo_config import cfg +from oslo_log import log as logging +from oslo_utils import importutils + from monasca_notification.plugins import email_notifier from monasca_notification.plugins import pagerduty_notifier from monasca_notification.plugins import webhook_notifier log = logging.getLogger(__name__) +CONF = cfg.CONF possible_notifiers = None configured_notifiers = None @@ -49,14 +54,16 @@ def init(statsd_obj): ] -def load_plugins(config): +def load_plugins(): global possible_notifiers - - for plugin_class in config.get("plugins", []): + for plugin_class in CONF.notification_types.enabled: try: - possible_notifiers.append(simport.load(plugin_class)(log)) + plugin_class = plugin_class.replace(':', '.') + clz = importutils.import_class(plugin_class) + possible_notifiers.append(clz(logging.getLogger(plugin_class))) except Exception: - log.exception("unable to load the class {0} , ignoring it".format(plugin_class)) + log.exception("unable to load the class %s , ignoring it" % + plugin_class) def enabled_notifications(): @@ -68,29 +75,20 @@ def enabled_notifications(): return results -def config(cfg): +@removals.remove( + message='Loading the plugin configuration has been moved to oslo, ' + 'This method will be fully deleted in future releases', + version='1.9.0', + removal_version='3.0.0' +) +def config(): global possible_notifiers, configured_notifiers, statsd_counter - formatted_config = {t.lower(): v for t, v in cfg.items()} for notifier in possible_notifiers: ntype = notifier.type.lower() - if ntype in formatted_config: - try: - notifier.config(formatted_config[ntype]) - configured_notifiers[ntype] = notifier - statsd_counter[ntype] = statsd.get_counter(notifier.statsd_name) - log.info("{} notification ready".format(ntype)) - except Exception: - log.exception("config exception for {}".format(ntype)) - else: - log.warn("No config data for type: {}".format(ntype)) - config_with_no_notifiers = set(formatted_config.keys()) - set(configured_notifiers.keys()) - # Plugins section contains only additional plugins and should not be - # considered as a separate plugin - if 'plugins' in config_with_no_notifiers: - config_with_no_notifiers.remove('plugins') - if config_with_no_notifiers: - log.warn("No notifiers found for {0}". format(", ".join(config_with_no_notifiers))) + configured_notifiers[ntype] = notifier + statsd_counter[ntype] = statsd.get_counter(notifier.statsd_name) + log.info("{} notification ready".format(ntype)) def send_notifications(notifications): diff --git a/monasca_notification/version.py b/monasca_notification/version.py new file mode 100644 index 0000000..6c44d83 --- /dev/null +++ b/monasca_notification/version.py @@ -0,0 +1,23 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from pbr import version + +__all__ = [ + 'version_info', + 'version_string' +] + +version_info = version.VersionInfo('monasca-notification') +version_string = version_info.version_string() diff --git a/requirements.txt b/requirements.txt index e34b634..63dd412 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,12 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -pbr!=2.1.0,>=2.0.0 # Apache-2.0 +pbr>=2.0.0,!=2.1.0 # Apache-2.0 +debtcollector>=1.2.0 # Apache-2.0 monasca-statsd>=1.1.0 # Apache-2.0 requests>=2.14.2 # Apache-2.0 -PyYAML>=3.10.0 # MIT +PyYAML>=3.10 # MIT six>=1.9.0 # MIT monasca-common>=1.4.0 # Apache-2.0 +oslo.config>=4.6.0 # Apache-2.0 +oslo.log>=3.30.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index 268218f..d40653c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,8 +13,10 @@ home-page = https://github.com/stackforge/monasca-notification license = Apache [entry_points] -console_scripts = +console_scripts = monasca-notification = monasca_notification.main:main +oslo.config.opts = + monasca_notification = monasca_notification.conf:list_opts [files] packages = monasca_notification diff --git a/test-requirements.txt b/test-requirements.txt index c8b2828..b5fb958 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,14 +2,16 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. # Hacking already pins down pep8, pyflakes and flake8 + bandit>=1.1.0 # Apache-2.0 +Babel>=2.3.4,!=2.4.0 # BSD hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0 -coverage!=4.4,>=4.0 # Apache-2.0 -mock>=2.0 # BSD -funcsigs>=0.4;python_version=='2.7' or python_version=='2.6' # Apache-2.0 -os-testr>=0.8.0 # Apache-2.0 +coverage>=4.0,!=4.4 # Apache-2.0 +mock>=2.0.0 # BSD +funcsigs>=1.0.0;python_version=='2.7' or python_version=='2.6' # Apache-2.0 +os-testr>=1.0.0 # Apache-2.0 oslotest>=1.10.0 # Apache-2.0 testrepository>=0.0.18 # Apache-2.0/BSD -SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT +SQLAlchemy>=1.0.10,!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8 # MIT PyMySQL>=0.7.6 # MIT License -psycopg2>=2.5 # LGPL/ZPL +psycopg2>=2.6.2 # LGPL/ZPL diff --git a/tests/base.py b/tests/base.py new file mode 100644 index 0000000..f39572a --- /dev/null +++ b/tests/base.py @@ -0,0 +1,83 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import fixtures +import mock + +from oslo_config import cfg +from oslo_config import fixture as oo_cfg +from oslotest import base as oslotest_base + +from monasca_notification import conf +from monasca_notification import config + + +class DisableStatsdFixture(fixtures.Fixture): + + def setUp(self): + super(DisableStatsdFixture, self).setUp() + statsd_patch = mock.patch('monascastatsd.Connection') + statsd_patch.start() + self.addCleanup(statsd_patch.stop) + + +class ConfigFixture(oo_cfg.Config): + """Mocks configuration""" + + def __init__(self): + super(ConfigFixture, self).__init__(config.CONF) + + def setUp(self): + super(ConfigFixture, self).setUp() + + self.addCleanup(self._clean_config_loaded_flag) + + conf.register_opts() + # prevent test from trying to load the yaml file + config.parse_args(argv=[], no_yaml=True) + + @staticmethod + def _clean_config_loaded_flag(): + config._CONF_LOADED = False + + +class BaseTestCase(oslotest_base.BaseTestCase): + + def setUp(self): + super(BaseTestCase, self).setUp() + self.useFixture(ConfigFixture()) + self.useFixture(DisableStatsdFixture()) + + @staticmethod + def conf_override(**kw): + """Override flag variables for a test.""" + group = kw.pop('group', None) + for k, v in kw.items(): + cfg.CONF.set_override(k, v, group) + + @staticmethod + def conf_default(**kw): + """Override flag variables for a test.""" + group = kw.pop('group', None) + for k, v in kw.items(): + cfg.CONF.set_default(k, v, group) + + +class PluginTestCase(BaseTestCase): + register_opts = None + + def setUp(self, register_opts=None): + super(PluginTestCase, self).setUp() + if register_opts: + register_opts(conf.CONF) diff --git a/tests/resources/notification.yaml b/tests/resources/notification.yaml new file mode 100644 index 0000000..cbc6a1f --- /dev/null +++ b/tests/resources/notification.yaml @@ -0,0 +1,125 @@ +kafka: + url: 127.0.0.1:9092 + group: a + alarm_topic: b + notification_topic: c + notification_retry_topic: d + periodic: + 60: e + max_offset_lag: 666 + +database: + repo_driver: mysql + orm: + url: 'postgres://a:b@127.0.0.1:9999/goo' + +mysql: + host: 100.99.100.99 + port: 3306 + user: goku + passwd: kame-ha-me-ha + db: planet_vegeta + +postgresql: + user: goku + password: kame-ha-me-ha + database: planet_vegeta + port: 9999 + host: 100.10.100.10 + +notification_types: + plugins: + - monasca_notification.plugins.hipchat_notifier:HipChatNotifier + - monasca_notification.plugins.slack_notifier:SlackNotifier + - monasca_notification.plugins.jira_notifier:JiraNotifier + + email: + server: 127.0.0.1 + port: 25 + user: + password: + timeout: 60 + from_addr: root@localhost + grafana_url: 'http://127.0.0.1:3000' + + webhook: + timeout: 123 + + pagerduty: + timeout: 231 + url: "https://a.b.c/d/e/f.json" + + hipchat: + timeout: 512 + ca_certs: "/a.crt" + insecure: True + proxy: https://myproxy.corp.invalid:9999 + + slack: + timeout: 512 + ca_certs: "/a.crt" + insecure: True + proxy: https://myproxy.corp.invalid:9999 + + jira: + user: username + password: password + timeout: 666 + custom_formatter: /some_yml.yml + proxy: www.example.org + +processors: + alarm: + number: 666 + ttl: 666 + notification: + number: 666 + +retry: + interval: 300 + max_attempts: 500 + +queues: + alarms_size: 1024 + finished_size: 1024 + notifications_size: 1024 + sent_notifications_size: 1024 + +zookeeper: + url: 127.0.0.1:2181 + notification_path: /foo/bar + notification_retry_path: /son/goku + periodic_path: + 666: /bu/666_bubu + +logging: + raise_exceptions: False + version: 1 + disable_existing_loggers: False + formatters: + default: + format: "%(asctime)s %(levelname)s %(name)s %(message)s" + handlers: + console: + class: logging.StreamHandler + formatter: default + file: + class : logging.handlers.RotatingFileHandler + filename: /tmp/notification.log + formatter: default + maxBytes: 10485760 # Rotate at file size ~10MB + backupCount: 5 # Keep 5 older logs around + loggers: + kazoo: + level: WARN + kafka: + level: WARN + statsd: + level: WARN + root: + handlers: + - console + level: DEBUG +statsd: + host: 'localhost' + port: 8125 diff --git a/tests/test_alarm_processor.py b/tests/test_alarm_processor.py index e6f32ae..c05691b 100644 --- a/tests/test_alarm_processor.py +++ b/tests/test_alarm_processor.py @@ -1,4 +1,5 @@ # (C) Copyright 2014-2016 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,17 +20,19 @@ import collections import json import mock import time -import unittest from monasca_notification import notification as m_notification from monasca_notification.processors import alarm_processor +from tests import base + alarm_tuple = collections.namedtuple('alarm_tuple', ['offset', 'message']) message_tuple = collections.namedtuple('message_tuple', ['value']) -class TestAlarmProcessor(unittest.TestCase): +class TestAlarmProcessor(base.BaseTestCase): def setUp(self): + super(TestAlarmProcessor, self).setUp() self.trap = [] def _create_raw_alarm(self, partition, offset, message): @@ -55,16 +58,13 @@ class TestAlarmProcessor(unittest.TestCase): mock_mysql.cursor.return_value = mock_mysql mock_mysql.__iter__.return_value = sql_response - config = {'mysql': {'ssl': None, - 'host': 'mysql_host', - 'port': 'mysql_port', - 'user': 'mysql_user', - 'db': 'dbname', - 'passwd': 'mysql_passwd'}, - 'statsd': {'host': 'localhost', - 'port': 8125}} - - processor = alarm_processor.AlarmProcessor(600, config) + self.conf_override(group='mysql', ssl=None, + host='localhost', port='3306', + user='mysql_user', db='dbname', + passwd='mysql_passwd') + self.conf_override(group='statsd', host='localhost', + port=8125) + processor = alarm_processor.AlarmProcessor() return processor.to_notification(alarm) diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..1d072b2 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,226 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +from oslo_config import cfg +from oslo_utils import importutils +import yaml + +from monasca_notification import conf +from monasca_notification import config +from tests import base + + +class TestConfig(base.BaseTestCase): + @mock.patch('monasca_notification.config.conf') + def test_should_load_deprecated_yaml(self, conf): + fake_config = """ + sayians: + - goku + - vegeta + """ + yaml_config = self.create_tempfiles( + files=[('notification', fake_config)], + ext='.yml' + )[0] + + self.conf_override(yaml_config=yaml_config) + + config.set_from_yaml() + + fake_yaml_config = { + 'sayians': ['goku', 'vegeta'] + } + conf.load_from_yaml.assert_called_once_with(fake_yaml_config) + + @mock.patch('monasca_notification.config.conf') + def test_should_not_load_deprecated_yaml(self, conf): + config.set_from_yaml() + conf.load_from_yaml.assert_not_called() + + +class TestYamlOverriding(base.BaseTestCase): + # TOP_LEVEL keys represents old groups in YAML file + VERIFIERS = { + 'statsd': { + 'groups': [ + ('statsd', { + 'host': 'localhost', + 'port': 8125 + }) + ] + }, + 'retry': { + 'groups': [ + ('retry_engine', { + 'interval': 300, + 'max_attempts': 500 + }) + ] + }, + 'queues': { + 'groups': [ + ('queues', { + 'alarms_size': 1024, + 'finished_size': 1024, + 'notifications_size': 1024, + 'sent_notifications_size': 1024 + }) + ] + }, + 'zookeeper': { + 'groups': [ + ('zookeeper', { + 'url': ['127.0.0.1:2181'], + 'notification_path': '/foo/bar', + 'periodic_path': { + 666: '/bu/666_bubu' + }, + }) + ] + }, + 'kafka': { + 'groups': [ + ('kafka', { + 'url': ['127.0.0.1:9092'], + 'group': 'a', + 'alarm_topic': 'b', + 'notification_topic': 'c', + 'notification_retry_topic': 'd', + 'periodic': { + 60: 'e' + }, + 'max_offset_lag': 666 + }) + ] + }, + 'processors': { + 'groups': [ + ('alarm_processor', {'number': 666, 'ttl': 666}), + ('notification_processor', {'number': 666}) + ] + }, + 'postgresql': { + 'groups': [ + ('postgresql', { + 'host': '100.10.100.10', + 'port': 9999, + 'user': 'goku', + 'password': 'kame-ha-me-ha', + 'database': 'planet_vegeta' + }) + ] + }, + 'mysql': { + 'groups': [ + ('mysql', { + 'host': '100.99.100.99', + 'port': 3306, + 'user': 'goku', + 'passwd': 'kame-ha-me-ha', + 'db': 'planet_vegeta', + 'ssl': {} + }) + ] + }, + 'database': { + 'groups': [ + ('database', {'repo_driver': importutils.import_class( + 'monasca_notification.common.repositories.mysql.' + 'mysql_repo.MysqlRepo')}), + ('orm', {'url': 'postgres://a:b@127.0.0.1:9999/goo'}) + ] + }, + 'notification_types': { + 'groups': [ + ('notification_types', { + 'enabled': [ + 'monasca_notification.plugins.hipchat_notifier:HipChatNotifier', + 'monasca_notification.plugins.slack_notifier:SlackNotifier', + 'monasca_notification.plugins.jira_notifier:JiraNotifier', + 'monasca_notification.plugins.email_notifier:EmailNotifier', + 'monasca_notification.plugins.pagerduty_notifier:PagerdutyNotifier', + 'monasca_notification.plugins.webhook_notifier:WebhookNotifier', + ] + }), + ('email_notifier', { + 'server': '127.0.0.1', + 'port': 25, + 'user': None, + 'password': None, + 'timeout': 60, + 'from_addr': 'root@localhost', + 'grafana_url': 'http://127.0.0.1:3000' + }), + ('webhook_notifier', {'timeout': 123}), + ('pagerduty_notifier', { + 'timeout': 231, + 'url': 'https://a.b.c/d/e/f.json' + }), + ('hipchat_notifier', { + 'timeout': 512, + 'ca_certs': "/a.crt", + 'insecure': True, + 'proxy': 'https://myproxy.corp.invalid:9999' + }), + ('slack_notifier', { + 'timeout': 512, + 'ca_certs': "/a.crt", + 'insecure': True, + 'proxy': 'https://myproxy.corp.invalid:9999' + }), + ('jira_notifier', { + 'user': 'username', + 'password': 'password', + 'timeout': 666, + 'custom_formatter': '/some_yml.yml', + 'proxy': 'www.example.org' + }) + ] + } + } + + def setUp(self): + super(TestYamlOverriding, self).setUp() + self.yaml_config = yaml.safe_load( + open('tests/resources/notification.yaml', 'rb') + ) + + def test_overriding(self): + + conf.load_from_yaml(yaml_config=self.yaml_config, conf=config.CONF) + opts = config.CONF + + for group in self.VERIFIERS.keys(): + verifier_details = self.VERIFIERS[group] + groups = verifier_details['groups'] + + for opt_group, opt_values in groups: + + for key, value in opt_values.items(): + try: + opt_value = opts[opt_group][key] + except (cfg.NoSuchOptError, cfg.NoSuchGroupError) as ex: + self.fail(str(ex)) + else: + msg = ('%s not overridden in group %s' + % (key, opt_group)) + + if (isinstance(value, list) and + isinstance(opt_value, list)): + for v in value: + self.assertIn(v, opt_value, msg) + continue + + self.assertEqual(value, opt_value, msg) diff --git a/tests/test_email_notification.py b/tests/test_email_notification.py index baef179..c5b1342 100644 --- a/tests/test_email_notification.py +++ b/tests/test_email_notification.py @@ -1,5 +1,6 @@ # coding=utf-8 # (C) Copyright 2014-2016 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,7 +21,6 @@ import mock import smtplib import socket import time -import unittest import six @@ -34,6 +34,7 @@ else: from monasca_notification.notification import Notification from monasca_notification.plugins import email_notifier +from tests import base UNICODE_CHAR = six.unichr(2344) UNICODE_CHAR_ENCODED = UNICODE_CHAR.encode("utf-8") @@ -113,20 +114,16 @@ class smtpStubException(object): raise smtplib.SMTPServerDisconnected -class TestEmail(unittest.TestCase): +class TestEmail(base.PluginTestCase): def setUp(self): + super(TestEmail, self).setUp(email_notifier.register_opts) + self.trap = [] - - self.email_config = {'server': 'my.smtp.server', - 'port': 25, - 'user': None, - 'password': None, - 'timeout': 60, - 'from_addr': 'hpcs.mon@hp.com', - 'grafana_url': 'http://127.0.0.1:3000'} - - def tearDown(self): - pass + self.conf_override(group='email_notifier', server='my.smtp.server', + port=25, user=None, + password=None, timeout=60, + from_addr='hpcs.mon@hp.com', + grafana_url='http://127.0.0.1:3000') def _smtpStub(self, *arg, **kwargs): return smtpStub(self.trap) @@ -143,7 +140,6 @@ class TestEmail(unittest.TestCase): mock_log.error = self.trap.append email = email_notifier.EmailNotifier(mock_log) - email.config(self.email_config) alarm_dict = alarm(metric) @@ -269,7 +265,7 @@ class TestEmail(unittest.TestCase): email = email_notifier.EmailNotifier(mock_log) - email.config(self.email_config) + email.config() alarm_dict = alarm(metrics) @@ -313,7 +309,7 @@ class TestEmail(unittest.TestCase): email = email_notifier.EmailNotifier(mock_log) - email.config(self.email_config) + email.config() del self.trap[:] @@ -324,8 +320,7 @@ class TestEmail(unittest.TestCase): email_result = email.send_notification(notification) self.assertFalse(email_result) - self.assertIn("Connecting to Email Server {}" - .format(self.email_config['server']), + self.assertIn("Connecting to Email Server my.smtp.server", self.trap) @mock.patch('monasca_notification.plugins.email_notifier.smtplib') @@ -358,7 +353,7 @@ class TestEmail(unittest.TestCase): email = email_notifier.EmailNotifier(mock_log) - email.config(self.email_config) + email.config() alarm_dict = alarm(metrics) @@ -398,7 +393,7 @@ class TestEmail(unittest.TestCase): email = email_notifier.EmailNotifier(mock_log) - email.config(self.email_config) + email.config() alarm_dict = alarm(metrics) @@ -438,7 +433,7 @@ class TestEmail(unittest.TestCase): email = email_notifier.EmailNotifier(mock_log) - email.config(self.email_config) + email.config() alarm_dict = alarm(metrics) @@ -471,7 +466,6 @@ class TestEmail(unittest.TestCase): mock_smtp.SMTPException = smtplib.SMTPException email = email_notifier.EmailNotifier(mock_log) - email.config(self.email_config) # Create alarm timestamp and timestamp for 'from' and 'to' dates in milliseconds. alarm_date = datetime.datetime(2017, 6, 7, 18, 0) diff --git a/tests/test_hipchat_notification.py b/tests/test_hipchat_notification.py index a9500e7..b5ef254 100644 --- a/tests/test_hipchat_notification.py +++ b/tests/test_hipchat_notification.py @@ -13,12 +13,12 @@ import json import mock -import unittest import six from monasca_notification import notification as m_notification from monasca_notification.plugins import hipchat_notifier +from tests import base if six.PY2: import Queue as queue @@ -47,12 +47,15 @@ class requestsResponse(object): self.status_code = status -class TestHipchat(unittest.TestCase): +class TestHipchat(base.PluginTestCase): def setUp(self): + super(TestHipchat, self).setUp(hipchat_notifier.register_opts) + self.conf_default(group='hipchat_notifier', timeout=50) + self.trap = queue.Queue() - self.hipchat_config = {'timeout': 50} def tearDown(self): + super(TestHipchat, self).tearDown() self.assertTrue(self.trap.empty()) def _http_post_200(self, url, data, **kwargs): @@ -72,7 +75,7 @@ class TestHipchat(unittest.TestCase): hipchat = hipchat_notifier.HipChatNotifier(mock_log) - hipchat.config(self.hipchat_config) + hipchat.config() metric = [] metric_data = {'dimensions': {'hostname': 'foo1', 'service': 'bar1'}} diff --git a/tests/test_jira_notification.py b/tests/test_jira_notification.py index 7f14ae5..58d7587 100644 --- a/tests/test_jira_notification.py +++ b/tests/test_jira_notification.py @@ -11,11 +11,11 @@ # the License. import mock -from oslotest import base import six from monasca_notification import notification as m_notification from monasca_notification.plugins import jira_notifier +from tests import base if six.PY2: import Queue as queue @@ -64,7 +64,7 @@ class RequestsResponse(object): self.status_code = status -class TestJira(base.BaseTestCase): +class TestJira(base.PluginTestCase): default_address = 'http://test.jira:3333/?project=MyProject' \ '&component=MyComponent' @@ -72,7 +72,14 @@ class TestJira(base.BaseTestCase): issue_status_resolved = 'resolved' def setUp(self): - super(TestJira, self).setUp() + super(TestJira, self).setUp( + jira_notifier.register_opts + ) + self.conf_override( + group='jira_notifier', + user='username', + password='password' + ) self._trap = queue.Queue() @@ -85,17 +92,12 @@ class TestJira(base.BaseTestCase): self._jr = jira_notifier.JiraNotifier(mock_log) - self._jira_config = {'user': 'username', - 'password': 'password'} - @mock.patch('monasca_notification.plugins.jira_notifier.jira') def _notify(self, transitions_value, issue_status, - jira_config, address, mock_jira): - self._jr.config(jira_config) alarm_dict = alarm() mock_jira_obj = mock.Mock() @@ -117,7 +119,6 @@ class TestJira(base.BaseTestCase): def test_send_notification_issue_status_resolved(self): mock_jira, mock_jira_obj, result = self._notify(TestJira.default_transitions_value, TestJira.issue_status_resolved, - self._jira_config, TestJira.default_address) self.assertTrue(result) mock_jira_obj.create_issue.assert_not_called() @@ -130,7 +131,6 @@ class TestJira(base.BaseTestCase): issue_status = 'closed' mock_jira, mock_jira_obj, result = self._notify(TestJira.default_transitions_value, issue_status, - self._jira_config, TestJira.default_address) self.assertTrue(result) mock_jira_obj.create_issue.assert_not_called() @@ -143,7 +143,6 @@ class TestJira(base.BaseTestCase): issue_status = 'progress' mock_jira, mock_jira_obj, result = self._notify(TestJira.default_transitions_value, issue_status, - self._jira_config, TestJira.default_address) self.assertTrue(result) mock_jira_obj.create_issue.assert_not_called() @@ -153,7 +152,6 @@ class TestJira(base.BaseTestCase): transitions_value = [[{'id': 100, 'name': 'not open'}]] mock_jira, mock_jira_obj, result = self._notify(transitions_value, TestJira.issue_status_resolved, - self._jira_config, TestJira.default_address) self.assertTrue(result) mock_jira_obj.create_issue.assert_not_called() @@ -163,7 +161,6 @@ class TestJira(base.BaseTestCase): issue_status = None mock_jira, mock_jira_obj, result = self._notify(TestJira.default_transitions_value, issue_status, - self._jira_config, TestJira.default_address) self.assertTrue(result) mock_jira_obj.transitions.assert_not_called() @@ -174,7 +171,6 @@ class TestJira(base.BaseTestCase): address = 'http://test.jira:3333/?project=MyProject' mock_jira, mock_jira_obj, result = self._notify(TestJira.default_transitions_value, issue_status, - self._jira_config, address) self.assertTrue(result) self.assertEqual(issue(component=False), mock_jira_obj.create_issue.call_args[1]) @@ -183,18 +179,18 @@ class TestJira(base.BaseTestCase): def test_send_notification_create_jira_object_args(self): mock_jira, mock_jira_obj, result = self._notify(TestJira.default_transitions_value, TestJira.issue_status_resolved, - self._jira_config, TestJira.default_address) self.assertEqual('http://test.jira:3333/', mock_jira.JIRA.call_args[0][0]) self.assertEqual(('username', 'password'), mock_jira.JIRA.call_args[1].get('basic_auth')) self.assertEqual(None, mock_jira.JIRA.call_args[1].get('proxies')) def test_send_notification_with_proxy(self): - jira_config = self._jira_config - jira_config.update({'proxy': 'http://yourid:password@proxyserver:8080'}) + self.conf_override( + proxy='http://yourid:password@proxyserver:8080', + group='jira_notifier' + ) mock_jira, mock_jira_obj, result = self._notify(TestJira.default_transitions_value, TestJira.issue_status_resolved, - jira_config, TestJira.default_address) self.assertTrue(result) self.assertEqual({'https': 'http://yourid:password@proxyserver:8080'}, @@ -202,45 +198,46 @@ class TestJira(base.BaseTestCase): def test_send_notification_custom_config_success(self): issue_status = None - jira_config = self._jira_config - jira_config.update( - {'custom_formatter': 'tests/resources/test_jiraformat.yml'}) + self.conf_override( + custom_formatter='tests/resources/test_jiraformat.yml', + group='jira_notifier' + ) mock_jira, mock_jira_obj, result = self._notify(TestJira.default_transitions_value, issue_status, - jira_config, TestJira.default_address) self.assertTrue(result) mock_jira_obj.transitions.assert_not_called() self.assertEqual(issue(custom_config=True), mock_jira_obj.create_issue.call_args[1]) def test_send_notification_custom_config_failed(self): - jira_config = self._jira_config - jira_config.update( - {'custom_formatter': 'tests/resources/test_jiraformat_without_summary.yml'}) + self.conf_override( + custom_formatter='tests/resources/test_jiraformat_without_summary.yml', + group='jira_notifier' + ) mock_jira, mock_jira_obj, result = self._notify(TestJira.default_transitions_value, TestJira.issue_status_resolved, - jira_config, TestJira.default_address) self.assertFalse(result) def test_send_notification_custom_config_without_comments(self): - jira_config = self._jira_config - jira_config.update( - {'custom_formatter': 'tests/resources/test_jiraformat_without_comments.yml'}) + self.conf_override( + custom_formatter='tests/resources/test_jiraformat_without_comments.yml', + group='jira_notifier' + ) mock_jira, mock_jira_obj, result = self._notify(TestJira.default_transitions_value, TestJira.issue_status_resolved, - jira_config, TestJira.default_address) self.assertTrue(result) mock_jira_obj.add_comment.assert_not_called() def test_send_notification_custom_config_exception(self): - jira_config = self._jira_config - jira_config.update({'custom_formatter': 'tests/resources/not_exist_file.yml'}) + self.conf_override( + custom_formatter='tests/resources/not_exist_file.yml', + group='jira_notifier' + ) self.assertRaises(Exception, self._notify, TestJira.default_transitions_value, TestJira.issue_status_resolved, - jira_config, TestJira.default_address) def test_type(self): @@ -248,6 +245,3 @@ class TestJira(base.BaseTestCase): def test_statsd_name(self): self.assertEqual('jira_notifier', self._jr.statsd_name) - - def test_config_exception(self): - self.assertRaises(Exception, self._jr.config, {}) diff --git a/tests/test_mysql_repo.py b/tests/test_mysql_repo.py index d1ae7bf..4e19fc7 100644 --- a/tests/test_mysql_repo.py +++ b/tests/test_mysql_repo.py @@ -1,4 +1,5 @@ # (C) Copyright 2015 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,15 +15,21 @@ # limitations under the License. import mock -import unittest import pymysql from monasca_notification.common.repositories import exceptions as exc from monasca_notification.common.repositories.mysql import mysql_repo +from tests import base -class TestMySqlRepo(unittest.TestCase): +class TestMySqlRepo(base.BaseTestCase): + def setUp(self): + super(TestMySqlRepo, self).setUp() + self.conf_default(group='mysql', host='localhost', + port=3306, user='bar', + passwd='1', db='2') + @mock.patch('monasca_notification.common.repositories.mysql.mysql_repo.pymysql') def testReconnect(self, mock_mysql): m = mock.MagicMock() @@ -32,13 +39,7 @@ class TestMySqlRepo(unittest.TestCase): mock_mysql.connect.return_value = m mock_mysql.Error = pymysql.Error - config = {'mysql': {'host': 'foo', - 'port': '3306', - 'user': 'bar', - 'passwd': '1', - 'db': '2'}} - - repo = mysql_repo.MysqlRepo(config) + repo = mysql_repo.MysqlRepo(base.config.CONF) alarm = {'alarmDefinitionId': 'foo', 'newState': 'bar'} diff --git a/tests/test_notification_processor.py b/tests/test_notification_processor.py index 5715f8b..605a949 100644 --- a/tests/test_notification_processor.py +++ b/tests/test_notification_processor.py @@ -1,4 +1,5 @@ # (C) Copyright 2014-2016 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,10 +18,11 @@ import mock import time -import unittest from monasca_notification import notification as m_notification -from monasca_notification.processors import notification_processor +from monasca_notification.plugins import email_notifier +from monasca_notification.processors import notification_processor as np +from tests import base class smtpStub(object): @@ -36,29 +38,23 @@ class requestsResponse(object): self.status_code = status -class TestNotificationProcessor(unittest.TestCase): +class TestNotificationProcessor(base.BaseTestCase): def setUp(self): + super(TestNotificationProcessor, self).setUp() self.trap = [] - self.email_config = {'server': 'my.smtp.server', - 'port': 25, - 'user': None, - 'password': None, - 'timeout': 60, - 'from_addr': 'hpcs.mon@hp.com'} - self.mysql_config = {'ssl': None, - 'host': 'mysql_host', - 'port': 'mysql_port', - 'user': 'mysql_user', - 'db': 'dbname', - 'passwd': 'mysql_passwd'} + email_notifier.register_opts(base.config.CONF) - self.statsd_config = {'host': 'localhost', - 'port': 8125} + self.conf_default(group='email_notifier', server='my.smtp.server', + port=25, user=None, password=None, + timeout=60, from_addr='hpcs.mon@hp.com') + self.conf_default(group='mysql', ssl=None, host='localhost', + port='3306', user='mysql_user', db='dbname', + passwd='mysql_passwd') + self.conf_default(group='statsd', host='localhost', port=8125) - def tearDown(self): - pass + self.conf_default(group='notification_types', enabled=[]) # ------------------------------------------------------------------------ # Test helper functions @@ -70,19 +66,15 @@ class TestNotificationProcessor(unittest.TestCase): def _start_processor(self, notifications, mock_log, mock_smtp, mock_statsd, mock_pymsql): """Start the processor with the proper mocks """ - # Since the log runs in another thread I can mock it directly, instead change the methods to put to a queue + # Since the log runs in another thread I can mock it directly, + # instead change the methods to put to a queue mock_log.warn = self.trap.append mock_log.error = self.trap.append mock_smtp.SMTP = self._smtpStub - config = {} - config["email"] = self.email_config - config["mysql"] = self.mysql_config - config["statsd"] = self.statsd_config - config["notification_types"] = {} - - processor = (notification_processor.NotificationProcessor(config)) + np.NotificationProcessor.insert_configured_plugins = mock.Mock() + processor = np.NotificationProcessor() processor.send(notifications) def _smtpStub(self, *arg, **kwargs): diff --git a/tests/test_notifiers.py b/tests/test_notifiers.py index ba952ce..ebd44d3 100644 --- a/tests/test_notifiers.py +++ b/tests/test_notifiers.py @@ -1,4 +1,5 @@ # (C) Copyright 2015-2016 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,10 +17,13 @@ import contextlib import mock import time -import unittest from monasca_notification import notification as m_notification +from monasca_notification.plugins import email_notifier +from monasca_notification.plugins import pagerduty_notifier +from monasca_notification.plugins import webhook_notifier from monasca_notification.types import notifiers +from tests import base def alarm(metrics): @@ -112,18 +116,23 @@ class StatsdCounter(object): self.counter += val -class TestInterface(unittest.TestCase): +class TestInterface(base.BaseTestCase): + def setUp(self): + super(TestInterface, self).setUp() self.trap = [] self.statsd = Statsd() - self.email_config = {'server': 'my.smtp.server', - 'port': 25, - 'user': None, - 'password': None, - 'timeout': 60, - 'from_addr': 'hpcs.mon@hp.com'} + + email_notifier.register_opts(base.config.CONF) + webhook_notifier.register_opts(base.config.CONF) + pagerduty_notifier.register_opts(base.config.CONF) + + self.conf_override(group='email_notifier', server='my.smtp.server', + port=25, user=None, password=None, + timeout=60, from_addr='hpcs.mon@hp.com') def tearDown(self): + super(TestInterface, self).tearDown() notifiers.possible_notifiers = [] notifiers.configured_notifiers = {} self.trap = [] @@ -143,70 +152,15 @@ class TestInterface(unittest.TestCase): @mock.patch('monasca_notification.types.notifiers.email_notifier.smtplib') @mock.patch('monasca_notification.types.notifiers.log') def test_enabled_notifications(self, mock_log, mock_smtp): - config_dict = {'email': self.email_config, - 'webhook': {'address': 'xyz.com'}, - 'pagerduty': {'address': 'xyz.com'}} - notifiers.init(self.statsd) - notifiers.config(config_dict) + notifiers.config() + notifications = notifiers.enabled_notifications() self.assertEqual(len(notifications), 3) self.assertEqual(sorted(notifications), ["EMAIL", "PAGERDUTY", "WEBHOOK"]) - @mock.patch('monasca_notification.types.notifiers.email_notifier.smtplib') - @mock.patch('monasca_notification.types.notifiers.log') - def test_config_missing_data(self, mock_log, mock_smtp): - mock_log.warn = self.trap.append - mock_log.error = self.trap.append - mock_log.info = self.trap.append - - config_dict = {'email': self.email_config, - 'webhook': {'address': 'xyz.com'}} - - notifiers.init(self.statsd) - notifiers.config(config_dict) - - self.assertIn("No config data for type: pagerduty", self.trap) - - @mock.patch('monasca_notification.types.notifiers.email_notifier') - @mock.patch('monasca_notification.types.notifiers.email_notifier.smtplib') - @mock.patch('monasca_notification.types.notifiers.log') - def test_config_exception(self, mock_log, mock_smtp, mock_email): - mock_log.warn = self.trap.append - mock_log.error = self.trap.append - mock_log.exception = self.trap.append - - mock_email.EmailNotifier = self._configExceptionStub - - config_dict = {'email': self.email_config, - 'webhook': {'address': 'xyz.com'}, - 'pagerduty': {'address': 'abc'}} - - notifiers.init(self.statsd) - notifiers.config(config_dict) - - self.assertIn("config exception for email", self.trap) - - @mock.patch('monasca_notification.types.notifiers.email_notifier.smtplib') - @mock.patch('monasca_notification.types.notifiers.log') - def test_config_correct(self, mock_log, mock_smtp): - mock_log.warn = self.trap.append - mock_log.error = self.trap.append - mock_log.info = self.trap.append - - config_dict = {'email': self.email_config, - 'webhook': {'address': 'xyz.com'}, - 'pagerduty': {'address': 'abc'}} - - notifiers.init(self.statsd) - notifiers.config(config_dict) - - self.assertIn("email notification ready", self.trap) - self.assertIn("webhook notification ready", self.trap) - self.assertIn("pagerduty notification ready", self.trap) - @mock.patch('monasca_notification.types.notifiers.email_notifier') @mock.patch('monasca_notification.types.notifiers.email_notifier.smtplib') @mock.patch('monasca_notification.types.notifiers.log') @@ -217,12 +171,8 @@ class TestInterface(unittest.TestCase): mock_email.EmailNotifier = self._sendExceptionStub - config_dict = {'email': self.email_config, - 'webhook': {'address': 'xyz.com'}, - 'pagerduty': {'address': 'abc'}} - notifiers.init(self.statsd) - notifiers.config(config_dict) + notifiers.config() notifications = [] notifications.append(m_notification.Notification(0, 'email', 'email notification', @@ -242,12 +192,8 @@ class TestInterface(unittest.TestCase): mock_email.EmailNotifier = self._sendFailureStub - config_dict = {'email': self.email_config, - 'webhook': {'address': 'xyz.com'}, - 'pagerduty': {'address': 'abc'}} - notifiers.init(self.statsd) - notifiers.config(config_dict) + notifiers.config() notifications = [] notifications.append(m_notification.Notification(0, 'email', 'email notification', @@ -259,36 +205,6 @@ class TestInterface(unittest.TestCase): self.assertEqual(len(failed), 1) self.assertEqual(invalid, []) - @mock.patch('monasca_notification.types.notifiers.email_notifier') - @mock.patch('monasca_notification.types.notifiers.email_notifier.smtplib') - @mock.patch('monasca_notification.types.notifiers.log') - def test_send_notification_unconfigured(self, mock_log, mock_smtp, mock_email): - mock_log.warn = self.trap.append - mock_log.error = self.trap.append - mock_log.info = self.trap.append - - mock_email.EmailNotifier = self._sendExceptionStub - - config_dict = {'email': self.email_config, - 'webhook': {'address': 'xyz.com'}} - - notifiers.init(self.statsd) - notifiers.config(config_dict) - - self.assertIn("No config data for type: pagerduty", self.trap) - - notifications = [] - notifications.append(m_notification.Notification(0, 'pagerduty', 'pagerduty notification', - 'me@here.com', 0, 0, alarm({}))) - - sent, failed, invalid = notifiers.send_notifications(notifications) - - self.assertEqual(sent, []) - self.assertEqual(failed, []) - self.assertEqual(len(invalid), 1) - - self.assertIn("attempting to send unconfigured notification: pagerduty", self.trap) - @mock.patch('monasca_notification.types.notifiers.time') @mock.patch('monasca_notification.types.notifiers.email_notifier') @mock.patch('monasca_notification.types.notifiers.email_notifier.smtplib') @@ -302,12 +218,8 @@ class TestInterface(unittest.TestCase): mock_time.time.return_value = 42 - config_dict = {'email': self.email_config, - 'webhook': {'address': 'xyz.com'}, - 'pagerduty': {'address': 'abc'}} - notifiers.init(self.statsd) - notifiers.config(config_dict) + notifiers.config() notifications = [] notifications.append(m_notification.Notification(0, 'email', 'email notification', @@ -335,12 +247,8 @@ class TestInterface(unittest.TestCase): mock_email.EmailNotifier = self._goodSendStub - config_dict = {'email': self.email_config, - 'webhook': {'address': 'xyz.com'}, - 'pagerduty': {'address': 'abc'}} - notifiers.init(self.statsd) - notifiers.config(config_dict) + notifiers.config() notifications = [] notifications.append(m_notification.Notification(0, 'email', 'email notification', @@ -357,11 +265,16 @@ class TestInterface(unittest.TestCase): self.assertEqual(self.statsd.counter.counter, 3) def test_plugin_load(self): - config_dict = {"plugins": ["monasca_notification.plugins.hipchat_notifier:HipChatNotifier", - "monasca_notification.plugins.slack_notifier:SlackNotifier"]} + self.conf_override( + group='notification_types', + enabled=[ + 'monasca_notification.plugins.hipchat_notifier:HipChatNotifier', + 'monasca_notification.plugins.slack_notifier:SlackNotifier' + ] + ) notifiers.init(self.statsd) - notifiers.load_plugins(config_dict) + notifiers.load_plugins() self.assertEqual(len(notifiers.possible_notifiers), 5) configured_plugins = ["email", "webhook", "pagerduty", "hipchat", "slack"] @@ -371,27 +284,19 @@ class TestInterface(unittest.TestCase): @mock.patch('monasca_notification.types.notifiers.log') def test_invalid_plugin_load_exception_ignored(self, mock_log): mock_log.exception = self.trap.append - config_dict = {"plugins": ["monasca_notification.plugins.hipchat_notifier:UnknownPlugin", - "monasca_notification.plugins.slack_notifier:SlackNotifier"]} + self.conf_override( + group='notification_types', + enabled=[ + 'monasca_notification.plugins.hipchat_notifier:UnknownPlugin', + 'monasca_notification.plugins.slack_notifier:SlackNotifier' + ] + ) notifiers.init(self.statsd) - notifiers.load_plugins(config_dict) + notifiers.load_plugins() self.assertEqual(len(notifiers.possible_notifiers), 4) self.assertEqual(len(self.trap), 1) configured_plugins = ["email", "webhook", "pagerduty", "slack"] for plugin in notifiers.configured_notifiers: self.asssertIn(plugin.type in configured_plugins) - - @mock.patch('monasca_notification.types.notifiers.log') - def test_no_plugins_keyword_in_possible_notifiers(self, mock_log): - mock_log.warning = self.trap.append - config_dict = {"plugins": ["monasca_notification.plugins.slack_notifier:SlackNotifier"], - "fake_notifier": ["monasca_notification.plugins.fake_notifier:FakeNotifier"], - "slack": {"timeout": 5, "ca_certs": "/etc/ssl/certs/ca-certificates.crt", "insecure": False} - } - notifiers.init(self.statsd) - notifiers.possible_notifiers = [] - notifiers.load_plugins(config_dict) - notifiers.config(config_dict) - self.assertEqual('No notifiers found for fake_notifier', mock_log.warn.call_args[0][0]) diff --git a/tests/test_orm_repo.py b/tests/test_orm_repo.py index 9ca37b4..383287a 100644 --- a/tests/test_orm_repo.py +++ b/tests/test_orm_repo.py @@ -13,17 +13,18 @@ import mock from monasca_notification.common.repositories.orm import orm_repo -from oslotest import base +from tests import base class TestOrmRepo(base.BaseTestCase): + @mock.patch('monasca_notification.common.repositories.orm.orm_repo.engine_from_config') def setUp(self, mock_sql_engine_from_config): super(TestOrmRepo, self).setUp() - config = {'database': - {'orm': - {'url': 'mysql+pymysql://user:password@hostname:3306/mon'}}} - self._rep = orm_repo.OrmRepo(config) + self.conf_default( + group='orm', url='mysql+pymysql://user:password@hostname:3306/mon' + ) + self._rep = orm_repo.OrmRepo() self.mock_conn = \ self._rep._orm_engine.connect.return_value.__enter__.return_value diff --git a/tests/test_pagerduty_notification.py b/tests/test_pagerduty_notification.py index a9a3583..cd1f126 100644 --- a/tests/test_pagerduty_notification.py +++ b/tests/test_pagerduty_notification.py @@ -1,4 +1,5 @@ # (C) Copyright 2014-2016 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,12 +18,12 @@ import json import mock import requests import time -import unittest import six from monasca_notification import notification as m_notification from monasca_notification.plugins import pagerduty_notifier +from tests import base if six.PY2: import Queue as queue @@ -49,12 +50,18 @@ class requestsResponse(object): self.status_code = status -class TestWebhook(unittest.TestCase): +class TestPagerduty(base.PluginTestCase): + def setUp(self): + super(TestPagerduty, self).setUp( + pagerduty_notifier.register_opts + ) + self.conf_override(group='pagerduty_notifier', timeout=50) + self.trap = queue.Queue() - self.pagerduty_config = {'timeout': 50, 'key': 'foobar'} def tearDown(self): + super(TestPagerduty, self).tearDown() self.assertTrue(self.trap.empty()) def _http_post_200(self, url, data, headers, **kwargs): @@ -142,8 +149,6 @@ class TestWebhook(unittest.TestCase): pagerduty = pagerduty_notifier.PagerdutyNotifier(mock_log) - pagerduty.config(self.pagerduty_config) - metric = [] metric_data = {'dimensions': {'hostname': 'foo1', 'service': 'bar1'}} metric.append(metric_data) @@ -275,7 +280,5 @@ class TestWebhook(unittest.TestCase): self.assertRegexpMatches(results, "Exception on pagerduty request") self.assertRegexpMatches(results, "key=") - self.assertRaises(requests.exceptions.Timeout) - return_value = self.trap.get() self.assertFalse(return_value) diff --git a/tests/test_slack_notification.py b/tests/test_slack_notification.py index 880ab44..517b317 100644 --- a/tests/test_slack_notification.py +++ b/tests/test_slack_notification.py @@ -14,12 +14,11 @@ import json import mock -from oslotest import base - import six from monasca_notification import notification as m_notification from monasca_notification.plugins import slack_notifier +from tests import base if six.PY2: import Queue as queue @@ -70,9 +69,16 @@ class RequestsResponse(object): return json.loads(self.text) -class TestSlack(base.BaseTestCase): +class TestSlack(base.PluginTestCase): + def setUp(self): - super(TestSlack, self).setUp() + super(TestSlack, self).setUp( + slack_notifier.register_opts + ) + self.conf_default(group='slack_notifier', timeout=50, + ca_certs='/etc/ssl/certs/ca-bundle.crt', + proxy='http://yourid:password@proxyserver:8080', + insecure=False) self._trap = queue.Queue() @@ -85,17 +91,10 @@ class TestSlack(base.BaseTestCase): self._slk = slack_notifier.SlackNotifier(mock_log) slack_notifier.SlackNotifier._raw_data_url_caches = [] - self._slack_config = {'timeout': 50, - 'ca_certs': '/etc/ssl/certs/ca-bundle.crt', - 'proxy': 'http://yourid:password@proxyserver:8080', - 'insecure': False} - @mock.patch('monasca_notification.plugins.slack_notifier.requests') - def _notify(self, response_list, slack_config, mock_requests): + def _notify(self, response_list, mock_requests): mock_requests.post = mock.Mock(side_effect=response_list) - self._slk.config(slack_config) - metric = [] metric_data = {'dimensions': {'hostname': 'foo1', 'service': 'bar1'}} metric.append(metric_data) @@ -123,7 +122,7 @@ class TestSlack(base.BaseTestCase): """ response_list = [RequestsResponse(200, 'ok', {'Content-Type': 'application/text'})] - mock_method, result = self._notify(response_list, self._slack_config) + mock_method, result = self._notify(response_list) self.assertTrue(result) mock_method.assert_called_once() self._validate_post_args(mock_method.call_args_list[0][1], 'json') @@ -137,7 +136,7 @@ class TestSlack(base.BaseTestCase): {'Content-Type': 'application/text'}), RequestsResponse(200, '{"ok":false,"error":"failure"}', {'Content-Type': 'application/json'})] - mock_method, result = self._notify(response_list, self._slack_config) + mock_method, result = self._notify(response_list) self.assertFalse(result) self._validate_post_args(mock_method.call_args_list[0][1], 'json') self._validate_post_args(mock_method.call_args_list[1][1], 'data') @@ -150,7 +149,7 @@ class TestSlack(base.BaseTestCase): {'Content-Type': 'application/json'}), RequestsResponse(200, '{"ok":true}', {'Content-Type': 'application/json'})] - mock_method, result = self._notify(response_list, self._slack_config) + mock_method, result = self._notify(response_list) self.assertTrue(result) self._validate_post_args(mock_method.call_args_list[0][1], 'json') self._validate_post_args(mock_method.call_args_list[1][1], 'data') @@ -165,7 +164,7 @@ class TestSlack(base.BaseTestCase): ['http://test.slack:3333']): response_list = [RequestsResponse(200, '{"ok":true}', {'Content-Type': 'application/json'})] - mock_method, result = self._notify(response_list, self._slack_config) + mock_method, result = self._notify(response_list) self.assertTrue(result) mock_method.assert_called_once() self._validate_post_args(mock_method.call_args_list[0][1], 'data') @@ -180,7 +179,7 @@ class TestSlack(base.BaseTestCase): ['http://test.slack:3333']): response_list = [RequestsResponse(200, '{"ok":false,"error":"failure"}', {'Content-Type': 'application/json'})] - mock_method, result = self._notify(response_list, self._slack_config) + mock_method, result = self._notify(response_list) self.assertFalse(result) mock_method.assert_called_once() self._validate_post_args(mock_method.call_args_list[0][1], 'data') @@ -190,9 +189,13 @@ class TestSlack(base.BaseTestCase): def test_slack_webhook_success_only_timeout(self): """slack success with only timeout config """ + self.conf_override(group='slack_notifier', timeout=50, + insecure=True, ca_certs=None, + proxy=None) + response_list = [RequestsResponse(200, 'ok', {'Content-Type': 'application/text'})] - mock_method, result = self._notify(response_list, {'timeout': 50}) + mock_method, result = self._notify(response_list) self.assertTrue(result) mock_method.assert_called_once() self.assertEqual(slack_notifier.SlackNotifier._raw_data_url_caches, []) @@ -205,16 +208,6 @@ class TestSlack(base.BaseTestCase): self.assertEqual('http://test.slack:3333', post_args.get('url')) self.assertFalse(post_args.get('verify')) - def test_slack_exception(self): - """exception occurs - """ - mock_method, result = self._notify(RuntimeError('exception'), - self._slack_config) - self.assertFalse(result) - - self._validate_post_args(mock_method.call_args_list[0][1], 'json') - self._validate_post_args(mock_method.call_args_list[1][1], 'data') - def test_slack_reponse_400(self): """slack returns 400 error """ @@ -222,7 +215,7 @@ class TestSlack(base.BaseTestCase): {'Content-Type': 'application/json'}), RequestsResponse(400, '{"ok":false,"error":"failure"}', {'Content-Type': 'application/json'})] - mock_method, result = self._notify(response_list, self._slack_config) + mock_method, result = self._notify(response_list) self.assertFalse(result) self._validate_post_args(mock_method.call_args_list[0][1], 'json') @@ -239,7 +232,7 @@ class TestSlack(base.BaseTestCase): {'Content-Type': 'application/json'}), RequestsResponse(200, '{"ok":true}', {'Content-Type': 'application/json'})] - mock_method, result = self._notify(response_list, self._slack_config) + mock_method, result = self._notify(response_list) self.assertTrue(result) self._validate_post_args(mock_method.call_args_list[0][1], 'json') self._validate_post_args(mock_method.call_args_list[1][1], 'data') @@ -247,13 +240,14 @@ class TestSlack(base.BaseTestCase): slack_notifier.SlackNotifier._raw_data_url_caches) def test_config_insecure_true_ca_certs(self): - slack_config = {'timeout': 50, - 'ca_certs': '/etc/ssl/certs/ca-bundle.crt', - 'insecure': True} + self.conf_override(group='slack_notifier', timeout=50, + insecure=True, + ca_certs='/etc/ssl/certs/ca-bundle.crt') + response_list = [RequestsResponse(200, 'ok', {'Content-Type': 'application/text'})] - mock_method, result = self._notify(response_list, slack_config) + mock_method, result = self._notify(response_list) self.assertTrue(result) mock_method.assert_called_once() self.assertEqual(slack_notifier.SlackNotifier._raw_data_url_caches, []) @@ -265,12 +259,14 @@ class TestSlack(base.BaseTestCase): self.assertEqual('/etc/ssl/certs/ca-bundle.crt', post_args.get('verify')) def test_config_insecure_true_no_ca_certs(self): - slack_config = {'timeout': 50, - 'insecure': True} + self.conf_override(group='slack_notifier', timeout=50, + insecure=True, + ca_certs=None) + response_list = [RequestsResponse(200, 'ok', {'Content-Type': 'application/text'})] - mock_method, result = self._notify(response_list, slack_config) + mock_method, result = self._notify(response_list) self.assertTrue(result) mock_method.assert_called_once() self.assertEqual(slack_notifier.SlackNotifier._raw_data_url_caches, []) @@ -282,12 +278,14 @@ class TestSlack(base.BaseTestCase): self.assertFalse(post_args.get('verify')) def test_config_insecure_false_no_ca_certs(self): - slack_config = {'timeout': 50, - 'insecure': False} + self.conf_override(group='slack_notifier', timeout=50, + insecure=False, + ca_certs=None) + response_list = [RequestsResponse(200, 'ok', {'Content-Type': 'application/text'})] - mock_method, result = self._notify(response_list, slack_config) + mock_method, result = self._notify(response_list) self.assertTrue(result) mock_method.assert_called_once() self.assertEqual(slack_notifier.SlackNotifier._raw_data_url_caches, []) diff --git a/tests/test_utils.py b/tests/test_utils.py index 32a0780..a98b3aa 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,4 @@ -# Copyright 2016 FUJITSU LIMITED +# Copyright 2016-2017 FUJITSU LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -13,41 +13,45 @@ # under the License. from mock import patch -import unittest from monasca_notification.common import utils +from tests import base -class TestStatsdConnection(unittest.TestCase): +class TestStatsdConnection(base.BaseTestCase): extra_dimensions = {'foo': 'bar'} base_name = 'monasca' def test_statsd_default_connection(self): - config = {} with patch( 'monasca_notification.common.utils.monascastatsd.Client') as c: - utils.get_statsd_client(config) + utils.get_statsd_client() c.assert_called_once_with(dimensions=utils.NOTIFICATION_DIMENSIONS, - name=self.base_name) + name=self.base_name, + host='127.0.0.1', + port=8125) def test_statsd_config_connection(self): port_number = 9999 hostname = 'www.example.org' - config = {'statsd': {'host': hostname, 'port': port_number}} + + self.conf_override(group='statsd', host=hostname, port=port_number) + with patch( 'monasca_notification.common.utils.monascastatsd.Client') as c: - utils.get_statsd_client(config) + utils.get_statsd_client() c.assert_called_once_with(dimensions=utils.NOTIFICATION_DIMENSIONS, name=self.base_name, port=port_number, host=hostname) def test_statsd_update_dimmensions(self): - config = {} expected_dimensions = utils.NOTIFICATION_DIMENSIONS.copy() expected_dimensions.update(self.extra_dimensions) with patch( 'monasca_notification.common.utils.monascastatsd.Client') as c: - utils.get_statsd_client(config, dimensions=self.extra_dimensions) + utils.get_statsd_client(dimensions=self.extra_dimensions) c.assert_called_once_with(dimensions=expected_dimensions, - name=self.base_name) + name=self.base_name, + host='127.0.0.1', + port=8125) diff --git a/tests/test_webhook_notification.py b/tests/test_webhook_notification.py index 381ecb9..f9effc7 100644 --- a/tests/test_webhook_notification.py +++ b/tests/test_webhook_notification.py @@ -1,4 +1,5 @@ # (C) Copyright 2014-2016 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,13 +16,13 @@ import mock import requests -import unittest import six import ujson as json from monasca_notification import notification as m_notification from monasca_notification.plugins import webhook_notifier +from tests import base if six.PY2: import Queue as queue @@ -50,12 +51,14 @@ class requestsResponse(object): self.status_code = status -class TestWebhook(unittest.TestCase): +class TestWebhook(base.PluginTestCase): + def setUp(self): + super(TestWebhook, self).setUp(webhook_notifier.register_opts) self.trap = queue.Queue() - self.webhook_config = {'timeout': 50} def tearDown(self): + super(TestWebhook, self).tearDown() self.assertTrue(self.trap.empty()) def _http_post_200(self, url, data, headers, **kwargs): @@ -84,8 +87,6 @@ class TestWebhook(unittest.TestCase): webhook = webhook_notifier.WebhookNotifier(mock_log) - webhook.config(self.webhook_config) - metric = [] metric_data = {'dimensions': {'hostname': 'foo1', 'service': 'bar1'}} metric.append(metric_data) @@ -140,6 +141,7 @@ class TestWebhook(unittest.TestCase): """webhook timeout exception """ + self.conf_override(group='webhook_notifier', timeout=50) self.notify(self._http_post_exception) result = self.trap.get() @@ -152,7 +154,6 @@ class TestWebhook(unittest.TestCase): self.assertNotRegexpMatches(result, "content-type.: .application/json") self.assertRegexpMatches(result, "Error trying to post on URL http://mock:3333/") - self.assertRaises(requests.exceptions.Timeout) return_value = self.trap.get() self.assertFalse(return_value) diff --git a/tox.ini b/tox.ini index f4b2305..9e2204c 100644 --- a/tox.ini +++ b/tox.ini @@ -72,6 +72,10 @@ commands = commands = bandit -r monasca_notification -n5 -x monasca_notification/tests +[testenv:genconfig] +description = Generates an example of monasca-notification configuration file +commands = oslo-config-generator --config-file={toxinidir}/config-generator/notification.conf + [flake8] max-line-length = 120 # TODO: ignored checks should be enabled in the future @@ -80,3 +84,7 @@ max-line-length = 120 # H405 multi line docstring summary not separated with an empty line ignore = F821,H201,H202,H405 exclude=.venv,.git,.tox,dist,*egg,build + +[hacking] +import_exceptions = + six.moves