Rename to monasca, setup for tox, removed legacy bits

Removed manual tests which are no longer valid with a modern mini-mon
Removed debian creation bits all distribution is with pypi now
Minor pep8 fixes

Change-Id: I1f2fc4d0ad6375f4c39446f9627247945066e4ad

.gitignore

@@ -1,2 +1,3 @@
 .idea
 *.pyc
+.tox

.gitreview

@@ -0,0 +1,4 @@
+[gerrit]
+host=review.openstack.org
+port=29418
+project=stackforge/monasca-notification.git

HACKING.rst

@@ -1,4 +1,4 @@
-Mon-notification Style Commandments
+Monasca-notification Style Commandments
 ===============================================
 
 Read the OpenStack Style Commandments http://docs.openstack.org/developer/hacking/

README.md

@@ -51,7 +51,7 @@ sent out multiple times. To minimize this risk a number of techniques are used:
   time has been set and exceeded when a new finished alarm comes in the offset is updated regardless of gaps.
 
 # Operation
-Yaml config file by default is in '/etc/mon/notification.yaml', a sample is in this project.
+Yaml config file by default is in '/etc/monasca/notification.yaml', a sample is in this project.
 
 ## Monitoring
 statsd is incorporated into the daemon and will send all stats to localhost on udp port 8125. In many cases the stats

debian/changelog

@@ -1,6 +0,0 @@
-mon-notification (0.0.1) precise; urgency=low
-
-  * Initial Package creation
-
- -- Tim Kuhlman <tim.kuhlman@hp.com>  Thu, 27 Feb 2014 04:12:44 -0600
-

debian/compat

@@ -1 +0,0 @@
-7

debian/control

@@ -1,16 +0,0 @@
-Source: mon-notification
-Section: python
-Priority: optional
-Maintainer: HPCloud Monitoring <hpcs-mon@hp.com>
-Build-Depends: debhelper (>= 7),
-               python (>= 2.6.6-3~),
-               python-setuptools
-Standards-Version: 3.9.3
-X-Python-Version: >= 2.6
-
-Package: mon-notification
-Architecture: all
-Section: python
-Depends: ${misc:Depends}, ${python:Depends}, libpython2.7, python-pkg-resources, kafka-python, python-yaml, python-mysqldb, python-kazoo, python-statsd
-Description: Notification engine for monitoring.
-  Consumes alarms from Kafka and sends notifications appropriately.

debian/copyright

@@ -1,4 +0,0 @@
-Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Files: *
-Copyright: Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
-License: Apache License, Version 2.0

debian/rules

@@ -1,4 +0,0 @@
-#!/usr/bin/make -f
-
-%:
-	dh $@ --with python2

monasca_notification/main.py

@@ -90,31 +90,31 @@ def main(argv=None):
         config_file = argv[1]
     elif len(argv) > 2:
         print("Usage: " + argv[0] + " <config_file>")
-        print("Config file defaults to /etc/mon/notification.yaml")
+        print("Config file defaults to /etc/monasca/notification.yaml")
         return 1
     else:
-        config_file = '/etc/mon/notification.yaml'
+        config_file = '/etc/monasca/notification.yaml'
 
     config = yaml.load(open(config_file, 'r'))
 
     # Setup logging
     logging.config.dictConfig(config['logging'])
 
-    #Create the queues
+    # Create the queues
     alarms = multiprocessing.Queue(config['queues']['alarms_size'])
     notifications = multiprocessing.Queue(config['queues']['notifications_size'])  # [notification_object, ]
     sent_notifications = multiprocessing.Queue(config['queues']['sent_notifications_size'])  # [notification_object, ]
     finished = multiprocessing.Queue(config['queues']['finished_size'])  # Data is of the form (partition, offset)
 
-    #State Tracker - Used for tracking the progress of fully processed alarms and the zookeeper lock
+    # State Tracker - Used for tracking the progress of fully processed alarms and the zookeeper lock
     global tracker  # Set to global for use in the cleanup function
     tracker = ZookeeperStateTracker(
         config['zookeeper']['url'], config['kafka']['alarm_topic'], finished, config['zookeeper']['max_offset_lag'])
     tracker.lock(clean_exit)  # Only begin if we have the processing lock
     tracker_thread = threading.Thread(target=tracker.run)
 
-    ## Define processors
+    # Define processors
-    #KafkaConsumer
+    # KafkaConsumer
     kafka = multiprocessing.Process(
         target=KafkaConsumer(
             alarms,
@@ -126,7 +126,7 @@ def main(argv=None):
     )
     processors.append(kafka)
 
-    #AlarmProcessors
+    # AlarmProcessors
     alarm_processors = []
     for i in range(config['processors']['alarm']['number']):
         alarm_processors.append(multiprocessing.Process(
@@ -143,7 +143,7 @@ def main(argv=None):
         )
     processors.extend(alarm_processors)
 
-    #NotificationProcessors
+    # NotificationProcessors
     notification_processors = []
     for i in range(config['processors']['notification']['number']):
         notification_processors.append(multiprocessing.Process(
@@ -156,7 +156,7 @@ def main(argv=None):
         )
     processors.extend(notification_processors)
 
-    #SentNotificationProcessor
+    # SentNotificationProcessor
     sent_notification_processor = multiprocessing.Process(
         target=SentNotificationProcessor(
             sent_notifications,
@@ -167,7 +167,7 @@ def main(argv=None):
     )
     processors.append(sent_notification_processor)
 
-    ## Start
+    # Start
     try:
         log.info('Starting processes')
         for process in processors:

monasca_notification/processors/__init__.py

@@ -12,5 +12,3 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from base import BaseProcessor

monasca_notification/processors/alarm_processor.py

@@ -20,9 +20,9 @@ import MySQLdb
 import statsd
 import time
 
-from mon_notification.notification import Notification
+from monasca_notification.notification import Notification
-from mon_notification.notification_exceptions import AlarmFormatError
+from monasca_notification.notification_exceptions import AlarmFormatError
-from mon_notification.processors import BaseProcessor
+from monasca_notification.processors.base import BaseProcessor
 
 
 log = logging.getLogger(__name__)
@@ -60,7 +60,7 @@ class AlarmProcessor(BaseProcessor):
         for field in expected_fields:
             if field not in alarm:
                 raise AlarmFormatError('Alarm data missing field %s' % field)
-        if (not 'tenantId' in alarm) or (not 'alarmId' in alarm):
+        if ('tenantId' not in alarm) or ('alarmId' not in alarm):
             raise AlarmFormatError
 
         return alarm

monasca_notification/processors/kafka_consumer.py

@@ -18,7 +18,7 @@ import kafka.consumer
 import logging
 import statsd
 
-from mon_notification.processors import BaseProcessor
+from monasca_notification.processors.base import BaseProcessor
 
 log = logging.getLogger(__name__)
 

monasca_notification/processors/notification_processor.py

@@ -20,7 +20,7 @@ import smtplib
 import statsd
 import time
 
-from mon_notification.processors import BaseProcessor
+from monasca_notification.processors.base import BaseProcessor
 
 log = logging.getLogger(__name__)
 

monasca_notification/processors/sent_notification_processor.py

@@ -18,7 +18,7 @@ import kafka.producer
 import logging
 import statsd
 
-from mon_notification.processors import BaseProcessor
+from monasca_notification.processors.base import BaseProcessor
 
 log = logging.getLogger(__name__)
 

monasca_notification/state_tracker.py

@@ -21,7 +21,7 @@ import Queue
 import statsd
 import time
 
-from mon_notification import notification_exceptions
+from monasca_notification import notification_exceptions
 
 log = logging.getLogger(__name__)
 
@@ -46,10 +46,10 @@ class ZookeeperStateTracker(object):
 
         self.zookeeper = kazoo.client.KazooClient(url)
         self.zookeeper.start()
-        self.topic_path = '/consumers/mon-notification/%s' % topic
+        self.topic_path = '/consumers/monasca-notification/%s' % topic
 
         self.lock_retry_time = 15  # number of seconds to wait for retrying for the lock
-        self.lock_path = '/locks/mon-notification/%s' % topic
+        self.lock_path = '/locks/monasca-notification/%s' % topic
 
         self._offsets = None
         # This is a dictionary of sets used for tracking finished offsets when there is a gap and the committed offset

notification.yaml

@@ -1,6 +1,6 @@
 kafka:
     url: 192.168.10.10:9092  # or comma seperated list of multiple hosts
-    group: mon-notification
+    group: monasca-notification
     alarm_topic: alarm-state-transitions
     notification_topic: alarm-notifications
 

setup.py

@@ -18,11 +18,11 @@
 import setuptools
 
 setuptools.setup(
-    name="mon-notification",
+    name="monasca-notification",
-    version="1.0.1",
+    version="1.0.0",
     author="Tim Kuhlman",
     author_email="tim.kuhlman@hp.com",
-    description="Notification engine used in the monitoring system",
+    description="Notification engine used in the monasca monitoring system",
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "License :: OSI Approved :: Apache Software License",
@@ -30,15 +30,15 @@ setuptools.setup(
     ],
     license="Apache",
     keywords="openstack monitoring email",
-    url="https://github.com/hpcloud-mon/mon-notification",
+    url="https://github.com/stackforge/monasca-notification",
     # possibly preferable to have the OS precompiled mysql version, python-mysqldb package on Ubuntu
     install_requires=["kafka-python>=0.9.0", "kazoo>=1.3", "MySQL-python", "python-statsd>=1.6.3", "PyYAML"],
     packages=setuptools.find_packages(exclude=['tests']),
     entry_points={
         'console_scripts': [
-            'mon-notification = mon_notification.main:main'
+            'monasca-notification = monasca_notification.main:main'
         ],
     },
-    scripts=['tools/mon_notification_offsets.py'],
+    scripts=['tools/monasca_notification_offsets.py'],
     test_suite='nose.collector'
 )

test-requirements.txt

@@ -1 +1,5 @@
+flake8
+pyflakes
+pep8
+nose
 mock>=1.0.1

tests/manual/README.md

@@ -1,14 +0,0 @@
-Basic testing.
-Note that mini-mon is set to use smtp3.hp.com which must be used from within the HP corporate network.
-
-# Load some notifications into the db.
-- First edit the last line of test_notifications.sql to add in your email
-- From the mysql vm run `mysql -uroot -ppassword mon < sample_notifications.sql`
- 
-# Feed alarm transition messages into kafka
-- It is helpful to watch the log file, `tail -f /var/log/mon-notification/notification.log`
-  - If desired edit /etc/mon/notification.yaml to change logging options.
-- Then for all of the json files or just some run
-  `/opt/kafka/bin/kafka-console-producer.sh --broker 192.168.10.10:9092 --topic alarm-state-transitions < *.json`
-  - Note with the alarm_ttl now implemented you may need to update the timestamp in the json files.
-    - `python -c "import time; print time.time()"` will give a current timestamp

tests/manual/invalid.json

@@ -1,2 +0,0 @@
-not json
-{ 'invalid_json': true }

tests/manual/no-op.json

@@ -1,3 +0,0 @@
-{"alarm-transitioned":{"tenantId":"0","alarmId":"4","alarmName":"test Alarm","oldState":"OK","newState":"ALARM","stateChangeReason":"I am alarming!","timestamp":1394641255}}
-{"alarm-transitioned":{"tenantId":"0","alarmId":"4","alarmName":"test Okay","oldState":"ALARM","newState":"OK","stateChangeReason":"Okay, I am not alarming.","timestamp":1394641255}}
-{"alarm-transitioned":{"tenantId":"0","alarmId":"4","alarmName":"test Undetermined","oldState":"OK","newState":"UNDETERMINED","stateChangeReason":"Well, maybe I am alarming.","timestamp":1394641255}}

tests/manual/notify.json

@@ -1,3 +0,0 @@
-{"alarm-transitioned":{"tenantId":"0","alarmId":"0","alarmName":"test Alarm","oldState":"OK","newState":"ALARM","stateChangeReason":"I am alarming!","timestamp":1395346830}}
-{"alarm-transitioned":{"tenantId":"0","alarmId":"1","alarmName":"test Okay","oldState":"ALARM","newState":"OK","stateChangeReason":"Okay, I am not alarming.","timestamp":1395346830}}
-{"alarm-transitioned":{"tenantId":"0","alarmId":"2","alarmName":"test Undetermined","oldState":"OK","newState":"UNDETERMINED","stateChangeReason":"Well, maybe I am alarming.","timestamp":1395346830}}

tests/manual/sample_notifications.sql

@@ -1,17 +0,0 @@
-INSERT INTO alarm (id, tenant_id, name, state, created_at, updated_at) VALUES ('0', '0', 'test Alarm', 'ALARM', NOW(), NOW());
-INSERT INTO alarm (id, tenant_id, name, state, created_at, updated_at) VALUES ('1', '0', 'test Okay', 'OK', NOW(), NOW());
-INSERT INTO alarm (id, tenant_id, name, state, created_at, updated_at) VALUES ('2', '0', 'test Undetermined', 'UNDETERMINED', NOW(), NOW());
-
-INSERT INTO notification_method (id, tenant_id, name, type, address, created_at, updated_at)
-  VALUES ('0', '0', 'test notification', 'EMAIL', 'me@here.com', NOW(), NOW());
-
-INSERT INTO alarm_action (alarm_id, alarm_state, action_id) VALUES ('0', 'OK', '0');
-INSERT INTO alarm_action (alarm_id, alarm_state, action_id) VALUES ('0', 'ALARM', '0');
-INSERT INTO alarm_action (alarm_id, alarm_state, action_id) VALUES ('0', 'UNDETERMINED', '0');
-INSERT INTO alarm_action (alarm_id, alarm_state, action_id) VALUES ('1', 'OK', '0');
-INSERT INTO alarm_action (alarm_id, alarm_state, action_id) VALUES ('1', 'ALARM', '0');
-INSERT INTO alarm_action (alarm_id, alarm_state, action_id) VALUES ('1', 'UNDETERMINED', '0');
-INSERT INTO alarm_action (alarm_id, alarm_state, action_id) VALUES ('2', 'OK', '0');
-INSERT INTO alarm_action (alarm_id, alarm_state, action_id) VALUES ('2', 'ALARM', '0');
-INSERT INTO alarm_action (alarm_id, alarm_state, action_id) VALUES ('2', 'UNDETERMINED', '0');
-

tests/test_alarm_processor.py

@@ -23,8 +23,8 @@ import Queue
 import time
 import unittest
 
-from mon_notification.notification import Notification
+from monasca_notification.notification import Notification
-from mon_notification.processors import alarm_processor
+from monasca_notification.processors import alarm_processor
 
 alarm_tuple = collections.namedtuple('alarm_tuple', ['offset', 'message'])
 message_tuple = collections.namedtuple('message_tuple', ['value'])
@@ -45,7 +45,7 @@ class TestAlarmProcessor(unittest.TestCase):
         return [partition, alarm_tuple(offset, msg_tuple)]
 
     @mock.patch('MySQLdb.connect')
-    @mock.patch('mon_notification.processors.alarm_processor.log')
+    @mock.patch('monasca_notification.processors.alarm_processor.log')
     def _run_alarm_processor(self, queue, sql_response, mock_log, mock_mysql):
         """Runs a mocked alarm processor reading from queue while running, returns (queue_message, log_message)
         """

tests/test_notification.py

@@ -16,7 +16,7 @@
 """Tests the notification class."""
 
 import json
-from mon_notification import notification
+from monasca_notification import notification
 
 
 def test_json():

tests/test_notification_processor.py

@@ -20,8 +20,8 @@ import multiprocessing
 import time
 import unittest
 
-from mon_notification.notification import Notification
+from monasca_notification.notification import Notification
-from mon_notification.processors import notification_processor
+from monasca_notification.processors import notification_processor
 
 
 class TestStateTracker(unittest.TestCase):
@@ -38,8 +38,8 @@ class TestStateTracker(unittest.TestCase):
                              'timeout': 60,
                              'from_addr': 'hpcs.mon@hp.com'}
 
-    @mock.patch('mon_notification.processors.notification_processor.smtplib')
+    @mock.patch('monasca_notification.processors.notification_processor.smtplib')
-    @mock.patch('mon_notification.processors.notification_processor.log')
+    @mock.patch('monasca_notification.processors.notification_processor.log')
     def _start_processor(self, mock_log, mock_smtp):
         """Start the processor with the proper mocks
         """

tests/test_state_tracker.py

@@ -21,7 +21,7 @@ import threading
 import time
 import unittest
 
-from mon_notification import state_tracker
+from monasca_notification import state_tracker
 
 
 class TestStateTracker(unittest.TestCase):

tools/monasca_notification_offsets.py

@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Used for manually querying/setting offsets associated with the mon_notification daemon.
+"""Used for manually querying/setting offsets associated with the monasca_notification daemon.
     This should only be used in unusual circumstances to force reprocessing or skip alarms.
 """
 
@@ -25,7 +25,7 @@ import logging
 import sys
 import yaml
 
-from mon_notification import state_tracker
+from monasca_notification import state_tracker
 
 
 def listener():
@@ -36,10 +36,10 @@ def listener():
 
 def main():
     # Parse args
-    parser = argparse.ArgumentParser(description="Query and set(DANGEROUS) mon_notification kafka consumer offsets\n")
+    parser = argparse.ArgumentParser(description="Query and set(DANGEROUS) monasca_notification kafka offsets\n")
-    parser.add_argument('--config', '-c', default='/etc/mon/notification.yaml', help='Configuration File')
+    parser.add_argument('--config', '-c', default='/etc/monasca/notification.yaml', help='Configuration File')
 
-    ## Either list or set not both
+    # Either list or set not both
     group = parser.add_mutually_exclusive_group(required=True)
     group.add_argument('--list', '-l', action='store_true')
     group.add_argument('--set-offsets', '-s',
@@ -48,7 +48,7 @@ def main():
 
     args = parser.parse_args()
 
-    # Silence most logging from mon_notification
+    # Silence most logging from monasca_notification
     logging.basicConfig(level=logging.CRITICAL)
 
     # Parse config and setup state tracker
@@ -63,9 +63,9 @@ def main():
         offsets = json.loads(args.set_offsets)
         raw_input("Warning setting offset will affect the behavior of the next notification engine to run.\n" +
                   "\tCtrl-C to exit, enter to continue")
-        print("All running mon_notification daemons must be shutdown to allow this process to grab the lock.")
+        print("All running monasca_notification daemons must be shutdown to allow this process to grab the lock.")
 
-        log = logging.getLogger('mon_notification.state_tracker')
+        log = logging.getLogger('monasca_notification.state_tracker')
         log.setLevel(logging.DEBUG)
 
         tracker.lock(listener)

tox.ini

@@ -0,0 +1,19 @@
+[tox]
+envlist = py27,pypy,pep8
+skipsdist = True
+
+[testenv]
+commands = nosetests
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+
+[testenv:pep8]
+commands = flake8
+
+[tox:jenkins]
+downloadcache = ~/cache/pip
+
+[flake8]
+max-line-length = 120
+ignore = F821
+exclude=.venv,.git,.tox,dist,*openstack/common*,*egg,build