Switch to using monasca-statsd
Switch to monasca-statsd so we can send additional dimensions for service and component with the statsd messages. Change-Id: Ic6ff3b67b4148c070ec9eec9f9f990680b5e9f4c
This commit is contained in:
parent
2fea98ec2b
commit
9025d8b0e4
4
.gitignore
vendored
4
.gitignore
vendored
@ -3,3 +3,7 @@
|
|||||||
.tox
|
.tox
|
||||||
AUTHORS
|
AUTHORS
|
||||||
ChangeLog
|
ChangeLog
|
||||||
|
.*project
|
||||||
|
build
|
||||||
|
dist
|
||||||
|
monasca_notification.egg-info
|
||||||
|
@ -15,9 +15,8 @@
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import multiprocessing
|
import monascastatsd as mstatsd
|
||||||
import MySQLdb
|
import MySQLdb
|
||||||
import statsd
|
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from monasca_notification.notification import Notification
|
from monasca_notification.notification import Notification
|
||||||
@ -36,7 +35,8 @@ class AlarmProcessor(BaseProcessor):
|
|||||||
self.alarm_ttl = alarm_ttl
|
self.alarm_ttl = alarm_ttl
|
||||||
self.notification_queue = notification_queue
|
self.notification_queue = notification_queue
|
||||||
self.finished_queue = finished_queue
|
self.finished_queue = finished_queue
|
||||||
|
self.monascastatsd = mstatsd.Client(name='monasca',
|
||||||
|
dimensions=BaseProcessor.dimensions)
|
||||||
try:
|
try:
|
||||||
self.mysql = MySQLdb.connect(host=mysql_host, user=mysql_user, passwd=mysql_passwd, db=dbname)
|
self.mysql = MySQLdb.connect(host=mysql_host, user=mysql_user, passwd=mysql_passwd, db=dbname)
|
||||||
self.mysql.autocommit(True)
|
self.mysql.autocommit(True)
|
||||||
@ -88,12 +88,12 @@ class AlarmProcessor(BaseProcessor):
|
|||||||
"""Check the notification setting for this project in mysql then create the appropriate notification or
|
"""Check the notification setting for this project in mysql then create the appropriate notification or
|
||||||
add to the finished_queue
|
add to the finished_queue
|
||||||
"""
|
"""
|
||||||
|
failed_parse_count = self.monascastatsd.get_counter(name='alarms_failed_parse_count')
|
||||||
|
no_notification_count = self.monascastatsd.get_counter(name='alarms_no_notification_count')
|
||||||
|
notification_count = self.monascastatsd.get_counter(name='created_count')
|
||||||
|
db_time = self.monascastatsd.get_timer()
|
||||||
|
|
||||||
cur = self.mysql.cursor()
|
cur = self.mysql.cursor()
|
||||||
pname = multiprocessing.current_process().name
|
|
||||||
failed_parse_count = statsd.Counter('AlarmsFailedParse-%s' % pname)
|
|
||||||
no_notification_count = statsd.Counter('AlarmsNoNotification-%s' % pname)
|
|
||||||
notification_count = statsd.Counter('NotificationsCreated-%s' % pname)
|
|
||||||
db_time = statsd.Timer('ConfigDBTime-%s' % pname)
|
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
raw_alarm = self.alarm_queue.get()
|
raw_alarm = self.alarm_queue.get()
|
||||||
@ -116,7 +116,7 @@ class AlarmProcessor(BaseProcessor):
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with db_time.time():
|
with db_time.time('config_db_time'):
|
||||||
cur.execute("""SELECT name, type, address
|
cur.execute("""SELECT name, type, address
|
||||||
FROM alarm_action as aa
|
FROM alarm_action as aa
|
||||||
JOIN notification_method as nm ON aa.action_id = nm.id
|
JOIN notification_method as nm ON aa.action_id = nm.id
|
||||||
|
@ -19,6 +19,9 @@ log = logging.getLogger(__name__)
|
|||||||
|
|
||||||
|
|
||||||
class BaseProcessor(object):
|
class BaseProcessor(object):
|
||||||
|
|
||||||
|
dimensions = {'service': 'monitoring', 'component': 'monasca-notification'}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _add_to_queue(queue, queue_name, msg):
|
def _add_to_queue(queue, queue_name, msg):
|
||||||
"""Warns on full queue then does a blocking push to the queue.
|
"""Warns on full queue then does a blocking push to the queue.
|
||||||
|
@ -17,7 +17,7 @@ import kafka.client
|
|||||||
import kafka.common
|
import kafka.common
|
||||||
import kafka.consumer
|
import kafka.consumer
|
||||||
import logging
|
import logging
|
||||||
import statsd
|
import monascastatsd as mstatsd
|
||||||
|
|
||||||
from monasca_notification.processors.base import BaseProcessor
|
from monasca_notification.processors.base import BaseProcessor
|
||||||
|
|
||||||
@ -26,6 +26,7 @@ log = logging.getLogger(__name__)
|
|||||||
|
|
||||||
class KafkaConsumer(BaseProcessor):
|
class KafkaConsumer(BaseProcessor):
|
||||||
"""Pull from the alarm topic and place alarm objects on the sent_queue.
|
"""Pull from the alarm topic and place alarm objects on the sent_queue.
|
||||||
|
|
||||||
No commit is being done until processing is finished and as the processing can take some time it is done in
|
No commit is being done until processing is finished and as the processing can take some time it is done in
|
||||||
another step.
|
another step.
|
||||||
|
|
||||||
@ -43,6 +44,8 @@ class KafkaConsumer(BaseProcessor):
|
|||||||
# No auto-commit so that commits only happen after the alarm is processed.
|
# No auto-commit so that commits only happen after the alarm is processed.
|
||||||
self.consumer = kafka.consumer.SimpleConsumer(self.kafka, group, topic, auto_commit=False)
|
self.consumer = kafka.consumer.SimpleConsumer(self.kafka, group, topic, auto_commit=False)
|
||||||
self.consumer.provide_partition_info() # Without this the partition is not provided in the response
|
self.consumer.provide_partition_info() # Without this the partition is not provided in the response
|
||||||
|
self.monascastatsd = mstatsd.Client(name='monasca',
|
||||||
|
dimensions=BaseProcessor.dimensions)
|
||||||
|
|
||||||
self._initialize_offsets(group, topic)
|
self._initialize_offsets(group, topic)
|
||||||
# After my pull request is merged I can remove _initialize_offsets and use
|
# After my pull request is merged I can remove _initialize_offsets and use
|
||||||
@ -84,10 +87,11 @@ class KafkaConsumer(BaseProcessor):
|
|||||||
def run(self):
|
def run(self):
|
||||||
"""Consume from kafka and place alarm objects on the sent_queue
|
"""Consume from kafka and place alarm objects on the sent_queue
|
||||||
"""
|
"""
|
||||||
counter = statsd.Counter('ConsumedFromKafka')
|
consumed_from_kafka = self.monascastatsd.get_counter(name='consumed_from_kafka')
|
||||||
|
|
||||||
try:
|
try:
|
||||||
for message in self.consumer:
|
for message in self.consumer:
|
||||||
counter += 1
|
consumed_from_kafka += 1
|
||||||
log.debug("Consuming message from kafka, partition %d, offset %d" % (message[0], message[1].offset))
|
log.debug("Consuming message from kafka, partition %d, offset %d" % (message[0], message[1].offset))
|
||||||
self._add_to_queue(self.queue, 'alarms', message)
|
self._add_to_queue(self.queue, 'alarms', message)
|
||||||
except Exception:
|
except Exception:
|
||||||
|
@ -15,9 +15,8 @@
|
|||||||
|
|
||||||
import email.mime.text
|
import email.mime.text
|
||||||
import logging
|
import logging
|
||||||
import multiprocessing
|
import monascastatsd as mstatsd
|
||||||
import smtplib
|
import smtplib
|
||||||
import statsd
|
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from monasca_notification.processors.base import BaseProcessor
|
from monasca_notification.processors.base import BaseProcessor
|
||||||
@ -38,6 +37,8 @@ class NotificationProcessor(BaseProcessor):
|
|||||||
|
|
||||||
# Types as key, method used to process that type as value
|
# Types as key, method used to process that type as value
|
||||||
self.notification_types = {'email': self._send_email}
|
self.notification_types = {'email': self._send_email}
|
||||||
|
self.monascastatsd = mstatsd.Client(name='monasca',
|
||||||
|
dimensions=BaseProcessor.dimensions)
|
||||||
|
|
||||||
def _send_email(self, notification):
|
def _send_email(self, notification):
|
||||||
"""Send the notification via email
|
"""Send the notification via email
|
||||||
@ -86,15 +87,10 @@ class NotificationProcessor(BaseProcessor):
|
|||||||
For each notification in a message it is sent according to its type.
|
For each notification in a message it is sent according to its type.
|
||||||
If all notifications fail the alarm partition/offset are added to the the finished queue
|
If all notifications fail the alarm partition/offset are added to the the finished queue
|
||||||
"""
|
"""
|
||||||
pname = multiprocessing.current_process().name
|
counters = {'email': self.monascastatsd.get_counter(name='sent_smtp_count')}
|
||||||
invalid_count = statsd.Counter('NotificationsInvalidType-%s' % pname)
|
timers = {'email': self.monascastatsd.get_timer()}
|
||||||
failed_count = statsd.Counter('NotificationsSentFailed-%s' % pname)
|
invalid_type_count = self.monascastatsd.get_counter(name='invalid_type_count')
|
||||||
|
sent_failed_count = self.monascastatsd.get_counter(name='sent_failed_count')
|
||||||
smtp_sent_count = statsd.Counter('NotificationsSentSMTP-%s' % pname)
|
|
||||||
counters = {'email': smtp_sent_count}
|
|
||||||
|
|
||||||
smtp_time = statsd.Timer('SMTPTime-%s' % pname)
|
|
||||||
timers = {'email': smtp_time}
|
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
notifications = self.notification_queue.get()
|
notifications = self.notification_queue.get()
|
||||||
@ -102,13 +98,14 @@ class NotificationProcessor(BaseProcessor):
|
|||||||
for notification in notifications:
|
for notification in notifications:
|
||||||
if notification.type not in self.notification_types:
|
if notification.type not in self.notification_types:
|
||||||
log.warn('Notification type %s is not a valid type' % notification.type)
|
log.warn('Notification type %s is not a valid type' % notification.type)
|
||||||
invalid_count += 1
|
invalid_type_count += 1
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
with timers[notification.type].time():
|
with timers[notification.type].time('smtp_time'):
|
||||||
sent = self.notification_types[notification.type](notification)
|
sent = self.notification_types[notification.type](notification)
|
||||||
|
|
||||||
if sent is None:
|
if sent is None:
|
||||||
failed_count += 1
|
sent_failed_count += 1
|
||||||
else:
|
else:
|
||||||
sent.notification_timestamp = time.time()
|
sent.notification_timestamp = time.time()
|
||||||
sent_notifications.append(sent)
|
sent_notifications.append(sent)
|
||||||
|
@ -16,7 +16,7 @@
|
|||||||
import kafka.client
|
import kafka.client
|
||||||
import kafka.producer
|
import kafka.producer
|
||||||
import logging
|
import logging
|
||||||
import statsd
|
import monascastatsd as mstatsd
|
||||||
|
|
||||||
from monasca_notification.processors.base import BaseProcessor
|
from monasca_notification.processors.base import BaseProcessor
|
||||||
|
|
||||||
@ -39,6 +39,8 @@ class SentNotificationProcessor(BaseProcessor):
|
|||||||
self.topic = topic
|
self.topic = topic
|
||||||
self.finished_queue = finished_queue
|
self.finished_queue = finished_queue
|
||||||
self.sent_queue = sent_queue
|
self.sent_queue = sent_queue
|
||||||
|
self.monascastatsd = mstatsd.Client(name='monasca',
|
||||||
|
dimensions=BaseProcessor.dimensions)
|
||||||
|
|
||||||
self.kafka = kafka.client.KafkaClient(url)
|
self.kafka = kafka.client.KafkaClient(url)
|
||||||
self.producer = kafka.producer.SimpleProducer(
|
self.producer = kafka.producer.SimpleProducer(
|
||||||
@ -52,12 +54,14 @@ class SentNotificationProcessor(BaseProcessor):
|
|||||||
"""Takes messages from the sent_queue, puts them on the kafka notification topic and then adds
|
"""Takes messages from the sent_queue, puts them on the kafka notification topic and then adds
|
||||||
partition/offset to the finished queue
|
partition/offset to the finished queue
|
||||||
"""
|
"""
|
||||||
published_count = statsd.Counter('PublishedToKafka')
|
published_to_kafka = self.monascastatsd.get_counter(name='published_to_kafka')
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
notifications = self.sent_queue.get()
|
notifications = self.sent_queue.get()
|
||||||
for notification in notifications:
|
for notification in notifications:
|
||||||
responses = self.producer.send_messages(self.topic, notification.to_json())
|
responses = self.producer.send_messages(self.topic, notification.to_json())
|
||||||
published_count += 1
|
published_to_kafka += 1
|
||||||
|
|
||||||
log.debug('Published to topic %s, message %s' % (self.topic, notification.to_json()))
|
log.debug('Published to topic %s, message %s' % (self.topic, notification.to_json()))
|
||||||
for resp in responses:
|
for resp in responses:
|
||||||
if resp.error != 0:
|
if resp.error != 0:
|
||||||
|
@ -20,10 +20,10 @@ import kazoo.client
|
|||||||
import kazoo.exceptions
|
import kazoo.exceptions
|
||||||
import logging
|
import logging
|
||||||
import Queue
|
import Queue
|
||||||
import statsd
|
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from monasca_notification import notification_exceptions
|
from monasca_notification import notification_exceptions
|
||||||
|
import monascastatsd as mstatsd
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@ -61,13 +61,16 @@ class KafkaStateTracker(object):
|
|||||||
self.lock_path = '/locks/monasca-notification/%s' % topic
|
self.lock_path = '/locks/monasca-notification/%s' % topic
|
||||||
|
|
||||||
self._offsets = None # Initialized in the beginning of the run
|
self._offsets = None # Initialized in the beginning of the run
|
||||||
|
self._dimensions = {'service': 'monitoring', 'component': 'monasca-notification'}
|
||||||
# This is a dictionary of sets used for tracking finished offsets when there is a gap and the committed offset
|
# This is a dictionary of sets used for tracking finished offsets when there is a gap and the committed offset
|
||||||
# can not yet be advanced
|
# can not yet be advanced
|
||||||
self._uncommitted_offsets = collections.defaultdict(set)
|
self._uncommitted_offsets = collections.defaultdict(set)
|
||||||
self._last_commit_time = collections.defaultdict(time.time)
|
self._last_commit_time = collections.defaultdict(time.time)
|
||||||
|
monascastatsd = mstatsd.Client(name='monasca',
|
||||||
self.zk_timer = statsd.Timer('OffsetCommitTime')
|
dimensions=self._dimensions)
|
||||||
self.offset_update_count = statsd.Counter('AlarmsOffsetUpdated')
|
self.offset_update_count = monascastatsd.get_counter(name='alarms_offset_update_count')
|
||||||
|
self.finished_count = monascastatsd.get_counter(name='alarms_finished_count')
|
||||||
|
self.kafka_timer = monascastatsd.get_timer()
|
||||||
|
|
||||||
def _drop_lock(self):
|
def _drop_lock(self):
|
||||||
"""Drop the lock file kept in zookeeper
|
"""Drop the lock file kept in zookeeper
|
||||||
@ -140,7 +143,6 @@ class KafkaStateTracker(object):
|
|||||||
if not self.has_lock:
|
if not self.has_lock:
|
||||||
raise notification_exceptions.NotificationException('Attempt to begin run without Zookeeper Lock')
|
raise notification_exceptions.NotificationException('Attempt to begin run without Zookeeper Lock')
|
||||||
|
|
||||||
finished_count = statsd.Counter('AlarmsFinished')
|
|
||||||
while True:
|
while True:
|
||||||
# If self.stop is True run the queue until it is empty, do final commits then exit
|
# If self.stop is True run the queue until it is empty, do final commits then exit
|
||||||
if self.stop and self.finished_queue.empty():
|
if self.stop and self.finished_queue.empty():
|
||||||
@ -161,7 +163,8 @@ class KafkaStateTracker(object):
|
|||||||
except Queue.Empty:
|
except Queue.Empty:
|
||||||
continue # This is non-blocking so the self.stop signal has a chance to take affect
|
continue # This is non-blocking so the self.stop signal has a chance to take affect
|
||||||
|
|
||||||
finished_count += 1
|
self.finished_count += 1
|
||||||
|
|
||||||
partition = int(msg[0])
|
partition = int(msg[0])
|
||||||
offset = int(msg[1])
|
offset = int(msg[1])
|
||||||
|
|
||||||
@ -203,12 +206,15 @@ class KafkaStateTracker(object):
|
|||||||
self.offsets
|
self.offsets
|
||||||
|
|
||||||
self.offset_update_count += value - self._offsets[partition]
|
self.offset_update_count += value - self._offsets[partition]
|
||||||
|
|
||||||
self._offsets[partition] = value
|
self._offsets[partition] = value
|
||||||
|
|
||||||
req = kafka.common.OffsetCommitRequest(self.topic, partition, value, None)
|
req = kafka.common.OffsetCommitRequest(self.topic, partition, value, None)
|
||||||
try:
|
try:
|
||||||
|
with self.kafka_timer.time('offset_commit_time_sec'):
|
||||||
responses = self.kafka.send_offset_commit_request(self.kafka_group, [req])
|
responses = self.kafka.send_offset_commit_request(self.kafka_group, [req])
|
||||||
kafka.common.check_error(responses[0])
|
kafka.common.check_error(responses[0])
|
||||||
|
|
||||||
log.debug('Updated committed offset for partition %s, offset %s' % (partition, value))
|
log.debug('Updated committed offset for partition %s, offset %s' % (partition, value))
|
||||||
except kafka.common.KafkaError:
|
except kafka.common.KafkaError:
|
||||||
log.exception('Error updating the committed offset in kafka, partition %s, value %s' % (partition, value))
|
log.exception('Error updating the committed offset in kafka, partition %s, value %s' % (partition, value))
|
||||||
|
@ -2,5 +2,5 @@ kafka-python>=0.9.1
|
|||||||
kazoo>=1.3
|
kazoo>=1.3
|
||||||
MySQL-python
|
MySQL-python
|
||||||
pbr>=0.6,<1.0
|
pbr>=0.6,<1.0
|
||||||
python-statsd>=1.6.3
|
monasca-statsd
|
||||||
PyYAML
|
PyYAML
|
||||||
|
Loading…
Reference in New Issue
Block a user