Switch to using monasca-statsd
Switch to monasca-statsd so we can send additional dimensions for service and component with the statsd messages. Change-Id: Ic6ff3b67b4148c070ec9eec9f9f990680b5e9f4c
This commit is contained in:
parent
2fea98ec2b
commit
9025d8b0e4
4
.gitignore
vendored
4
.gitignore
vendored
@ -3,3 +3,7 @@
|
||||
.tox
|
||||
AUTHORS
|
||||
ChangeLog
|
||||
.*project
|
||||
build
|
||||
dist
|
||||
monasca_notification.egg-info
|
||||
|
@ -15,9 +15,8 @@
|
||||
|
||||
import json
|
||||
import logging
|
||||
import multiprocessing
|
||||
import monascastatsd as mstatsd
|
||||
import MySQLdb
|
||||
import statsd
|
||||
import time
|
||||
|
||||
from monasca_notification.notification import Notification
|
||||
@ -36,7 +35,8 @@ class AlarmProcessor(BaseProcessor):
|
||||
self.alarm_ttl = alarm_ttl
|
||||
self.notification_queue = notification_queue
|
||||
self.finished_queue = finished_queue
|
||||
|
||||
self.monascastatsd = mstatsd.Client(name='monasca',
|
||||
dimensions=BaseProcessor.dimensions)
|
||||
try:
|
||||
self.mysql = MySQLdb.connect(host=mysql_host, user=mysql_user, passwd=mysql_passwd, db=dbname)
|
||||
self.mysql.autocommit(True)
|
||||
@ -88,12 +88,12 @@ class AlarmProcessor(BaseProcessor):
|
||||
"""Check the notification setting for this project in mysql then create the appropriate notification or
|
||||
add to the finished_queue
|
||||
"""
|
||||
failed_parse_count = self.monascastatsd.get_counter(name='alarms_failed_parse_count')
|
||||
no_notification_count = self.monascastatsd.get_counter(name='alarms_no_notification_count')
|
||||
notification_count = self.monascastatsd.get_counter(name='created_count')
|
||||
db_time = self.monascastatsd.get_timer()
|
||||
|
||||
cur = self.mysql.cursor()
|
||||
pname = multiprocessing.current_process().name
|
||||
failed_parse_count = statsd.Counter('AlarmsFailedParse-%s' % pname)
|
||||
no_notification_count = statsd.Counter('AlarmsNoNotification-%s' % pname)
|
||||
notification_count = statsd.Counter('NotificationsCreated-%s' % pname)
|
||||
db_time = statsd.Timer('ConfigDBTime-%s' % pname)
|
||||
|
||||
while True:
|
||||
raw_alarm = self.alarm_queue.get()
|
||||
@ -116,7 +116,7 @@ class AlarmProcessor(BaseProcessor):
|
||||
continue
|
||||
|
||||
try:
|
||||
with db_time.time():
|
||||
with db_time.time('config_db_time'):
|
||||
cur.execute("""SELECT name, type, address
|
||||
FROM alarm_action as aa
|
||||
JOIN notification_method as nm ON aa.action_id = nm.id
|
||||
|
@ -19,6 +19,9 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BaseProcessor(object):
|
||||
|
||||
dimensions = {'service': 'monitoring', 'component': 'monasca-notification'}
|
||||
|
||||
@staticmethod
|
||||
def _add_to_queue(queue, queue_name, msg):
|
||||
"""Warns on full queue then does a blocking push to the queue.
|
||||
|
@ -17,7 +17,7 @@ import kafka.client
|
||||
import kafka.common
|
||||
import kafka.consumer
|
||||
import logging
|
||||
import statsd
|
||||
import monascastatsd as mstatsd
|
||||
|
||||
from monasca_notification.processors.base import BaseProcessor
|
||||
|
||||
@ -26,6 +26,7 @@ log = logging.getLogger(__name__)
|
||||
|
||||
class KafkaConsumer(BaseProcessor):
|
||||
"""Pull from the alarm topic and place alarm objects on the sent_queue.
|
||||
|
||||
No commit is being done until processing is finished and as the processing can take some time it is done in
|
||||
another step.
|
||||
|
||||
@ -43,6 +44,8 @@ class KafkaConsumer(BaseProcessor):
|
||||
# No auto-commit so that commits only happen after the alarm is processed.
|
||||
self.consumer = kafka.consumer.SimpleConsumer(self.kafka, group, topic, auto_commit=False)
|
||||
self.consumer.provide_partition_info() # Without this the partition is not provided in the response
|
||||
self.monascastatsd = mstatsd.Client(name='monasca',
|
||||
dimensions=BaseProcessor.dimensions)
|
||||
|
||||
self._initialize_offsets(group, topic)
|
||||
# After my pull request is merged I can remove _initialize_offsets and use
|
||||
@ -84,10 +87,11 @@ class KafkaConsumer(BaseProcessor):
|
||||
def run(self):
|
||||
"""Consume from kafka and place alarm objects on the sent_queue
|
||||
"""
|
||||
counter = statsd.Counter('ConsumedFromKafka')
|
||||
consumed_from_kafka = self.monascastatsd.get_counter(name='consumed_from_kafka')
|
||||
|
||||
try:
|
||||
for message in self.consumer:
|
||||
counter += 1
|
||||
consumed_from_kafka += 1
|
||||
log.debug("Consuming message from kafka, partition %d, offset %d" % (message[0], message[1].offset))
|
||||
self._add_to_queue(self.queue, 'alarms', message)
|
||||
except Exception:
|
||||
|
@ -15,9 +15,8 @@
|
||||
|
||||
import email.mime.text
|
||||
import logging
|
||||
import multiprocessing
|
||||
import monascastatsd as mstatsd
|
||||
import smtplib
|
||||
import statsd
|
||||
import time
|
||||
|
||||
from monasca_notification.processors.base import BaseProcessor
|
||||
@ -38,6 +37,8 @@ class NotificationProcessor(BaseProcessor):
|
||||
|
||||
# Types as key, method used to process that type as value
|
||||
self.notification_types = {'email': self._send_email}
|
||||
self.monascastatsd = mstatsd.Client(name='monasca',
|
||||
dimensions=BaseProcessor.dimensions)
|
||||
|
||||
def _send_email(self, notification):
|
||||
"""Send the notification via email
|
||||
@ -86,15 +87,10 @@ class NotificationProcessor(BaseProcessor):
|
||||
For each notification in a message it is sent according to its type.
|
||||
If all notifications fail the alarm partition/offset are added to the the finished queue
|
||||
"""
|
||||
pname = multiprocessing.current_process().name
|
||||
invalid_count = statsd.Counter('NotificationsInvalidType-%s' % pname)
|
||||
failed_count = statsd.Counter('NotificationsSentFailed-%s' % pname)
|
||||
|
||||
smtp_sent_count = statsd.Counter('NotificationsSentSMTP-%s' % pname)
|
||||
counters = {'email': smtp_sent_count}
|
||||
|
||||
smtp_time = statsd.Timer('SMTPTime-%s' % pname)
|
||||
timers = {'email': smtp_time}
|
||||
counters = {'email': self.monascastatsd.get_counter(name='sent_smtp_count')}
|
||||
timers = {'email': self.monascastatsd.get_timer()}
|
||||
invalid_type_count = self.monascastatsd.get_counter(name='invalid_type_count')
|
||||
sent_failed_count = self.monascastatsd.get_counter(name='sent_failed_count')
|
||||
|
||||
while True:
|
||||
notifications = self.notification_queue.get()
|
||||
@ -102,13 +98,14 @@ class NotificationProcessor(BaseProcessor):
|
||||
for notification in notifications:
|
||||
if notification.type not in self.notification_types:
|
||||
log.warn('Notification type %s is not a valid type' % notification.type)
|
||||
invalid_count += 1
|
||||
invalid_type_count += 1
|
||||
continue
|
||||
else:
|
||||
with timers[notification.type].time():
|
||||
with timers[notification.type].time('smtp_time'):
|
||||
sent = self.notification_types[notification.type](notification)
|
||||
|
||||
if sent is None:
|
||||
failed_count += 1
|
||||
sent_failed_count += 1
|
||||
else:
|
||||
sent.notification_timestamp = time.time()
|
||||
sent_notifications.append(sent)
|
||||
|
@ -16,7 +16,7 @@
|
||||
import kafka.client
|
||||
import kafka.producer
|
||||
import logging
|
||||
import statsd
|
||||
import monascastatsd as mstatsd
|
||||
|
||||
from monasca_notification.processors.base import BaseProcessor
|
||||
|
||||
@ -39,6 +39,8 @@ class SentNotificationProcessor(BaseProcessor):
|
||||
self.topic = topic
|
||||
self.finished_queue = finished_queue
|
||||
self.sent_queue = sent_queue
|
||||
self.monascastatsd = mstatsd.Client(name='monasca',
|
||||
dimensions=BaseProcessor.dimensions)
|
||||
|
||||
self.kafka = kafka.client.KafkaClient(url)
|
||||
self.producer = kafka.producer.SimpleProducer(
|
||||
@ -52,12 +54,14 @@ class SentNotificationProcessor(BaseProcessor):
|
||||
"""Takes messages from the sent_queue, puts them on the kafka notification topic and then adds
|
||||
partition/offset to the finished queue
|
||||
"""
|
||||
published_count = statsd.Counter('PublishedToKafka')
|
||||
published_to_kafka = self.monascastatsd.get_counter(name='published_to_kafka')
|
||||
|
||||
while True:
|
||||
notifications = self.sent_queue.get()
|
||||
for notification in notifications:
|
||||
responses = self.producer.send_messages(self.topic, notification.to_json())
|
||||
published_count += 1
|
||||
published_to_kafka += 1
|
||||
|
||||
log.debug('Published to topic %s, message %s' % (self.topic, notification.to_json()))
|
||||
for resp in responses:
|
||||
if resp.error != 0:
|
||||
|
@ -20,10 +20,10 @@ import kazoo.client
|
||||
import kazoo.exceptions
|
||||
import logging
|
||||
import Queue
|
||||
import statsd
|
||||
import time
|
||||
|
||||
from monasca_notification import notification_exceptions
|
||||
import monascastatsd as mstatsd
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@ -61,13 +61,16 @@ class KafkaStateTracker(object):
|
||||
self.lock_path = '/locks/monasca-notification/%s' % topic
|
||||
|
||||
self._offsets = None # Initialized in the beginning of the run
|
||||
self._dimensions = {'service': 'monitoring', 'component': 'monasca-notification'}
|
||||
# This is a dictionary of sets used for tracking finished offsets when there is a gap and the committed offset
|
||||
# can not yet be advanced
|
||||
self._uncommitted_offsets = collections.defaultdict(set)
|
||||
self._last_commit_time = collections.defaultdict(time.time)
|
||||
|
||||
self.zk_timer = statsd.Timer('OffsetCommitTime')
|
||||
self.offset_update_count = statsd.Counter('AlarmsOffsetUpdated')
|
||||
monascastatsd = mstatsd.Client(name='monasca',
|
||||
dimensions=self._dimensions)
|
||||
self.offset_update_count = monascastatsd.get_counter(name='alarms_offset_update_count')
|
||||
self.finished_count = monascastatsd.get_counter(name='alarms_finished_count')
|
||||
self.kafka_timer = monascastatsd.get_timer()
|
||||
|
||||
def _drop_lock(self):
|
||||
"""Drop the lock file kept in zookeeper
|
||||
@ -140,7 +143,6 @@ class KafkaStateTracker(object):
|
||||
if not self.has_lock:
|
||||
raise notification_exceptions.NotificationException('Attempt to begin run without Zookeeper Lock')
|
||||
|
||||
finished_count = statsd.Counter('AlarmsFinished')
|
||||
while True:
|
||||
# If self.stop is True run the queue until it is empty, do final commits then exit
|
||||
if self.stop and self.finished_queue.empty():
|
||||
@ -161,7 +163,8 @@ class KafkaStateTracker(object):
|
||||
except Queue.Empty:
|
||||
continue # This is non-blocking so the self.stop signal has a chance to take affect
|
||||
|
||||
finished_count += 1
|
||||
self.finished_count += 1
|
||||
|
||||
partition = int(msg[0])
|
||||
offset = int(msg[1])
|
||||
|
||||
@ -203,12 +206,15 @@ class KafkaStateTracker(object):
|
||||
self.offsets
|
||||
|
||||
self.offset_update_count += value - self._offsets[partition]
|
||||
|
||||
self._offsets[partition] = value
|
||||
|
||||
req = kafka.common.OffsetCommitRequest(self.topic, partition, value, None)
|
||||
try:
|
||||
with self.kafka_timer.time('offset_commit_time_sec'):
|
||||
responses = self.kafka.send_offset_commit_request(self.kafka_group, [req])
|
||||
kafka.common.check_error(responses[0])
|
||||
|
||||
log.debug('Updated committed offset for partition %s, offset %s' % (partition, value))
|
||||
except kafka.common.KafkaError:
|
||||
log.exception('Error updating the committed offset in kafka, partition %s, value %s' % (partition, value))
|
||||
|
@ -2,5 +2,5 @@ kafka-python>=0.9.1
|
||||
kazoo>=1.3
|
||||
MySQL-python
|
||||
pbr>=0.6,<1.0
|
||||
python-statsd>=1.6.3
|
||||
monasca-statsd
|
||||
PyYAML
|
||||
|
Loading…
Reference in New Issue
Block a user