From 091aae8a6ddbe0bbe31df3ead927248496c363c6 Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Fri, 22 Jul 2011 15:43:44 +0000 Subject: [PATCH] Add notifications for uploads, updates and deletes Change-Id: I372f77fe2d1a575f2108c9b8d1f69301c0d5eb5e --- Authors | 1 + doc/source/configuring.rst | 57 +++++++++++ doc/source/notifications.rst | 85 +++++++++++++++++ etc/glance-api.conf | 16 ++++ glance/api/v1/images.py | 11 +++ glance/common/exception.py | 4 + glance/common/notifier.py | 147 +++++++++++++++++++++++++++++ glance/tests/unit/test_notifier.py | 123 ++++++++++++++++++++++++ tools/pip-requires | 1 + 9 files changed, 445 insertions(+) create mode 100644 doc/source/notifications.rst create mode 100644 glance/common/notifier.py create mode 100644 glance/tests/unit/test_notifier.py diff --git a/Authors b/Authors index 92ab57cf4e..e918827c0c 100644 --- a/Authors +++ b/Authors @@ -1,4 +1,5 @@ Andrey Brindeyev +Brian Lamar Brian Waldon Christopher MacGown Cory Wright diff --git a/doc/source/configuring.rst b/doc/source/configuring.rst index cdf42d238a..1b00431e8c 100644 --- a/doc/source/configuring.rst +++ b/doc/source/configuring.rst @@ -331,3 +331,60 @@ Can only be specified in configuration files. Sets the number of seconds after which SQLAlchemy should reconnect to the datastore if no activity has been made on the connection. + +Configuring Notifications +------------------------- + +Glance can optionally generate notifications to be logged or sent to +a RabbitMQ queue. The configuration options are specified in the +``glance-api.conf`` config file in the section ``[DEFAULT]``. + +* ``notifier_strategy`` + +Optional. Default: ``noop`` + +Sets the strategy used for notifications. Options are ``logging``, +``rabbit`` and ``noop``. +For more information :doc:`Glance notifications ` + +* ``rabbit_host`` + +Optional. Default: ``localhost`` + +Host to connect to when using ``rabbit`` strategy. + +* ``rabbit_port`` + +Optional. Default: ``5672`` + +Port to connect to when using ``rabbit`` strategy. + +* ``rabbit_use_ssl`` + +Optional. Default: ``false`` + +Boolean to use SSL for connecting when using ``rabbit`` strategy. + +* ``rabbit_userid`` + +Optional. Default: ``guest`` + +Userid to use for connection when using ``rabbit`` strategy. + +* ``rabbit_password`` + +Optional. Default: ``guest`` + +Password to use for connection when using ``rabbit`` strategy. + +* ``rabbit_virtual_host`` + +Optional. Default: ``/`` + +Virtual host to use for connection when using ``rabbit`` strategy. + +* ``rabbit_notification_topic`` + +Optional. Default: ``glance_notifications`` + +Topic to use for connection when using ``rabbit`` strategy. diff --git a/doc/source/notifications.rst b/doc/source/notifications.rst new file mode 100644 index 0000000000..1ac2af8c06 --- /dev/null +++ b/doc/source/notifications.rst @@ -0,0 +1,85 @@ +.. + Copyright 2011 OpenStack, LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Notifications +============= + +Notifications can be generated for each upload, update or delete image +event. These can be used for auditing, troubleshooting, etc. + +Strategies +---------- + +* logging + + This strategy uses the standard Python logging infrastructure with + the notifications ending up in file specificed by the log_file + configuration directive. + +* rabbit + + This strategy sends notifications to a rabbitmq queue. This can then + be processed by other services or applications. + +* noop + + This strategy produces no notifications. It is the default strategy. + +Content +------- + +Every message contains a handful of attributes. + +* message_id + + UUID identifying the message. + +* publisher_id + + The hostname of the glance instance that generated the message. + +* event_type + + Event that generated the message. + +* priority + + One of WARN, INFO or ERROR. + +* timestamp + + UTC timestamp of when event was generated. + +* payload + + Data specific to the event type. + +Payload +------- + +WARN and ERROR events contain a text message in the payload. + +* image.upload + + For INFO events, it is the image metadata. + +* image.update + + For INFO events, it is the image metadata. + +* image.delete + + For INFO events, it is the image id. diff --git a/etc/glance-api.conf b/etc/glance-api.conf index f3f6f3b468..a920ea523d 100644 --- a/etc/glance-api.conf +++ b/etc/glance-api.conf @@ -29,6 +29,22 @@ log_file = /var/log/glance/api.log # Send logs to syslog (/dev/log) instead of to file specified by `log_file` use_syslog = False +# Notifications can be sent when images are create, updated or deleted. +# There are three methods of sending notifications, logging (via the +# log_file directive), rabbit (via a rabbitmq queue) or noop (no +# notifications sent, the default) +notifier_strategy = noop + +# Configuration options if sending notifications via rabbitmq (these are +# the defaults) +rabbit_host = localhost +rabbit_port = 5672 +rabbit_use_ssl = false +rabbit_userid = guest +rabbit_password = guest +rabbit_virtual_host = / +rabbit_notification_topic = glance_notifications + # ============ Filesystem Store Options ======================== # Directory that the Filesystem backend store diff --git a/glance/api/v1/images.py b/glance/api/v1/images.py index 3e32279ed8..efd81f20d9 100644 --- a/glance/api/v1/images.py +++ b/glance/api/v1/images.py @@ -34,6 +34,7 @@ from webob.exc import (HTTPNotFound, from glance import api from glance import image_cache from glance.common import exception +from glance.common import notifier from glance.common import wsgi import glance.store import glance.store.filesystem @@ -79,6 +80,7 @@ class Controller(api.BaseController): def __init__(self, options): self.options = options glance.store.create_stores(options) + self.notifier = notifier.Notifier(options) def index(self, req): """ @@ -365,6 +367,7 @@ class Controller(api.BaseController): image_id, {'checksum': checksum, 'size': size}) + self.notifier.info('image.upload', image_meta) return location @@ -372,12 +375,14 @@ class Controller(api.BaseController): msg = ("Attempt to upload duplicate image: %s") % e logger.error(msg) self._safe_kill(req, image_id) + self.notifier.error('image.upload', msg) raise HTTPConflict(msg, request=req) except exception.NotAuthorized, e: msg = ("Unauthorized upload attempt: %s") % e logger.error(msg) self._safe_kill(req, image_id) + self.notifier.error('image.upload', msg) raise HTTPForbidden(msg, request=req, content_type='text/plain') @@ -389,6 +394,8 @@ class Controller(api.BaseController): msg = ("Error uploading image: (%s): '%s") % ( e.__class__.__name__, str(e)) + + self.notifier.error('image.upload', msg) raise HTTPBadRequest(msg, request=req) def _activate(self, req, image_id, location): @@ -539,7 +546,10 @@ class Controller(api.BaseController): % locals()) for line in msg.split('\n'): logger.error(line) + self.notifier.error('image.update', msg) raise HTTPBadRequest(msg, request=req, content_type="text/plain") + else: + self.notifier.info('image.update', image_meta) return {'image_meta': image_meta} @@ -571,6 +581,7 @@ class Controller(api.BaseController): schedule_delete_from_backend(image['location'], self.options, req.context, id) registry.delete_image_metadata(self.options, req.context, id) + self.notifier.info('image.delete', id) def get_store_or_400(self, request, store_name): """ diff --git a/glance/common/exception.py b/glance/common/exception.py index 5a471a0166..911ec427c2 100644 --- a/glance/common/exception.py +++ b/glance/common/exception.py @@ -159,3 +159,7 @@ class StoreDeleteNotSupported(GlanceException): class StoreAddDisabled(GlanceException): message = _("Configuration for store failed. Adding images to this " "store is disabled.") + + +class InvalidNotifierStrategy(GlanceException): + message = "'%(strategy)s' is not an available notifier strategy." diff --git a/glance/common/notifier.py b/glance/common/notifier.py new file mode 100644 index 0000000000..2cc6082319 --- /dev/null +++ b/glance/common/notifier.py @@ -0,0 +1,147 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime +import logging +import socket +import uuid + +import kombu.connection + +from glance.common import config +from glance.common import exception + + +class NoopStrategy(object): + """A notifier that does nothing when called.""" + + def __init__(self, options): + pass + + def warn(self, msg): + pass + + def info(self, msg): + pass + + def error(self, msg): + pass + + +class LoggingStrategy(object): + """A notifier that calls logging when called.""" + + def __init__(self, options): + self.logger = logging.getLogger('glance.notifier.logging_notifier') + + def warn(self, msg): + self.logger.warn(msg) + + def info(self, msg): + self.logger.info(msg) + + def error(self, msg): + self.logger.error(msg) + + +class RabbitStrategy(object): + """A notifier that puts a message on a queue when called.""" + + def __init__(self, options): + """Initialize the rabbit notification strategy.""" + self._options = options + host = self._get_option('rabbit_host', 'str', 'localhost') + port = self._get_option('rabbit_port', 'int', 5672) + use_ssl = self._get_option('rabbit_use_ssl', 'bool', False) + userid = self._get_option('rabbit_userid', 'str', 'guest') + password = self._get_option('rabbit_password', 'str', 'guest') + virtual_host = self._get_option('rabbit_virtual_host', 'str', '/') + + self.connection = kombu.connection.BrokerConnection( + hostname=host, + userid=userid, + password=password, + virtual_host=virtual_host, + ssl=use_ssl) + + self.topic = self._get_option('rabbit_notification_topic', + 'str', + 'glance_notifications') + + def _get_option(self, name, datatype, default): + """Retrieve a configuration option.""" + return config.get_option(self._options, + name, + type=datatype, + default=default) + + def _send_message(self, message, priority): + topic = "%s.%s" % (self.topic, priority) + queue = self.connection.SimpleQueue(topic) + queue.put(message, serializer="json") + queue.close() + + def warn(self, msg): + self._send_message(msg, "WARN") + + def info(self, msg): + self._send_message(msg, "INFO") + + def error(self, msg): + self._send_message(msg, "ERROR") + + +class Notifier(object): + """Uses a notification strategy to send out messages about events.""" + + STRATEGIES = { + "logging": LoggingStrategy, + "rabbit": RabbitStrategy, + "noop": NoopStrategy, + "default": NoopStrategy, + } + + def __init__(self, options, strategy=None): + strategy = config.get_option(options, "notifier_strategy", + type="str", default="default") + try: + self.strategy = self.STRATEGIES[strategy](options) + except KeyError: + raise exception.InvalidNotifierStrategy(strategy=strategy) + + @staticmethod + def generate_message(event_type, priority, payload): + return { + "message_id": str(uuid.uuid4()), + "publisher_id": socket.gethostname(), + "event_type": event_type, + "priority": priority, + "payload": payload, + "timestamp": str(datetime.datetime.utcnow()), + } + + def warn(self, event_type, payload): + msg = self.generate_message(event_type, "WARN", payload) + self.strategy.warn(msg) + + def info(self, event_type, payload): + msg = self.generate_message(event_type, "INFO", payload) + self.strategy.info(msg) + + def error(self, event_type, payload): + msg = self.generate_message(event_type, "ERROR", payload) + self.strategy.error(msg) diff --git a/glance/tests/unit/test_notifier.py b/glance/tests/unit/test_notifier.py new file mode 100644 index 0000000000..246800a361 --- /dev/null +++ b/glance/tests/unit/test_notifier.py @@ -0,0 +1,123 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import unittest + +from glance.common import exception +from glance.common import notifier + + +class TestInvalidNotifier(unittest.TestCase): + """Test that notifications are generated appropriately""" + + def test_cannot_create(self): + options = {"notifier_strategy": "invalid_notifier"} + self.assertRaises(exception.InvalidNotifierStrategy, + notifier.Notifier, + options) + + +class TestLoggingNotifier(unittest.TestCase): + """Test the logging notifier is selected and works properly.""" + + def setUp(self): + options = {"notifier_strategy": "logging"} + self.called = False + self.logger = logging.getLogger("glance.notifier.logging_notifier") + self.notifier = notifier.Notifier(options) + + def _called(self, msg): + self.called = msg + + def test_warn(self): + self.logger.warn = self._called + self.notifier.warn("test_event", "test_message") + if self.called is False: + self.fail("Did not call logging library correctly.") + + def test_info(self): + self.logger.info = self._called + self.notifier.info("test_event", "test_message") + if self.called is False: + self.fail("Did not call logging library correctly.") + + def test_erorr(self): + self.logger.error = self._called + self.notifier.error("test_event", "test_message") + if self.called is False: + self.fail("Did not call logging library correctly.") + + +class TestNoopNotifier(unittest.TestCase): + """Test that the noop notifier works...and does nothing?""" + + def setUp(self): + options = {"notifier_strategy": "noop"} + self.notifier = notifier.Notifier(options) + + def test_warn(self): + self.notifier.warn("test_event", "test_message") + + def test_info(self): + self.notifier.info("test_event", "test_message") + + def test_error(self): + self.notifier.error("test_event", "test_message") + + +class TestRabbitNotifier(unittest.TestCase): + """Test AMQP/Rabbit notifier works.""" + + def setUp(self): + notifier.RabbitStrategy._send_message = self._send_message + self.called = False + options = {"notifier_strategy": "rabbit"} + self.notifier = notifier.Notifier(options) + + def _send_message(self, message, priority): + self.called = { + "message": message, + "priority": priority, + } + + def test_warn(self): + self.notifier.warn("test_event", "test_message") + + if self.called is False: + self.fail("Did not call _send_message properly.") + + self.assertEquals("test_message", self.called["message"]["payload"]) + self.assertEquals("WARN", self.called["message"]["priority"]) + + def test_info(self): + self.notifier.info("test_event", "test_message") + + if self.called is False: + self.fail("Did not call _send_message properly.") + + self.assertEquals("test_message", self.called["message"]["payload"]) + self.assertEquals("INFO", self.called["message"]["priority"]) + + def test_error(self): + self.notifier.error("test_event", "test_message") + + if self.called is False: + self.fail("Did not call _send_message properly.") + + self.assertEquals("test_message", self.called["message"]["payload"]) + self.assertEquals("ERROR", self.called["message"]["priority"]) diff --git a/tools/pip-requires b/tools/pip-requires index badf2f781b..f928299d40 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -20,3 +20,4 @@ sqlalchemy-migrate>=0.6,<0.7 bzr httplib2 xattr>=0.6.0 +kombu