Add notifications for uploads, updates and deletes

Change-Id: I372f77fe2d1a575f2108c9b8d1f69301c0d5eb5e

Authors

@@ -1,4 +1,5 @@
 Andrey Brindeyev <abrindeyev@griddynamics.com>
+Brian Lamar <brian.lamar@rackspace.com>
 Brian Waldon <brian.waldon@rackspace.com>
 Christopher MacGown <chris@slicehost.com>
 Cory Wright <corywright@gmail.com>

doc/source/configuring.rst

@@ -331,3 +331,60 @@ Can only be specified in configuration files.
 
 Sets the number of seconds after which SQLAlchemy should reconnect to the
 datastore if no activity has been made on the connection.
+
+Configuring Notifications
+-------------------------
+
+Glance can optionally generate notifications to be logged or sent to
+a RabbitMQ queue. The configuration options are specified in the
+``glance-api.conf`` config file in the section ``[DEFAULT]``.
+
+* ``notifier_strategy``
+
+Optional. Default: ``noop``
+
+Sets the strategy used for notifications. Options are ``logging``,
+``rabbit`` and ``noop``.
+For more information :doc:`Glance notifications <notifications>`
+
+* ``rabbit_host``
+
+Optional. Default: ``localhost``
+
+Host to connect to when using ``rabbit`` strategy.
+
+* ``rabbit_port``
+
+Optional. Default: ``5672``
+
+Port to connect to when using ``rabbit`` strategy.
+
+* ``rabbit_use_ssl``
+
+Optional. Default: ``false``
+
+Boolean to use SSL for connecting when using ``rabbit`` strategy.
+
+* ``rabbit_userid``
+
+Optional. Default: ``guest``
+
+Userid to use for connection when using ``rabbit`` strategy.
+
+* ``rabbit_password``
+
+Optional. Default: ``guest``
+
+Password to use for connection when using ``rabbit`` strategy.
+
+* ``rabbit_virtual_host``
+
+Optional. Default: ``/``
+
+Virtual host to use for connection when using ``rabbit`` strategy.
+
+* ``rabbit_notification_topic``
+
+Optional. Default: ``glance_notifications``
+
+Topic to use for connection when using ``rabbit`` strategy.

doc/source/notifications.rst

@@ -0,0 +1,85 @@
+..
+      Copyright 2011 OpenStack, LLC
+      All Rights Reserved.
+
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+Notifications
+=============
+
+Notifications can be generated for each upload, update or delete image
+event. These can be used for auditing, troubleshooting, etc.
+
+Strategies
+----------
+
+* logging
+
+  This strategy uses the standard Python logging infrastructure with
+  the notifications ending up in file specificed by the log_file
+  configuration directive.
+
+* rabbit
+
+  This strategy sends notifications to a rabbitmq queue. This can then
+  be processed by other services or applications.
+
+* noop
+
+  This strategy produces no notifications. It is the default strategy.
+
+Content
+-------
+
+Every message contains a handful of attributes.
+
+* message_id
+
+  UUID identifying the message.
+
+* publisher_id
+
+  The hostname of the glance instance that generated the message.
+
+* event_type
+
+  Event that generated the message.
+
+* priority
+
+  One of WARN, INFO or ERROR.
+
+* timestamp
+
+  UTC timestamp of when event was generated.
+
+* payload
+
+  Data specific to the event type.
+
+Payload
+-------
+
+WARN and ERROR events contain a text message in the payload.
+
+* image.upload
+
+  For INFO events, it is the image metadata.
+
+* image.update
+
+  For INFO events, it is the image metadata.
+
+* image.delete
+
+  For INFO events, it is the image id.

etc/glance-api.conf

@@ -29,6 +29,22 @@ log_file = /var/log/glance/api.log
 # Send logs to syslog (/dev/log) instead of to file specified by `log_file`
 use_syslog = False
 
+# Notifications can be sent when images are create, updated or deleted.
+# There are three methods of sending notifications, logging (via the
+# log_file directive), rabbit (via a rabbitmq queue) or noop (no
+# notifications sent, the default)
+notifier_strategy = noop
+
+# Configuration options if sending notifications via rabbitmq (these are
+# the defaults)
+rabbit_host = localhost
+rabbit_port = 5672
+rabbit_use_ssl = false
+rabbit_userid = guest
+rabbit_password = guest
+rabbit_virtual_host = /
+rabbit_notification_topic = glance_notifications
+
 # ============ Filesystem Store Options ========================
 
 # Directory that the Filesystem backend store

glance/api/v1/images.py

@@ -34,6 +34,7 @@ from webob.exc import (HTTPNotFound,
 from glance import api
 from glance import image_cache
 from glance.common import exception
+from glance.common import notifier
 from glance.common import wsgi
 import glance.store
 import glance.store.filesystem
@@ -79,6 +80,7 @@ class Controller(api.BaseController):
     def __init__(self, options):
         self.options = options
         glance.store.create_stores(options)
+        self.notifier = notifier.Notifier(options)
 
     def index(self, req):
         """
@@ -365,6 +367,7 @@ class Controller(api.BaseController):
                                            image_id,
                                            {'checksum': checksum,
                                             'size': size})
+            self.notifier.info('image.upload', image_meta)
 
             return location
 
@@ -372,12 +375,14 @@ class Controller(api.BaseController):
             msg = ("Attempt to upload duplicate image: %s") % e
             logger.error(msg)
             self._safe_kill(req, image_id)
+            self.notifier.error('image.upload', msg)
             raise HTTPConflict(msg, request=req)
 
         except exception.NotAuthorized, e:
             msg = ("Unauthorized upload attempt: %s") % e
             logger.error(msg)
             self._safe_kill(req, image_id)
+            self.notifier.error('image.upload', msg)
             raise HTTPForbidden(msg, request=req,
                                 content_type='text/plain')
 
@@ -389,6 +394,8 @@ class Controller(api.BaseController):
 
             msg = ("Error uploading image: (%s): '%s") % (
                 e.__class__.__name__, str(e))
+
+            self.notifier.error('image.upload', msg)
             raise HTTPBadRequest(msg, request=req)
 
     def _activate(self, req, image_id, location):
@@ -539,7 +546,10 @@ class Controller(api.BaseController):
                    % locals())
             for line in msg.split('\n'):
                 logger.error(line)
+            self.notifier.error('image.update', msg)
             raise HTTPBadRequest(msg, request=req, content_type="text/plain")
+        else:
+            self.notifier.info('image.update', image_meta)
 
         return {'image_meta': image_meta}
 
@@ -571,6 +581,7 @@ class Controller(api.BaseController):
             schedule_delete_from_backend(image['location'], self.options,
                                          req.context, id)
         registry.delete_image_metadata(self.options, req.context, id)
+        self.notifier.info('image.delete', id)
 
     def get_store_or_400(self, request, store_name):
         """

glance/common/exception.py

@@ -159,3 +159,7 @@ class StoreDeleteNotSupported(GlanceException):
 class StoreAddDisabled(GlanceException):
     message = _("Configuration for store failed. Adding images to this "
                "store is disabled.")
+
+
+class InvalidNotifierStrategy(GlanceException):
+    message = "'%(strategy)s' is not an available notifier strategy."

glance/common/notifier.py

@@ -0,0 +1,147 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import datetime
+import logging
+import socket
+import uuid
+
+import kombu.connection
+
+from glance.common import config
+from glance.common import exception
+
+
+class NoopStrategy(object):
+    """A notifier that does nothing when called."""
+
+    def __init__(self, options):
+        pass
+
+    def warn(self, msg):
+        pass
+
+    def info(self, msg):
+        pass
+
+    def error(self, msg):
+        pass
+
+
+class LoggingStrategy(object):
+    """A notifier that calls logging when called."""
+
+    def __init__(self, options):
+        self.logger = logging.getLogger('glance.notifier.logging_notifier')
+
+    def warn(self, msg):
+        self.logger.warn(msg)
+
+    def info(self, msg):
+        self.logger.info(msg)
+
+    def error(self, msg):
+        self.logger.error(msg)
+
+
+class RabbitStrategy(object):
+    """A notifier that puts a message on a queue when called."""
+
+    def __init__(self, options):
+        """Initialize the rabbit notification strategy."""
+        self._options = options
+        host = self._get_option('rabbit_host', 'str', 'localhost')
+        port = self._get_option('rabbit_port', 'int', 5672)
+        use_ssl = self._get_option('rabbit_use_ssl', 'bool', False)
+        userid = self._get_option('rabbit_userid', 'str', 'guest')
+        password = self._get_option('rabbit_password', 'str', 'guest')
+        virtual_host = self._get_option('rabbit_virtual_host', 'str', '/')
+
+        self.connection = kombu.connection.BrokerConnection(
+            hostname=host,
+            userid=userid,
+            password=password,
+            virtual_host=virtual_host,
+            ssl=use_ssl)
+
+        self.topic = self._get_option('rabbit_notification_topic',
+                                      'str',
+                                      'glance_notifications')
+
+    def _get_option(self, name, datatype, default):
+        """Retrieve a configuration option."""
+        return config.get_option(self._options,
+                                 name,
+                                 type=datatype,
+                                 default=default)
+
+    def _send_message(self, message, priority):
+        topic = "%s.%s" % (self.topic, priority)
+        queue = self.connection.SimpleQueue(topic)
+        queue.put(message, serializer="json")
+        queue.close()
+
+    def warn(self, msg):
+        self._send_message(msg, "WARN")
+
+    def info(self, msg):
+        self._send_message(msg, "INFO")
+
+    def error(self, msg):
+        self._send_message(msg, "ERROR")
+
+
+class Notifier(object):
+    """Uses a notification strategy to send out messages about events."""
+
+    STRATEGIES = {
+        "logging": LoggingStrategy,
+        "rabbit": RabbitStrategy,
+        "noop": NoopStrategy,
+        "default": NoopStrategy,
+    }
+
+    def __init__(self, options, strategy=None):
+        strategy = config.get_option(options, "notifier_strategy",
+                                     type="str", default="default")
+        try:
+            self.strategy = self.STRATEGIES[strategy](options)
+        except KeyError:
+            raise exception.InvalidNotifierStrategy(strategy=strategy)
+
+    @staticmethod
+    def generate_message(event_type, priority, payload):
+        return {
+            "message_id": str(uuid.uuid4()),
+            "publisher_id": socket.gethostname(),
+            "event_type": event_type,
+            "priority": priority,
+            "payload": payload,
+            "timestamp": str(datetime.datetime.utcnow()),
+        }
+
+    def warn(self, event_type, payload):
+        msg = self.generate_message(event_type, "WARN", payload)
+        self.strategy.warn(msg)
+
+    def info(self, event_type, payload):
+        msg = self.generate_message(event_type, "INFO", payload)
+        self.strategy.info(msg)
+
+    def error(self, event_type, payload):
+        msg = self.generate_message(event_type, "ERROR", payload)
+        self.strategy.error(msg)

glance/tests/unit/test_notifier.py

@@ -0,0 +1,123 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack, LLC
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging
+import unittest
+
+from glance.common import exception
+from glance.common import notifier
+
+
+class TestInvalidNotifier(unittest.TestCase):
+    """Test that notifications are generated appropriately"""
+
+    def test_cannot_create(self):
+        options = {"notifier_strategy": "invalid_notifier"}
+        self.assertRaises(exception.InvalidNotifierStrategy,
+                          notifier.Notifier,
+                          options)
+
+
+class TestLoggingNotifier(unittest.TestCase):
+    """Test the logging notifier is selected and works properly."""
+
+    def setUp(self):
+        options = {"notifier_strategy": "logging"}
+        self.called = False
+        self.logger = logging.getLogger("glance.notifier.logging_notifier")
+        self.notifier = notifier.Notifier(options)
+
+    def _called(self, msg):
+        self.called = msg
+
+    def test_warn(self):
+        self.logger.warn = self._called
+        self.notifier.warn("test_event", "test_message")
+        if self.called is False:
+            self.fail("Did not call logging library correctly.")
+
+    def test_info(self):
+        self.logger.info = self._called
+        self.notifier.info("test_event", "test_message")
+        if self.called is False:
+            self.fail("Did not call logging library correctly.")
+
+    def test_erorr(self):
+        self.logger.error = self._called
+        self.notifier.error("test_event", "test_message")
+        if self.called is False:
+            self.fail("Did not call logging library correctly.")
+
+
+class TestNoopNotifier(unittest.TestCase):
+    """Test that the noop notifier works...and does nothing?"""
+
+    def setUp(self):
+        options = {"notifier_strategy": "noop"}
+        self.notifier = notifier.Notifier(options)
+
+    def test_warn(self):
+        self.notifier.warn("test_event", "test_message")
+
+    def test_info(self):
+        self.notifier.info("test_event", "test_message")
+
+    def test_error(self):
+        self.notifier.error("test_event", "test_message")
+
+
+class TestRabbitNotifier(unittest.TestCase):
+    """Test AMQP/Rabbit notifier works."""
+
+    def setUp(self):
+        notifier.RabbitStrategy._send_message = self._send_message
+        self.called = False
+        options = {"notifier_strategy": "rabbit"}
+        self.notifier = notifier.Notifier(options)
+
+    def _send_message(self, message, priority):
+        self.called = {
+            "message": message,
+            "priority": priority,
+        }
+
+    def test_warn(self):
+        self.notifier.warn("test_event", "test_message")
+
+        if self.called is False:
+            self.fail("Did not call _send_message properly.")
+
+        self.assertEquals("test_message", self.called["message"]["payload"])
+        self.assertEquals("WARN", self.called["message"]["priority"])
+
+    def test_info(self):
+        self.notifier.info("test_event", "test_message")
+
+        if self.called is False:
+            self.fail("Did not call _send_message properly.")
+
+        self.assertEquals("test_message", self.called["message"]["payload"])
+        self.assertEquals("INFO", self.called["message"]["priority"])
+
+    def test_error(self):
+        self.notifier.error("test_event", "test_message")
+
+        if self.called is False:
+            self.fail("Did not call _send_message properly.")
+
+        self.assertEquals("test_message", self.called["message"]["payload"])
+        self.assertEquals("ERROR", self.called["message"]["priority"])

tools/pip-requires

@@ -20,3 +20,4 @@ sqlalchemy-migrate>=0.6,<0.7
 bzr
 httplib2
 xattr>=0.6.0
+kombu