From fe164f59d0f22f96b3c976cd07d59e6654dce740 Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Tue, 27 May 2014 22:57:46 +0000 Subject: [PATCH] Inital commit. --- .gitignore | 30 ++ LICENSE | 202 +++++++++ MANIFEST.in | 2 + README.md | 84 ++++ bin/test-distiller.py | 128 ++++++ doc/event_definitions_config.md | 4 + etc/event_definitions.yaml | 63 +++ requirements.txt | 6 + setup.py | 37 ++ stackdistiller/__init__.py | 0 stackdistiller/condenser.py | 102 +++++ stackdistiller/distiller.py | 343 +++++++++++++++ stackdistiller/trait_plugins.py | 162 +++++++ test-requirements.txt | 3 + tests/__init__.py | 0 tests/test_distiller.py | 725 ++++++++++++++++++++++++++++++++ tests/test_trait_plugins.py | 120 ++++++ tox.ini | 15 + 18 files changed, 2026 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 MANIFEST.in create mode 100644 README.md create mode 100755 bin/test-distiller.py create mode 100644 doc/event_definitions_config.md create mode 100644 etc/event_definitions.yaml create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 stackdistiller/__init__.py create mode 100644 stackdistiller/condenser.py create mode 100644 stackdistiller/distiller.py create mode 100644 stackdistiller/trait_plugins.py create mode 100644 test-requirements.txt create mode 100644 tests/__init__.py create mode 100644 tests/test_distiller.py create mode 100644 tests/test_trait_plugins.py create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a735f8b --- /dev/null +++ b/.gitignore @@ -0,0 +1,30 @@ +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +var +sdist +develop-eggs +.installed.cfg +lib +lib64 +__pycache__ + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml + +# Translations +*.mo diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e06d208 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..8afbefe --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include README.md +include requirements.txt diff --git a/README.md b/README.md new file mode 100644 index 0000000..a45eeeb --- /dev/null +++ b/README.md @@ -0,0 +1,84 @@ +stackdistiller +============== + +A data extraction and transformation library for OpenStack notifications. + +Stackdistiller is designed to extract data from openstack notifications +and convert it into a form relevant to the application consuming the +notification. It consists of two components, the Distiller, which extracts +data from notifications according to a YAML config file, and the Condenser, +which receives the data extracted by the Distiller, and formats it into +an application-specific object, referred to as an Event. This could be a +simple python dictionary, an XML document tree, or a set of ORM model +objects. + +## Distiller + +The Distiller reads a YAML config file to determine what data to extract +from each notification, according to it's event type. event types can be +wildcarded using shell glob syntax. The distiller will extract two types of +data from each notification: + +* Metadata from the notifications envelope, including the event type, + message id (uuid of the notification) and the timestamp showing when + notification was generated by the source system. +* A series of data items extracted from the notification's body. These + are called Traits. Traits are basically just typed name-value pairs. + +The distiller can also do some basic data massaging on traits extracted +from the notification, such as splitting a value from a string. This is +handled by trait plugins. These are just classes that implement the +TraitPluginBase interface. They are referred to by name in the config, and +are looked up in a trait\_plugin\_map passed to the distiller on init. +The plugin map is just a dictionary, or dictionary-like object (such as a +plugin manager) that maps names to plugin classes. If no map is passed to +the distiller, it will use a default that just contains the builtin plugins +bundled with stackdistiller. + +If a notification does not match any event definition in the distiller's +config file, the distiller's to\_event method will return None, indicating +it cannot extract that notification. This may be what you want (i.e. your +application may only be interested in certain notifications.), but if you +want to record basic informaton from *any* event type, you can pass +"catchall=True" to the distiller, and it will generate a minimal event from +any notification. + +## Condenser + +The Condenser receives the data extracted from the notification by the +Distiller and formats it into an appropriate type of Event object. +An instance of a Condenser class is passed, along with the raw, +deserialized notification, to the distiller object's to\_event method. +To create your own type of Event from the data extracted by the distiller, +you just need to create a Condenser class to receive the data. +Condenser classes don't have to subclass any particular class, as long as +they implement the methods defined in stackdistiller.condenser.CondenserBase. +If you do not pass a condenser to the distiller when you call to\_event, +it will create an instance of the default DictionaryCondenser for you. +This just formats the event as a plain python dictionary. + +## Example: + + import json + from stackdistiller import distiller + from stackdistiller import condenser + + config_file_name = "events_i_want.yaml" + notification_string = open('a_notification_here.json', 'r').read() + + notification = json.loads(notification_string) + config = distiller.load_config(config_file_name) + + d = distiller.Distiller(config, catchall=False) + + #this is the default condenser. + cond = condenser.DictionaryCondenser() + + if dist.to_event(notification, cond): + # What get_event() returns is up to the condenser class. In this + # case, it's a dictionary. + event = cond.get_event() + print "Yay! An Event: %s" % str(event) + else: + print "Not something we are interested in. Ignoring." + diff --git a/bin/test-distiller.py b/bin/test-distiller.py new file mode 100755 index 0000000..0e38d3f --- /dev/null +++ b/bin/test-distiller.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# +# Copyright © 2014 Rackspace Hosting. +# +# Author: Monsyne Dragon +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Command line tool help you debug your distiller event definitions. + +Feed it a list of test notifications in json format, and it will show +you what events will be generated. +""" + +import argparse +import json +import sys + +from stackdistiller import distiller +from stackdistiller import condenser + + +class TestCondenser(condenser.CondenserBase): + def __init__(self): + self.clear() + + def add_trait(self, name, trait_type, value): + self.traits.append(distiller.Trait(name, trait_type, value)) + + def add_envelope_info(self, event_type, message_id, when): + self.event_type = event_type + self.message_id = message_id + self.when = when + + def get_event(self): + return self + + def clear(self): + self.event_type = None + self.message_id = None + self.when = None + self.traits = [] + + def __str__(self): + text = ["Event: %s (id: %s) at %s" % (self.event_type, + self.message_id, + self.when)] + for trait in sorted(self.traits): + text.append(" Trait: name: %s, type: %s, value: %s" % trait) + text.append('') + return "\n".join(text) + + +def test_data(args): + if not args.test_data: + n = json.load(sys.stdin) + if args.list: + for notif in n: + yield notif + else: + yield n + else: + for f in args.test_data: + with open(f, 'r') as data: + n = json.load(data) + if args.list: + for notif in n: + yield notif + else: + yield n + + +parser = argparse.ArgumentParser(description="Test Distiller configuration") +parser.add_argument('-c', '--config', + default='event_definitions.yaml', + help='Name of event definitions file ' + 'to test (Default: %(default)s)') +parser.add_argument('-l', '--list', action='store_true', + help='Test data files contain JSON list of notifications.' + ' (By default data files should contain a single ' + 'notification.)') +parser.add_argument('-d', '--add_default_definition', action='store_true', + help='Add default event definition. Normally, ' + 'notifications are dropped if there is no event ' + 'definition for their event_type. Setting this adds a ' + '"catchall" that converts unknown notifications to Events' + ' with a few basic traits.') +parser.add_argument('-o', '--output', type=argparse.FileType('w'), + default=sys.stdout, help="Output file. Default stdout") +parser.add_argument('test_data', nargs='*', metavar='JSON_FILE', + help="Test notifications in JSON format. Defaults to stdin") +args = parser.parse_args() + + +config = distiller.load_config(args.config) + +out = args.output +out.write("Definitions file: %s\n" % args.config) +notifications = test_data(args) + +dist = distiller.Distiller(config, catchall=args.add_default_definition) +nct = 0 +drops = 0 +cond = TestCondenser() +for notification in notifications: + cond.clear() + nct +=1 + if dist.to_event(notification, cond) is None: + out.write("Dropped notification: %s\n" % + notification['message_id']) + drops += 1 + else: + event = cond.get_event() + out.write(str(event)) + out.write("--\n") + +out.write("Notifications tested: %s (%s dropped)\n" % (nct, drops)) diff --git a/doc/event_definitions_config.md b/doc/event_definitions_config.md new file mode 100644 index 0000000..8b24d4c --- /dev/null +++ b/doc/event_definitions_config.md @@ -0,0 +1,4 @@ +Event Definitions YAML Config Format +==================================== + +Documentation to go here soon. diff --git a/etc/event_definitions.yaml b/etc/event_definitions.yaml new file mode 100644 index 0000000..b4f00d3 --- /dev/null +++ b/etc/event_definitions.yaml @@ -0,0 +1,63 @@ +--- +- event_type: compute.instance.* + traits: &instance_traits + tenant_id: + fields: payload.tenant_id + user_id: + fields: payload.user_id + instance_id: + fields: payload.instance_id + host: + fields: publisher_id + plugin: + name: split + parameters: + segment: 1 + max_split: 1 + service: + fields: publisher_id + plugin: split + memory_mb: + type: int + fields: payload.memory_mb + disk_gb: + type: int + fields: payload.disk_gb + root_gb: + type: int + fields: payload.root_gb + ephemeral_gb: + type: int + fields: payload.ephemeral_gb + vcpus: + type: int + fields: payload.vcpus + instance_type_id: + type: int + fields: payload.instance_type_id + instance_type: + fields: payload.instance_type + state: + fields: payload.state + os_architecture: + fields: payload.image_meta.'org.openstack__1__architecture' + os_version: + fields: payload.image_meta.'org.openstack__1__os_version' + os_distro: + fields: payload.image_meta.'org.openstack__1__os_distro' + launched_at: + type: datetime + fields: payload.launched_at + deleted_at: + type: datetime + fields: payload.deleted_at +- event_type: compute.instance.exists + traits: + <<: *instance_traits + audit_period_beginning: + type: datetime + fields: payload.audit_period_beginning + audit_period_ending: + type: datetime + fields: payload.audit_period_ending + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ddae1fb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +enum34>=1.0 +iso8601>=0.1.10 +jsonpath-rw>=1.2.0,<2.0 +PyYAML>=3.1.0 +six>=1.5.2 +stevedore>=0.14 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..4be730c --- /dev/null +++ b/setup.py @@ -0,0 +1,37 @@ +import os +from pip.req import parse_requirements +from setuptools import setup, find_packages + + +def read(fname): + return open(os.path.join(os.path.dirname(__file__), fname)).read() + + +req_file = os.path.join(os.path.dirname(__file__), "requirements.txt") +install_reqs = [str(r.req) for r in parse_requirements(req_file)] + + +setup( + name='stackdistiller', + version='0.10', + author='Monsyne Dragon', + author_email='mdragon@rackspace.com', + description=("A data extraction and transformation library for " + "OpenStack notifications"), + license='Apache License (2.0)', + keywords='OpenStack notifications events extraction transformation', + packages=find_packages(exclude=['tests']), + classifiers=[ + 'Development Status :: 3 - Alpha', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + ], + url='https://github.com/StackTach/stackdistiller', + scripts=['bin/test-distiller.py'], + long_description=read('README.md'), + install_requires=install_reqs, + + zip_safe=False +) diff --git a/stackdistiller/__init__.py b/stackdistiller/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/stackdistiller/condenser.py b/stackdistiller/condenser.py new file mode 100644 index 0000000..eb5f7ad --- /dev/null +++ b/stackdistiller/condenser.py @@ -0,0 +1,102 @@ +# -*- encoding: utf-8 -*- +# +# Copyright © 2014 Rackspace Hosting. +# +# Author: Monsyne Dragon +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class CondenserBase(object): + """Base class for Condenser objects that collect data extracted from a + Notification by the Distiller, and format it into a usefull datastructure. + + A simple Condenser may just colect all the traits received into a dictionary. + More complex ones may build collections of application or ORM model objects, + or XML document trees. + + Condensers also have hooks for verification logic, to check that all needed + traits are present.""" + + def __init__(self, **kw): + """Setup the condenser. A new instance of the condenser is passed to the + distiller for each notification extracted. + + :param kw: keyword parameters for condenser. + + """ + super(CondenserBase, self).__init__() + + @abc.abstractmethod + def add_trait(self, name, trait_type, value): + """Add a trait to the Event datastructure being built by this + condenser. The distiller will call this for each extracted trait. + + :param name: (string) name of the trait + :param trait_type: (distiller.Datatype) data type of the trait. + :param value: Value of the trait (of datatype indicated by trait_type) + + """ + + @abc.abstractmethod + def add_envelope_info(self, event_type, message_id, when): + """Add the metadata for this event, extracted from the notification's + envelope. The distiller will call this once. + + :param event_type: (string) Type of event, as a dotted string such as + "compute.instance.update". + :param message_id: (string) UUID of notification. + :param when: (datetime) Timestamp of notification from source system. + + """ + + @abc.abstractmethod + def get_event(self): + """Return the Event datastructure constructed by this condenser.""" + + @abc.abstractmethod + def clear(self): + """Clear condenser state.""" + + def validate(self): + """Check Event against whatever validation logic this condenser may have + + :returns: (bool) True if valid. + + """ + return True + + +class DictionaryCondenser(CondenserBase): + """Return event data as a simple python dictionary""" + def __init__(self, **kw): + self.clear() + super(DictionaryCondenser, self).__init__(**kw) + + def get_event(self): + return self.event + + def clear(self): + self.event = dict() + + def add_envelope_info(self, event_type, message_id, when): + self.event['event_type'] = event_type + self.event['message_id'] = message_id + self.event['when'] = when + + def add_trait(self, name, trait_type, value): + self.event[name] = value diff --git a/stackdistiller/distiller.py b/stackdistiller/distiller.py new file mode 100644 index 0000000..d834ef0 --- /dev/null +++ b/stackdistiller/distiller.py @@ -0,0 +1,343 @@ +# -*- encoding: utf-8 -*- +# +# Copyright © 2013 Rackspace Hosting. +# +# Author: Monsyne Dragon +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +import datetime +import fnmatch +import logging +import os + +from enum import Enum +import iso8601 +import jsonpath_rw +import six +import yaml + +from stackdistiller.condenser import DictionaryCondenser +from stackdistiller.trait_plugins import DEFAULT_PLUGINMAP + + +logger = logging.getLogger(__name__) + + +def utcnow(): + # defined here so the call can be mocked out in unittests. + dt = datetime.datetime.utcnow() + return dt.replace(tzinfo=iso8601.iso8601.UTC) + + +def convert_datetime(value): + value = iso8601.parse_date(value) + tz = iso8601.iso8601.UTC + return value.astimezone(tz) + + +def load_config(filename): + """Load the event definitions from yaml config file.""" + logger.debug("Event Definitions configuration file: %s", filename) + + with open(filename, 'r') as cf: + config = cf.read() + + try: + events_config = yaml.safe_load(config) + except yaml.YAMLError as err: + if hasattr(err, 'problem_mark'): + mark = err.problem_mark + errmsg = ("Invalid YAML syntax in Event Definitions file " + "%(file)s at line: %(line)s, column: %(column)s." + % dict(file=filename, + line=mark.line + 1, + column=mark.column + 1)) + else: + errmsg = ("YAML error reading Event Definitions file " + "%(file)s" + % dict(file=filename)) + logger.error(errmsg) + raise + + logger.info("Event Definitions: %s", events_config) + return events_config + + +class EventDefinitionException(Exception): + def __init__(self, message, definition_cfg): + super(EventDefinitionException, self).__init__(message) + self.definition_cfg = definition_cfg + + def __str__(self): + return '%s %s: %s' % (self.__class__.__name__, + self.definition_cfg, self.message) + + +class Datatype(Enum): + text = (1, str) + int = (2, int) + float = (3, float) + datetime = (4, convert_datetime) + + def convert(self, value): + f = self.value[1] + return f(value) + + +Trait = collections.namedtuple('Trait', ('name', 'trait_type', 'value')) + + +class TraitDefinition(object): + + def __init__(self, name, trait_cfg, plugin_map): + self.cfg = trait_cfg + self.name = name + + type_name = trait_cfg.get('type', 'text') + + if 'plugin' in trait_cfg: + plugin_cfg = trait_cfg['plugin'] + if isinstance(plugin_cfg, six.string_types): + plugin_name = plugin_cfg + plugin_params = {} + else: + try: + plugin_name = plugin_cfg['name'] + except KeyError: + raise EventDefinitionException( + 'Plugin specified, but no plugin name supplied for ' + 'trait %s' % name, self.cfg) + plugin_params = plugin_cfg.get('parameters') + if plugin_params is None: + plugin_params = {} + try: + plugin_class = plugin_map[plugin_name] + except KeyError: + raise EventDefinitionException( + 'No plugin named %(plugin)s available for ' + 'trait %(trait)s' % dict(plugin=plugin_name, + trait=name), self.cfg) + self.plugin = plugin_class(**plugin_params) + else: + self.plugin = None + + if 'fields' not in trait_cfg: + raise EventDefinitionException( + "Required field in trait definition not specified: " + "'%s'" % 'fields', + self.cfg) + + fields = trait_cfg['fields'] + if not isinstance(fields, six.string_types): + # NOTE(mdragon): if not a string, we assume a list. + if len(fields) == 1: + fields = fields[0] + else: + fields = '|'.join('(%s)' % path for path in fields) + try: + self.fields = jsonpath_rw.parse(fields) + except Exception as e: + raise EventDefinitionException( + "Parse error in JSONPath specification " + "'%(jsonpath)s' for %(trait)s: %(err)s" + % dict(jsonpath=fields, trait=name, err=e), self.cfg) + try: + self.trait_type = Datatype[type_name] + except KeyError: + raise EventDefinitionException( + "Invalid trait type '%(type)s' for trait %(trait)s" + % dict(type=type_name, trait=name), self.cfg) + + def _get_path(self, match): + if match.context is not None: + for path_element in self._get_path(match.context): + yield path_element + yield str(match.path) + + def to_trait(self, notification_body): + values = [match for match in self.fields.find(notification_body) + if match.value is not None] + + if self.plugin is not None: + value_map = [('.'.join(self._get_path(match)), match.value) for + match in values] + value = self.plugin.trait_value(value_map) + else: + value = values[0].value if values else None + + if value is None: + return None + + # NOTE(mdragon): some openstack projects (mostly Nova) emit '' + # for null fields for things like dates. + if self.trait_type != Datatype.text and value == '': + return None + + value = self.trait_type.convert(value) + return Trait(self.name, self.trait_type, value) + + +class EventDefinition(object): + + DEFAULT_TRAITS = dict( + service=dict(type='text', fields='publisher_id'), + request_id=dict(type='text', fields='_context_request_id'), + tenant_id=dict(type='text', fields=['payload.tenant_id', + '_context_tenant']), + ) + + def __init__(self, definition_cfg, trait_plugin_map): + self._included_types = [] + self._excluded_types = [] + self.traits = dict() + self.cfg = definition_cfg + + try: + event_type = definition_cfg['event_type'] + traits = definition_cfg['traits'] + except KeyError as err: + raise EventDefinitionException( + "Required field %s not specified" % err.args[0], self.cfg) + + if isinstance(event_type, six.string_types): + event_type = [event_type] + + for t in event_type: + if t.startswith('!'): + self._excluded_types.append(t[1:]) + else: + self._included_types.append(t) + + if self._excluded_types and not self._included_types: + self._included_types.append('*') + + for trait_name in self.DEFAULT_TRAITS: + self.traits[trait_name] = TraitDefinition( + trait_name, + self.DEFAULT_TRAITS[trait_name], + trait_plugin_map) + for trait_name in traits: + self.traits[trait_name] = TraitDefinition( + trait_name, + traits[trait_name], + trait_plugin_map) + + def included_type(self, event_type): + for t in self._included_types: + if fnmatch.fnmatch(event_type, t): + return True + return False + + def excluded_type(self, event_type): + for t in self._excluded_types: + if fnmatch.fnmatch(event_type, t): + return True + return False + + def match_type(self, event_type): + return (self.included_type(event_type) + and not self.excluded_type(event_type)) + + @property + def is_catchall(self): + return '*' in self._included_types and not self._excluded_types + + @staticmethod + def _extract_when(body): + """Extract the generated datetime from the notification. + """ + # NOTE: I am keeping the logic the same as it was in openstack + # code, However, *ALL* notifications should have a 'timestamp' + # field, it's part of the notification envelope spec. If this was + # put here because some openstack project is generating notifications + # without a timestamp, then that needs to be filed as a bug with the + # offending project (mdragon) + when = body.get('timestamp', body.get('_context_timestamp')) + if when: + return Datatype.datetime.convert(when) + return utcnow() + + def to_event(self, notification_body, condenser): + event_type = notification_body['event_type'] + message_id = notification_body['message_id'] + when = self._extract_when(notification_body) + + condenser.add_envelope_info(event_type, message_id, when) + for t in self.traits: + trait_info = self.traits[t].to_trait(notification_body) + # Only accept non-None value traits ... + if trait_info is not None: + condenser.add_trait(*trait_info) + return condenser + + +class Distiller(object): + """Distiller + + The Distiller extracts relevent information from an OpenStack Notification, + with optional minor data massaging, and hands the information to a + Condenser object, which formats it into an Event which only contains + information you need, in a format relevent to your application. + + The extraction is handled according to event definitions in a config file. + + :param events_config: event-definitions configuration deserializerd + from the YAML config-file. + :param trait_plugin_map: Dictionary, or dictionary-like object mapping + names to plugin classes. Defaults to default + map of builtin plugins. + :param catchall: Boolean. Add basic event definition to cover + any notifications not otherwise described by + the loaded config. The basic event definion + only has envelope metadata, and a few really + basic traits. Defaults to False. + + """ + + def __init__(self, events_config, trait_plugin_map=None, catchall=False): + if trait_plugin_map is None: + trait_plugin_map = DEFAULT_PLUGINMAP + self.definitions = [ + EventDefinition(event_def, trait_plugin_map) + for event_def in reversed(events_config)] + self.catchall = catchall + if catchall and not any(d.is_catchall for d in self.definitions): + event_def = dict(event_type='*', traits={}) + self.definitions.append(EventDefinition(event_def, + trait_plugin_map)) + + def to_event(self, notification_body, condenser=None): + if condenser is None: + condenser = DictionaryCondenser() + event_type = notification_body['event_type'] + message_id = notification_body['message_id'] + edef = None + for d in self.definitions: + if d.match_type(event_type): + edef = d + break + + if edef is None: + msg = ('Dropping Notification %(type)s (uuid:%(msgid)s)' + % dict(type=event_type, msgid=message_id)) + if self.catchall: + # If catchall is True, this should + # never happen. (mdragon) + logger.error(msg) + else: + logger.debug(msg) + return None + + return edef.to_event(notification_body, condenser) diff --git a/stackdistiller/trait_plugins.py b/stackdistiller/trait_plugins.py new file mode 100644 index 0000000..0b8b96e --- /dev/null +++ b/stackdistiller/trait_plugins.py @@ -0,0 +1,162 @@ +# -*- encoding: utf-8 -*- +# +# Copyright © 2013 Rackspace Hosting. +# +# Author: Monsyne Dragon +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class TraitPluginBase(object): + """Base class for plugins that convert notification fields to + Trait values. + """ + + def __init__(self, **kw): + """Setup the trait plugin. + + For each Trait definition a plugin is used on in a conversion + definition, a new instance of the plugin will be created, and + initialized with the parameters (if any) specified in the + config file. + + :param kw: the parameters specified in the event definitions file. + + """ + super(TraitPluginBase, self).__init__() + + @abc.abstractmethod + def trait_value(self, match_list): + """Convert a set of fields to a Trait value. + + This method is called each time a trait is attempted to be extracted + from a notification. It will be called *even if* no matching fields + are found in the notification (in that case, the match_list will be + empty). If this method returns None, the trait *will not* be added to + the event. Any other value returned by this method will be used as + the value for the trait. Values returned will be coerced to the + appropriate type for the trait. + + :param match_list: A list (may be empty if no matches) of *tuples*. + Each tuple is (field_path, value) where field_path + is the jsonpath for that specific field, + + Example: + trait's fields definition: ['payload.foobar', + 'payload.baz', + 'payload.thing.*'] + notification body: + { + 'message_id': '12345', + 'publisher': 'someservice.host', + 'payload': { + 'foobar': 'test', + 'thing': { + 'bar': 12, + 'boing': 13, + } + } + } + match_list will be: [('payload.foobar','test'), + ('payload.thing.bar',12), + ('payload.thing.boing',13)] + + Here is a plugin that emulates the default (no plugin) behavior: + + class DefaultPlugin(TraitPluginBase): + "Plugin that returns the first field value" + + def __init__(self, **kw): + super(DefaultPlugin, self).__init__() + + def trait_value(self, match_list): + if not match_list: + return None + return match_list[0][1] + """ + + +class SplitterTraitPlugin(TraitPluginBase): + """Plugin that splits a piece off of a string value.""" + + def __init__(self, separator=".", segment=0, max_split=None, **kw): + """Setup how do split the field. + + :param separator: String to split on. default "." + :param segment: Which segment to return. (int) default 0 + :param max_split: Limit number of splits. Default: None (no limit) + """ + self.separator = separator + self.segment = segment + self.max_split = max_split + super(SplitterTraitPlugin, self).__init__(**kw) + + def trait_value(self, match_list): + if not match_list: + return None + value = str(match_list[0][1]) + if self.max_split is not None: + values = value.split(self.separator, self.max_split) + else: + values = value.split(self.separator) + try: + return values[self.segment] + except IndexError: + return None + + +class BitfieldTraitPlugin(TraitPluginBase): + """Plugin to set flags on a bitfield.""" + def __init__(self, initial_bitfield=0, flags=None, **kw): + """Setup bitfield trait. + + :param initial_bitfield: (int) initial value for the bitfield + Flags that are set will be OR'ed with this. + :param flags: List of dictionaries defining bitflags to set depending + on data in the notification. Each one has the following + keys: + path: jsonpath of field to match. + bit: (int) number of bit to set (lsb is bit 0) + value: set bit if corresponding field's value + matches this. If value is not provided, + bit will be set if the field exists (and + is non-null), regardless of it's value. + + """ + self.initial_bitfield = initial_bitfield + if flags is None: + flags = [] + self.flags = flags + super(BitfieldTraitPlugin, self).__init__(**kw) + + def trait_value(self, match_list): + matches = dict(match_list) + bitfield = self.initial_bitfield + for flagdef in self.flags: + path = flagdef['path'] + bit = 2 ** int(flagdef['bit']) + if path in matches: + if 'value' in flagdef: + if matches[path] == flagdef['value']: + bitfield |= bit + else: + bitfield |= bit + return bitfield + + +DEFAULT_PLUGINMAP = dict(split=SplitterTraitPlugin, + bitfield=BitfieldTraitPlugin) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..02e37d7 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,3 @@ +mock>=1.0 +nose +unittest2 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_distiller.py b/tests/test_distiller.py new file mode 100644 index 0000000..66d6729 --- /dev/null +++ b/tests/test_distiller.py @@ -0,0 +1,725 @@ +# -*- encoding: utf-8 -*- +# +# Copyright © 2013 Rackspace Hosting. +# +# Author: Monsyne Dragon +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime + +#for Python2.6 compatability. +import unittest2 as unittest + +import iso8601 +import jsonpath_rw +import mock +import six + +from stackdistiller import distiller + + +class TestCondenser(object): + def __init__(self): + self.event_type = None + self.message_id = None + self.when = None + self.traits = [] + + def add_trait(self, name, trait_type, value): + self.traits.append(distiller.Trait(name, trait_type, value)) + + def add_envelope_info(self, event_type, message_id, when): + self.event_type = event_type + self.message_id = message_id + self.when = when + + def get_event(self): + return self + + +class DistillerTestBase(unittest.TestCase): + def _create_test_notification(self, event_type, message_id, **kw): + return dict(event_type=event_type, + message_id=message_id, + priority="INFO", + publisher_id="compute.host-1-2-3", + timestamp="2013-08-08 21:06:37.803826", + payload=kw, + ) + + def assertIsValidEvent(self, event, notification): + self.assertIsNot( + None, event, + "Notification dropped unexpectedly:" + " %s" % str(notification)) + + def assertIsNotValidEvent(self, event, notification): + self.assertIs( + None, event, + "Notification NOT dropped when expected to be dropped:" + " %s" % str(notification)) + + def assertHasTrait(self, event, name, value=None, trait_type=None): + traits = [trait for trait in event.traits if trait.name == name] + self.assertTrue( + len(traits) > 0, + "Trait %s not found in event %s" % (name, event)) + trait = traits[0] + if value is not None: + self.assertEqual(value, trait.value) + if trait_type is not None: + self.assertEqual(trait_type, trait.trait_type) + if trait_type == distiller.Datatype.int: + self.assertIsInstance(trait.value, int) + elif trait_type == distiller.Datatype.float: + self.assertIsInstance(trait.value, float) + elif trait_type == distiller.Datatype.datetime: + self.assertIsInstance(trait.value, datetime.datetime) + elif trait_type == distiller.Datatype.text: + self.assertIsInstance(trait.value, six.string_types) + + def assertDoesNotHaveTrait(self, event, name): + traits = [trait for trait in event.traits if trait.name == name] + self.assertEqual( + len(traits), 0, + "Extra Trait %s found in event %s" % (name, event)) + + def assertHasDefaultTraits(self, event): + text = distiller.Datatype.text + self.assertHasTrait(event, 'service', trait_type=text) + + def _cmp_tree(self, this, other): + if hasattr(this, 'right') and hasattr(other, 'right'): + return (self._cmp_tree(this.right, other.right) and + self._cmp_tree(this.left, other.left)) + if not hasattr(this, 'right') and not hasattr(other, 'right'): + return this == other + return False + + def assertPathsEqual(self, path1, path2): + self.assertTrue(self._cmp_tree(path1, path2), + 'JSONPaths not equivalent %s %s' % (path1, path2)) + + +class TestTraitDefinition(DistillerTestBase): + + def setUp(self): + super(TestTraitDefinition, self).setUp() + self.n1 = self._create_test_notification( + "test.thing", + "uuid-for-notif-0001", + instance_uuid="uuid-for-instance-0001", + instance_id="id-for-instance-0001", + instance_uuid2=None, + instance_id2=None, + host='host-1-2-3', + bogus_date='', + image_meta=dict( + disk_gb='20', + thing='whatzit'), + foobar=50) + + self.test_plugin_class = mock.MagicMock(name='mock_test_plugin') + self.test_plugin = self.test_plugin_class() + self.test_plugin.trait_value.return_value = 'foobar' + self.test_plugin_class.reset_mock() + + self.nothing_plugin_class = mock.MagicMock(name='mock_nothing_plugin') + self.nothing_plugin = self.nothing_plugin_class() + self.nothing_plugin.trait_value.return_value = None + self.nothing_plugin_class.reset_mock() + + self.fake_plugin_map = dict(test=self.test_plugin_class, + nothing=self.nothing_plugin_class) + + def test_to_trait_with_plugin(self): + cfg = dict(type='text', + fields=['payload.instance_id', 'payload.instance_uuid'], + plugin=dict(name='test')) + + tdef = distiller.TraitDefinition('test_trait', cfg, + self.fake_plugin_map) + tname, trait_type, value = tdef.to_trait(self.n1) + self.assertEqual('test_trait', tname) + self.assertEqual(distiller.Datatype.text, trait_type) + self.assertEqual('foobar', value) + self.test_plugin_class.assert_called_once_with() + self.test_plugin.trait_value.assert_called_once_with([ + ('payload.instance_id', 'id-for-instance-0001'), + ('payload.instance_uuid', 'uuid-for-instance-0001')]) + + def test_to_trait_null_match_with_plugin(self): + cfg = dict(type='text', + fields=['payload.nothere', 'payload.bogus'], + plugin=dict(name='test')) + + tdef = distiller.TraitDefinition('test_trait', cfg, + self.fake_plugin_map) + tname, trait_type, value = tdef.to_trait(self.n1) + self.assertEqual('test_trait', tname) + self.assertEqual(distiller.Datatype.text, trait_type) + self.assertEqual('foobar', value) + self.test_plugin_class.assert_called_once_with() + self.test_plugin.trait_value.assert_called_once_with([]) + + def test_to_trait_with_plugin_null(self): + cfg = dict(type='text', + fields=['payload.instance_id', 'payload.instance_uuid'], + plugin=dict(name='nothing')) + + tdef = distiller.TraitDefinition('test_trait', cfg, + self.fake_plugin_map) + t = tdef.to_trait(self.n1) + self.assertIs(None, t) + self.nothing_plugin_class.assert_called_once_with() + self.nothing_plugin.trait_value.assert_called_once_with([ + ('payload.instance_id', 'id-for-instance-0001'), + ('payload.instance_uuid', 'uuid-for-instance-0001')]) + + def test_to_trait_with_plugin_with_parameters(self): + cfg = dict(type='text', + fields=['payload.instance_id', 'payload.instance_uuid'], + plugin=dict(name='test', parameters=dict(a=1, b='foo'))) + + tdef = distiller.TraitDefinition('test_trait', cfg, + self.fake_plugin_map) + tname, trait_type, value = tdef.to_trait(self.n1) + self.assertEqual('test_trait', tname) + self.assertEqual(distiller.Datatype.text, trait_type) + self.assertEqual('foobar', value) + self.test_plugin_class.assert_called_once_with(a=1, b='foo') + self.test_plugin.trait_value.assert_called_once_with([ + ('payload.instance_id', 'id-for-instance-0001'), + ('payload.instance_uuid', 'uuid-for-instance-0001')]) + + def test_to_trait(self): + cfg = dict(type='text', fields='payload.instance_id') + tdef = distiller.TraitDefinition('test_trait', cfg, + self.fake_plugin_map) + t = tdef.to_trait(self.n1) + self.assertEqual('test_trait', t.name) + self.assertEqual(distiller.Datatype.text, t.trait_type) + self.assertEqual('id-for-instance-0001', t.value) + + cfg = dict(type='int', fields='payload.image_meta.disk_gb') + tdef = distiller.TraitDefinition('test_trait', cfg, + self.fake_plugin_map) + t = tdef.to_trait(self.n1) + self.assertEqual('test_trait', t.name) + self.assertEqual(distiller.Datatype.int, t.trait_type) + self.assertEqual(20, t.value) + + def test_to_trait_multiple(self): + cfg = dict(type='text', fields=['payload.instance_id', + 'payload.instance_uuid']) + tdef = distiller.TraitDefinition('test_trait', cfg, + self.fake_plugin_map) + t = tdef.to_trait(self.n1) + self.assertEqual('id-for-instance-0001', t.value) + + cfg = dict(type='text', fields=['payload.instance_uuid', + 'payload.instance_id']) + tdef = distiller.TraitDefinition('test_trait', cfg, + self.fake_plugin_map) + t = tdef.to_trait(self.n1) + self.assertEqual('uuid-for-instance-0001', t.value) + + def test_to_trait_multiple_different_nesting(self): + cfg = dict(type='int', fields=['payload.foobar', + 'payload.image_meta.disk_gb']) + tdef = distiller.TraitDefinition('test_trait', cfg, + self.fake_plugin_map) + t = tdef.to_trait(self.n1) + self.assertEqual(50, t.value) + + cfg = dict(type='int', fields=['payload.image_meta.disk_gb', + 'payload.foobar']) + tdef = distiller.TraitDefinition('test_trait', cfg, + self.fake_plugin_map) + t = tdef.to_trait(self.n1) + self.assertEqual(20, t.value) + + def test_to_trait_some_null_multiple(self): + cfg = dict(type='text', fields=['payload.instance_id2', + 'payload.instance_uuid']) + tdef = distiller.TraitDefinition('test_trait', cfg, + self.fake_plugin_map) + t = tdef.to_trait(self.n1) + self.assertEqual('uuid-for-instance-0001', t.value) + + def test_to_trait_some_missing_multiple(self): + cfg = dict(type='text', fields=['payload.not_here_boss', + 'payload.instance_uuid']) + tdef = distiller.TraitDefinition('test_trait', cfg, + self.fake_plugin_map) + t = tdef.to_trait(self.n1) + self.assertEqual('uuid-for-instance-0001', t.value) + + def test_to_trait_missing(self): + cfg = dict(type='text', fields='payload.not_here_boss') + tdef = distiller.TraitDefinition('test_trait', cfg, + self.fake_plugin_map) + t = tdef.to_trait(self.n1) + self.assertIs(None, t) + + def test_to_trait_null(self): + cfg = dict(type='text', fields='payload.instance_id2') + tdef = distiller.TraitDefinition('test_trait', cfg, + self.fake_plugin_map) + t = tdef.to_trait(self.n1) + self.assertIs(None, t) + + def test_to_trait_empty_nontext(self): + cfg = dict(type='datetime', fields='payload.bogus_date') + tdef = distiller.TraitDefinition('test_trait', cfg, + self.fake_plugin_map) + t = tdef.to_trait(self.n1) + self.assertIs(None, t) + + def test_to_trait_multiple_null_missing(self): + cfg = dict(type='text', fields=['payload.not_here_boss', + 'payload.instance_id2']) + tdef = distiller.TraitDefinition('test_trait', cfg, + self.fake_plugin_map) + t = tdef.to_trait(self.n1) + self.assertIs(None, t) + + def test_missing_fields_config(self): + self.assertRaises(distiller.EventDefinitionException, + distiller.TraitDefinition, + 'bogus_trait', + dict(), + self.fake_plugin_map) + + def test_string_fields_config(self): + cfg = dict(fields='payload.test') + t = distiller.TraitDefinition('test_trait', cfg, self.fake_plugin_map) + self.assertPathsEqual(t.fields, jsonpath_rw.parse('payload.test')) + + def test_list_fields_config(self): + cfg = dict(fields=['payload.test', 'payload.other']) + t = distiller.TraitDefinition('test_trait', cfg, self.fake_plugin_map) + self.assertPathsEqual( + t.fields, + jsonpath_rw.parse('(payload.test)|(payload.other)')) + + def test_invalid_path_config(self): + #test invalid jsonpath... + cfg = dict(fields='payload.bogus(') + self.assertRaises(distiller.EventDefinitionException, + distiller.TraitDefinition, + 'bogus_trait', + cfg, + self.fake_plugin_map) + + def test_invalid_plugin_config(self): + #test invalid jsonpath... + cfg = dict(fields='payload.test', plugin=dict(bogus="true")) + self.assertRaises(distiller.EventDefinitionException, + distiller.TraitDefinition, + 'test_trait', + cfg, + self.fake_plugin_map) + + def test_unknown_plugin(self): + #test invalid jsonpath... + cfg = dict(fields='payload.test', plugin=dict(name='bogus')) + self.assertRaises(distiller.EventDefinitionException, + distiller.TraitDefinition, + 'test_trait', + cfg, + self.fake_plugin_map) + + def test_type_config(self): + cfg = dict(type='text', fields='payload.test') + t = distiller.TraitDefinition('test_trait', cfg, self.fake_plugin_map) + self.assertEqual(distiller.Datatype.text, t.trait_type) + + cfg = dict(type='int', fields='payload.test') + t = distiller.TraitDefinition('test_trait', cfg, self.fake_plugin_map) + self.assertEqual(distiller.Datatype.int, t.trait_type) + + cfg = dict(type='float', fields='payload.test') + t = distiller.TraitDefinition('test_trait', cfg, self.fake_plugin_map) + self.assertEqual(distiller.Datatype.float, t.trait_type) + + cfg = dict(type='datetime', fields='payload.test') + t = distiller.TraitDefinition('test_trait', cfg, self.fake_plugin_map) + self.assertEqual(distiller.Datatype.datetime, t.trait_type) + + def test_invalid_type_config(self): + #test invalid jsonpath... + cfg = dict(type='bogus', fields='payload.test') + self.assertRaises(distiller.EventDefinitionException, + distiller.TraitDefinition, + 'bogus_trait', + cfg, + self.fake_plugin_map) + + +class TestEventDefinition(DistillerTestBase): + + def setUp(self): + super(TestEventDefinition, self).setUp() + + self.traits_cfg = { + 'instance_id': { + 'type': 'text', + 'fields': ['payload.instance_uuid', + 'payload.instance_id'], + }, + 'host': { + 'type': 'text', + 'fields': 'payload.host', + }, + } + + self.test_notification1 = self._create_test_notification( + "test.thing", + "uuid-for-notif-0001", + instance_id="uuid-for-instance-0001", + host='host-1-2-3') + + self.test_notification2 = self._create_test_notification( + "test.thing", + "uuid-for-notif-0002", + instance_id="uuid-for-instance-0002") + + self.test_notification3 = self._create_test_notification( + "test.thing", + "uuid-for-notif-0003", + instance_id="uuid-for-instance-0003", + host=None) + self.fake_plugin_map = {} + self.condenser = TestCondenser() + + def test_to_event(self): + trait_type = distiller.Datatype.text + cfg = dict(event_type='test.thing', traits=self.traits_cfg) + edef = distiller.EventDefinition(cfg, self.fake_plugin_map) + + e = edef.to_event(self.test_notification1, self.condenser) + self.assertTrue(e is self.condenser) + self.assertEqual('test.thing', e.event_type) + self.assertEqual(datetime.datetime(2013, 8, 8, 21, 6, 37, 803826, iso8601.iso8601.UTC), + e.when) + + self.assertHasDefaultTraits(e) + self.assertHasTrait(e, 'host', value='host-1-2-3', trait_type=trait_type) + self.assertHasTrait(e, 'instance_id', + value='uuid-for-instance-0001', + trait_type=trait_type) + + def test_to_event_missing_trait(self): + trait_type = distiller.Datatype.text + cfg = dict(event_type='test.thing', traits=self.traits_cfg) + edef = distiller.EventDefinition(cfg, self.fake_plugin_map) + + e = edef.to_event(self.test_notification2, self.condenser) + + self.assertHasDefaultTraits(e) + self.assertHasTrait(e, 'instance_id', + value='uuid-for-instance-0002', + trait_type=trait_type) + self.assertDoesNotHaveTrait(e, 'host') + + def test_to_event_null_trait(self): + trait_type = distiller.Datatype.text + cfg = dict(event_type='test.thing', traits=self.traits_cfg) + edef = distiller.EventDefinition(cfg, self.fake_plugin_map) + + e = edef.to_event(self.test_notification3, self.condenser) + + self.assertHasDefaultTraits(e) + self.assertHasTrait(e, 'instance_id', + value='uuid-for-instance-0003', + trait_type=trait_type) + self.assertDoesNotHaveTrait(e, 'host') + + def test_bogus_cfg_no_traits(self): + bogus = dict(event_type='test.foo') + self.assertRaises(distiller.EventDefinitionException, + distiller.EventDefinition, + bogus, + self.fake_plugin_map) + + def test_bogus_cfg_no_type(self): + bogus = dict(traits=self.traits_cfg) + self.assertRaises(distiller.EventDefinitionException, + distiller.EventDefinition, + bogus, + self.fake_plugin_map) + + def test_included_type_string(self): + cfg = dict(event_type='test.thing', traits=self.traits_cfg) + edef = distiller.EventDefinition(cfg, self.fake_plugin_map) + self.assertEqual(1, len(edef._included_types)) + self.assertEqual('test.thing', edef._included_types[0]) + self.assertEqual(0, len(edef._excluded_types)) + self.assertTrue(edef.included_type('test.thing')) + self.assertFalse(edef.excluded_type('test.thing')) + self.assertTrue(edef.match_type('test.thing')) + self.assertFalse(edef.match_type('random.thing')) + + def test_included_type_list(self): + cfg = dict(event_type=['test.thing', 'other.thing'], + traits=self.traits_cfg) + edef = distiller.EventDefinition(cfg, self.fake_plugin_map) + self.assertEqual(2, len(edef._included_types)) + self.assertEqual(0, len(edef._excluded_types)) + self.assertTrue(edef.included_type('test.thing')) + self.assertTrue(edef.included_type('other.thing')) + self.assertFalse(edef.excluded_type('test.thing')) + self.assertTrue(edef.match_type('test.thing')) + self.assertTrue(edef.match_type('other.thing')) + self.assertFalse(edef.match_type('random.thing')) + + def test_excluded_type_string(self): + cfg = dict(event_type='!test.thing', traits=self.traits_cfg) + edef = distiller.EventDefinition(cfg, self.fake_plugin_map) + self.assertEqual(1, len(edef._included_types)) + self.assertEqual('*', edef._included_types[0]) + self.assertEqual('test.thing', edef._excluded_types[0]) + self.assertEqual(1, len(edef._excluded_types)) + self.assertEqual('test.thing', edef._excluded_types[0]) + self.assertTrue(edef.excluded_type('test.thing')) + self.assertTrue(edef.included_type('random.thing')) + self.assertFalse(edef.match_type('test.thing')) + self.assertTrue(edef.match_type('random.thing')) + + def test_excluded_type_list(self): + cfg = dict(event_type=['!test.thing', '!other.thing'], + traits=self.traits_cfg) + edef = distiller.EventDefinition(cfg, self.fake_plugin_map) + self.assertEqual(1, len(edef._included_types)) + self.assertEqual(2, len(edef._excluded_types)) + self.assertTrue(edef.excluded_type('test.thing')) + self.assertTrue(edef.excluded_type('other.thing')) + self.assertFalse(edef.excluded_type('random.thing')) + self.assertFalse(edef.match_type('test.thing')) + self.assertFalse(edef.match_type('other.thing')) + self.assertTrue(edef.match_type('random.thing')) + + def test_mixed_type_list(self): + cfg = dict(event_type=['*.thing', '!test.thing', '!other.thing'], + traits=self.traits_cfg) + edef = distiller.EventDefinition(cfg, self.fake_plugin_map) + self.assertEqual(1, len(edef._included_types)) + self.assertEqual(2, len(edef._excluded_types)) + self.assertTrue(edef.excluded_type('test.thing')) + self.assertTrue(edef.excluded_type('other.thing')) + self.assertFalse(edef.excluded_type('random.thing')) + self.assertFalse(edef.match_type('test.thing')) + self.assertFalse(edef.match_type('other.thing')) + self.assertFalse(edef.match_type('random.whatzit')) + self.assertTrue(edef.match_type('random.thing')) + + def test_catchall(self): + cfg = dict(event_type=['*.thing', '!test.thing', '!other.thing'], + traits=self.traits_cfg) + edef = distiller.EventDefinition(cfg, self.fake_plugin_map) + self.assertFalse(edef.is_catchall) + + cfg = dict(event_type=['!other.thing'], + traits=self.traits_cfg) + edef = distiller.EventDefinition(cfg, self.fake_plugin_map) + self.assertFalse(edef.is_catchall) + + cfg = dict(event_type=['other.thing'], + traits=self.traits_cfg) + edef = distiller.EventDefinition(cfg, self.fake_plugin_map) + self.assertFalse(edef.is_catchall) + + cfg = dict(event_type=['*', '!other.thing'], + traits=self.traits_cfg) + edef = distiller.EventDefinition(cfg, self.fake_plugin_map) + self.assertFalse(edef.is_catchall) + + cfg = dict(event_type=['*'], + traits=self.traits_cfg) + edef = distiller.EventDefinition(cfg, self.fake_plugin_map) + self.assertTrue(edef.is_catchall) + + cfg = dict(event_type=['*', 'foo'], + traits=self.traits_cfg) + edef = distiller.EventDefinition(cfg, self.fake_plugin_map) + self.assertTrue(edef.is_catchall) + + @mock.patch('stackdistiller.distiller.utcnow') + def test_extract_when(self, mock_utcnow): + now = datetime.datetime.utcnow().replace(tzinfo=iso8601.iso8601.UTC) + modified = now + datetime.timedelta(minutes=1) + mock_utcnow.return_value = now + + body = {"timestamp": str(modified)} + when = distiller.EventDefinition._extract_when(body) + self.assertEqual(modified, when) + + body = {"_context_timestamp": str(modified)} + when = distiller.EventDefinition._extract_when(body) + self.assertEqual(modified, when) + + then = now + datetime.timedelta(hours=1) + body = {"timestamp": str(modified), "_context_timestamp": str(then)} + when = distiller.EventDefinition._extract_when(body) + self.assertEqual(modified, when) + + when = distiller.EventDefinition._extract_when({}) + self.assertEqual(now, when) + + def test_default_traits(self): + cfg = dict(event_type='test.thing', traits={}) + edef = distiller.EventDefinition(cfg, self.fake_plugin_map) + default_traits = distiller.EventDefinition.DEFAULT_TRAITS.keys() + traits = set(edef.traits.keys()) + for dt in default_traits: + self.assertIn(dt, traits) + self.assertEqual(len(distiller.EventDefinition.DEFAULT_TRAITS), + len(edef.traits)) + + def test_traits(self): + cfg = dict(event_type='test.thing', traits=self.traits_cfg) + edef = distiller.EventDefinition(cfg, self.fake_plugin_map) + default_traits = distiller.EventDefinition.DEFAULT_TRAITS.keys() + traits = set(edef.traits.keys()) + for dt in default_traits: + self.assertIn(dt, traits) + self.assertIn('host', traits) + self.assertIn('instance_id', traits) + self.assertEqual(len(distiller.EventDefinition.DEFAULT_TRAITS) + 2, + len(edef.traits)) + + +class TestDistiller(DistillerTestBase): + + def setUp(self): + super(TestDistiller, self).setUp() + + self.valid_event_def1 = [{ + 'event_type': 'compute.instance.create.*', + 'traits': { + 'instance_id': { + 'type': 'text', + 'fields': ['payload.instance_uuid', + 'payload.instance_id'], + }, + 'host': { + 'type': 'text', + 'fields': 'payload.host', + }, + }, + }] + + self.test_notification1 = self._create_test_notification( + "compute.instance.create.start", + "uuid-for-notif-0001", + instance_id="uuid-for-instance-0001", + host='host-1-2-3') + self.test_notification2 = self._create_test_notification( + "bogus.notification.from.mars", + "uuid-for-notif-0002", + weird='true', + host='cydonia') + self.fake_plugin_map = {} + + @mock.patch('stackdistiller.distiller.utcnow') + def test_distiller_missing_keys(self, mock_utcnow): + # test a malformed notification + now = datetime.datetime.utcnow().replace(tzinfo=iso8601.iso8601.UTC) + mock_utcnow.return_value = now + c = distiller.Distiller( + [], + self.fake_plugin_map, + catchall=True) + message = {'event_type': "foo", + 'message_id': "abc", + 'publisher_id': "1"} + e = c.to_event(message, TestCondenser()) + self.assertIsValidEvent(e, message) + self.assertEqual(1, len(e.traits)) + self.assertEqual("foo", e.event_type) + self.assertEqual(now, e.when) + + def test_distiller_with_catchall(self): + c = distiller.Distiller( + self.valid_event_def1, + self.fake_plugin_map, + catchall=True) + self.assertEqual(2, len(c.definitions)) + e = c.to_event(self.test_notification1, TestCondenser()) + self.assertIsValidEvent(e, self.test_notification1) + self.assertEqual(3, len(e.traits)) + self.assertHasDefaultTraits(e) + self.assertHasTrait(e, 'instance_id') + self.assertHasTrait(e, 'host') + + e = c.to_event(self.test_notification2, TestCondenser()) + self.assertIsValidEvent(e, self.test_notification2) + self.assertEqual(1, len(e.traits), + "Wrong number of traits %s: %s" % (len(e.traits), e.traits)) + self.assertHasDefaultTraits(e) + self.assertDoesNotHaveTrait(e, 'instance_id') + self.assertDoesNotHaveTrait(e, 'host') + + def test_distiller_without_catchall(self): + c = distiller.Distiller( + self.valid_event_def1, + self.fake_plugin_map, + catchall=False) + self.assertEqual(1, len(c.definitions)) + e = c.to_event(self.test_notification1, TestCondenser()) + self.assertIsValidEvent(e, self.test_notification1) + self.assertEqual(3, len(e.traits)) + self.assertHasDefaultTraits(e) + self.assertHasTrait(e, 'instance_id') + self.assertHasTrait(e, 'host') + + e = c.to_event(self.test_notification2, TestCondenser()) + self.assertIsNotValidEvent(e, self.test_notification2) + + def test_distiller_empty_cfg_with_catchall(self): + c = distiller.Distiller( + [], + self.fake_plugin_map, + catchall=True) + self.assertEqual(1, len(c.definitions)) + e = c.to_event(self.test_notification1, TestCondenser()) + self.assertIsValidEvent(e, self.test_notification1) + self.assertEqual(1, len(e.traits)) + self.assertHasDefaultTraits(e) + + e = c.to_event(self.test_notification2, TestCondenser()) + self.assertIsValidEvent(e, self.test_notification2) + self.assertEqual(1, len(e.traits)) + self.assertHasDefaultTraits(e) + + def test_distiller_empty_cfg_without_catchall(self): + c = distiller.Distiller( + [], + self.fake_plugin_map, + catchall=False) + self.assertEqual(0, len(c.definitions)) + e = c.to_event(self.test_notification1, TestCondenser()) + self.assertIsNotValidEvent(e, self.test_notification1) + + e = c.to_event(self.test_notification2, TestCondenser()) + self.assertIsNotValidEvent(e, self.test_notification2) + + def test_default_config(self): + d = distiller.Distiller([], catchall=True) + self.assertEqual(1, len(d.definitions)) + self.assertTrue(d.definitions[0].is_catchall) + + d = distiller.Distiller([], catchall=False) + self.assertEqual(0, len(d.definitions)) diff --git a/tests/test_trait_plugins.py b/tests/test_trait_plugins.py new file mode 100644 index 0000000..d0c386a --- /dev/null +++ b/tests/test_trait_plugins.py @@ -0,0 +1,120 @@ +# -*- encoding: utf-8 -*- +# +# Copyright © 2013 Rackspace Hosting. +# +# Author: Monsyne Dragon +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +#for Python2.6 compatability. +import unittest2 as unittest + +from stackdistiller import trait_plugins + + +class TestSplitterPlugin(unittest.TestCase): + + def setUp(self): + super(TestSplitterPlugin, self).setUp() + self.pclass = trait_plugins.SplitterTraitPlugin + + def test_split(self): + param = dict(separator='-', segment=0) + plugin = self.pclass(**param) + match_list = [('test.thing', 'test-foobar-baz')] + value = plugin.trait_value(match_list) + self.assertEqual('test', value) + + param = dict(separator='-', segment=1) + plugin = self.pclass(**param) + match_list = [('test.thing', 'test-foobar-baz')] + value = plugin.trait_value(match_list) + self.assertEqual('foobar', value) + + param = dict(separator='-', segment=1, max_split=1) + plugin = self.pclass(**param) + match_list = [('test.thing', 'test-foobar-baz')] + value = plugin.trait_value(match_list) + self.assertEqual('foobar-baz', value) + + def test_no_sep(self): + param = dict(separator='-', segment=0) + plugin = self.pclass(**param) + match_list = [('test.thing', 'test.foobar.baz')] + value = plugin.trait_value(match_list) + self.assertEqual('test.foobar.baz', value) + + def test_no_segment(self): + param = dict(separator='-', segment=5) + plugin = self.pclass(**param) + match_list = [('test.thing', 'test-foobar-baz')] + value = plugin.trait_value(match_list) + self.assertIs(None, value) + + def test_no_match(self): + param = dict(separator='-', segment=0) + plugin = self.pclass(**param) + match_list = [] + value = plugin.trait_value(match_list) + self.assertIs(None, value) + + +class TestBitfieldPlugin(unittest.TestCase): + + def setUp(self): + super(TestBitfieldPlugin, self).setUp() + self.pclass = trait_plugins.BitfieldTraitPlugin + self.init = 0 + self.params = dict(initial_bitfield=self.init, + flags=[dict(path='payload.foo', bit=0, value=42), + dict(path='payload.foo', bit=1, value=12), + dict(path='payload.thud', bit=1, value=23), + dict(path='thingy.boink', bit=4), + dict(path='thingy.quux', bit=6, + value="wokka"), + dict(path='payload.bar', bit=10, + value='test')]) + + def test_bitfield(self): + match_list = [('payload.foo', 12), + ('payload.bar', 'test'), + ('thingy.boink', 'testagain')] + + plugin = self.pclass(**self.params) + value = plugin.trait_value(match_list) + self.assertEqual(0x412, value) + + def test_initial(self): + match_list = [('payload.foo', 12), + ('payload.bar', 'test'), + ('thingy.boink', 'testagain')] + self.params['initial_bitfield'] = 0x2000 + plugin = self.pclass(**self.params) + value = plugin.trait_value(match_list) + self.assertEqual(0x2412, value) + + def test_no_match(self): + match_list = [] + plugin = self.pclass(**self.params) + value = plugin.trait_value(match_list) + self.assertEqual(self.init, value) + + def test_multi(self): + match_list = [('payload.foo', 12), + ('payload.thud', 23), + ('payload.bar', 'test'), + ('thingy.boink', 'testagain')] + + plugin = self.pclass(**self.params) + value = plugin.trait_value(match_list) + self.assertEqual(0x412, value) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..9afa955 --- /dev/null +++ b/tox.ini @@ -0,0 +1,15 @@ +[tox] +envlist = py26,py27 + +[testenv] +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt + +setenv = VIRTUAL_ENV={envdir} + +commands = + nosetests tests + +sitepackages = False +