x/stacktach-stackdistiller
Code Issues Proposed changes

Add PEP8 check and fix related issues

Browse Source 
- Add PEP8 section to tox.ini
- Add hacking to requirements to enforce OpenStack style requirements
- Change setup.py to use PBR
- Add setup.cfg
- Fix formatting issues flagged by flake8 check
- Add copyright notices to all remaining files
- Update .gitignore file
- Bump version number

Change-Id: If32d332d3b7800f66fe6ad0f815f178bda739036
This commit is contained in:
Levi Blackstone 2015-05-05 09:37:31 -05:00
parent 2a005502e0
commit b13bcb7513
11 changed files with 152 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -28,3 +28,8 @@ nosetests.xml
# Translations
*.mo
# IDE Project Files
*.project
*.pydev*
*.idea

48
bin/test-distiller.py
View File

@ -3,8 +3,6 @@
#
# Copyright © 2014 Rackspace Hosting.
#
# Author: Monsyne Dragon <mdragon@rackspace.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
@ -27,8 +25,8 @@ import argparse
import json
import sys
from stackdistiller import distiller
from stackdistiller import condenser
from stackdistiller import distiller
class TestCondenser(condenser.CondenserBase):
@ -81,28 +79,32 @@ def test_data(args):
yield n
parser = argparse.ArgumentParser(description="Test Distiller configuration")
parser.add_argument('-c', '--config',
default='event_definitions.yaml',
help='Name of event definitions file '
'to test (Default: %(default)s)')
parser.add_argument('-l', '--list', action='store_true',
help='Test data files contain JSON list of notifications.'
' (By default data files should contain a single '
'notification.)')
parser.add_argument('-d', '--add_default_definition', action='store_true',
help='Add default event definition. Normally, '
'notifications are dropped if there is no event '
'definition for their event_type. Setting this adds a '
'"catchall" that converts unknown notifications to Events'
' with a few basic traits.')
parser.add_argument('-o', '--output', type=argparse.FileType('w'),
parser = argparse.ArgumentParser(
description="Test Distiller configuration")
parser.add_argument(
'-c', '--config',
default='event_definitions.yaml',
help='Name of event definitions file '
'to test (Default: %(default)s)')
parser.add_argument(
'-l', '--list', action='store_true',
help='Test data files contain JSON list of notifications.'
' (By default data files should contain a single '
'notification.)')
parser.add_argument(
'-d', '--add_default_definition', action='store_true',
help='Add default event definition. Normally, '
'notifications are dropped if there is no event '
'definition for their event_type. Setting this adds a '
'"catchall" that converts unknown notifications to Events'
' with a few basic traits.')
parser.add_argument('-o', '--output', type=argparse.FileType('w'),
default=sys.stdout, help="Output file. Default stdout")
parser.add_argument('test_data', nargs='*', metavar='JSON_FILE',
help="Test notifications in JSON format. Defaults to stdin")
parser.add_argument(
'test_data', nargs='*', metavar='JSON_FILE',
help="Test notifications in JSON format. Defaults to stdin")
args = parser.parse_args()
config = distiller.load_config(args.config)
out = args.output
@ -115,7 +117,7 @@ drops = 0
cond = TestCondenser()
for notification in notifications:
cond.clear()
nct +=1
nct += 1
if dist.to_event(notification, cond) is None:
out.write("Dropped notification: %s\n" %
notification['message_id'])

7
requirements.txt
View File

@ -1 +1,6 @@
-e .
hacking>=0.10.0,<0.11
enum34>=1.0
iso8601>=0.1.10
jsonpath-rw>=1.2.0, < 2.0
PyYAML>=3.1.0
six>=1.5.2

26
setup.cfg Normal file
View File

@ -0,0 +1,26 @@
[metadata]
description-file = README.md
name = stackdistiller
version = 0.12
author = Monsyne Dragon
author_email = mdragon@rackspace.com
summary = A data extraction and transformation library for OpenStack notifications
license = Apache-2
keywords =
OpenStack
notifications
events
extraction
transformation
classifiers =
Development Status :: 3 - Alpha
License :: OSI Approved :: Apache Software License
Operating System :: POSIX :: Linux
Programming Language :: Python :: 2.6
Programming Language :: Python :: 2.7
home-page = https://github.com/stackforge/stacktach-stackdistiller
[files]
packages =
stackdistiller

37
setup.py
View File

@ -1,37 +1,8 @@
import os
from setuptools import setup, find_packages
#!/usr/bin/env python
def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read()
from setuptools import setup
setup(
name='stackdistiller',
version='0.11',
author='Monsyne Dragon',
author_email='mdragon@rackspace.com',
description=("A data extraction and transformation library for "
"OpenStack notifications"),
license='Apache License (2.0)',
keywords='OpenStack notifications events extraction transformation',
packages=find_packages(exclude=['tests']),
classifiers=[
'Development Status :: 3 - Alpha',
'License :: OSI Approved :: Apache Software License',
'Operating System :: POSIX :: Linux',
'Programming Language :: Python :: 2.6',
'Programming Language :: Python :: 2.7',
],
url='https://github.com/stackforge/stacktach-stackdistiller',
scripts=['bin/test-distiller.py'],
long_description=read('README.md'),
install_requires=[
"enum34 >= 1.0",
"iso8601 >= 0.1.10",
"jsonpath-rw >= 1.2.0, < 2.0",
"PyYAML >= 3.1.0",
"six >= 1.5.2",
],
zip_safe=False
setup_requires=['pbr'],
pbr=True,
)

36
stackdistiller/condenser.py
View File

@ -2,8 +2,6 @@
#
# Copyright © 2014 Rackspace Hosting.
#
# Author: Monsyne Dragon <mdragon@rackspace.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
@ -22,18 +20,23 @@ import six
@six.add_metaclass(abc.ABCMeta)
class CondenserBase(object):
"""Base class for Condenser objects that collect data extracted from a
Notification by the Distiller, and format it into a usefull datastructure.
"""Base class for Condenser objects
A simple Condenser may just colect all the traits received into a dictionary.
More complex ones may build collections of application or ORM model objects,
or XML document trees.
Collect data extracted from a Notification by the Distiller, and
format it into a useful data structure.
Condensers also have hooks for verification logic, to check that all needed
traits are present."""
A simple Condenser may just colect all the traits received into
a dictionary. More complex ones may build collections of application
or ORM model objects, or XML document trees.
Condensers also have hooks for verification logic, to check that
all needed traits are present.
"""
def __init__(self, **kw):
"""Setup the condenser. A new instance of the condenser is passed to the
"""Set up the condenser.
A new instance of the condenser is passed to the
distiller for each notification extracted.
:param kw: keyword parameters for condenser.
@ -43,7 +46,9 @@ class CondenserBase(object):
@abc.abstractmethod
def add_trait(self, name, trait_type, value):
"""Add a trait to the Event datastructure being built by this
"""Add a trait
Add a trait to the Event data structure being built by this
condenser. The distiller will call this for each extracted trait.
:param name: (string) name of the trait
@ -54,7 +59,9 @@ class CondenserBase(object):
@abc.abstractmethod
def add_envelope_info(self, event_type, message_id, when):
"""Add the metadata for this event, extracted from the notification's
"""Add the metadata for this event
Add metadata extracted from the notification's
envelope. The distiller will call this once.
:param event_type: (string) Type of event, as a dotted string such as
@ -66,14 +73,14 @@ class CondenserBase(object):
@abc.abstractmethod
def get_event(self):
"""Return the Event datastructure constructed by this condenser."""
"""Return the Event data structure constructed by this condenser."""
@abc.abstractmethod
def clear(self):
"""Clear condenser state."""
def validate(self):
"""Check Event against whatever validation logic this condenser may have
"""Check Event against whatever validation logic this condenser has
:returns: (bool) True if valid.
@ -83,6 +90,7 @@ class CondenserBase(object):
class DictionaryCondenser(CondenserBase):
"""Return event data as a simple python dictionary"""
def __init__(self, **kw):
self.clear()
super(DictionaryCondenser, self).__init__(**kw)

27
stackdistiller/distiller.py
View File

@ -2,8 +2,6 @@
#
# Copyright © 2013 Rackspace Hosting.
#
# Author: Monsyne Dragon <mdragon@rackspace.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
@ -20,7 +18,6 @@ import collections
import datetime
import fnmatch
import logging
import os
from enum import Enum
import iso8601
@ -60,13 +57,13 @@ def load_config(filename):
if hasattr(err, 'problem_mark'):
mark = err.problem_mark
errmsg = ("Invalid YAML syntax in Event Definitions file "
"%(file)s at line: %(line)s, column: %(column)s."
"%(file)s at line: %(line)s, column: %(column)s."
% dict(file=filename,
line=mark.line + 1,
column=mark.column + 1))
else:
errmsg = ("YAML error reading Event Definitions file "
"%(file)s"
"%(file)s"
% dict(file=filename))
logger.error(errmsg)
raise
@ -100,7 +97,6 @@ Trait = collections.namedtuple('Trait', ('name', 'trait_type', 'value'))
class TraitDefinition(object):
def __init__(self, name, trait_cfg, plugin_map):
self.cfg = trait_cfg
self.name = name
@ -110,8 +106,8 @@ class TraitDefinition(object):
type_name = trait_cfg.get('type', 'text')
except AttributeError as e:
raise EventDefinitionException(
"Unable to get type for '%s'" % trait_cfg,
self.cfg)
"Unable to get type for '%s'" % trait_cfg,
self.cfg)
if 'plugin' in trait_cfg:
plugin_cfg = trait_cfg['plugin']
@ -124,7 +120,7 @@ class TraitDefinition(object):
except KeyError:
raise EventDefinitionException(
'Plugin specified, but no plugin name supplied for '
'trait %s' % name, self.cfg)
'trait %s' % name, self.cfg)
plugin_params = plugin_cfg.get('parameters')
if plugin_params is None:
plugin_params = {}
@ -133,8 +129,8 @@ class TraitDefinition(object):
except KeyError:
raise EventDefinitionException(
'No plugin named %(plugin)s available for '
'trait %(trait)s' % dict(plugin=plugin_name,
trait=name), self.cfg)
'trait %(trait)s' % dict(plugin=plugin_name,
trait=name), self.cfg)
self.plugin = plugin_class(**plugin_params)
else:
self.plugin = None
@ -142,7 +138,7 @@ class TraitDefinition(object):
if 'fields' not in trait_cfg:
raise EventDefinitionException(
"Required field in trait definition not specified: "
"'%s'" % 'fields',
"'%s'" % 'fields',
self.cfg)
fields = trait_cfg['fields']
@ -157,7 +153,7 @@ class TraitDefinition(object):
except Exception as e:
raise EventDefinitionException(
"Parse error in JSONPath specification "
"'%(jsonpath)s' for %(trait)s: %(err)s"
"'%(jsonpath)s' for %(trait)s: %(err)s"
% dict(jsonpath=fields, trait=name, err=e), self.cfg)
try:
self.trait_type = Datatype[type_name]
@ -196,7 +192,6 @@ class TraitDefinition(object):
class EventDefinition(object):
DEFAULT_TRAITS = dict(
service=dict(type='text', fields='publisher_id'),
request_id=dict(type='text', fields='_context_request_id'),
@ -262,8 +257,8 @@ class EventDefinition(object):
@staticmethod
def _extract_when(body):
"""Extract the generated datetime from the notification.
"""
"""Extract the generated datetime from the notification."""
# NOTE: I am keeping the logic the same as it was in openstack
# code, However, *ALL* notifications should have a 'timestamp'
# field, it's part of the notification envelope spec. If this was

6
stackdistiller/trait_plugins.py
View File

@ -2,8 +2,6 @@
#
# Copyright © 2013 Rackspace Hosting.
#
# Author: Monsyne Dragon <mdragon@rackspace.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
@ -22,9 +20,7 @@ import six
@six.add_metaclass(abc.ABCMeta)
class TraitPluginBase(object):
"""Base class for plugins that convert notification fields to
Trait values.
"""
"""Base class for plugins that convert notification fields to Traits"""
def __init__(self, **kw):
"""Setup the trait plugin.

92
tests/test_distiller.py
View File

@ -2,13 +2,11 @@
#
# Copyright © 2013 Rackspace Hosting.
#
# Author: Monsyne Dragon <mdragon@rackspace.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
@ -18,7 +16,7 @@
import datetime
#for Python2.6 compatability.
# for Python2.6 compatability.
import unittest2 as unittest
import iso8601
@ -50,13 +48,14 @@ class TestCondenser(object):
class DistillerTestBase(unittest.TestCase):
def _create_test_notification(self, event_type, message_id, **kw):
return dict(event_type=event_type,
message_id=message_id,
priority="INFO",
publisher_id="compute.host-1-2-3",
timestamp="2013-08-08 21:06:37.803826",
payload=kw,
)
return dict(
event_type=event_type,
message_id=message_id,
priority="INFO",
publisher_id="compute.host-1-2-3",
timestamp="2013-08-08 21:06:37.803826",
payload=kw,
)
def assertIsValidEvent(self, event, notification):
self.assertIsNot(
@ -113,7 +112,6 @@ class DistillerTestBase(unittest.TestCase):
class TestTraitDefinition(DistillerTestBase):
def setUp(self):
super(TestTraitDefinition, self).setUp()
self.n1 = self._create_test_notification(
@ -126,8 +124,8 @@ class TestTraitDefinition(DistillerTestBase):
host='host-1-2-3',
bogus_date='',
image_meta=dict(
disk_gb='20',
thing='whatzit'),
disk_gb='20',
thing='whatzit'),
foobar=50)
self.test_plugin_class = mock.MagicMock(name='mock_test_plugin')
@ -244,14 +242,14 @@ class TestTraitDefinition(DistillerTestBase):
def test_to_trait_multiple_different_nesting(self):
cfg = dict(type='int', fields=['payload.foobar',
'payload.image_meta.disk_gb'])
'payload.image_meta.disk_gb'])
tdef = distiller.TraitDefinition('test_trait', cfg,
self.fake_plugin_map)
t = tdef.to_trait(self.n1)
self.assertEqual(50, t.value)
cfg = dict(type='int', fields=['payload.image_meta.disk_gb',
'payload.foobar'])
'payload.foobar'])
tdef = distiller.TraitDefinition('test_trait', cfg,
self.fake_plugin_map)
t = tdef.to_trait(self.n1)
@ -322,7 +320,7 @@ class TestTraitDefinition(DistillerTestBase):
jsonpath_rw.parse('(payload.test)|(payload.other)'))
def test_invalid_path_config(self):
#test invalid jsonpath...
# test invalid jsonpath...
cfg = dict(fields='payload.bogus(')
self.assertRaises(distiller.EventDefinitionException,
distiller.TraitDefinition,
@ -331,7 +329,7 @@ class TestTraitDefinition(DistillerTestBase):
self.fake_plugin_map)
def test_invalid_plugin_config(self):
#test invalid jsonpath...
# test invalid jsonpath...
cfg = dict(fields='payload.test', plugin=dict(bogus="true"))
self.assertRaises(distiller.EventDefinitionException,
distiller.TraitDefinition,
@ -340,7 +338,7 @@ class TestTraitDefinition(DistillerTestBase):
self.fake_plugin_map)
def test_unknown_plugin(self):
#test invalid jsonpath...
# test invalid jsonpath...
cfg = dict(fields='payload.test', plugin=dict(name='bogus'))
self.assertRaises(distiller.EventDefinitionException,
distiller.TraitDefinition,
@ -366,7 +364,7 @@ class TestTraitDefinition(DistillerTestBase):
self.assertEqual(distiller.Datatype.datetime, t.trait_type)
def test_invalid_type_config(self):
#test invalid jsonpath...
# test invalid jsonpath...
cfg = dict(type='bogus', fields='payload.test')
self.assertRaises(distiller.EventDefinitionException,
distiller.TraitDefinition,
@ -376,7 +374,6 @@ class TestTraitDefinition(DistillerTestBase):
class TestEventDefinition(DistillerTestBase):
def setUp(self):
super(TestEventDefinition, self).setUp()
@ -419,11 +416,13 @@ class TestEventDefinition(DistillerTestBase):
e = edef.to_event(self.test_notification1, self.condenser)
self.assertTrue(e is self.condenser)
self.assertEqual('test.thing', e.event_type)
self.assertEqual(datetime.datetime(2013, 8, 8, 21, 6, 37, 803826, iso8601.iso8601.UTC),
self.assertEqual(datetime.datetime(2013, 8, 8, 21, 6, 37, 803826,
iso8601.iso8601.UTC),
e.when)
self.assertHasDefaultTraits(e)
self.assertHasTrait(e, 'host', value='host-1-2-3', trait_type=trait_type)
self.assertHasTrait(e, 'host', value='host-1-2-3',
trait_type=trait_type)
self.assertHasTrait(e, 'instance_id',
value='uuid-for-instance-0001',
trait_type=trait_type)
@ -609,24 +608,25 @@ class TestEventDefinition(DistillerTestBase):
class TestDistiller(DistillerTestBase):
def setUp(self):
super(TestDistiller, self).setUp()
self.valid_event_def1 = [{
'event_type': 'compute.instance.create.*',
'traits': {
'instance_id': {
'type': 'text',
'fields': ['payload.instance_uuid',
'payload.instance_id'],
self.valid_event_def1 = [
{
'event_type': 'compute.instance.create.*',
'traits': {
'instance_id': {
'type': 'text',
'fields': ['payload.instance_uuid',
'payload.instance_id'],
},
'host': {
'type': 'text',
'fields': 'payload.host',
},
},
'host': {
'type': 'text',
'fields': 'payload.host',
},
},
}]
}
]
self.test_notification1 = self._create_test_notification(
"compute.instance.create.start",
@ -645,10 +645,7 @@ class TestDistiller(DistillerTestBase):
# test a malformed notification
now = datetime.datetime.utcnow().replace(tzinfo=iso8601.iso8601.UTC)
mock_utcnow.return_value = now
c = distiller.Distiller(
[],
self.fake_plugin_map,
catchall=True)
c = distiller.Distiller([], self.fake_plugin_map, catchall=True)
message = {'event_type': "foo",
'message_id': "abc",
'publisher_id': "1"}
@ -674,7 +671,8 @@ class TestDistiller(DistillerTestBase):
e = c.to_event(self.test_notification2, TestCondenser())
self.assertIsValidEvent(e, self.test_notification2)
self.assertEqual(1, len(e.traits),
"Wrong number of traits %s: %s" % (len(e.traits), e.traits))
"Wrong number of traits %s: %s" % (
len(e.traits), e.traits))
self.assertHasDefaultTraits(e)
self.assertDoesNotHaveTrait(e, 'instance_id')
self.assertDoesNotHaveTrait(e, 'host')
@ -696,10 +694,7 @@ class TestDistiller(DistillerTestBase):
self.assertIsNotValidEvent(e, self.test_notification2)
def test_distiller_empty_cfg_with_catchall(self):
c = distiller.Distiller(
[],
self.fake_plugin_map,
catchall=True)
c = distiller.Distiller([], self.fake_plugin_map, catchall=True)
self.assertEqual(1, len(c.definitions))
e = c.to_event(self.test_notification1, TestCondenser())
self.assertIsValidEvent(e, self.test_notification1)
@ -712,10 +707,7 @@ class TestDistiller(DistillerTestBase):
self.assertHasDefaultTraits(e)
def test_distiller_empty_cfg_without_catchall(self):
c = distiller.Distiller(
[],
self.fake_plugin_map,
catchall=False)
c = distiller.Distiller([], self.fake_plugin_map, catchall=False)
self.assertEqual(0, len(c.definitions))
e = c.to_event(self.test_notification1, TestCondenser())
self.assertIsNotValidEvent(e, self.test_notification1)

6
tests/test_trait_plugins.py
View File

@ -2,8 +2,6 @@
#
# Copyright © 2013 Rackspace Hosting.
#
# Author: Monsyne Dragon <mdragon@rackspace.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
@ -16,14 +14,13 @@
# License for the specific language governing permissions and limitations
# under the License.
#for Python2.6 compatability.
# for Python2.6 compatability.
import unittest2 as unittest
from stackdistiller import trait_plugins
class TestSplitterPlugin(unittest.TestCase):
def setUp(self):
super(TestSplitterPlugin, self).setUp()
self.pclass = trait_plugins.SplitterTraitPlugin
@ -70,7 +67,6 @@ class TestSplitterPlugin(unittest.TestCase):
class TestBitfieldPlugin(unittest.TestCase):
def setUp(self):
super(TestBitfieldPlugin, self).setUp()
self.pclass = trait_plugins.BitfieldTraitPlugin

10
tox.ini
View File

@ -1,5 +1,5 @@
[tox]
envlist = py26,py27
envlist = py26,py27,pep8
[testenv]
deps =
@ -13,3 +13,11 @@ commands =
sitepackages = False
[testenv:pep8]
commands =
flake8
[flake8]
ignore =
exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg
show-source = True