diff --git a/.gitignore b/.gitignore index ded6067..f637812 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ pip-log.txt .coverage .tox nosetests.xml +test_temp/* # Translations *.mo @@ -34,3 +35,8 @@ nosetests.xml .mr.developer.cfg .project .pydevproject + +# IDE Project Files +*.project +*.pydev* +*.idea diff --git a/requirements.txt b/requirements.txt index cc17005..70bf758 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +hacking>=0.10.0,<0.11 notigen notification_utils python-dateutil diff --git a/shoebox/archive.py b/shoebox/archive.py index d95c37f..dd493f0 100644 --- a/shoebox/archive.py +++ b/shoebox/archive.py @@ -30,7 +30,9 @@ class Archive(object): class ArchiveWriter(Archive): """The active Archive for appending. + """ + def __init__(self, filename): super(ArchiveWriter, self).__init__(filename) self._open_file(filename) @@ -47,7 +49,9 @@ class ArchiveWriter(Archive): class ArchiveReader(Archive): """The active Archive for consuming. + """ + def __init__(self, filename): super(ArchiveReader, self).__init__(filename) self._open_file(filename) diff --git a/shoebox/disk_storage.py b/shoebox/disk_storage.py index 6234152..d8f21cc 100644 --- a/shoebox/disk_storage.py +++ b/shoebox/disk_storage.py @@ -1,6 +1,18 @@ -import calendar -import datetime -import json +# Copyright (c) 2014 Dark Secret Software Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import struct @@ -61,7 +73,7 @@ class Version1(Version0): # *s = raw data # EXAMPLE - #-------- + # -------- # With above Event and Metadata # # Header schema: "iii" @@ -84,16 +96,16 @@ class Version1(Version0): self.header_size = struct.calcsize(self.header_schema) def _encode(self, s): - if isinstance(s, unicode): + if isinstance(s, unicode): return s.encode('utf-8') - return s + return s def pack(self, notification, metadata): nsize = len(notification) raw_block_schema = "i%ds" % nsize raw_block = struct.pack(raw_block_schema, nsize, notification) - metadata_items = ["i"] # appended with N "%ds"'s + metadata_items = ["i"] # appended with N "%ds"'s metadata_values = [len(metadata) * 4] # [n]=key, [n+1]=value for key, value in metadata.iteritems(): key = self._encode(key) @@ -142,7 +154,7 @@ class Version1(Version0): offset += struct.calcsize(lengths_schema) key_values = struct.unpack_from(key_value_schema, metadata_bytes, offset=offset) - metadata = dict((key_values[n], key_values[n+1]) + metadata = dict((key_values[n], key_values[n + 1]) for n in range(len(key_values))[::2]) raw = file_handle.read(header[1]) @@ -156,6 +168,7 @@ class Version1(Version0): VERSIONS = {1: Version1()} CURRENT_VERSION = 1 + def get_version_handler(version=CURRENT_VERSION): global VERSIONS diff --git a/shoebox/handlers.py b/shoebox/handlers.py index 477a51d..259ad05 100644 --- a/shoebox/handlers.py +++ b/shoebox/handlers.py @@ -34,6 +34,7 @@ class ArchiveCallback(object): def on_close(self, filename): """Called when an Archive is closed. + If you move/change the file/name return the new location so subsequent callbacks will have the right location. @@ -49,9 +50,11 @@ class CallbackList(ArchiveCallback): callback_str = self.config.get('callback_list', "") callback_str_list = [x.strip() for x in callback_str.split(",")] self.callbacks = [simport.load(c)(**self.config) - for c in callback_str_list] + for c in callback_str_list] + # TODO(Sandy): Need some exception handling around these. # The failure of one shouldn't stop processing. + def on_open(self, filename): for c in self.callbacks: c.on_open(filename) @@ -63,6 +66,7 @@ class CallbackList(ArchiveCallback): class ChangeExtensionCallback(ArchiveCallback): """filename.dat becomes filename.dat.done""" + def __init__(self, **kwargs): super(ChangeExtensionCallback, self).__init__(**kwargs) self.new_extension = kwargs.get('new_extension', '.done') diff --git a/shoebox/roll_manager.py b/shoebox/roll_manager.py index 9356a14..e621037 100644 --- a/shoebox/roll_manager.py +++ b/shoebox/roll_manager.py @@ -64,20 +64,20 @@ class RollManager(object): class ReadingRollManager(RollManager): def __init__(self, filename_template, directory=".", destination_directory=".", - archive_class = archive.ArchiveReader, + archive_class=archive.ArchiveReader, archive_callback=None, roll_size_mb=1000): super(ReadingRollManager, self).__init__( - filename_template, - directory=directory, - archive_callback=archive_callback, - archive_class=archive_class) + filename_template, + directory=directory, + archive_callback=archive_callback, + archive_class=archive_class) self.files_to_read = self._get_matching_files(directory, filename_template) def _get_matching_files(self, directory, filename_template): files = [os.path.join(directory, f) - for f in os.listdir(self.directory) - if os.path.isfile(os.path.join(directory, f))] + for f in os.listdir(self.directory) + if os.path.isfile(os.path.join(directory, f))] return sorted(fnmatch.filter(files, filename_template)) def read(self): @@ -107,15 +107,17 @@ class WritingRollManager(RollManager): archive_class=archive.ArchiveWriter, archive_callback=None, roll_size_mb=1000): super(WritingRollManager, self).__init__( - filename_template, - directory=directory, - archive_callback=archive_callback, - archive_class=archive_class) + filename_template, + directory=directory, + archive_callback=archive_callback, + archive_class=archive_class) self.roll_checker = roll_checker def write(self, metadata, payload): - """metadata is string:string dict. - payload must be encoded as string. + """Write metadata + + metadata is string:string dict. + payload must be encoded as string. """ a = self.get_active_archive() a.write(metadata, payload) @@ -143,11 +145,15 @@ class WritingRollManager(RollManager): class WritingJSONRollManager(object): - """No archiver. No roll checker. Just write 1 file line per json payload. - Once the file gets big enough, gzip the file and move - into the destination_directory. - Expects an external tool like rsync to move the file. - A SHA-256 of the payload may be included in the archive filename.""" + """Simple event archival. + + No archiver. No roll checker. Just write 1 file line per json payload. + Once the file gets big enough, gzip the file and move + into the destination_directory. + Expects an external tool like rsync to move the file. + A SHA-256 of the payload may be included in the archive filename. + """ + def __init__(self, *args, **kwargs): self.filename_template = args[0] self.directory = kwargs.get('directory', '.') @@ -193,7 +199,7 @@ class WritingJSONRollManager(object): self._get_time() >= (self.start_time + self.roll_after))) def _get_file_sha(self, filename): - block_size=2**20 + block_size = 2 ** 20 sha256 = hashlib.sha256() # no context manager, just to keep testing simple. f = open(filename, "r") diff --git a/test/integration/test_json_tarball.py b/test/integration/test_json_tarball.py index 83304ed..3ba3fa7 100644 --- a/test/integration/test_json_tarball.py +++ b/test/integration/test_json_tarball.py @@ -1,8 +1,21 @@ +# Copyright (c) 2014 Dark Secret Software Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import datetime import gzip -import hashlib import json -import mock import os import shutil import unittest @@ -26,10 +39,10 @@ class TestDirectory(unittest.TestCase): def test_size_rolling(self): manager = roll_manager.WritingJSONRollManager( - "%Y_%m_%d_%X_%f_[[CRC]].event", - directory=TEMPDIR, - destination_directory=DESTDIR, - roll_size_mb=10) + "%Y_%m_%d_%X_%f_[[CRC]].event", + directory=TEMPDIR, + destination_directory=DESTDIR, + roll_size_mb=10) g = notigen.EventGenerator("test/integration/templates") entries = {} @@ -39,8 +52,9 @@ class TestDirectory(unittest.TestCase): if events: for event in events: metadata = {} - json_event = json.dumps(event, - cls=notification_utils.DateTimeEncoder) + json_event = json.dumps( + event, + cls=notification_utils.DateTimeEncoder) manager.write(metadata, json_event) msg_id = event['message_id'] entries[msg_id] = json_event @@ -49,7 +63,7 @@ class TestDirectory(unittest.TestCase): manager.close() manager._archive_working_files() - print "Starting entries:", len(entries) + print("Starting entries:", len(entries)) actual = len(entries) @@ -68,8 +82,8 @@ class TestDirectory(unittest.TestCase): num = len(file_content) - 1 total += num - print "In %s: %d of %d Remaining: %d" % (f, num, actual, - actual - total) + print("In %s: %d of %d Remaining: %d" + % (f, num, actual, actual - total)) if actual != total: raise Exception("Num generated != actual") diff --git a/test/integration/test_rolling.py b/test/integration/test_rolling.py index 4ad53be..8f860c3 100644 --- a/test/integration/test_rolling.py +++ b/test/integration/test_rolling.py @@ -1,6 +1,20 @@ +# Copyright (c) 2014 Dark Secret Software Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import datetime import json -import mock import os import shutil import unittest @@ -8,7 +22,6 @@ import unittest import notification_utils import notigen -from shoebox import disk_storage from shoebox import roll_checker from shoebox import roll_manager @@ -64,14 +77,16 @@ class TestSizeRolling(unittest.TestCase): events = g.generate(now) if events: for event in events: - metadata = {'event': event['event_type'], - 'request_id': event['_context_request_id'], - 'generated': str(event['timestamp']), - 'uuid': event.get('payload', {} - ).get("instance_id", ""), - } - json_event = json.dumps(event, - cls=notification_utils.DateTimeEncoder) + metadata = { + 'event': event['event_type'], + 'request_id': event['_context_request_id'], + 'generated': str(event['timestamp']), + 'uuid': event.get('payload', {}).get( + "instance_id", ""), + } + json_event = json.dumps( + event, + cls=notification_utils.DateTimeEncoder) manager.write(metadata, json_event) entries.append((metadata, json_event)) diff --git a/test/test_callbacks.py b/test/test_callbacks.py index 8b7e3eb..531479b 100644 --- a/test/test_callbacks.py +++ b/test/test_callbacks.py @@ -1,5 +1,19 @@ -import unittest +# Copyright (c) 2014 Dark Secret Software Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest from shoebox import handlers @@ -24,6 +38,5 @@ class TestCallbackList(unittest.TestCase): # (the 'test' module). self.assertTrue("FooCallback" in str(type(c.callbacks[0]))) self.assertTrue(isinstance(c.callbacks[1], - handlers.ChangeExtensionCallback)) + handlers.ChangeExtensionCallback)) self.assertTrue(isinstance(c.callbacks[2], BlahCallback)) - diff --git a/test/test_disk_storage.py b/test/test_disk_storage.py index 5083f15..89d6ee8 100644 --- a/test/test_disk_storage.py +++ b/test/test_disk_storage.py @@ -1,20 +1,32 @@ -import datetime -import mock +# Copyright (c) 2014 Dark Secret Software Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json +import mock import struct import unittest -import dateutil.tz - from shoebox import disk_storage class TestVersion0(unittest.TestCase): def setUp(self): - self.v0 = disk_storage.Version0() + self.v0 = disk_storage.Version0() def test_make_preamble(self): - self.assertEqual(6, len(self.v0.make_preamble(99))) + self.assertEqual(6, len(self.v0.make_preamble(99))) def test_load_preamble_bad_bor(self): file_handle = mock.Mock() @@ -24,14 +36,16 @@ class TestVersion0(unittest.TestCase): def test_load_preamble(self): file_handle = mock.Mock() - file_handle.read.return_value = struct.pack("ih", - disk_storage.BOR_MAGIC_NUMBER, 99) + file_handle.read.return_value = struct.pack( + "ih", + disk_storage.BOR_MAGIC_NUMBER, + 99) self.assertEqual(99, self.v0.load_preamble(file_handle)) class TestVersion1(unittest.TestCase): def setUp(self): - self.v1 = disk_storage.Version1() + self.v1 = disk_storage.Version1() def test_no_metadata(self): metadata = {} @@ -76,7 +90,7 @@ class TestVersion1(unittest.TestCase): blocks = blocks[1:] # Remove preamble # break the EOR marker - print len(blocks[0]) + print(len(blocks[0])) newblock = blocks[0][:8] + '\x00\x00\x01\x02' blocks = list(blocks) blocks[0] = newblock @@ -110,8 +124,10 @@ class TestHelpers(unittest.TestCase): def test_unpack_notification(self): file_handle = mock.Mock() - file_handle.read.return_value = struct.pack("ih", - disk_storage.BOR_MAGIC_NUMBER, 99) + file_handle.read.return_value = struct.pack( + "ih", + disk_storage.BOR_MAGIC_NUMBER, + 99) with mock.patch('shoebox.disk_storage.get_version_handler') as h: fake_handler = mock.Mock() diff --git a/test/test_roll_checker.py b/test/test_roll_checker.py index 15cbb31..2ea793c 100644 --- a/test/test_roll_checker.py +++ b/test/test_roll_checker.py @@ -1,3 +1,18 @@ +# Copyright (c) 2014 Dark Secret Software Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import datetime import mock import unittest @@ -33,7 +48,7 @@ class TestRollChecker(unittest.TestCase): self.assertFalse(x.check(None)) with mock.patch.object(notification_utils, 'now') as dt: - dt.return_value = now + one_hour - datetime.timedelta(seconds = 1) + dt.return_value = now + one_hour - datetime.timedelta(seconds=1) self.assertFalse(x.check(None)) def test_size_roll_checker_end(self): diff --git a/test/test_roll_manager.py b/test/test_roll_manager.py index 2db62d3..56b9a76 100644 --- a/test/test_roll_manager.py +++ b/test/test_roll_manager.py @@ -1,10 +1,23 @@ +# Copyright (c) 2014 Dark Secret Software Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import datetime -import mock -import os import unittest +import mock import notification_utils - from shoebox import archive from shoebox import roll_checker from shoebox import roll_manager @@ -53,7 +66,7 @@ class TestWritingRollManager(unittest.TestCase): def test_correct_archiver(self): x = roll_manager.WritingRollManager("foo", None) - print x.archive_class + print(x.archive_class) self.assertEqual(x.archive_class, archive.ArchiveWriter) def test_get_active_archive(self): @@ -63,8 +76,8 @@ class TestWritingRollManager(unittest.TestCase): x = roll_manager.WritingRollManager(filename_template, checker, archive_callback=callback, archive_class=FakeArchive) - with mock.patch("shoebox.archive.ArchiveWriter._open_file") as of: - arc = x.get_active_archive() + with mock.patch("shoebox.archive.ArchiveWriter._open_file"): + x.get_active_archive() self.assertTrue(checker.start.called) self.assertTrue(callback.on_open.called) @@ -105,7 +118,7 @@ class TestJSONRollManager(unittest.TestCase): td.return_value = 123.45 dt.return_value = now x = roll_manager.WritingJSONRollManager( - "%Y%m%d [[TIMESTAMP]] [[CRC]].foo") + "%Y%m%d [[TIMESTAMP]] [[CRC]].foo") fn = x._make_filename("mycrc", "foo") self.assertEqual("foo/20140201_123.45_mycrc.foo", fn) @@ -124,10 +137,11 @@ class TestJSONRollManager(unittest.TestCase): def test_should_roll(self, awf): rm = roll_manager.WritingJSONRollManager("template.foo") rm.roll_size_mb = 10 - self.assertFalse(rm._should_roll(9*1048576)) - self.assertTrue(rm._should_roll(10*1048576)) + self.assertFalse(rm._should_roll(9 * 1048576)) + self.assertTrue(rm._should_roll(10 * 1048576)) - rm = roll_manager.WritingJSONRollManager("template.foo", roll_minutes=10) + rm = roll_manager.WritingJSONRollManager("template.foo", + roll_minutes=10) self.assertFalse(rm._should_roll(0)) self.assertFalse(rm._should_roll(1)) with mock.patch.object(rm, "_get_time") as gt: diff --git a/tox.ini b/tox.ini index 8e4c1a8..1520e68 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27 +envlist = py26, py27, pep8 [testenv] deps = @@ -12,3 +12,12 @@ deps = simport commands = nosetests -d -v --with-coverage --cover-inclusive --cover-package shoebox [] + +[testenv:pep8] +commands = + flake8 + +[flake8] +ignore = +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg +show-source = True