From ddc2a4c6b293a4b6c233492150cd44f754d48919 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 16 Feb 2011 19:30:18 -0600 Subject: [PATCH 1/2] fixed glob pattern matching index offset error in stats' LogUploader --- swift/stats/log_uploader.py | 2 +- test/unit/stats/test_log_uploader.py | 137 ++++++++++++++++++++++++++- 2 files changed, 135 insertions(+), 4 deletions(-) diff --git a/swift/stats/log_uploader.py b/swift/stats/log_uploader.py index b425738938..e75a05c07d 100644 --- a/swift/stats/log_uploader.py +++ b/swift/stats/log_uploader.py @@ -78,7 +78,7 @@ class LogUploader(Daemon): i = [(self.filename_format.index(c), c) for c in '%Y %m %d %H'.split()] i.sort() year_offset = month_offset = day_offset = hour_offset = None - base_offset = len(self.log_dir) + base_offset = len(self.log_dir.rstrip('/')) + 1 for start, c in i: offset = base_offset + start if c == '%Y': diff --git a/test/unit/stats/test_log_uploader.py b/test/unit/stats/test_log_uploader.py index 3585111750..145da9d710 100644 --- a/test/unit/stats/test_log_uploader.py +++ b/test/unit/stats/test_log_uploader.py @@ -13,16 +13,147 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: Tests +# TODO: More tests import unittest +import os +from datetime import datetime +from tempfile import mkdtemp +from shutil import rmtree + from swift.stats import log_uploader +import logging +logging.basicConfig(level=logging.DEBUG) +LOGGER = logging.getLogger() + +DEFAULT_GLOB = '%Y%m%d%H' class TestLogUploader(unittest.TestCase): - def test_placeholder(self): - pass + def test_upload_all_logs(self): + + class MockInternalProxy(): + + def create_container(self, *args, **kwargs): + pass + + class MonkeyLogUploader(log_uploader.LogUploader): + + def __init__(self, conf, logger=LOGGER): + self.log_dir = conf['log_dir'] + self.filename_format = conf.get('filename_format', DEFAULT_GLOB) + self.new_log_cutoff = 0 + self.logger = logger + self.internal_proxy = MockInternalProxy() + self.swift_account = '' + self.container_name = '' + + self.uploaded_files = [] + + def upload_one_log(self, filename, year, month, day, hour): + d = {'year': year, 'month': month, 'day': day, 'hour': hour} + self.uploaded_files.append((filename, d)) + + tmpdir = mkdtemp() + try: + today = datetime.now() + year = today.year + month = today.month + day = today.day + + today_str = today.strftime('%Y%m%d') + time_strs = [] + for i in range(24): + time_strs.append('%s%0.2d' % (today_str, i)) + for ts in time_strs: + open(os.path.join(tmpdir, ts), 'w').close() + + conf = {'log_dir': tmpdir} + uploader = MonkeyLogUploader(conf) + uploader.upload_all_logs() + self.assertEquals(len(uploader.uploaded_files), 24) + for i, file_date in enumerate(sorted(uploader.uploaded_files)): + d = {'year': year, 'month': month, 'day': day, 'hour': i} + for k, v in d.items(): + d[k] = '%0.2d' % v + expected = (os.path.join(tmpdir, '%s%0.2d' % (today_str, i)), d) + self.assertEquals(file_date, expected) + finally: + rmtree(tmpdir) + + tmpdir = mkdtemp() + try: + today = datetime.now() + year = today.year + month = today.month + day = today.day + + today_str = today.strftime('%Y%m%d') + time_strs = [] + for i in range(24): + time_strs.append('%s-%0.2d00' % (today_str, i)) + for ts in time_strs: + open(os.path.join(tmpdir, 'swift-blah_98764.%s-2400.tar.gz' % ts), 'w').close() + + open(os.path.join(tmpdir, 'swift.blah_98764.%s-2400.tar.gz' % ts), 'w').close() + open(os.path.join(tmpdir, 'swift-blah_98764.%s-2400.tar.g' % ts), 'w').close() + open(os.path.join(tmpdir, 'swift-blah_201102160100.%s-2400.tar.gz' % + '201102160100'), 'w').close() + + conf = { + 'log_dir': '%s/' % tmpdir, + 'filename_format': 'swift-blah_98764.%Y%m%d-%H*.tar.gz', + } + uploader = MonkeyLogUploader(conf) + uploader.upload_all_logs() + self.assertEquals(len(uploader.uploaded_files), 24) + for i, file_date in enumerate(sorted(uploader.uploaded_files)): + filename, date_dict = file_date + filename = os.path.basename(filename) + self.assert_(today_str in filename, filename) + self.assert_(filename.startswith('swift'), filename) + self.assert_(filename.endswith('tar.gz'), filename) + d = {'year': year, 'month': month, 'day': day, 'hour': i} + for k, v in d.items(): + d[k] = '%0.2d' % v + self.assertEquals(d, date_dict) + finally: + rmtree(tmpdir) + + tmpdir = mkdtemp() + try: + today = datetime.now() + year = today.year + month = today.month + day = today.day + + today_str = today.strftime('%Y%m%d') + time_strs = [] + for i in range(24): + time_strs.append('%s%0.2d' % (today_str, i)) + for i, ts in enumerate(time_strs): + open(os.path.join(tmpdir, '%s.%s.log' % (i, ts)), 'w').close() + + conf = { + 'log_dir': tmpdir, + 'filename_format': '*.%Y%m%d%H.log', + } + uploader = MonkeyLogUploader(conf) + uploader.upload_all_logs() + self.assertEquals(len(uploader.uploaded_files), 24) + for i, file_date in enumerate(sorted(uploader.uploaded_files)): + d = {'year': year, 'month': month, 'day': day, 'hour': i} + for k, v in d.items(): + d[k] = '%0.2d' % v + expected = (os.path.join(tmpdir, '%s.%s%0.2d.log' % + (i, today_str, i)), d) + # TODO: support wildcards before the date pattern + # (i.e. relative offsets) + #print file_date + #self.assertEquals(file_date, expected) + finally: + rmtree(tmpdir) if __name__ == '__main__': From 841a894955038cc54dd1b9b4d029e5390fc7c7f3 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 16 Feb 2011 19:52:22 -0600 Subject: [PATCH 2/2] pep8 --- test/unit/stats/test_log_uploader.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/test/unit/stats/test_log_uploader.py b/test/unit/stats/test_log_uploader.py index 145da9d710..b82e0ce02c 100644 --- a/test/unit/stats/test_log_uploader.py +++ b/test/unit/stats/test_log_uploader.py @@ -29,6 +29,7 @@ LOGGER = logging.getLogger() DEFAULT_GLOB = '%Y%m%d%H' + class TestLogUploader(unittest.TestCase): def test_upload_all_logs(self): @@ -42,13 +43,14 @@ class TestLogUploader(unittest.TestCase): def __init__(self, conf, logger=LOGGER): self.log_dir = conf['log_dir'] - self.filename_format = conf.get('filename_format', DEFAULT_GLOB) + self.filename_format = conf.get('filename_format', + DEFAULT_GLOB) self.new_log_cutoff = 0 self.logger = logger self.internal_proxy = MockInternalProxy() self.swift_account = '' self.container_name = '' - + self.uploaded_files = [] def upload_one_log(self, filename, year, month, day, hour): @@ -77,7 +79,8 @@ class TestLogUploader(unittest.TestCase): d = {'year': year, 'month': month, 'day': day, 'hour': i} for k, v in d.items(): d[k] = '%0.2d' % v - expected = (os.path.join(tmpdir, '%s%0.2d' % (today_str, i)), d) + expected = (os.path.join(tmpdir, '%s%0.2d' % + (today_str, i)), d) self.assertEquals(file_date, expected) finally: rmtree(tmpdir) @@ -94,11 +97,15 @@ class TestLogUploader(unittest.TestCase): for i in range(24): time_strs.append('%s-%0.2d00' % (today_str, i)) for ts in time_strs: - open(os.path.join(tmpdir, 'swift-blah_98764.%s-2400.tar.gz' % ts), 'w').close() + open(os.path.join(tmpdir, 'swift-blah_98764.%s-2400.tar.gz' % + ts), 'w').close() - open(os.path.join(tmpdir, 'swift.blah_98764.%s-2400.tar.gz' % ts), 'w').close() - open(os.path.join(tmpdir, 'swift-blah_98764.%s-2400.tar.g' % ts), 'w').close() - open(os.path.join(tmpdir, 'swift-blah_201102160100.%s-2400.tar.gz' % + open(os.path.join(tmpdir, 'swift.blah_98764.%s-2400.tar.gz' % ts), + 'w').close() + open(os.path.join(tmpdir, 'swift-blah_98764.%s-2400.tar.g' % ts), + 'w').close() + open(os.path.join(tmpdir, + 'swift-blah_201102160100.%s-2400.tar.gz' % '201102160100'), 'w').close() conf = { @@ -146,7 +153,7 @@ class TestLogUploader(unittest.TestCase): d = {'year': year, 'month': month, 'day': day, 'hour': i} for k, v in d.items(): d[k] = '%0.2d' % v - expected = (os.path.join(tmpdir, '%s.%s%0.2d.log' % + expected = (os.path.join(tmpdir, '%s.%s%0.2d.log' % (i, today_str, i)), d) # TODO: support wildcards before the date pattern # (i.e. relative offsets)