diff --git a/swift/stats/log_uploader.py b/swift/stats/log_uploader.py index d87d799324..c88eeb9c0a 100644 --- a/swift/stats/log_uploader.py +++ b/swift/stats/log_uploader.py @@ -79,7 +79,7 @@ class LogUploader(Daemon): i = [(self.filename_format.index(c), c) for c in '%Y %m %d %H'.split()] i.sort() year_offset = month_offset = day_offset = hour_offset = None - base_offset = len(self.log_dir) + base_offset = len(self.log_dir.rstrip('/')) + 1 for start, c in i: offset = base_offset + start if c == '%Y': diff --git a/test/unit/stats/test_log_uploader.py b/test/unit/stats/test_log_uploader.py index 3585111750..b82e0ce02c 100644 --- a/test/unit/stats/test_log_uploader.py +++ b/test/unit/stats/test_log_uploader.py @@ -13,16 +13,154 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: Tests +# TODO: More tests import unittest +import os +from datetime import datetime +from tempfile import mkdtemp +from shutil import rmtree + from swift.stats import log_uploader +import logging +logging.basicConfig(level=logging.DEBUG) +LOGGER = logging.getLogger() + +DEFAULT_GLOB = '%Y%m%d%H' + class TestLogUploader(unittest.TestCase): - def test_placeholder(self): - pass + def test_upload_all_logs(self): + + class MockInternalProxy(): + + def create_container(self, *args, **kwargs): + pass + + class MonkeyLogUploader(log_uploader.LogUploader): + + def __init__(self, conf, logger=LOGGER): + self.log_dir = conf['log_dir'] + self.filename_format = conf.get('filename_format', + DEFAULT_GLOB) + self.new_log_cutoff = 0 + self.logger = logger + self.internal_proxy = MockInternalProxy() + self.swift_account = '' + self.container_name = '' + + self.uploaded_files = [] + + def upload_one_log(self, filename, year, month, day, hour): + d = {'year': year, 'month': month, 'day': day, 'hour': hour} + self.uploaded_files.append((filename, d)) + + tmpdir = mkdtemp() + try: + today = datetime.now() + year = today.year + month = today.month + day = today.day + + today_str = today.strftime('%Y%m%d') + time_strs = [] + for i in range(24): + time_strs.append('%s%0.2d' % (today_str, i)) + for ts in time_strs: + open(os.path.join(tmpdir, ts), 'w').close() + + conf = {'log_dir': tmpdir} + uploader = MonkeyLogUploader(conf) + uploader.upload_all_logs() + self.assertEquals(len(uploader.uploaded_files), 24) + for i, file_date in enumerate(sorted(uploader.uploaded_files)): + d = {'year': year, 'month': month, 'day': day, 'hour': i} + for k, v in d.items(): + d[k] = '%0.2d' % v + expected = (os.path.join(tmpdir, '%s%0.2d' % + (today_str, i)), d) + self.assertEquals(file_date, expected) + finally: + rmtree(tmpdir) + + tmpdir = mkdtemp() + try: + today = datetime.now() + year = today.year + month = today.month + day = today.day + + today_str = today.strftime('%Y%m%d') + time_strs = [] + for i in range(24): + time_strs.append('%s-%0.2d00' % (today_str, i)) + for ts in time_strs: + open(os.path.join(tmpdir, 'swift-blah_98764.%s-2400.tar.gz' % + ts), 'w').close() + + open(os.path.join(tmpdir, 'swift.blah_98764.%s-2400.tar.gz' % ts), + 'w').close() + open(os.path.join(tmpdir, 'swift-blah_98764.%s-2400.tar.g' % ts), + 'w').close() + open(os.path.join(tmpdir, + 'swift-blah_201102160100.%s-2400.tar.gz' % + '201102160100'), 'w').close() + + conf = { + 'log_dir': '%s/' % tmpdir, + 'filename_format': 'swift-blah_98764.%Y%m%d-%H*.tar.gz', + } + uploader = MonkeyLogUploader(conf) + uploader.upload_all_logs() + self.assertEquals(len(uploader.uploaded_files), 24) + for i, file_date in enumerate(sorted(uploader.uploaded_files)): + filename, date_dict = file_date + filename = os.path.basename(filename) + self.assert_(today_str in filename, filename) + self.assert_(filename.startswith('swift'), filename) + self.assert_(filename.endswith('tar.gz'), filename) + d = {'year': year, 'month': month, 'day': day, 'hour': i} + for k, v in d.items(): + d[k] = '%0.2d' % v + self.assertEquals(d, date_dict) + finally: + rmtree(tmpdir) + + tmpdir = mkdtemp() + try: + today = datetime.now() + year = today.year + month = today.month + day = today.day + + today_str = today.strftime('%Y%m%d') + time_strs = [] + for i in range(24): + time_strs.append('%s%0.2d' % (today_str, i)) + for i, ts in enumerate(time_strs): + open(os.path.join(tmpdir, '%s.%s.log' % (i, ts)), 'w').close() + + conf = { + 'log_dir': tmpdir, + 'filename_format': '*.%Y%m%d%H.log', + } + uploader = MonkeyLogUploader(conf) + uploader.upload_all_logs() + self.assertEquals(len(uploader.uploaded_files), 24) + for i, file_date in enumerate(sorted(uploader.uploaded_files)): + d = {'year': year, 'month': month, 'day': day, 'hour': i} + for k, v in d.items(): + d[k] = '%0.2d' % v + expected = (os.path.join(tmpdir, '%s.%s%0.2d.log' % + (i, today_str, i)), d) + # TODO: support wildcards before the date pattern + # (i.e. relative offsets) + #print file_date + #self.assertEquals(file_date, expected) + finally: + rmtree(tmpdir) if __name__ == '__main__':