openstack/swift
Code Issues Proposed changes

added lazy load of internal proxy to log processor (you were right clay)

Browse Source
This commit is contained in:
John Dickinson
2010-10-08 17:20:43 -05:00
parent 86faf56ee1
commit 9572fa62f5
2 changed files with 20 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
swift/stats/log_processor.py
View File

@@ -39,21 +39,13 @@ class LogProcessor(object):
"""Load plugins, process logs""" """Load plugins, process logs"""
def __init__(self, conf, logger): def __init__(self, conf, logger):
stats_conf = conf.get('log-processor', {})
if isinstance(logger, tuple): if isinstance(logger, tuple):
self.logger = get_logger(*logger) self.logger = get_logger(*logger)
else: else:
self.logger = logger self.logger = logger
proxy_server_conf_loc = stats_conf.get('proxy_server_conf', self.conf = conf
'/etc/swift/proxy-server.conf') self._internal_proxy = None
self.proxy_server_conf = appconfig(
'config:%s' % proxy_server_conf_loc,
name='proxy-server')
self.internal_proxy = InternalProxy(self.proxy_server_conf,
self.logger,
retries=3)
# load the processing plugins # load the processing plugins
self.plugins = {} self.plugins = {}
@@ -69,6 +61,21 @@ class LogProcessor(object):
self.plugins[plugin_name]['instance'] = klass(plugin_conf) self.plugins[plugin_name]['instance'] = klass(plugin_conf)
self.logger.debug('Loaded plugin "%s"' % plugin_name) self.logger.debug('Loaded plugin "%s"' % plugin_name)
@property
def internal_proxy(self):
if self._internal_proxy is None:
stats_conf = self.conf.get('log-processor', {})
proxy_server_conf_loc = stats_conf.get('proxy_server_conf',
'/etc/swift/proxy-server.conf')
proxy_server_conf = appconfig(
'config:%s' % proxy_server_conf_loc,
name='proxy-server')
self._internal_proxy = InternalProxy(proxy_server_conf,
self.logger,
retries=3)
else:
return self._internal_proxy
def process_one_file(self, plugin_name, account, container, object_name): def process_one_file(self, plugin_name, account, container, object_name):
self.logger.info('Processing %s/%s/%s with plugin "%s"' % (account, self.logger.info('Processing %s/%s/%s with plugin "%s"' % (account,
container, container,

6
test/unit/stats/test_log_processor.py
View File

@@ -127,7 +127,7 @@ class TestLogProcessor(unittest.TestCase):
def test_get_container_listing(self): def test_get_container_listing(self):
p = log_processor.LogProcessor(self.proxy_config, DumbLogger()) p = log_processor.LogProcessor(self.proxy_config, DumbLogger())
p.internal_proxy = DumbInternalProxy() p._internal_proxy = DumbInternalProxy()
result = p.get_container_listing('a', 'foo') result = p.get_container_listing('a', 'foo')
expected = ['2010/03/14/13/obj1'] expected = ['2010/03/14/13/obj1']
self.assertEquals(result, expected) self.assertEquals(result, expected)
@@ -148,7 +148,7 @@ class TestLogProcessor(unittest.TestCase):
def test_get_object_data(self): def test_get_object_data(self):
p = log_processor.LogProcessor(self.proxy_config, DumbLogger()) p = log_processor.LogProcessor(self.proxy_config, DumbLogger())
p.internal_proxy = DumbInternalProxy() p._internal_proxy = DumbInternalProxy()
result = list(p.get_object_data('a', 'c', 'o', False)) result = list(p.get_object_data('a', 'c', 'o', False))
expected = ['obj','data'] expected = ['obj','data']
self.assertEquals(result, expected) self.assertEquals(result, expected)
@@ -163,7 +163,7 @@ class TestLogProcessor(unittest.TestCase):
'swift.stats.stats_processor.StatsLogProcessor' 'swift.stats.stats_processor.StatsLogProcessor'
}}) }})
p = log_processor.LogProcessor(stats_proxy_config, DumbLogger()) p = log_processor.LogProcessor(stats_proxy_config, DumbLogger())
p.internal_proxy = DumbInternalProxy() p._internal_proxy = DumbInternalProxy()
def get_object_data(*a,**kw): def get_object_data(*a,**kw):
return [self.stats_test_line] return [self.stats_test_line]
p.get_object_data = get_object_data p.get_object_data = get_object_data