documentation clarification and pep8 fixes

This commit is contained in:
John Dickinson 2010-09-30 15:48:56 -05:00
parent b44296d146
commit 880f5af2ff
4 changed files with 44 additions and 23 deletions

View File

@ -4,7 +4,7 @@ Swift stats system
The swift stats system is composed of three parts parts: log creation, log
uploading, and log processing. The system handles two types of logs (access
and storage stats), but it can be extended to handle other types of logs.
and account stats), but it can be extended to handle other types of logs.
---------
Log Types
@ -20,21 +20,21 @@ is made on August 4, 2010 at 12:37 gets logged in a file named 2010080412.
This allows easy log rotation and easy per-hour log processing.
******************
Storage stats logs
Account stats logs
******************
Storage logs (also referred to as stats logs) are generated by a stats system
process. swift-account-stats-logger runs on each account server (via cron) and
walks the filesystem looking for account databases. When an account database
is found, the logger selects the account hash, bytes_used, container_count,
and object_count. These values are then written out as one line in a csv file.
One csv file is produced for every run of swift-account-stats-logger. This
means that, system wide, one csv file is produced for every storage node.
Rackspace runs the account stats logger every hour. Therefore, in a cluster of
ten account servers, ten csv files are produced every hour. Also, every
account will have one entry for every replica in the system. On average, there
will be three copies of each account in the aggregate of all account stat csv
files created in one system-wide run.
Account stats logs are generated by a stats system process.
swift-account-stats-logger runs on each account server (via cron) and walks
the filesystem looking for account databases. When an account database is
found, the logger selects the account hash, bytes_used, container_count, and
object_count. These values are then written out as one line in a csv file. One
csv file is produced for every run of swift-account-stats-logger. This means
that, system wide, one csv file is produced for every storage node. Rackspace
runs the account stats logger every hour. Therefore, in a cluster of ten
account servers, ten csv files are produced every hour. Also, every account
will have one entry for every replica in the system. On average, there will be
three copies of each account in the aggregate of all account stat csv files
created in one system-wide run.
----------------------
Log Processing plugins

View File

@ -16,6 +16,7 @@
import zlib
import struct
class CompressingFileReader(object):
'''
Wraps a file object and provides a read method that returns gzip'd data.
@ -35,6 +36,7 @@ class CompressingFileReader(object):
:param file_obj: File object to read from
:param compresslevel: compression level
'''
def __init__(self, file_obj, compresslevel=9):
self._f = file_obj
self._compressor = zlib.compressobj(compresslevel,

View File

@ -20,13 +20,27 @@ from json import loads as json_loads
from swift.common.compressing_file_reader import CompressingFileReader
from swift.proxy.server import BaseApplication
class MemcacheStub(object):
def get(self, *a, **kw): return None
def set(self, *a, **kw): return None
def incr(self, *a, **kw): return 0
def delete(self, *a, **kw): return None
def set_multi(self, *a, **kw): return None
def get_multi(self, *a, **kw): return []
def get(self, *a, **kw):
return None
def set(self, *a, **kw):
return None
def incr(self, *a, **kw):
return 0
def delete(self, *a, **kw):
return None
def set_multi(self, *a, **kw):
return None
def get_multi(self, *a, **kw):
return []
class InternalProxy(object):
"""
@ -38,6 +52,7 @@ class InternalProxy(object):
:param logger: logger to log requests to
:param retries: number of times to retry each request
"""
def __init__(self, proxy_server_conf=None, logger=None, retries=0):
self.upload_app = BaseApplication(proxy_server_conf,
memcache=MemcacheStub(),
@ -56,6 +71,7 @@ class InternalProxy(object):
:param object_name: name of object being uploaded
:param compress: if True, compresses object as it is uploaded
:param content_type: content-type of object
:param etag: etag for object to check successful upload
:returns: True if successful, False otherwise
"""
log_create_pattern = '/v1/%s/%s/%s' % (account, container, object_name)
@ -72,7 +88,8 @@ class InternalProxy(object):
if hasattr(source_file, 'read'):
compressed_file = CompressingFileReader(source_file)
else:
compressed_file = CompressingFileReader(open(source_file, 'rb'))
compressed_file = CompressingFileReader(
open(source_file, 'rb'))
req.body_file = compressed_file
else:
if not hasattr(source_file, 'read'):

View File

@ -552,6 +552,7 @@ def cache_from_env(env):
"""
return item_from_env(env, 'swift.cache')
def readconf(conf, section_name, log_name=None):
"""
Read config file and return config items as a dict
@ -571,7 +572,8 @@ def readconf(conf, section_name, log_name=None):
if c.has_section(section_name):
conf = dict(c.items(section_name))
else:
print "Unable to find %s config section in %s" % (section_name, conf)
print "Unable to find %s config section in %s" % (section_name,
conf)
sys.exit(1)
if "log_name" not in conf:
if log_name is not None: