x/stackalytics
Code Issues Proposed changes

Fix dump utility and improve its logging

Browse Source 
Change-Id: I04b795743290c1d7554334e18dd35efa1d7d9887
This commit is contained in:
Ilya Shakhat
2014-06-11 17:10:20 +04:00
parent 74af0d8feb
commit f4458ec5ae
1 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
stackalytics/processor/dump.py
View File

@@ -35,7 +35,7 @@ OPTS = [
help='Restore data into memcached'), help='Restore data into memcached'),
cfg.StrOpt('file', cfg.StrOpt('file',
short='f', short='f',
help='File name where to store data'), help='The name of file to store data'),
cfg.StrOpt('min-compress-len', default=0, cfg.StrOpt('min-compress-len', default=0,
short='m', short='m',
help='The threshold length to kick in auto-compression'), help='The threshold length to kick in auto-compression'),
@@ -60,6 +60,7 @@ def read_records_from_fd(fd):
def store_bucket(memcached_inst, bucket): def store_bucket(memcached_inst, bucket):
LOG.debug('Store bucket of records into memcached')
res = memcached_inst.set_multi(bucket, res = memcached_inst.set_multi(bucket,
min_compress_len=cfg.CONF.min_compress_len) min_compress_len=cfg.CONF.min_compress_len)
if res: if res:
@@ -71,11 +72,11 @@ def import_data(memcached_inst, fd):
LOG.info('Importing data into memcached') LOG.info('Importing data into memcached')
bucket = {} bucket = {}
for key, value in read_records_from_fd(fd): for key, value in read_records_from_fd(fd):
if len(bucket) < BULK_READ_SIZE: LOG.debug('Reading record key %s, value %s', key, value)
bucket[key] = value if len(bucket) == BULK_READ_SIZE:
else:
store_bucket(memcached_inst, bucket) store_bucket(memcached_inst, bucket)
bucket = {} bucket = {}
bucket[key] = value
if bucket: if bucket:
store_bucket(memcached_inst, bucket) store_bucket(memcached_inst, bucket)