x/stackalytics
Code Issues Proposed changes

Minimize memory consumption in dashboard

Browse Source 
Memory consumption optimized from 2.7G down to 1.5G on full dataset:
 * Record index is not necessary in runtime storage since it it
   needed for update operations only
 * Store compact records as tuples instead of dicts

Co-author: Yuriy Taraday <yorik.sar@gmail.com>

Change-Id: I9a361e9373b5ed4fada527959a604cf09a02e5b9
This commit is contained in:
Ilya Shakhat
2014-06-09 16:19:28 +04:00
parent aeb6c67498
commit edd80c26f0
2 changed files with 33 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
stackalytics/processor/runtime_storage.py
View File

@@ -65,17 +65,19 @@ class MemcachedStorage(RuntimeStorage):
if stripped:
storage_uri = stripped.split(',')
self.memcached = memcache.Client(storage_uri)
self._build_index()
self._init_user_count()
self.record_index = {}
else:
raise Exception('Invalid storage uri %s' % uri)
def _build_index(self):
self.record_index = {}
def _build_index_lazily(self):
if self.record_index:
return
for record in self.get_all_records():
self.record_index[record['primary_key']] = record['record_id']
def set_records(self, records_iterator, merge_handler=None):
self._build_index_lazily()
for record in records_iterator:
if record['primary_key'] in self.record_index:
# update
@@ -103,6 +105,7 @@ class MemcachedStorage(RuntimeStorage):
self._commit_update(record_id)
def apply_corrections(self, corrections_iterator):
self._build_index_lazily()
for correction in corrections_iterator:
if correction['primary_key'] not in self.record_index:
continue