x/stackalytics
Code Issues Proposed changes

Fixed bunch deletion from memcached

Browse Source 
Closes bug 1209211

Change-Id: Ia0cf06e29e1e5e8e9bfb217b20b4b83120b54264
This commit is contained in:
Ilya Shakhat
2013-08-07 16:37:44 +04:00
parent 6779de9dc2
commit 164448cff1
2 changed files with 10 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
etc/test_default_data.json
View File

@@ -45,7 +45,7 @@
} }
], ],
"project_sources": [ "x-project_sources": [
{ {
"organization": "openstack-dev", "organization": "openstack-dev",
"project_type": "openstack", "project_type": "openstack",

15
stackalytics/processor/runtime_storage.py
View File

@@ -22,6 +22,7 @@ from stackalytics.openstack.common import log as logging
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
BULK_READ_SIZE = 64 BULK_READ_SIZE = 64
BULK_DELETE_SIZE = 4096
RECORD_ID_PREFIX = 'record:' RECORD_ID_PREFIX = 'record:'
UPDATE_ID_PREFIX = 'update:' UPDATE_ID_PREFIX = 'update:'
MEMCACHED_URI_PREFIX = r'^memcached:\/\/' MEMCACHED_URI_PREFIX = r'^memcached:\/\/'
@@ -51,7 +52,6 @@ class RuntimeStorage(object):
class MemcachedStorage(RuntimeStorage): class MemcachedStorage(RuntimeStorage):
def __init__(self, uri): def __init__(self, uri):
super(MemcachedStorage, self).__init__(uri) super(MemcachedStorage, self).__init__(uri)
@@ -155,8 +155,11 @@ class MemcachedStorage(RuntimeStorage):
min_update = n min_update = n
first_valid_update = self.memcached.get('first_valid_update') or 0 first_valid_update = self.memcached.get('first_valid_update') or 0
self.memcached.delete_multi(range(first_valid_update, min_update), for delete_id_set in self._make_range(first_valid_update, min_update,
key_prefix=UPDATE_ID_PREFIX) BULK_DELETE_SIZE):
if not self.memcached.delete_multi(delete_id_set,
key_prefix=UPDATE_ID_PREFIX):
raise Exception('Failed to delete from memcache')
self.memcached.set('first_valid_update', min_update) self.memcached.set('first_valid_update', min_update)
def _get_update_count(self): def _get_update_count(self):
@@ -180,10 +183,10 @@ class MemcachedStorage(RuntimeStorage):
def _make_range(self, start, stop, step): def _make_range(self, start, stop, step):
i = start i = start
for i in range(start, stop, step): for i in xrange(start, stop, step):
yield range(i, i + step) yield xrange(i, i + step)
if (stop - start) % step > 0: if (stop - start) % step > 0:
yield range(i, stop) yield xrange(i, stop)
def get_all_records(self): def get_all_records(self):
for record_id_set in self._make_range(0, self._get_record_count(), for record_id_set in self._make_range(0, self._get_record_count(),