x/stackalytics
Code Issues Proposed changes

Merge "Fixed bunch deletion from memcached"

Browse Source
This commit is contained in:
Jenkins
2013-08-08 14:57:39 +00:00
committed by Gerrit Code Review
parent 661c1ec71a 164448cff1
commit 07c2103924
2 changed files with 10 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
etc/test_default_data.json
View File

@@ -45,7 +45,7 @@
} }
], ],
"project_sources": [ "x-project_sources": [
{ {
"organization": "openstack-dev", "organization": "openstack-dev",
"project_type": "openstack", "project_type": "openstack",

15
stackalytics/processor/runtime_storage.py
View File

@@ -22,6 +22,7 @@ from stackalytics.openstack.common import log as logging
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
BULK_READ_SIZE = 64 BULK_READ_SIZE = 64
BULK_DELETE_SIZE = 4096
RECORD_ID_PREFIX = 'record:' RECORD_ID_PREFIX = 'record:'
UPDATE_ID_PREFIX = 'update:' UPDATE_ID_PREFIX = 'update:'
MEMCACHED_URI_PREFIX = r'^memcached:\/\/' MEMCACHED_URI_PREFIX = r'^memcached:\/\/'
@@ -51,7 +52,6 @@ class RuntimeStorage(object):
class MemcachedStorage(RuntimeStorage): class MemcachedStorage(RuntimeStorage):
def __init__(self, uri): def __init__(self, uri):
super(MemcachedStorage, self).__init__(uri) super(MemcachedStorage, self).__init__(uri)
@@ -156,8 +156,11 @@ class MemcachedStorage(RuntimeStorage):
min_update = n min_update = n
first_valid_update = self.memcached.get('first_valid_update') or 0 first_valid_update = self.memcached.get('first_valid_update') or 0
self.memcached.delete_multi(range(first_valid_update, min_update), for delete_id_set in self._make_range(first_valid_update, min_update,
key_prefix=UPDATE_ID_PREFIX) BULK_DELETE_SIZE):
if not self.memcached.delete_multi(delete_id_set,
key_prefix=UPDATE_ID_PREFIX):
raise Exception('Failed to delete from memcache')
self.memcached.set('first_valid_update', min_update) self.memcached.set('first_valid_update', min_update)
def _get_update_count(self): def _get_update_count(self):
@@ -181,10 +184,10 @@ class MemcachedStorage(RuntimeStorage):
def _make_range(self, start, stop, step): def _make_range(self, start, stop, step):
i = start i = start
for i in range(start, stop, step): for i in xrange(start, stop, step):
yield range(i, i + step) yield xrange(i, i + step)
if (stop - start) % step > 0: if (stop - start) % step > 0:
yield range(i, stop) yield xrange(i, stop)
def get_all_records(self): def get_all_records(self):
for record_id_set in self._make_range(0, self._get_record_count(), for record_id_set in self._make_range(0, self._get_record_count(),