x/stackalytics
Code Issues Proposed changes

Poll open reviews by chunks instead of one by one

Browse Source 
Closes bug 1213358

Change-Id: Ibcf0e3a2d48e249af68e7d2d1ac27b6507510e47
This commit is contained in:
Ilya Shakhat
2013-08-19 18:07:51 +04:00
parent 5a7da39e7d
commit 42aaa1d77e
2 changed files with 32 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
stackalytics/processor/main.py
View File

@@ -73,7 +73,7 @@ def _record_typer(record_iterator, record_type):
yield record
def process_repo(repo, runtime_storage, record_processor_inst, open_reviews):
def process_repo(repo, runtime_storage, record_processor_inst):
uri = repo['uri']
LOG.debug('Processing repo uri %s' % uri)
@@ -104,7 +104,7 @@ def process_repo(repo, runtime_storage, record_processor_inst, open_reviews):
rcs_key = 'rcs:' + str(urllib.quote_plus(uri) + ':' + branch)
last_id = runtime_storage.get_by_key(rcs_key)
review_iterator = rcs_inst.log(branch, last_id, open_reviews)
review_iterator = rcs_inst.log(branch, last_id)
review_iterator_typed = _record_typer(review_iterator, 'review')
processed_review_iterator = record_processor_inst.process(
review_iterator_typed)
@@ -114,35 +114,13 @@ def process_repo(repo, runtime_storage, record_processor_inst, open_reviews):
runtime_storage.set_by_key(rcs_key, last_id)
def _open_reviews(runtime_storage_inst):
LOG.debug('Collecting list of open reviews from')
open_reviews = {}
for record in runtime_storage_inst.get_all_records():
if record['record_type'] == 'review':
if record['open']:
module = record['module']
if module not in open_reviews:
open_reviews[module] = set([record['sortKey']])
else:
open_reviews[module].add(record['sortKey'])
return open_reviews
def update_repos(runtime_storage_inst):
repos = runtime_storage_inst.get_by_key('repos')
record_processor_inst = record_processor.RecordProcessor(
runtime_storage_inst)
open_reviews = _open_reviews(runtime_storage_inst)
for repo in repos:
module = repo['module']
open_reviews_repo = set()
if module in open_reviews:
open_reviews_repo = open_reviews[module]
process_repo(repo, runtime_storage_inst, record_processor_inst,
open_reviews_repo)
process_repo(repo, runtime_storage_inst, record_processor_inst)
def apply_corrections(uri, runtime_storage_inst):