x/stackalytics
Code Issues Proposed changes

Fixed lost record update with updated user profile

Browse Source 
Resolves bug 1220691

Change-Id: I910f6c7de3a35efa5f4bb83670f10ebe6e8f07a6
This commit is contained in:
Ilya Shakhat
2013-09-04 17:52:44 +04:00
parent f60297474e
commit a09e002107
2 changed files with 21 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
stackalytics/processor/default_data_processor.py
View File

@@ -142,9 +142,8 @@ def process(runtime_storage_inst, default_data, sources_root, force_update):
# need to iterate over full view of records and generate valid # need to iterate over full view of records and generate valid
# users profiles # users profiles
LOG.debug('Iterate all records to create valid users profiles') LOG.debug('Iterate all records to create valid users profiles')
for record in record_processor_inst.update( for record in runtime_storage_inst.get_all_records():
runtime_storage_inst.get_all_records(), release_index): record_processor_inst.update_user(record)
pass
# update records according to generated users profiles # update records according to generated users profiles
LOG.debug('Update all records according to users profiles') LOG.debug('Update all records according to users profiles')
updated_records = record_processor_inst.update( updated_records = record_processor_inst.update(

38
stackalytics/processor/record_processor.py
View File

@@ -93,7 +93,7 @@ class RecordProcessor(object):
def _get_independent(self): def _get_independent(self):
return self.domains_index[''] return self.domains_index['']
def _update_user(self, user, email): def _update_user_profile(self, user, email):
LOG.debug('Add email %s to user %s', email, user['user_id']) LOG.debug('Add email %s to user %s', email, user['user_id'])
user['emails'].append(email) user['emails'].append(email)
company_name = self._get_company_by_email(email) company_name = self._get_company_by_email(email)
@@ -104,25 +104,22 @@ class RecordProcessor(object):
user['companies'][0]['company_name'] = company_name user['companies'][0]['company_name'] = company_name
self.updated_users.add(user['user_id']) self.updated_users.add(user['user_id'])
def _update_record_and_user(self, record): def update_user(self, record):
email = record['author_email'].lower() email = record['author_email']
record['author_email'] = email
if email in self.users_index: if email in self.users_index:
user = self.users_index[email] user = self.users_index[email]
record['launchpad_id'] = user['launchpad_id']
else: else:
if ('launchpad_id' in record) and (record['launchpad_id']): if ('launchpad_id' in record) and (record['launchpad_id']):
launchpad_id = record['launchpad_id'] launchpad_id = record['launchpad_id']
user_name = record['author_name'] user_name = record['author_name']
else: else:
launchpad_id, user_name = self._get_lp_info(email) launchpad_id, user_name = self._get_lp_info(email)
record['launchpad_id'] = launchpad_id
if (launchpad_id) and (launchpad_id in self.users_index): if (launchpad_id) and (launchpad_id in self.users_index):
# merge emails # merge emails
user = self.users_index[launchpad_id] user = self.users_index[launchpad_id]
self._update_user(user, email) self._update_user_profile(user, email)
else: else:
# create new # create new
if not user_name: if not user_name:
@@ -134,24 +131,27 @@ class RecordProcessor(object):
if user['launchpad_id']: if user['launchpad_id']:
self.users_index[user['launchpad_id']] = user self.users_index[user['launchpad_id']] = user
record['user_id'] = user['user_id'] return user
company_by_user = self._find_company(user['companies'], record['date']) def _update_record_and_user(self, record):
if company_by_user == '*robots': user = self.update_user(record)
# don't map robots by email
company = company_by_user record['user_id'] = user['user_id']
else: record['launchpad_id'] = user['launchpad_id']
company = self._get_company_by_email(email)
if not company:
company = company_by_user
record['company_name'] = company
if ('user_name' in user) and (user['user_name']): if ('user_name' in user) and (user['user_name']):
record['author_name'] = user['user_name'] record['author_name'] = user['user_name']
company = self._find_company(user['companies'], record['date'])
if company != '*robots':
company = (self._get_company_by_email(record['author_email'])
or company)
record['company_name'] = company
def _process_commit(self, record): def _process_commit(self, record):
record['primary_key'] = record['commit_id'] record['primary_key'] = record['commit_id']
record['loc'] = record['lines_added'] + record['lines_deleted'] record['loc'] = record['lines_added'] + record['lines_deleted']
record['author_email'] = record['author_email'].lower()
self._update_record_and_user(record) self._update_record_and_user(record)
@@ -169,7 +169,7 @@ class RecordProcessor(object):
review['primary_key'] = review['id'] review['primary_key'] = review['id']
review['launchpad_id'] = owner['username'] review['launchpad_id'] = owner['username']
review['author_name'] = owner['name'] review['author_name'] = owner['name']
review['author_email'] = owner['email'] review['author_email'] = owner['email'].lower()
review['date'] = record['createdOn'] review['date'] = record['createdOn']
self._update_record_and_user(review) self._update_record_and_user(review)
@@ -199,7 +199,7 @@ class RecordProcessor(object):
mark['type']) mark['type'])
mark['launchpad_id'] = reviewer['username'] mark['launchpad_id'] = reviewer['username']
mark['author_name'] = reviewer['name'] mark['author_name'] = reviewer['name']
mark['author_email'] = reviewer['email'] mark['author_email'] = reviewer['email'].lower()
mark['module'] = module mark['module'] = module
mark['review_id'] = review_id mark['review_id'] = review_id