x/stackalytics
Code Issues Proposed changes

Merge user profiles during default data update

Browse Source 
* Existing user profiles are updated with data read from default-data
* Refactoring of default data processor
* Removed unused code from normalizer (validity is enforced by schema)
* Add more restrictions into the schema

Fixes bug 1260696

Change-Id: Iab1f190ff8a7e2a0c0800712174565437149e119
This commit is contained in:
Ilya Shakhat
2013-12-19 18:47:31 +04:00
parent 0ff9761a64
commit 82600d8805
7 changed files with 94 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
stackalytics/processor/normalizer.py
View File

@@ -19,20 +19,9 @@ from stackalytics.processor import utils
LOG = logging.getLogger(__name__)
def get_user_id(launchpad_id, email):
return launchpad_id or email
def normalize_user(user):
user['emails'] = [email.lower() for email in user['emails']]
if user['launchpad_id']:
user['launchpad_id'] = user['launchpad_id'].lower()
def _normalize_user(user):
for c in user['companies']:
end_date_numeric = 0
if c['end_date']:
end_date_numeric = utils.date_to_timestamp(c['end_date'])
c['end_date'] = end_date_numeric
c['end_date'] = utils.date_to_timestamp(c['end_date'])
# sort companies by end_date
def end_date_comparator(x, y):
@@ -44,19 +33,12 @@ def normalize_user(user):
return cmp(x["end_date"], y["end_date"])
user['companies'].sort(cmp=end_date_comparator)
if user['emails']:
user['user_id'] = get_user_id(user['launchpad_id'], user['emails'][0])
else:
user['user_id'] = user['launchpad_id']
user['user_id'] = user['launchpad_id']
def _normalize_users(users):
for user in users:
if ('launchpad_id' not in user) or ('emails' not in user):
LOG.warn('Skipping invalid user: %s', user)
continue
normalize_user(user)
_normalize_user(user)
def _normalize_releases(releases):