Completed processing of blueprints data

* Update blueprints with number of mentions (emails, commits, reviews)
 * Improved understandability of record processor tests (significantly refactored)
 * Extracted LP specific functions into launchpad_utils
 * Refactored activity template in UI

Implements bp metric-by-bugs-blueprints

Change-Id: I9008a84ef1960e54be6e61f0ef3d69cd2cc1d9e4

dashboard/static/css/style.css

@@ -216,3 +216,94 @@ a[href^="https://launchpad"]:after {
 .review_mark {
     font-weight: bold;
 }
+
+.specstatusApproved, .specstatusApproved a {
+    color: #008000;
+}
+.specstatusPendingApproval, .specstatusPendingApproval a, .specstatusPendingView, .specstatusPendingView a {
+    color: #FF0099;
+}
+.specstatusDraft, .specstatusDraft a, .specstatusDiscussion, .specstatusDiscussion a {
+    color: #993300;
+}
+.specstatusNew, .specstatusNew a {
+    color: #FF0000;
+}
+.specstatusSuperseded, .specstatusSuperseded a, .specstatusObsolete, .specstatusObsolete a, .specpriorityUndefined, .specpriorityUndefined a {
+    color: #808080;
+}
+.specpriorityLow, .specpriorityLow a {
+    color: #000000;
+}
+.specpriorityMedium, .specpriorityMedium a {
+    color: #FF6600;
+}
+.specpriorityHigh, .specpriorityHigh a, .specpriorityEssential, .specpriorityEssential a {
+    color: #FF0000;
+}
+.specdeliveryUnknown, .specdeliveryUnknown a, .specdeliveryNotStarted, .specdeliveryNotStarted a {
+    color: #808080;
+}
+.specdeliveryDeferred, .specdeliveryDeferred a, .specdeliveryNeendsInfrastructure, .specdeliveryNeendsInfrastructure a, .specdeliveryBlocked, .specdeliveryBlocked a {
+    color: #FF0000;
+}
+.specdeliveryStarted, .specdeliveryStarted a, .specdeliveryGood, .specdeliveryGood a {
+    color: #0000FF;
+}
+.specdeliverySlow, .specdeliverySlow a {
+    color: #FF0000;
+}
+.specdeliveryBeta, .specdeliveryBeta a {
+    color: #FF6600;
+}
+.specdeliveryNEEDSREVIEW, .specdeliveryNEEDSREVIEW a {
+    color: #800080;
+}
+.specdeliveryAWAITINGDEPLOYMENT, .specdeliveryAWAITINGDEPLOYMENT a {
+    color: #FF0000;
+}
+.specdeliveryImplemented, .specdeliveryImplemented a, .specdeliveryINFORMATIONAL, .specdeliveryINFORMATIONAL a {
+    color: #008000;
+}
+.bug-activity {
+    color: #555555;
+}
+.statusNew, .statusNew a {
+    color: #993300;
+}
+.statusIncomplete, .statusIncomplete a, .statusConfirmed, .statusConfirmed a {
+    color: #FF0000;
+}
+.statusTriaged, .statusTriaged a {
+    color: #FF6600;
+}
+.statusInProgress, .statusInProgress a {
+    color: #000000;
+}
+.statusFixCommitted, .statusFixCommitted a {
+    color: #005500;
+}
+.statusFixReleased, .statusFixReleased a {
+    color: #008000;
+}
+.statusInvalid, .statusInvalid a, .statusWontFix, .statusWontFix a {
+    color: #555555;
+}
+.importanceCritical, .importanceCritical a {
+    color: #FF0000;
+}
+.importanceHigh, .importanceHigh a {
+    color: #FF6600;
+}
+.importanceMedium, .importanceMedium a {
+    color: #008000;
+}
+.importanceLow, .importanceLow a {
+    color: #000000;
+}
+.importanceWishlist, .importanceWishlist a {
+    color: #0000FF;
+}
+.importanceUndecided, .importanceUndecided a {
+    color: #999999;
+}

dashboard/templates/overview.html

@@ -6,7 +6,7 @@
 {% set show_user_activity = (user_id) %}
 {% set show_module_activity = (module) and (not user_id) %}
 {% set show_activity = (show_user_activity) or (show_module_activity) %}
-{% set show_user_contribution = (user_id) %}
+{% set show_user_contribution = (user_id) or (company) %}
 {% set show_module_contribution = (module) and (not user_id) %}
 {% set show_contribution = (show_user_contribution) or (show_module_contribution) %}
 {% set show_user_profile = (user_id) %}
@@ -117,10 +117,24 @@
             </a>
             {%/if%}
             </div>
-        {%elif ((record_type == "bp_draft") || (record_type == "bp_implementation")) %}
+            {%if blueprint_id_count %}
+            <div>Mentions blueprints:
+            {%each( index, value ) blueprint_id %}
+                ${value}
+            {%/each%}
+            </div>
+            {%/if%}
+        {%elif ((record_type == "bpd") || (record_type == "bpc")) %}
             <div style='font-weight: bold;'>${title} (<a href='${web_link}'>${name}</a>)</div>
             <div style='white-space: pre-wrap;'>${summary}</div>
 
+            <div>Priority: <span class="specpriority${priority}">${priority}</span></div>
+            <div>Status: <span class="status${lifecycle_status}">${lifecycle_status}</span>
+                (<span class="specstatus${definition_status}">${definition_status}</span>,
+                <span class="specdelivery${implementation_status}">${implementation_status}</span>)</div>
+            {%if mention_count %}
+            <div><b>Mention count: ${mention_count}, last mention on ${mention_date_str}</b></div>
+            {%/if%}
         {%/if%}
     </div>
     </div>
@@ -144,37 +158,14 @@
 
 <script id="contribution_template" type="text/x-jquery-tmpl">
     {% raw %}
-<h3>Contribution overview</h3>
+<h2>Contribution Summary</h2>
-{%if blueprints.length > 0 %}
-    <div>Blueprints:
-        <ol>
-            {%each(i,rec) blueprints %}
-                <li>
-                    <a href="https://blueprints.launchpad.net/${rec.module}/+spec/${rec.id}">${rec.id}</a>
-                    <small>${rec.module}</small>
-                </li>
-            {%/each%}
-        </ol>
-    </div>
-{%/if%}
-
-{%if bugs.length > 0 %}
-    <div>Bugs:
-        <ol>
-            {%each(i,rec) bugs %}
-                <li>
-                    <a href="https://bugs.launchpad.net/bugs/${rec.id}">${rec.id}</a>
-                    <small>${rec.module}</small>
-                </li>
-            {%/each%}
-        </ol>
-    </div>
-{%/if%}
-
 <div>Total commits: <b>${commit_count}</b></div>
 <div>Total LOC: <b>${loc}</b></div>
 <div>Review stat (-2, -1, +1, +2): <b>${marks["-2"]}, ${marks["-1"]}, ${marks["1"]}, ${marks["2"]}</b></div>
     {% endraw %}
+<div>Draft Blueprints: <b>${new_blueprint_count}</b></div>
+<div>Completed Blueprints: <b>${competed_blueprint_count}</b></div>
+<div>Emails: <b>${email_count}</b></div>
 </script>
 
 {% endblock %}
@@ -230,7 +221,7 @@
         <div id="user_profile_container" style="margin-bottom: 2em;"></div>
     {% endif %}
     {% if show_user_activity %}
-        <h3 id="activity_header">Activity log</h3>
+        <h2 id="activity_header">Activity log</h2>
         <div id="activity_container"></div>
 
         <div style="height: 44px;">
@@ -274,7 +265,7 @@
     {% endif %}
 
     {% if show_module_activity %}
-        <h3 id="activity_header">Activity log</h3>
+        <h2 id="activity_header">Activity log</h2>
         <div id="activity_container"></div>
 
         <div style="height: 44px;">

dashboard/web.py

@@ -49,8 +49,8 @@ METRIC_LABELS = {
     'commits': 'Commits',
     'marks': 'Reviews',
     'emails': 'Emails',
-    'bp_draft': 'New Blueprints',
+    'bpd': 'New Blueprints',
-    'bp_implementation': 'Completed Blueprints',
+    'bpc': 'Completed Blueprints',
 }
 
 METRIC_TO_RECORD_TYPE = {
@@ -58,8 +58,8 @@ METRIC_TO_RECORD_TYPE = {
     'commits': 'commit',
     'marks': 'mark',
     'emails': 'email',
-    'bp_draft': 'bp_draft',
+    'bpd': 'bpd',
-    'bp_implementation': 'bp_implementation',
+    'bpc': 'bpc',
 }
 
 DEFAULT_RECORDS_LIMIT = 10
@@ -406,8 +406,8 @@ def aggregate_filter():
                 'loc': (loc_filter, None),
                 'marks': (mark_filter, mark_finalize),
                 'emails': (incremental_filter, None),
-                'bp_draft': (incremental_filter, None),
+                'bpd': (incremental_filter, None),
-                'bp_implementation': (incremental_filter, None),
+                'bpc': (incremental_filter, None),
             }
             if metric not in metric_to_filters_map:
                 raise Exception('Invalid metric %s' % metric)
@@ -554,44 +554,6 @@ def page_not_found(e):
     pass
 
 
-def contribution_details(records):
-    blueprints_map = {}
-    bugs_map = {}
-    marks = dict((m, 0) for m in [-2, -1, 0, 1, 2])
-    commit_count = 0
-    loc = 0
-
-    for record in records:
-        if 'blueprint_id' in record:
-            for bp in record['blueprint_id']:
-                blueprints_map[bp] = record
-        if 'bug_id' in record:
-            for bug in record['bug_id']:
-                bugs_map[bug] = record
-
-        if record['record_type'] == 'mark':
-            marks[int(record['value'])] += 1
-        elif record['record_type'] == 'commit':
-            commit_count += 1
-            loc += record['loc']
-
-    blueprints = sorted([{'id': key, 'module': value['module']}
-                         for key, value in blueprints_map.iteritems()],
-                        key=lambda x: x['id'])
-    bugs = sorted([{'id': key, 'module': value['module']}
-                   for key, value in bugs_map.iteritems()],
-                  key=lambda x: int(x['id']))
-
-    result = {
-        'blueprints': blueprints,
-        'bugs': bugs,
-        'commit_count': commit_count,
-        'loc': loc,
-        'marks': marks,
-    }
-    return result
-
-
 # AJAX Handlers ---------
 
 def _get_aggregated_stats(records, metric_filter, keys, param_id,
@@ -653,7 +615,9 @@ def _extend_record(record):
     record['company_link'] = make_link(
         record['company_name'], '/',
         {'company': record['company_name'], 'user_id': ''})
-    record['gravatar'] = gravatar(record['author_email'])
+    record['gravatar'] = gravatar(record.get('author_email', 'stackalytics'))
+    record['blueprint_id_count'] = len(record.get('blueprint_id', []))
+    record['bug_id_count'] = len(record.get('bug_id', []))
 
 
 @app.route('/api/1.0/activity')
@@ -693,10 +657,13 @@ def get_activity_json(records):
             _extend_record(email)
             email['email_link'] = email.get('email_link') or ''
             result.append(email)
-        elif ((record['record_type'] == 'bp_draft') or
+        elif ((record['record_type'] == 'bpd') or
-             (record['record_type'] == 'bp_implementation')):
+             (record['record_type'] == 'bpc')):
             blueprint = record.copy()
             _extend_record(blueprint)
+            if 'mention_date' in record:
+                record['mention_date_str'] = format_datetime(
+                    record['mention_date'])
             result.append(blueprint)
 
     result.sort(key=lambda x: x['date'], reverse=True)
@@ -708,7 +675,36 @@ def get_activity_json(records):
 @exception_handler()
 @record_filter(ignore='metric')
 def get_contribution_json(records):
-    return contribution_details(records)
+    marks = dict((m, 0) for m in [-2, -1, 0, 1, 2])
+    commit_count = 0
+    loc = 0
+    new_blueprint_count = 0
+    competed_blueprint_count = 0
+    email_count = 0
+
+    for record in records:
+        record_type = record['record_type']
+        if record_type == 'commit':
+            commit_count += 1
+            loc += record['loc']
+        elif record['record_type'] == 'mark':
+            marks[int(record['value'])] += 1
+        elif record['record_type'] == 'email':
+            email_count += 1
+        elif record['record_type'] == 'bpd':
+            new_blueprint_count += 1
+        elif record['record_type'] == 'bpc':
+            competed_blueprint_count += 1
+
+    result = {
+        'new_blueprint_count': new_blueprint_count,
+        'competed_blueprint_count': competed_blueprint_count,
+        'commit_count': commit_count,
+        'email_count': email_count,
+        'loc': loc,
+        'marks': marks,
+    }
+    return result
 
 
 @app.route('/api/1.0/companies')
@@ -817,7 +813,10 @@ def get_user(user_id):
             company_name, '/', {'company': company_name, 'user_id': ''})
     else:
         user['company_link'] = ''
+    if user['emails']:
         user['gravatar'] = gravatar(user['emails'][0])
+    else:
+        user['gravatar'] = gravatar('stackalytics')
     return user
 
 

etc/test_default_data.json

@@ -119,7 +119,7 @@
     "releases": [
         {
             "release_name": "prehistory",
-            "end_date": "2011-Apr-21"
+            "end_date": "2011-Apr-22"
         },
         {
             "release_name": "Diablo",

stackalytics/processor/launchpad_utils.py

@@ -0,0 +1,66 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import httplib
+import urlparse
+
+from stackalytics.openstack.common import log as logging
+from stackalytics.processor import utils
+
+LOG = logging.getLogger(__name__)
+
+
+LP_URI_V1 = 'https://api.launchpad.net/1.0/%s'
+LP_URI_DEVEL = 'https://api.launchpad.net/devel/%s'
+
+
+def lp_profile_by_launchpad_id(launchpad_id):
+    LOG.debug('Lookup user id %s at Launchpad', launchpad_id)
+    uri = LP_URI_V1 % ('~' + launchpad_id)
+    return utils.read_json_from_uri(uri)
+
+
+def lp_profile_by_email(email):
+    LOG.debug('Lookup user email %s at Launchpad', email)
+    uri = LP_URI_V1 % ('people/?ws.op=getByEmail&email=' + email)
+    return utils.read_json_from_uri(uri)
+
+
+def lp_module_exists(module):
+    uri = LP_URI_DEVEL % module
+    parsed_uri = urlparse.urlparse(uri)
+    conn = httplib.HTTPConnection(parsed_uri.netloc)
+    conn.request('GET', parsed_uri.path)
+    res = conn.getresponse()
+    LOG.debug('Checked uri: %(uri)s, status: %(status)s',
+              {'uri': uri, 'status': res.status})
+    conn.close()
+    return res.status != 404
+
+
+def lp_blueprint_generator(module):
+    uri = LP_URI_DEVEL % (module + '/all_specifications')
+    while uri:
+        LOG.debug('Reading chunk from uri %s', uri)
+        chunk = utils.read_json_from_uri(uri)
+
+        if not chunk:
+            LOG.warn('No data was read from uri %s', uri)
+            break
+
+        for record in chunk['entries']:
+            yield record
+
+        uri = chunk.get('next_collection_link')

stackalytics/processor/lp.py

@@ -13,10 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import httplib
-import urlparse
-
 from stackalytics.openstack.common import log as logging
+from stackalytics.processor import launchpad_utils
 from stackalytics.processor import utils
 
 
@@ -27,18 +25,6 @@ LINK_FIELDS = ['owner', 'drafter', 'starter', 'completer',
 DATE_FIELDS = ['date_created', 'date_completed', 'date_started']
 
 
-def _module_exists(module):
-    uri = 'https://api.launchpad.net/devel/%s' % module
-    parsed_uri = urlparse.urlparse(uri)
-    conn = httplib.HTTPConnection(parsed_uri.netloc)
-    conn.request('GET', parsed_uri.path)
-    res = conn.getresponse()
-    LOG.debug('Checked uri: %(uri)s, status: %(status)s',
-              {'uri': uri, 'status': res.status})
-    conn.close()
-    return res.status != 404
-
-
 def _link_to_launchpad_id(link):
     return link[link.find('~') + 1:]
 
@@ -47,19 +33,11 @@ def log(repo):
     module = repo['module']
     LOG.debug('Retrieving list of blueprints for module: %s', module)
 
-    if not _module_exists(module):
+    if not launchpad_utils.lp_module_exists(module):
         LOG.debug('Module %s not exist at Launchpad', module)
         return
 
-    uri = 'https://api.launchpad.net/devel/%s/all_specifications' % module
+    for record in launchpad_utils.lp_blueprint_generator(module):
-    while True:
-        LOG.debug('Reading chunk from uri %s', uri)
-        chunk = utils.read_json_from_uri(uri)
-        if 'next_collection_link' not in chunk:
-            break
-        uri = chunk['next_collection_link']
-
-        for record in chunk['entries']:
         for field in LINK_FIELDS:
             link = record[field + '_link']
             if link:

stackalytics/processor/normalizer.py

@@ -44,7 +44,10 @@ def normalize_user(user):
             return cmp(x["end_date"], y["end_date"])
 
     user['companies'].sort(cmp=end_date_comparator)
+    if user['emails']:
         user['user_id'] = get_user_id(user['launchpad_id'], user['emails'][0])
+    else:
+        user['user_id'] = user['launchpad_id']
 
 
 def _normalize_users(users):

stackalytics/processor/record_processor.py

@@ -16,6 +16,7 @@
 import bisect
 
 from stackalytics.openstack.common import log as logging
+from stackalytics.processor import launchpad_utils
 from stackalytics.processor import normalizer
 from stackalytics.processor import utils
 
@@ -66,6 +67,9 @@ class RecordProcessor(object):
         return companies[-1]['company_name']
 
     def _get_company_by_email(self, email):
+        if not email:
+            return None
+
         name, at, domain = email.partition('@')
         if domain:
             parts = domain.split('.')
@@ -81,13 +85,16 @@ class RecordProcessor(object):
         user = {
             'user_id': normalizer.get_user_id(launchpad_id, email),
             'launchpad_id': launchpad_id,
-            'user_name': user_name,
+            'user_name': user_name or '',
-            'emails': [email],
             'companies': [{
                 'company_name': company,
                 'end_date': 0,
             }],
         }
+        if email:
+            user['emails'] = [email]
+        else:
+            user['emails'] = []
         normalizer.normalize_user(user)
         LOG.debug('Create new user: %s', user)
         return user
@@ -95,12 +102,9 @@ class RecordProcessor(object):
     def _get_lp_info(self, email):
         lp_profile = None
         if not utils.check_email_validity(email):
-            LOG.debug('User email is not valid %s' % email)
+            LOG.debug('User email is not valid %s', email)
         else:
-            LOG.debug('Lookup user email %s at Launchpad' % email)
+            lp_profile = launchpad_utils.lp_profile_by_email(email)
-            uri = ('https://api.launchpad.net/1.0/people/?'
-                   'ws.op=getByEmail&email=%s' % email)
-            lp_profile = utils.read_json_from_uri(uri)
 
         if not lp_profile:
             LOG.debug('User with email %s not found', email)
@@ -109,6 +113,18 @@ class RecordProcessor(object):
         LOG.debug('Email is mapped to launchpad user: %s', lp_profile['name'])
         return lp_profile['name'], lp_profile['display_name']
 
+    def _get_lp_user_name(self, launchpad_id):
+        if not launchpad_id:
+            return None
+
+        lp_profile = launchpad_utils.lp_profile_by_launchpad_id(launchpad_id)
+
+        if not lp_profile:
+            LOG.debug('User with id %s not found', launchpad_id)
+            return launchpad_id
+
+        return lp_profile['display_name']
+
     def _get_independent(self):
         return self.domains_index['']
 
@@ -124,14 +140,14 @@ class RecordProcessor(object):
             self.updated_users.add(user['user_id'])
 
     def update_user(self, record):
-        email = record['author_email']
+        email = record.get('author_email')
 
         if email in self.users_index:
             user = self.users_index[email]
         else:
-            if ('launchpad_id' in record) and (record['launchpad_id']):
+            if record.get('launchpad_id'):
-                launchpad_id = record['launchpad_id']
+                launchpad_id = record.get('launchpad_id')
-                user_name = record['author_name']
+                user_name = record.get('author_name')
             else:
                 launchpad_id, user_name = self._get_lp_info(email)
 
@@ -142,10 +158,13 @@ class RecordProcessor(object):
             else:
                 # create new
                 if not user_name:
-                    user_name = record['author_name']
+                    user_name = record.get('author_name')
+                if not user_name:
+                    user_name = self._get_lp_user_name(launchpad_id)
                 user = self._create_user(launchpad_id, email, user_name)
 
             utils.store_user(self.runtime_storage_inst, user)
+            if email:
                 self.users_index[email] = user
             if user['launchpad_id']:
                 self.users_index[user['launchpad_id']] = user
@@ -158,12 +177,12 @@ class RecordProcessor(object):
         record['user_id'] = user['user_id']
         record['launchpad_id'] = user['launchpad_id']
 
-        if ('user_name' in user) and (user['user_name']):
+        if user.get('user_name'):
             record['author_name'] = user['user_name']
 
         company = self._find_company(user['companies'], record['date'])
         if company != '*robots':
-            company = (self._get_company_by_email(record['author_email'])
+            company = (self._get_company_by_email(record.get('author_email'))
                        or company)
         record['company_name'] = company
 
@@ -199,7 +218,7 @@ class RecordProcessor(object):
         review_id = record['id']
         module = record['module']
 
-        for patch in record['patchSets']:
+        for patch in record.get('patchSets', []):
             if 'approvals' not in patch:
                 continue  # not reviewed by anyone
             for approval in patch['approvals']:
@@ -265,40 +284,28 @@ class RecordProcessor(object):
         yield record
 
     def _process_blueprint(self, record):
-        if record.get('drafter'):
+        bpd_author = record.get('drafter') or record.get('owner')
-            bp_draft = dict([(k, v) for k, v in record.iteritems()])
-            bp_draft['primary_key'] = 'bpd:' + record['self_link']
 
-            drafter = utils.load_user(self.runtime_storage_inst,
+        bpd = dict([(k, v) for k, v in record.iteritems()])
-                                      record['drafter'])
+        bpd['record_type'] = 'bpd'
-            if drafter and record['date_created']:
+        bpd['primary_key'] = 'bpd:' + record['self_link']
-                bp_draft['record_type'] = 'bp_draft'
+        bpd['launchpad_id'] = bpd_author
-                bp_draft['author_name'] = drafter['user_name']
+        bpd['date'] = record['date_created']
-                bp_draft['author_email'] = drafter['emails'][0]
-                bp_draft['launchpad_id'] = record['drafter']
-                bp_draft['date'] = record['date_created']
 
-                self._update_record_and_user(bp_draft)
+        self._update_record_and_user(bpd)
 
-                yield bp_draft
+        yield bpd
 
-        if record.get('assignee'):
+        if record.get('assignee') and record['date_completed']:
-            bp_implementation = dict([(k, v) for k, v in record.iteritems()])
+            bpc = dict([(k, v) for k, v in record.iteritems()])
-            bp_implementation['primary_key'] = 'bpi:' + record['self_link']
+            bpc['record_type'] = 'bpc'
+            bpc['primary_key'] = 'bpc:' + record['self_link']
+            bpc['launchpad_id'] = record['assignee']
+            bpc['date'] = record['date_completed']
 
-            assignee = utils.load_user(self.runtime_storage_inst,
+            self._update_record_and_user(bpc)
-                                       record['assignee'])
-            if assignee and record['date_completed']:
-                bp_implementation['record_type'] = 'bp_implementation'
-                bp_implementation['author_name'] = assignee['user_name']
-                bp_implementation['author_email'] = assignee['emails'][0]
-                bp_implementation['launchpad_id'] = record['assignee']
-                bp_implementation['date'] = record['date_completed']
 
-                if bp_implementation['author_email']:
+            yield bpc
-                    self._update_record_and_user(bp_implementation)
-
-                yield bp_implementation
 
     def _apply_type_based_processing(self, record):
         if record['record_type'] == 'commit':
@@ -359,15 +366,68 @@ class RecordProcessor(object):
         self.runtime_storage_inst.set_by_key('users', self.users_index)
 
     def _get_records_for_users_to_update(self):
+        valid_blueprints = {}
+        mentioned_blueprints = {}
         for record in self.runtime_storage_inst.get_all_records():
+            for bp in record.get('blueprint_id', []):
+                if bp in mentioned_blueprints:
+                    mentioned_blueprints[bp]['count'] += 1
+                    if record['date'] > mentioned_blueprints[bp]['date']:
+                        mentioned_blueprints[bp]['date'] = record['date']
+                else:
+                    mentioned_blueprints[bp] = {
+                        'count': 1,
+                        'date': record['date']
+                    }
+            if record['record_type'] in ['bpd', 'bpi']:
+                valid_blueprints[record['name']] = {
+                    'primary_key': record['primary_key'],
+                    'count': 0,
+                    'date': record['date']
+                }
+
+        for bp in valid_blueprints.keys():
+            if bp in mentioned_blueprints:
+                valid_blueprints[bp]['count'] = (
+                    mentioned_blueprints[bp]['count'])
+                valid_blueprints[bp]['date'] = (
+                    mentioned_blueprints[bp]['date'])
+
+        for record in self.runtime_storage_inst.get_all_records():
+
+            need_update = False
+
             user_id = record['user_id']
             if user_id in self.updated_users:
                 user = self.users_index[user_id]
                 user_company_name = user['companies'][0]['company_name']
                 if record['company_name'] != user_company_name:
-                    LOG.debug('Record company will be changed to: %s',
+                    LOG.debug('Update record %s: company changed to: %s',
-                              user_company_name)
+                              record['primary_key'], user_company_name)
                     record['company_name'] = user_company_name
+                    need_update = True
+
+            valid_bp = set([])
+            for bp in record.get('blueprint_id', []):
+                if bp in valid_blueprints:
+                    valid_bp.add(bp)
+                else:
+                    LOG.debug('Update record %s: removed invalid bp: %s',
+                              record['primary_key'], bp)
+                    need_update = True
+            record['blueprint_id'] = list(valid_bp)
+
+            if record['record_type'] in ['bpd', 'bpi']:
+                bp = valid_blueprints[record['name']]
+                if ((record.get('mention_count') != bp['count']) or
+                        (record.get('mention_date') != bp['date'])):
+                    record['mention_count'] = bp['count']
+                    record['mention_date'] = bp['date']
+                    LOG.debug('Update record %s: mention stats: (%s:%s)',
+                              record['primary_key'], bp['count'], bp['date'])
+                    need_update = True
+
+            if need_update:
                 yield record
 
     def finalize(self):

stackalytics/processor/utils.py

@@ -63,7 +63,10 @@ def read_uri(uri):
 
 
 def read_json_from_uri(uri):
+    try:
         return json.loads(read_uri(uri))
+    except Exception as e:
+        LOG.warn('Error parsing json: %s' % e)
 
 
 def make_range(start, stop, step):

tests/unit/test_record_processor.py

@@ -21,38 +21,6 @@ from stackalytics.processor import runtime_storage
 from stackalytics.processor import utils
 
 
-LP_URI = 'https://api.launchpad.net/1.0/people/?ws.op=getByEmail&email=%s'
-
-COMPANIES = [
-    {
-        'company_name': 'SuperCompany',
-        'domains': ['super.com', 'super.no']
-    },
-    {
-        "domains": ["nec.com", "nec.co.jp"],
-        "company_name": "NEC"
-    },
-    {
-        'company_name': '*independent',
-        'domains': ['']
-    },
-]
-
-USERS = [
-    {
-        'user_id': 'john_doe',
-        'launchpad_id': 'john_doe',
-        'user_name': 'John Doe',
-        'emails': ['johndoe@gmail.com', 'jdoe@super.no'],
-        'companies': [
-            {'company_name': '*independent',
-             'end_date': 1234567890},
-            {'company_name': 'SuperCompany',
-             'end_date': 0},
-        ]
-    }
-]
-
 RELEASES = [
     {
         'release_name': 'prehistory',
@@ -84,164 +52,423 @@ class TestRecordProcessor(testtools.TestCase):
         self.read_json_from_uri_patch = mock.patch(
             'stackalytics.processor.utils.read_json_from_uri')
         self.read_launchpad = self.read_json_from_uri_patch.start()
+        self.lp_profile_by_launchpad_id_patch = mock.patch(
+            'stackalytics.processor.launchpad_utils.'
+            'lp_profile_by_launchpad_id')
+        self.lp_profile_by_launchpad_id = (
+            self.lp_profile_by_launchpad_id_patch.start())
+        self.lp_profile_by_launchpad_id.return_value = None
+        self.lp_profile_by_email_patch = mock.patch(
+            'stackalytics.processor.launchpad_utils.lp_profile_by_email')
+        self.lp_profile_by_email = (
+            self.lp_profile_by_email_patch.start())
+        self.lp_profile_by_email.return_value = None
 
     def tearDown(self):
         super(TestRecordProcessor, self).tearDown()
         self.read_json_from_uri_patch.stop()
+        self.lp_profile_by_launchpad_id_patch.stop()
+        self.lp_profile_by_email_patch.stop()
+
+    # get_company_by_email
 
     def test_get_company_by_email_mapped(self):
-        record_processor_inst = make_record_processor()
+        record_processor_inst = self.make_record_processor(
-        email = 'jdoe@super.no'
+            companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}]
+        )
+        email = 'jdoe@ibm.com'
         res = record_processor_inst._get_company_by_email(email)
-        self.assertEquals('SuperCompany', res)
+        self.assertEquals('IBM', res)
 
     def test_get_company_by_email_with_long_suffix_mapped(self):
-        record_processor_inst = make_record_processor()
+        record_processor_inst = self.make_record_processor(
+            companies=[{'company_name': 'NEC', 'domains': ['nec.co.jp']}]
+        )
         email = 'man@mxw.nes.nec.co.jp'
         res = record_processor_inst._get_company_by_email(email)
         self.assertEquals('NEC', res)
 
     def test_get_company_by_email_with_long_suffix_mapped_2(self):
-        record_processor_inst = make_record_processor()
+        record_processor_inst = self.make_record_processor(
+            companies=[{'company_name': 'NEC',
+                        'domains': ['nec.co.jp', 'nec.com']}]
+        )
         email = 'man@mxw.nes.nec.com'
         res = record_processor_inst._get_company_by_email(email)
         self.assertEquals('NEC', res)
 
     def test_get_company_by_email_not_mapped(self):
-        record_processor_inst = make_record_processor()
+        record_processor_inst = self.make_record_processor()
         email = 'foo@boo.com'
         res = record_processor_inst._get_company_by_email(email)
         self.assertEquals(None, res)
 
-    def test_update_commit_existing_user(self):
+    # get_lp_info
-        record_processor_inst = make_record_processor()
-        commit_generator = generate_commits()
-        commit = list(record_processor_inst.process(commit_generator))[0]
 
-        self.assertEquals('SuperCompany', commit['company_name'])
+    def test_get_lp_info_invalid_email(self):
-        self.assertEquals('john_doe', commit['launchpad_id'])
+        self.read_launchpad.return_value = None
+        record_processor_inst = self.make_record_processor(users=[])
+        self.assertEquals((None, None),
+                          record_processor_inst._get_lp_info('error.root'))
 
-    def test_update_commit_existing_user_old_job(self):
+    # commit processing
-        record_processor_inst = make_record_processor()
-        commit_generator = generate_commits(date=1000000000)
-        commit = list(record_processor_inst.process(commit_generator))[0]
 
-        self.assertEquals('*independent', commit['company_name'])
+    def test_process_commit_existing_user(self):
-        self.assertEquals('john_doe', commit['launchpad_id'])
+        record_processor_inst = self.make_record_processor(
-
+            users=[
-    def test_update_commit_existing_user_new_email_known_company(self):
+                {
-        # User is known to LP, his email is new to us, and maps to other
-        # company. Should return other company instead of those mentioned
-        # in user db
-        email = 'johndoe@nec.co.jp'
-        commit_generator = generate_commits(email=email)
-        launchpad_id = 'john_doe'
-        self.read_launchpad.return_value = {'name': launchpad_id,
-                                            'display_name': launchpad_id}
-        user = make_user()
-        record_processor_inst = make_record_processor(
-            make_runtime_storage(users=[user]))
-
-        commit = list(record_processor_inst.process(commit_generator))[0]
-
-        self.read_launchpad.assert_called_once_with(LP_URI % email)
-        self.assertIn(email, user['emails'])
-        self.assertEquals('NEC', commit['company_name'])
-        self.assertEquals(launchpad_id, commit['launchpad_id'])
-
-    def test_update_commit_existing_user_new_email_unknown_company(self):
-        # User is known to LP, but his email is new to us. Should match
-        # the user and return current company
-        email = 'johndoe@yahoo.com'
-        commit_generator = generate_commits(email=email)
-        launchpad_id = 'john_doe'
-        self.read_launchpad.return_value = {'name': launchpad_id,
-                                            'display_name': launchpad_id}
-        user = make_user()
-        record_processor_inst = make_record_processor(
-            make_runtime_storage(users=[user]))
-
-        commit = list(record_processor_inst.process(commit_generator))[0]
-
-        self.read_launchpad.assert_called_once_with(LP_URI % email)
-        self.assertIn(email, user['emails'])
-        self.assertEquals('SuperCompany', commit['company_name'])
-        self.assertEquals(launchpad_id, commit['launchpad_id'])
-
-    def test_update_commit_existing_user_new_email_known_company_update(self):
-        # User is known to LP, his email is new to us and belongs to company B.
-        # Should match the user and return company B and update user
-        email = 'johndoe@nec.com'
-        commit_generator = generate_commits(email=email)
-        launchpad_id = 'john_doe'
-        self.read_launchpad.return_value = {'name': launchpad_id,
-                                            'display_name': launchpad_id}
-        user = {
                     'user_id': 'john_doe',
-            'launchpad_id': launchpad_id,
+                    'launchpad_id': 'john_doe',
                     'user_name': 'John Doe',
-            'emails': ['johndoe@gmail.com'],
+                    'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
                     'companies': [
-                {'company_name': '*independent', 'end_date': 0}
+                        {'company_name': '*independent',
+                         'end_date': 1234567890},
+                        {'company_name': 'NEC',
+                         'end_date': 0},
                     ]
                 }
-        record_processor_inst = make_record_processor(
+            ])
-            make_runtime_storage(users=[user]))
 
-        commit = list(record_processor_inst.process(commit_generator))[0]
+        processed_commit = list(record_processor_inst.process(
+            generate_commits(author_email='johndoe@gmail.com',
+                             author_name='John Doe')))[0]
 
-        self.read_launchpad.assert_called_once_with(LP_URI % email)
+        expected_commit = {
-        self.assertIn(email, user['emails'])
+            'launchpad_id': 'john_doe',
-        self.assertEquals('NEC', user['companies'][0]['company_name'],
+            'author_email': 'johndoe@gmail.com',
+            'author_name': 'John Doe',
+            'company_name': 'NEC',
+        }
+
+        self.assertRecordsMatch(expected_commit, processed_commit)
+
+    def test_process_commit_existing_user_old_job(self):
+        record_processor_inst = self.make_record_processor(
+            users=[
+                {
+                    'user_id': 'john_doe',
+                    'launchpad_id': 'john_doe',
+                    'user_name': 'John Doe',
+                    'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
+                    'companies': [
+                        {'company_name': '*independent',
+                         'end_date': 1234567890},
+                        {'company_name': 'NEC',
+                         'end_date': 0},
+                    ]
+                }
+            ])
+
+        processed_commit = list(record_processor_inst.process(
+            generate_commits(author_email='johndoe@gmail.com',
+                             author_name='John Doe',
+                             date=1000000000)))[0]
+
+        expected_commit = {
+            'launchpad_id': 'john_doe',
+            'author_email': 'johndoe@gmail.com',
+            'author_name': 'John Doe',
+            'company_name': '*independent',
+        }
+
+        self.assertRecordsMatch(expected_commit, processed_commit)
+
+    def test_process_commit_existing_user_new_email_known_company(self):
+        # User is known to LP, his email is new to us, and maps to other
+        # company. Should return other company instead of those mentioned
+        # in user profile
+        record_processor_inst = self.make_record_processor(
+            users=[
+                {'user_id': 'john_doe',
+                 'launchpad_id': 'john_doe',
+                 'user_name': 'John Doe',
+                 'emails': ['johndoe@nec.co.jp'],
+                 'companies': [{'company_name': 'NEC', 'end_date': 0}]}
+            ],
+            companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
+            lp_info={'johndoe@ibm.com':
+                     {'name': 'john_doe', 'display_name': 'John Doe'}})
+
+        processed_commit = list(record_processor_inst.process(
+            generate_commits(author_email='johndoe@ibm.com',
+                             author_name='John Doe')))[0]
+
+        expected_commit = {
+            'launchpad_id': 'john_doe',
+            'author_email': 'johndoe@ibm.com',
+            'author_name': 'John Doe',
+            'company_name': 'IBM',
+        }
+
+        self.assertRecordsMatch(expected_commit, processed_commit)
+        self.assertIn('johndoe@ibm.com',
+                      record_processor_inst.users_index['john_doe']['emails'])
+
+    def test_process_commit_existing_user_new_email_unknown_company(self):
+        # User is known to LP, but his email is new to us. Should match
+        # the user and return company from user profile
+        record_processor_inst = self.make_record_processor(
+            users=[
+                {'user_id': 'john_doe',
+                 'launchpad_id': 'john_doe',
+                 'user_name': 'John Doe',
+                 'emails': ['johndoe@nec.co.jp'],
+                 'companies': [{'company_name': 'NEC', 'end_date': 0}]}
+            ],
+            companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
+            lp_info={'johndoe@gmail.com':
+                     {'name': 'john_doe', 'display_name': 'John Doe'}})
+
+        processed_commit = list(record_processor_inst.process(
+            generate_commits(author_email='johndoe@gmail.com',
+                             author_name='John Doe')))[0]
+
+        expected_commit = {
+            'launchpad_id': 'john_doe',
+            'author_email': 'johndoe@gmail.com',
+            'author_name': 'John Doe',
+            'company_name': 'NEC',
+        }
+
+        self.assertRecordsMatch(expected_commit, processed_commit)
+        self.assertIn('johndoe@gmail.com',
+                      record_processor_inst.users_index['john_doe']['emails'])
+
+    def test_process_commit_existing_user_new_email_known_company_update(self):
+        record_processor_inst = self.make_record_processor(
+            users=[
+                {'user_id': 'john_doe',
+                 'launchpad_id': 'john_doe',
+                 'user_name': 'John Doe',
+                 'emails': ['johndoe@gmail.com'],
+                 'companies': [{'company_name': '*independent',
+                                'end_date': 0}]}
+            ],
+            companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
+            lp_info={'johndoe@ibm.com':
+                     {'name': 'john_doe', 'display_name': 'John Doe'}})
+
+        processed_commit = list(record_processor_inst.process(
+            generate_commits(author_email='johndoe@ibm.com',
+                             author_name='John Doe')))[0]
+
+        expected_commit = {
+            'launchpad_id': 'john_doe',
+            'author_email': 'johndoe@ibm.com',
+            'author_name': 'John Doe',
+            'company_name': 'IBM',
+        }
+
+        self.assertRecordsMatch(expected_commit, processed_commit)
+        user = record_processor_inst.users_index['john_doe']
+        self.assertIn('johndoe@gmail.com', user['emails'])
+        self.assertEquals('IBM', user['companies'][0]['company_name'],
                           message='User affiliation should be updated')
-        self.assertEquals('NEC', commit['company_name'])
-        self.assertEquals(launchpad_id, commit['launchpad_id'])
 
-    def test_update_commit_new_user(self):
+    def test_process_commit_new_user(self):
         # User is known to LP, but new to us
         # Should add new user and set company depending on email
-        email = 'smith@nec.com'
+        record_processor_inst = self.make_record_processor(
-        commit_generator = generate_commits(email=email)
+            companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
-        launchpad_id = 'smith'
+            lp_info={'johndoe@ibm.com':
-        self.read_launchpad.return_value = {'name': launchpad_id,
+                     {'name': 'john_doe', 'display_name': 'John Doe'}})
-                                            'display_name': 'Smith'}
-        record_processor_inst = make_record_processor(
-            make_runtime_storage(users=[]))
 
-        commit = list(record_processor_inst.process(commit_generator))[0]
+        processed_commit = list(record_processor_inst.process(
+            generate_commits(author_email='johndoe@ibm.com',
+                             author_name='John Doe')))[0]
 
-        self.read_launchpad.assert_called_once_with(LP_URI % email)
+        expected_commit = {
-        self.assertEquals('NEC', commit['company_name'])
+            'launchpad_id': 'john_doe',
-        self.assertEquals(launchpad_id, commit['launchpad_id'])
+            'author_email': 'johndoe@ibm.com',
+            'author_name': 'John Doe',
+            'company_name': 'IBM',
+        }
 
-    def test_update_commit_new_user_unknown_to_lb(self):
+        self.assertRecordsMatch(expected_commit, processed_commit)
+        user = record_processor_inst.users_index['john_doe']
+        self.assertIn('johndoe@ibm.com', user['emails'])
+        self.assertEquals('IBM', user['companies'][0]['company_name'])
+
+    def test_process_commit_new_user_unknown_to_lb(self):
         # User is new to us and not known to LP
         # Should set user name and empty LPid
-        email = 'inkognito@avs.com'
+        record_processor_inst = self.make_record_processor(
-        commit_generator = generate_commits(email=email)
+            companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}])
-        self.read_launchpad.return_value = None
-        record_processor_inst = make_record_processor(
-            make_runtime_storage(users=[]))
 
-        commit = list(record_processor_inst.process(commit_generator))[0]
+        processed_commit = list(record_processor_inst.process(
+            generate_commits(author_email='johndoe@ibm.com',
+                             author_name='John Doe')))[0]
 
-        self.read_launchpad.assert_called_once_with(LP_URI % email)
+        expected_commit = {
-        self.assertEquals('*independent', commit['company_name'])
+            'launchpad_id': None,
-        self.assertEquals(None, commit['launchpad_id'])
+            'author_email': 'johndoe@ibm.com',
+            'author_name': 'John Doe',
+            'company_name': 'IBM',
+        }
 
-    def test_update_commit_invalid_email(self):
+        self.assertRecordsMatch(expected_commit, processed_commit)
-        # User's email is malformed
+        self.assertEquals(1, len(record_processor_inst.users_index))
-        email = 'error.root'
+        user = record_processor_inst.users_index['johndoe@ibm.com']
-        commit_generator = generate_commits(email=email)
+        self.assertIn('johndoe@ibm.com', user['emails'])
-        self.read_launchpad.return_value = None
+        self.assertEquals('IBM', user['companies'][0]['company_name'])
-        record_processor_inst = make_record_processor(
+        self.assertEquals(None, user['launchpad_id'])
-            make_runtime_storage(users=[]))
 
-        commit = list(record_processor_inst.process(commit_generator))[0]
+    # process records complex scenarios
 
-        self.assertEquals(0, self.read_launchpad.called)
+    def test_process_blueprint_one_draft_spawned_lp_doesnt_know_user(self):
-        self.assertEquals('*independent', commit['company_name'])
+        # In: blueprint record
-        self.assertEquals(None, commit['launchpad_id'])
+        #     LP doesn't know user
+        # Out: blueprint-draft record
+        #      new user profile created
+        record_processor_inst = self.make_record_processor()
+
+        processed_records = list(record_processor_inst.process([
+            {'record_type': 'bp',
+             'self_link': 'http://launchpad.net/blueprint',
+             'owner': 'john_doe',
+             'date_created': 1234567890}
+        ]))
+
+        self.assertRecordsMatch(
+            {'record_type': 'bpd',
+             'launchpad_id': 'john_doe',
+             'author_name': 'john_doe',
+             'company_name': '*independent'},
+            processed_records[0])
+
+        self.assertEquals(1, len(record_processor_inst.users_index))
+        user = record_processor_inst.users_index['john_doe']
+        self.assertEquals({
+            'user_id': 'john_doe',
+            'launchpad_id': 'john_doe',
+            'user_name': 'john_doe',
+            'emails': [],
+            'companies': [{'company_name': '*independent', 'end_date': 0}]
+        }, user)
+
+    def test_process_blueprint_one_draft_spawned_lp_knows_user(self):
+        # In: blueprint record
+        #     LP knows user
+        # Out: blueprint-draft record
+        #      new user profile created, name is taken from LP profile
+        record_processor_inst = self.make_record_processor(
+            lp_user_name={
+                'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}})
+
+        processed_records = list(record_processor_inst.process([
+            {'record_type': 'bp',
+             'self_link': 'http://launchpad.net/blueprint',
+             'owner': 'john_doe',
+             'date_created': 1234567890}
+        ]))
+
+        self.assertRecordsMatch(
+            {'record_type': 'bpd',
+             'launchpad_id': 'john_doe',
+             'author_name': 'John Doe',
+             'company_name': '*independent'},
+            processed_records[0])
+
+        self.assertEquals(1, len(record_processor_inst.users_index))
+        user = record_processor_inst.users_index['john_doe']
+        self.assertEquals({
+            'user_id': 'john_doe',
+            'launchpad_id': 'john_doe',
+            'user_name': 'John Doe',
+            'emails': [],
+            'companies': [{'company_name': '*independent', 'end_date': 0}]
+        }, user)
+
+    def test_process_blueprint_then_review(self):
+        record_processor_inst = self.make_record_processor(
+            lp_user_name={
+                'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}})
+
+        processed_records = list(record_processor_inst.process([
+            {'record_type': 'bp',
+             'self_link': 'http://launchpad.net/blueprint',
+             'owner': 'john_doe',
+             'date_created': 1234567890},
+            {'record_type': 'review',
+             'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
+             'subject': 'Fix AttributeError in Keypair._add_details()',
+             'owner': {'name': 'John Doe',
+                       'email': 'john_doe@gmail.com',
+                       'username': 'john_doe'},
+             'createdOn': 1379404951,
+             'module': 'nova'}
+        ]))
+
+        self.assertRecordsMatch(
+            {'record_type': 'bpd',
+             'launchpad_id': 'john_doe',
+             'author_name': 'John Doe',
+             'company_name': '*independent'},
+            processed_records[0])
+
+        self.assertRecordsMatch(
+            {'record_type': 'review',
+             'launchpad_id': 'john_doe',
+             'author_name': 'John Doe',
+             'author_email': 'john_doe@gmail.com',
+             'company_name': '*independent'},
+            processed_records[1])
+
+        user = {'user_id': 'john_doe',
+                'launchpad_id': 'john_doe',
+                'user_name': 'John Doe',
+                'emails': ['john_doe@gmail.com'],
+                'companies': [{'company_name': '*independent', 'end_date': 0}]}
+        self.assertEquals({'john_doe': user, 'john_doe@gmail.com': user},
+                          record_processor_inst.users_index)
+
+    def test_process_blueprint_then_commit(self):
+        record_processor_inst = self.make_record_processor(
+            lp_user_name={
+                'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}},
+            lp_info={'john_doe@gmail.com':
+                     {'name': 'john_doe', 'display_name': 'John Doe'}})
+
+        processed_records = list(record_processor_inst.process([
+            {'record_type': 'bp',
+             'self_link': 'http://launchpad.net/blueprint',
+             'owner': 'john_doe',
+             'date_created': 1234567890},
+            {'record_type': 'commit',
+             'commit_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1',
+             'author_name': 'John Doe',
+             'author_email': 'john_doe@gmail.com',
+             'date': 1234567890,
+             'lines_added': 25,
+             'lines_deleted': 9,
+             'release_name': 'havana'}
+        ]))
+
+        self.assertRecordsMatch(
+            {'record_type': 'bpd',
+             'launchpad_id': 'john_doe',
+             'author_name': 'John Doe',
+             'company_name': '*independent'},
+            processed_records[0])
+
+        self.assertRecordsMatch(
+            {'record_type': 'commit',
+             'launchpad_id': 'john_doe',
+             'author_name': 'John Doe',
+             'author_email': 'john_doe@gmail.com',
+             'company_name': '*independent'},
+            processed_records[1])
+
+        user = {'user_id': 'john_doe',
+                'launchpad_id': 'john_doe',
+                'user_name': 'John Doe',
+                'emails': ['john_doe@gmail.com'],
+                'companies': [{'company_name': '*independent', 'end_date': 0}]}
+        self.assertEquals({'john_doe': user, 'john_doe@gmail.com': user},
+                          record_processor_inst.users_index)
+
+    # update records
 
     def _generate_record_commit(self):
         yield {'commit_id': u'0afdc64bfd041b03943ceda7849c4443940b6053',
@@ -272,68 +499,187 @@ class TestRecordProcessor(testtools.TestCase):
     def test_update_record_no_changes(self):
         commit_generator = self._generate_record_commit()
         release_index = {'0afdc64bfd041b03943ceda7849c4443940b6053': 'havana'}
-        record_processor_inst = make_record_processor(
+        record_processor_inst = self.make_record_processor(
-            make_runtime_storage(users=[]))
+            users=[],
+            companies=[{'company_name': 'SuperCompany',
+                        'domains': ['super.no']}])
 
         updated = list(record_processor_inst.update(commit_generator,
                                                     release_index))
 
         self.assertEquals(0, len(updated))
 
-    def test_process_mail(self):
+    # mail processing
-        record_processor_inst = make_record_processor()
-        commit_generator = generate_emails(
-            subject='[openstack-dev] [Stackalytics] Configuration files')
-        commit = list(record_processor_inst.process(commit_generator))[0]
 
-        self.assertEquals('SuperCompany', commit['company_name'])
+    def test_process_mail(self):
-        self.assertEquals('john_doe', commit['launchpad_id'])
+        record_processor_inst = self.make_record_processor(
-        self.assertEquals('stackalytics', commit['module'])
+            users=[
+                {
+                    'user_id': 'john_doe',
+                    'launchpad_id': 'john_doe',
+                    'user_name': 'John Doe',
+                    'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
+                    'companies': [
+                        {'company_name': 'NEC', 'end_date': 0},
+                    ]
+                }
+            ],
+            repos=[{"module": "stackalytics"}]
+        )
+
+        processed_commit = list(record_processor_inst.process(
+            generate_emails(
+                author_email='johndoe@gmail.com',
+                author_name='John Doe',
+                subject='[openstack-dev] [Stackalytics] Configuration files')
+        ))[0]
+
+        expected_commit = {
+            'launchpad_id': 'john_doe',
+            'author_email': 'johndoe@gmail.com',
+            'author_name': 'John Doe',
+            'company_name': 'NEC',
+            'module': 'stackalytics',
+        }
+
+        self.assertRecordsMatch(expected_commit, processed_commit)
 
     def test_process_mail_guessed(self):
-        runtime_storage_inst = make_runtime_storage(
+        record_processor_inst = self.make_record_processor(
-            repos=[{'module': 'nova'}, {'module': 'neutron'}])
+            users=[
-        record_processor_inst = make_record_processor(runtime_storage_inst)
+                {
-        commit_generator = generate_emails(
+                    'user_id': 'john_doe',
-            subject='[openstack-dev] [Neutron] [Nova] Integration issue')
+                    'launchpad_id': 'john_doe',
-        commit = list(record_processor_inst.process(commit_generator))[0]
+                    'user_name': 'John Doe',
+                    'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
+                    'companies': [
+                        {'company_name': 'NEC', 'end_date': 0},
+                    ]
+                }
+            ],
+            repos=[{'module': 'nova'}, {'module': 'neutron'}]
+        )
 
-        self.assertEquals('neutron', commit['module'])
+        processed_commit = list(record_processor_inst.process(
+            generate_emails(
+                author_email='johndoe@gmail.com',
+                author_name='John Doe',
+                subject='[openstack-dev] [Neutron] [Nova] Integration issue')
+        ))[0]
+
+        expected_commit = {
+            'launchpad_id': 'john_doe',
+            'author_email': 'johndoe@gmail.com',
+            'author_name': 'John Doe',
+            'company_name': 'NEC',
+            'module': 'neutron',
+        }
+
+        self.assertRecordsMatch(expected_commit, processed_commit)
 
     def test_process_mail_guessed_module_in_body_override(self):
-        runtime_storage_inst = make_runtime_storage(
+        record_processor_inst = self.make_record_processor(
-            repos=[{'module': 'nova'}, {'module': 'heat'}])
+            users=[
-        record_processor_inst = make_record_processor(runtime_storage_inst)
+                {
-        commit_generator = generate_emails(
+                    'user_id': 'john_doe',
-            subject='[openstack-dev] [heat] Comments/questions on the',
+                    'launchpad_id': 'john_doe',
-            module='nova')
+                    'user_name': 'John Doe',
-        commit = list(record_processor_inst.process(commit_generator))[0]
+                    'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
+                    'companies': [
+                        {'company_name': 'NEC', 'end_date': 0},
+                    ]
+                }
+            ],
+            repos=[{'module': 'nova'}, {'module': 'neutron'}]
+        )
 
-        self.assertEquals('heat', commit['module'])
+        processed_commit = list(record_processor_inst.process(
+            generate_emails(
+                author_email='johndoe@gmail.com',
+                author_name='John Doe',
+                module='nova',
+                subject='[openstack-dev] [neutron] Comments/questions on the')
+        ))[0]
+
+        expected_commit = {
+            'launchpad_id': 'john_doe',
+            'author_email': 'johndoe@gmail.com',
+            'author_name': 'John Doe',
+            'company_name': 'NEC',
+            'module': 'neutron',
+        }
+
+        self.assertRecordsMatch(expected_commit, processed_commit)
 
     def test_process_mail_guessed_module_in_body(self):
-        runtime_storage_inst = make_runtime_storage(
+        record_processor_inst = self.make_record_processor(
-            repos=[{'module': 'nova'}, {'module': 'heat'}])
+            users=[
-        record_processor_inst = make_record_processor(runtime_storage_inst)
+                {
-        commit_generator = generate_emails(
+                    'user_id': 'john_doe',
-            subject='[openstack-dev] Comments/questions on the heat',
+                    'launchpad_id': 'john_doe',
-            module='nova')
+                    'user_name': 'John Doe',
-        commit = list(record_processor_inst.process(commit_generator))[0]
+                    'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
+                    'companies': [
+                        {'company_name': 'NEC', 'end_date': 0},
+                    ]
+                }
+            ],
+            repos=[{'module': 'nova'}, {'module': 'neutron'}]
+        )
 
-        self.assertEquals('nova', commit['module'])
+        processed_commit = list(record_processor_inst.process(
+            generate_emails(
+                author_email='johndoe@gmail.com',
+                author_name='John Doe',
+                module='nova',
+                subject='[openstack-dev] Comments/questions on the')
+        ))[0]
+
+        expected_commit = {
+            'launchpad_id': 'john_doe',
+            'author_email': 'johndoe@gmail.com',
+            'author_name': 'John Doe',
+            'company_name': 'NEC',
+            'module': 'nova',
+        }
+
+        self.assertRecordsMatch(expected_commit, processed_commit)
 
     def test_process_mail_unmatched(self):
-        record_processor_inst = make_record_processor()
+        record_processor_inst = self.make_record_processor(
-        commit_generator = generate_emails(
+            users=[
-            subject='[openstack-dev] [Photon] Configuration files')
+                {
-        commit = list(record_processor_inst.process(commit_generator))[0]
+                    'user_id': 'john_doe',
+                    'launchpad_id': 'john_doe',
+                    'user_name': 'John Doe',
+                    'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
+                    'companies': [
+                        {'company_name': 'NEC', 'end_date': 0},
+                    ]
+                }
+            ],
+            repos=[{'module': 'nova'}, {'module': 'neutron'}]
+        )
 
-        self.assertEquals('SuperCompany', commit['company_name'])
+        processed_commit = list(record_processor_inst.process(
-        self.assertEquals('john_doe', commit['launchpad_id'])
+            generate_emails(
-        self.assertEquals('unknown', commit['module'])
+                author_email='johndoe@gmail.com',
+                author_name='John Doe',
+                subject='[openstack-dev] Comments/questions on the')
+        ))[0]
+
+        expected_commit = {
+            'launchpad_id': 'john_doe',
+            'author_email': 'johndoe@gmail.com',
+            'author_name': 'John Doe',
+            'company_name': 'NEC',
+            'module': 'unknown',
+        }
+
+        self.assertRecordsMatch(expected_commit, processed_commit)
 
     def test_get_modules(self):
-        record_processor_inst = make_record_processor()
+        record_processor_inst = self.make_record_processor()
         with mock.patch('stackalytics.processor.utils.load_repos') as patch:
             patch.return_value = [{'module': 'nova'},
                                   {'module': 'python-novaclient'},
@@ -341,15 +687,36 @@ class TestRecordProcessor(testtools.TestCase):
             modules = record_processor_inst._get_modules()
             self.assertEqual(set(['nova', 'neutron']), set(modules))
 
+    def assertRecordsMatch(self, expected, actual):
+        for key, value in expected.iteritems():
+            self.assertEquals(value, actual[key],
+                              'Values for key %s do not match' % key)
 
-# Helpers
+    # Helpers
 
-def generate_commits(email='johndoe@gmail.com', date=1999999999):
+    def make_record_processor(self, users=None, companies=None, releases=None,
+                              repos=None, lp_info=None, lp_user_name=None):
+        rp = record_processor.RecordProcessor(make_runtime_storage(
+            users=users, companies=companies, releases=releases, repos=repos))
+
+        if lp_info is not None:
+            self.lp_profile_by_email.side_effect = (
+                lambda x: lp_info.get(x))
+
+        if lp_user_name is not None:
+            self.lp_profile_by_launchpad_id.side_effect = (
+                lambda x: lp_user_name.get(x))
+
+        return rp
+
+
+def generate_commits(author_name='John Doe', author_email='johndoe@gmail.com',
+                     date=1999999999):
     yield {
         'record_type': 'commit',
         'commit_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1',
-        'author_name': 'John Doe',
+        'author_name': author_name,
-        'author_email': email,
+        'author_email': author_email,
         'date': date,
         'lines_added': 25,
         'lines_deleted': 9,
@@ -357,13 +724,13 @@ def generate_commits(email='johndoe@gmail.com', date=1999999999):
     }
 
 
-def generate_emails(email='johndoe@gmail.com', date=1999999999,
+def generate_emails(author_name='John Doe', author_email='johndoe@gmail.com',
-                    subject='[openstack-dev]', module=None):
+                    date=1999999999, subject='[openstack-dev]', module=None):
     yield {
         'record_type': 'email',
         'message_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1',
-        'author_name': 'John Doe',
+        'author_name': author_name,
-        'author_email': email,
+        'author_email': author_email,
         'date': date,
         'subject': subject,
         'module': module,
@@ -374,9 +741,11 @@ def make_runtime_storage(users=None, companies=None, releases=None,
                          repos=None):
     def get_by_key(collection):
         if collection == 'companies':
-            return _make_companies(companies or COMPANIES)
+            return _make_companies(companies or [
+                {"company_name": "*independent", "domains": [""]},
+            ])
         elif collection == 'users':
-            return _make_users(users or USERS)
+            return _make_users(users or [])
         elif collection == 'releases':
             return releases or RELEASES
         elif collection == 'repos':
@@ -389,26 +758,6 @@ def make_runtime_storage(users=None, companies=None, releases=None,
     return rs
 
 
-def make_record_processor(runtime_storage_inst=None):
-    return record_processor.RecordProcessor(runtime_storage_inst or
-                                            make_runtime_storage())
-
-
-def make_user():
-    return {
-        'user_id': 'john_doe',
-        'launchpad_id': 'john_doe',
-        'user_name': 'John Doe',
-        'emails': ['johndoe@gmail.com', 'jdoe@super.no'],
-        'companies': [
-            {'company_name': '*independent',
-             'end_date': 1234567890},
-            {'company_name': 'SuperCompany',
-             'end_date': 0},
-        ]
-    }
-
-
 def _make_users(users):
     users_index = {}
     for user in users: