Finished implementation of Co-Authored support

* VCS parser yields 1 record per commit and fills co-authors into
corresponding field
* Record processor yields one record for every author from the list
and extend every author with info about company, user_id
* List of authors is added into record view in activity log

Change-Id: I717d68484d7b677fb6a4168b5c87a3fe30e48bbf

dashboard/helpers.py

@@ -30,21 +30,28 @@ INFINITY_HTML = '∞'
 gravatar = gravatar_ext.Gravatar(None, size=64, rating='g', default='wavatar')
 
 
-def _extend_record_common_fields(record):
+def _extend_author_fields(record):
-    record['date_str'] = format_datetime(record['date'])
     record['author_link'] = make_link(
         record['author_name'], '/',
         {'user_id': record['user_id'], 'company': ''})
     record['company_link'] = make_link(
         record['company_name'], '/',
         {'company': record['company_name'], 'user_id': ''})
+    record['gravatar'] = gravatar(record.get('author_email', 'stackalytics'))
+
+
+def _extend_record_common_fields(record):
+    _extend_author_fields(record)
+    record['date_str'] = format_datetime(record['date'])
     record['module_link'] = make_link(
         record['module'], '/',
         {'module': record['module'], 'company': '', 'user_id': ''})
-    record['gravatar'] = gravatar(record.get('author_email', 'stackalytics'))
     record['blueprint_id_count'] = len(record.get('blueprint_id', []))
     record['bug_id_count'] = len(record.get('bug_id', []))
 
+    for coauthor in record.get('coauthor') or []:
+        _extend_author_fields(coauthor)
+
 
 def extend_record(record):
     record = record.copy()

dashboard/templates/_macros/activity_log.html

@@ -72,6 +72,15 @@ show_record_type=True, show_user_gravatar=True, gravatar_size=32, show_all=True)
         <div class="header">{%html author_link %} ({%html company_link %})</div>
         <div class="header">${date_str} in {%html module_link%}</div>
 
+        {%if coauthor %}
+        <div class="header">Co-Authors:
+            {%each(index,value) coauthor %}
+                {%if index>0 %},{%/if%}
+                {%html value.author_link %} ({%html value.company_link %})
+            {%/each%}
+        </div>
+        {%/if%}
+
         {%if record_type == "commit" %}
             <div class="header">Commit &ldquo;${subject}&rdquo;</div>
             <div class="message">{%html message %}</div>

stackalytics/processor/record_processor.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import bisect
+import copy
 import time
 
 import six
@@ -221,10 +222,25 @@ class RecordProcessor(object):
         record['author_email'] = record['author_email'].lower()
         record['commit_date'] = record['date']
 
-        self._update_record_and_user(record)
+        coauthors = record.get('coauthor')
+        if not coauthors:
+            self._update_record_and_user(record)
 
-        if record['company_name'] != '*robots':
+            if record['company_name'] != '*robots':
-            yield record
+                yield record
+        else:
+            coauthors.append({'author_name': record['author_name'],
+                              'author_email': record['author_email']})
+            for coauthor in coauthors:
+                coauthor['date'] = record['date']
+                self._update_record_and_user(coauthor)
+
+            for coauthor in coauthors:
+                new_record = copy.deepcopy(record)
+                new_record.update(coauthor)
+                new_record['primary_key'] += coauthor['author_email']
+
+                yield new_record
 
     def _spawn_review(self, record):
         # copy everything except patchsets and flatten user data

stackalytics/processor/vcs.py

@@ -73,12 +73,12 @@ MESSAGE_PATTERNS = {
     'blueprint_id': re.compile(r'\b(?:blueprint|bp)\b[ \t]*[#:]?[ \t]*'
                                r'(?P<id>[a-z0-9-]+)', re.IGNORECASE),
     'change_id': re.compile('Change-Id: (?P<id>I[0-9a-f]{40})', re.IGNORECASE),
-    'co-author': re.compile(r'(?:Co-Authored|Also)-By:'
+    'coauthor': re.compile(r'(?:Co-Authored|Also)-By:'
-                            r'\s*(?P<id>.*)\s', re.IGNORECASE)
+                           r'\s*(?P<id>.*)\s', re.IGNORECASE)
 }
 
 CO_AUTHOR_PATTERN = re.compile(
-    r'(?P<author_name>.+)\s*<(?P<author_email>.+)>', re.IGNORECASE)
+    r'(?P<author_name>.+?)\s*<(?P<author_email>.+)>', re.IGNORECASE)
 
 
 class Git(Vcs):
@@ -86,7 +86,7 @@ class Git(Vcs):
     def __init__(self, repo, sources_root):
         super(Git, self).__init__(repo, sources_root)
         uri = self.repo['uri']
-        match = re.search(r'([^\/]+)\.git$', uri)
+        match = re.search(r'([^/]+)\.git$', uri)
         if match:
             self.folder = os.path.normpath(self.sources_root + '/' +
                                            match.group(1))
@@ -212,7 +212,8 @@ class Git(Vcs):
                 collection = set()
                 for item in re.finditer(pattern, commit['message']):
                     collection.add(item.group('id'))
-                commit[pattern_name] = list(collection)
+                if collection:
+                    commit[pattern_name] = list(collection)
 
             commit['date'] = int(commit['date'])
             commit['module'] = self.repo['module']
@@ -227,16 +228,16 @@ class Git(Vcs):
                                           for bp_name
                                           in commit['blueprint_id']]
 
-            yield commit
+            coauthors = []
+            for coauthor in commit.get('coauthor') or []:
+                m = re.match(CO_AUTHOR_PATTERN, coauthor)
+                if utils.check_email_validity(m.group("author_email")):
+                    coauthors.append(m.groupdict())
 
-            # Handles co-authors in the commit message. According to the bp
+            if coauthors:
-            # we want to count contribution for authors and co-authors.
+                commit['coauthor'] = coauthors
-            if 'co-author' in commit:
+
-                for coauthor in commit['co-author']:
+            yield commit
-                    m = re.match(CO_AUTHOR_PATTERN, coauthor)
-                    if utils.check_email_validity(m.group("author_email")):
-                        commit.update(m.groupdict())
-                        yield commit
 
     def get_last_id(self, branch):
         LOG.debug('Get head commit for repo uri: %s', self.repo['uri'])

tests/unit/test_record_processor.py

@@ -734,6 +734,44 @@ class TestRecordProcessor(testtools.TestCase):
         self.assertEqual(user_2, utils.load_user(runtime_storage_inst,
                                                  'homer'))
 
+    def test_process_commit_with_coauthors(self):
+        record_processor_inst = self.make_record_processor(
+            lp_info={'jimi.hendrix@openstack.com':
+                     {'name': 'jimi', 'display_name': 'Jimi Hendrix'},
+                     'tupac.shakur@openstack.com':
+                     {'name': 'tupac', 'display_name': 'Tupac Shakur'},
+                     'bob.dylan@openstack.com':
+                     {'name': 'bob', 'display_name': 'Bob Dylan'}})
+        processed_commits = list(record_processor_inst.process([
+            {'record_type': 'commit',
+             'commit_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1',
+             'author_name': 'Jimi Hendrix',
+             'author_email': 'jimi.hendrix@openstack.com', 'date': 1234567890,
+             'lines_added': 25, 'lines_deleted': 9, 'release_name': 'havana',
+             'coauthor': [{'author_name': 'Tupac Shakur',
+                           'author_email': 'tupac.shakur@openstack.com'},
+                          {'author_name': 'Bob Dylan',
+                           'author_email': 'bob.dylan@openstack.com'}]}]))
+
+        self.assertEqual(3, len(processed_commits))
+
+        self.assertRecordsMatch({
+            'launchpad_id': 'tupac',
+            'author_email': 'tupac.shakur@openstack.com',
+            'author_name': 'Tupac Shakur',
+        }, processed_commits[0])
+        self.assertRecordsMatch({
+            'launchpad_id': 'jimi',
+            'author_email': 'jimi.hendrix@openstack.com',
+            'author_name': 'Jimi Hendrix',
+        }, processed_commits[2])
+        self.assertEqual('tupac',
+                         processed_commits[0]['coauthor'][0]['user_id'])
+        self.assertEqual('bob',
+                         processed_commits[0]['coauthor'][1]['user_id'])
+        self.assertEqual('jimi',
+                         processed_commits[0]['coauthor'][2]['user_id'])
+
     # record post-processing
 
     def test_blueprint_mention_count(self):

tests/unit/test_vcs.py

@@ -117,7 +117,7 @@ diff_stat:
             '''
 
         commits = list(self.git.log('dummy', 'dummy'))
-        commits_expected = 6 + 2  # authors + co-authors
+        commits_expected = 6
         self.assertEqual(commits_expected, len(commits))
 
         self.assertEqual(21, commits[0]['files_changed'])
@@ -144,8 +144,11 @@ diff_stat:
         self.assertEqual(0, commits[4]['files_changed'])
         self.assertEqual(0, commits[4]['lines_added'])
         self.assertEqual(0, commits[4]['lines_deleted'])
+        self.assertFalse('coauthor' in commits[4])
 
         self.assertEqual(
-            ['Tupac Shakur <tupac.shakur@openstack.com>',
+            [{'author_name': 'Tupac Shakur',
-             'Bob Dylan <bob.dylan@openstack.com>'],
+              'author_email': 'tupac.shakur@openstack.com'},
-            commits[5]['co-author'])
+             {'author_name': 'Bob Dylan',
+              'author_email': 'bob.dylan@openstack.com'}],
+            commits[5]['coauthor'])