Improve processing of bugs assigned to multiple projects

* Correct issue when bugs assigned to multiple projects get mapped
  to the latest author
* Process bugs assigned to milestones (treat them as branches)
* Improve UI: highlight the link, use colors for statuses

Closes bug 1473638

Change-Id: I5e583053cb9250b967988a4cfc57c23739742e47

stackalytics/dashboard/helpers.py

@@ -95,6 +95,10 @@ def extend_record(record):
                 record['mention_date'])
         record['blueprint_link'] = make_blueprint_link(record['module'],
                                                        record['name'])
+    elif record['record_type'] in ['bugr', 'bugf']:
+        record['number'] = record['web_link'].split('/')[-1]
+        record['title'] = filter_bug_title(record['title'])
+        record['status_class'] = re.sub('\s+', '', record['status'])
 
     return record
 
@@ -307,3 +311,7 @@ def make_page_title(project_type_inst, release, module_inst, company,
         if release != 'all':
             s += ' during OpenStack %s release' % release.capitalize()
     return s
+
+
+def filter_bug_title(title):
+    return re.sub(r'^(?:Bug #\d+.+:\s+)"(.*)"', r'\1', title)

stackalytics/dashboard/templates/_macros/activity_log.html

@@ -151,9 +151,9 @@ show_record_type=True, show_user_gravatar=True, gravatar_size=32, show_all=True)
             <div><b>Mention count: ${mention_count}, last mention on ${mention_date_str}</b></div>
             {%/if%}
         {%elif ((record_type == "bugf") || (record_type == "bugr")) %}
-            <div class="header">&ldquo;${title}&rdquo;</div>
+            <div class="header">Bug &ldquo;${title}&rdquo; (<a href="${web_link}" class="ext_link">${number}</a>)</div>
-            <div>Status: <span class="status${status}">${status}</span></div>
+            <div>Status: <span class="status${status_class}">${status}</span></div>
-            <div>Importance: ${importance}</div>
+            <div>Importance: <span class="importance${importance}">${importance}</span></div>
         {%elif record_type == "ci_vote" %}
             <div class="header">New CI vote in change request ${review_number}
                  {%if is_merged %}(<span style="color: green;">Merged</span>){%/if%}</div>

stackalytics/processor/bps.py

@@ -41,8 +41,18 @@ def log(repo, modified_since):
     for record_draft in launchpad_utils.lp_bug_generator(module,
                                                          modified_since):
 
+        # record_draft can be a bug or bug target and
+        # in the latter case it can be from a different module
+        bug_target = record_draft['bug_target_name'].split('/')
+        target_module = bug_target[0]
+        if target_module != module:
+            continue  # ignore foreigners
+
         record = {}
 
+        if len(bug_target) == 2:
+            record['release'] = bug_target[1]  # treat target as release
+
         for field in LINK_FIELDS:
             link = record_draft[field + '_link']
             if link:
@@ -58,7 +68,7 @@ def log(repo, modified_since):
 
         bug_id = _get_bug_id(record_draft['web_link'])
         record['module'] = module
-        record['id'] = utils.get_bug_id(module, bug_id)
+        record['id'] = utils.make_bug_id(bug_id, module, record.get('release'))
 
         LOG.debug('New bug: %s', record)
         yield record

stackalytics/processor/utils.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import calendar
 import cgi
 import datetime
 import gzip
@@ -69,7 +70,7 @@ def member_date_to_timestamp(d):
 
 
 def iso8601_to_timestamp(s):
-    return int(time.mktime(iso8601.parse_date(s).timetuple()))
+    return calendar.timegm(iso8601.parse_date(s).utctimetuple())
 
 
 def timestamp_to_date(timestamp):
@@ -232,8 +233,11 @@ def get_blueprint_id(module, name):
     return module + ':' + name
 
 
-def get_bug_id(module, bug_id):
+def make_bug_id(bug_id, module, release=None):
-    return module + '/' + bug_id
+    if release:
+        return '/'.join([module, release, bug_id])
+    else:
+        return '/'.join([module, bug_id])
 
 
 def get_patch_id(review_id, patch_number):

tests/unit/test_bps.py

@@ -0,0 +1,221 @@
+# Copyright (c) 2015 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import mock
+import testtools
+
+from stackalytics.processor import bps
+
+
+BUG = json.loads("""
+{
+    "date_closed": "2015-06-02T17:31:05.820479+00:00",
+    "date_assigned": "2015-06-02T17:31:44.957976+00:00",
+    "title": "Bug #1458945 in Sahara: \\\"Use graduated oslo.policy\\\"",
+    "bug_link": "https://api.launchpad.net/devel/bugs/1458945",
+    "bug_watch_link": null,
+    "milestone_link": null,
+    "date_left_closed": null,
+    "date_fix_committed": "2015-06-02T17:31:05.820479+00:00",
+    "date_fix_released": "2015-06-02T17:31:05.820479+00:00",
+    "date_in_progress": "2015-06-02T17:31:05.820479+00:00",
+    "resource_type_link": "https://api.launchpad.net/devel/#bug_task",
+    "status": "Fix Released",
+    "bug_target_name": "sahara",
+    "importance": "Medium",
+    "assignee_link": "https://api.launchpad.net/devel/~slukjanov",
+    "date_triaged": "2015-06-02T17:31:05.820479+00:00",
+    "self_link": "https://api.launchpad.net/devel/sahara/+bug/1458945",
+    "target_link": "https://api.launchpad.net/devel/sahara",
+    "bug_target_display_name": "Sahara",
+    "related_tasks_collection_link":
+      "https://api.launchpad.net/devel/sahara/+bug/1458945/related_tasks",
+    "date_confirmed": "2015-06-02T17:31:05.820479+00:00",
+    "date_left_new": "2015-06-02T17:31:05.820479+00:00",
+    "web_link": "https://bugs.launchpad.net/sahara/+bug/1458945",
+    "owner_link": "https://api.launchpad.net/devel/~samueldmq",
+    "date_created": "2015-06-02T13:35:54.101235+00:00",
+    "date_incomplete": null,
+    "is_complete": true
+}
+""")
+
+ANOTHER_MILESTONE_BUG = json.loads("""
+{
+    "date_closed": "2015-06-02T17:31:05.820479+00:00",
+    "date_assigned": "2015-06-02T17:31:44.957976+00:00",
+    "title": "Bug #1458945 in Sahara Kilo: \\\"Use graduated oslo.policy\\\"",
+    "bug_link": "https://api.launchpad.net/devel/bugs/1458945",
+    "bug_watch_link": null,
+    "milestone_link": null,
+    "date_left_closed": null,
+    "date_fix_committed": "2015-06-02T17:31:05.820479+00:00",
+    "date_fix_released": "2015-06-02T17:31:05.820479+00:00",
+    "date_in_progress": "2015-06-02T17:31:05.820479+00:00",
+    "resource_type_link": "https://api.launchpad.net/devel/#bug_task",
+    "status": "Fix Released",
+    "bug_target_name": "sahara/kilo",
+    "importance": "Medium",
+    "assignee_link": "https://api.launchpad.net/devel/~slukjanov",
+    "date_triaged": "2015-06-02T17:31:05.820479+00:00",
+    "self_link": "https://api.launchpad.net/devel/sahara/kilo/+bug/1458945",
+    "target_link": "https://api.launchpad.net/devel/sahara/kilo",
+    "bug_target_display_name": "Sahara Kilo",
+    "related_tasks_collection_link":
+      "https://api.launchpad.net/devel/sahara/kilo/+bug/1458945/related_tasks",
+    "date_confirmed": "2015-06-02T17:31:05.820479+00:00",
+    "date_left_new": "2015-06-02T17:31:05.820479+00:00",
+    "web_link": "https://bugs.launchpad.net/sahara/kilo/+bug/1458945",
+    "owner_link": "https://api.launchpad.net/devel/~samueldmq",
+    "date_created": "2015-06-02T13:35:54.101235+00:00",
+    "date_incomplete": null,
+    "is_complete": true
+}
+""")
+
+LINKED_BUG = json.loads("""
+{
+    "date_closed": "2015-06-24T20:59:57.982386+00:00",
+    "date_assigned": "2015-06-18T06:46:03.741208+00:00",
+    "title": "Bug #1458945 in Barbican: \\\"Use graduated oslo.policy\\\"",
+    "bug_link": "https://api.launchpad.net/devel/bugs/1458945",
+    "bug_watch_link": null,
+    "milestone_link":
+      "https://api.launchpad.net/devel/barbican/+milestone/liberty-1",
+    "date_left_closed": null,
+    "date_fix_committed": "2015-06-18T06:45:39.997949+00:00",
+    "date_fix_released": "2015-06-24T20:59:57.982386+00:00",
+    "date_in_progress": "2015-06-18T06:45:39.997949+00:00",
+    "resource_type_link": "https://api.launchpad.net/devel/#bug_task",
+    "status": "Fix Released",
+    "bug_target_name": "barbican",
+    "importance": "Medium",
+    "assignee_link": "https://api.launchpad.net/devel/~juan-osorio-robles",
+    "date_triaged": "2015-06-18T06:45:39.997949+00:00",
+    "self_link": "https://api.launchpad.net/devel/barbican/+bug/1458945",
+    "target_link": "https://api.launchpad.net/devel/barbican",
+    "bug_target_display_name": "Barbican",
+    "related_tasks_collection_link":
+      "https://api.launchpad.net/devel/barbican/+bug/1458945/related_tasks",
+    "date_confirmed": "2015-06-18T06:45:39.997949+00:00",
+    "date_left_new": "2015-06-18T06:45:39.997949+00:00",
+    "web_link": "https://bugs.launchpad.net/barbican/+bug/1458945",
+    "owner_link": "https://api.launchpad.net/devel/~samueldmq",
+    "date_created": "2015-05-26T17:47:32.438795+00:00",
+    "date_incomplete": null,
+    "is_complete": true
+}
+""")
+
+
+class TestBps(testtools.TestCase):
+    def setUp(self):
+        super(TestBps, self).setUp()
+        p_module_exists = mock.patch(
+            'stackalytics.processor.launchpad_utils.lp_module_exists')
+        m_module_exists = p_module_exists.start()
+        m_module_exists.return_value = True
+
+    @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
+    def test_log(self, lp_bug_generator):
+        repo = {
+            'module': 'sahara'
+        }
+        modified_since = 1234567890
+        lp_bug_generator.return_value = iter([BUG])
+
+        expected = [{
+            'assignee': 'slukjanov',
+            'date_created': 1433252154,
+            'date_fix_committed': 1433266265,
+            'id': 'sahara/1458945',
+            'importance': 'Medium',
+            'module': 'sahara',
+            'owner': 'samueldmq',
+            'status': 'Fix Released',
+            'title': 'Bug #1458945 in Sahara: "Use graduated oslo.policy"',
+            'web_link': 'https://bugs.launchpad.net/sahara/+bug/1458945'
+        }]
+
+        actual = list(bps.log(repo, modified_since))
+
+        self.assertEqual(expected, actual)
+
+    @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
+    def test_log_additional_module(self, lp_bug_generator):
+        # bug linked to another project should not appear
+        repo = {
+            'module': 'sahara'
+        }
+        modified_since = 1234567890
+        lp_bug_generator.return_value = iter([BUG, LINKED_BUG])
+
+        expected = [{
+            'assignee': 'slukjanov',
+            'date_created': 1433252154,
+            'date_fix_committed': 1433266265,
+            'id': 'sahara/1458945',
+            'importance': 'Medium',
+            'module': 'sahara',
+            'owner': 'samueldmq',
+            'status': 'Fix Released',
+            'title': 'Bug #1458945 in Sahara: "Use graduated oslo.policy"',
+            'web_link': 'https://bugs.launchpad.net/sahara/+bug/1458945'
+        }]
+
+        actual = list(bps.log(repo, modified_since))
+
+        self.assertEqual(expected, actual)
+
+    @mock.patch('stackalytics.processor.launchpad_utils.lp_bug_generator')
+    def test_log_additional_milestone(self, lp_bug_generator):
+        # bug linked to different milestone should be mapped to the release
+        repo = {
+            'module': 'sahara'
+        }
+        modified_since = 1234567890
+        lp_bug_generator.return_value = iter([BUG, ANOTHER_MILESTONE_BUG])
+
+        expected = [{
+            'assignee': 'slukjanov',
+            'date_created': 1433252154,
+            'date_fix_committed': 1433266265,
+            'id': 'sahara/1458945',
+            'importance': 'Medium',
+            'module': 'sahara',
+            'owner': 'samueldmq',
+            'status': 'Fix Released',
+            'title': 'Bug #1458945 in Sahara: "Use graduated oslo.policy"',
+            'web_link': 'https://bugs.launchpad.net/sahara/+bug/1458945'
+        }, {
+            'assignee': 'slukjanov',
+            'date_created': 1433252154,
+            'date_fix_committed': 1433266265,
+            'id': 'sahara/kilo/1458945',
+            'importance': 'Medium',
+            'module': 'sahara',
+            'release': 'kilo',
+            'owner': 'samueldmq',
+            'status': 'Fix Released',
+            'title': 'Bug #1458945 in Sahara Kilo: '
+                     '"Use graduated oslo.policy"',
+            'web_link': 'https://bugs.launchpad.net/sahara/kilo/+bug/1458945'
+
+        }]
+
+        actual = list(bps.log(repo, modified_since))
+
+        self.assertEqual(expected, actual)

tests/unit/test_web_utils.py

@@ -150,3 +150,12 @@ Implements Blueprint ''' + (
         get_default.side_effect = make({'param': 'foo'})
         self.assertEqual(['foo'], parameters.get_parameter({}, 'param'))
         self.assertEqual([], parameters.get_parameter({}, 'other'))
+
+    def test_filter_bug_title(self):
+        bug_title = ('Bug #1459454 in Barbican: "Stored key certificate '
+                     'order does not set PK on generated container"')
+        expected = ('Stored key certificate order does not set PK '
+                    'on generated container')
+
+        actual = helpers.filter_bug_title(bug_title)
+        self.assertEqual(expected, actual)