Added new metrics "bug_filed" and "bug_resolved"

implements bp metric-bugs

Added new metrics 'bugs' by companies and users
Bugs are retrieved using launchpad-api
Bugs are shown in UI as other metrics as commits, emails, etc.

Change-Id: Ia9d9d8ca2fcbbe0fa257d90585ab1c56403f2419

dashboard/decorators.py

@@ -310,6 +310,8 @@ def aggregate_filter():
                 'emails': (incremental_filter, None),
                 'bpd': (incremental_filter, None),
                 'bpc': (incremental_filter, None),
+                'filed-bugs': (incremental_filter, None),
+                'resolved-bugs': (incremental_filter, None),
                 'members': (incremental_filter, None),
                 'person-day': (person_day_filter, None),
             }

dashboard/helpers.py

@@ -136,6 +136,8 @@ def get_contribution_summary(records):
     drafted_blueprint_count = 0
     completed_blueprint_count = 0
     email_count = 0
+    filed_bug_count = 0
+    resolved_bug_count = 0
 
     for record in records:
         record_type = record['record_type']
@@ -153,6 +155,10 @@ def get_contribution_summary(records):
             drafted_blueprint_count += 1
         elif record['record_type'] == 'bpc':
             completed_blueprint_count += 1
+        elif record['record_type'] == 'bugf':
+            filed_bug_count += 1
+        elif record['record_type'] == 'bugr':
+            resolved_bug_count += 1
 
     result = {
         'drafted_blueprint_count': drafted_blueprint_count,
@@ -161,6 +167,8 @@ def get_contribution_summary(records):
         'email_count': email_count,
         'loc': loc,
         'marks': marks,
+        'filed_bug_count': filed_bug_count,
+        'resolved_bug_count': resolved_bug_count,
     }
     return result
 

dashboard/parameters.py

@@ -35,6 +35,8 @@ METRIC_LABELS = {
     'emails': 'Emails',
     'bpd': 'Drafted Blueprints',
     'bpc': 'Completed Blueprints',
+    'filed-bugs': 'Filed Bugs',
+    'resolved-bugs': 'Resolved Bugs',
     # 'person-day': "Person-day effort"
 }
 
@@ -45,6 +47,8 @@ METRIC_TO_RECORD_TYPE = {
     'emails': 'email',
     'bpd': 'bpd',
     'bpc': 'bpc',
+    'filed-bugs': 'bugf',
+    'resolved-bugs': 'bugr',
     'members': 'member',
 }
 

dashboard/templates/_macros/activity_log.html

@@ -145,6 +145,10 @@ show_record_type=True, show_user_gravatar=True, gravatar_size=32, show_all=True)
             {%if mention_count %}
             <div><b>Mention count: ${mention_count}, last mention on ${mention_date_str}</b></div>
             {%/if%}
+        {%elif ((record_type == "bugf") || (record_type == "bugr")) %}
+            <div class="header">&ldquo;${title}&rdquo;</div>
+            <div>Status: <span class="status${status}">${status}</span></div>
+            <div>Importance: ${importance}</div>
         {%/if%}
     </div>
     </div>

dashboard/templates/_macros/contribution_summary.html

@@ -34,6 +34,8 @@
 <div>Review stat (-2, -1, +1, +2, A): <b>${marks["-2"]}, ${marks["-1"]}, ${marks["1"]}, ${marks["2"]}, ${marks["A"]}</b></div>
 <div>Draft Blueprints: <b>${drafted_blueprint_count}</b></div>
 <div>Completed Blueprints: <b>${completed_blueprint_count}</b></div>
+<div>Filed Bugs: <b>${filed_bug_count}</b></div>
+<div>Resolved Bugs: <b>${resolved_bug_count}</b></div>
 <div>Emails: <b>${email_count}</b></div>
 {% endraw %}
 </script>

stackalytics/processor/bps.py

@@ -0,0 +1,63 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from stackalytics.openstack.common import log as logging
+from stackalytics.processor import launchpad_utils
+from stackalytics.processor import utils
+
+
+LOG = logging.getLogger(__name__)
+
+LINK_FIELDS = ['owner', 'assignee']
+BUG_FIELDS = ['web_link', 'status', 'title', 'importance']
+DATE_FIELDS = ['date_created', 'date_fix_committed']
+
+
+def _get_bug_id(web_link):
+    return web_link[web_link.rfind('/') + 1:]
+
+
+def log(repo, last_bug_date):
+    module = repo['module']
+    LOG.debug('Retrieving list of bugs for module: %s', module)
+
+    if not launchpad_utils.lp_module_exists(module):
+        LOG.debug('Module %s does not exist at Launchpad', module)
+        return
+
+    for record_draft in launchpad_utils.lp_bug_generator(module,
+                                                         last_bug_date):
+
+        record = {}
+
+        for field in LINK_FIELDS:
+            link = record_draft[field + '_link']
+            if link:
+                record[field] = launchpad_utils.link_to_launchpad_id(link)
+
+        for field in BUG_FIELDS:
+            record[field] = record_draft[field]
+
+        for field in DATE_FIELDS:
+            date = record_draft[field]
+            if date:
+                record[field] = utils.iso8601_to_timestamp(date)
+
+        bug_id = _get_bug_id(record_draft['web_link'])
+        record['module'] = module
+        record['id'] = utils.get_bug_id(module, bug_id)
+
+        LOG.debug('New bug: %s', record)
+        yield record

stackalytics/processor/launchpad_utils.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import six
 from six.moves import http_client
 from six.moves.urllib import parse
 
@@ -22,11 +23,19 @@ from stackalytics.processor import utils
 
 LOG = logging.getLogger(__name__)
 
-
+BUG_STATUSES = ['New', 'Incomplete', 'Opinion', 'Invalid', 'Won\'t Fix',
+                'Expired', 'Confirmed', 'Triaged', 'In Progress',
+                'Fix Committed', 'Fix Released',
+                'Incomplete (with response)',
+                'Incomplete (without response)']
 LP_URI_V1 = 'https://api.launchpad.net/1.0/%s'
 LP_URI_DEVEL = 'https://api.launchpad.net/devel/%s'
 
 
+def link_to_launchpad_id(link):
+    return link[link.find('~') + 1:]
+
+
 def lp_profile_by_launchpad_id(launchpad_id):
     LOG.debug('Lookup user id %s at Launchpad', launchpad_id)
     uri = LP_URI_V1 % ('~' + launchpad_id)
@@ -65,3 +74,24 @@ def lp_blueprint_generator(module):
             yield record
 
         uri = chunk.get('next_collection_link')
+
+
+def lp_bug_generator(module, last_bug_date):
+    uri = LP_URI_DEVEL % (module + '?ws.op=searchTasks')
+    for status in BUG_STATUSES:
+        uri += '&status=' + six.moves.urllib.parse.quote_plus(status)
+    if last_bug_date:
+        uri += '&modified_since=' + last_bug_date
+
+    while uri:
+        LOG.debug('Reading chunk from uri %s', uri)
+        chunk = utils.read_json_from_uri(uri)
+
+        if not chunk:
+            LOG.warn('No data was read from uri %s', uri)
+            break
+
+        for record in chunk['entries']:
+            yield record
+
+        uri = chunk.get('next_collection_link')

stackalytics/processor/lp.py

@@ -25,23 +25,19 @@ LINK_FIELDS = ['owner', 'drafter', 'starter', 'completer',
 DATE_FIELDS = ['date_created', 'date_completed', 'date_started']
 
 
-def _link_to_launchpad_id(link):
-    return link[link.find('~') + 1:]
-
-
 def log(repo):
     module = repo['module']
     LOG.debug('Retrieving list of blueprints for module: %s', module)
 
     if not launchpad_utils.lp_module_exists(module):
-        LOG.debug('Module %s not exist at Launchpad', module)
+        LOG.debug('Module %s does not exist at Launchpad', module)
         return
 
     for record in launchpad_utils.lp_blueprint_generator(module):
         for field in LINK_FIELDS:
             link = record[field + '_link']
             if link:
-                record[field] = _link_to_launchpad_id(link)
+                record[field] = launchpad_utils.link_to_launchpad_id(link)
                 del record[field + '_link']
         for field in DATE_FIELDS:
             date = record[field]

stackalytics/processor/main.py

@@ -19,9 +19,11 @@ from oslo.config import cfg
 import psutil
 import six
 from six.moves.urllib import parse
+import time
 import yaml
 
 from stackalytics.openstack.common import log as logging
+from stackalytics.processor import bps
 from stackalytics.processor import config
 from stackalytics.processor import default_data_processor
 from stackalytics.processor import lp
@@ -76,7 +78,8 @@ def _record_typer(record_iterator, record_type):
         yield record
 
 
-def process_repo(repo, runtime_storage_inst, record_processor_inst):
+def process_repo(repo, runtime_storage_inst, record_processor_inst,
+                 last_bug_date):
     uri = repo['uri']
     LOG.debug('Processing repo uri %s' % uri)
 
@@ -87,6 +90,13 @@ def process_repo(repo, runtime_storage_inst, record_processor_inst):
     runtime_storage_inst.set_records(processed_bp_iterator,
                                      utils.merge_records)
 
+    bug_iterator = bps.log(repo, last_bug_date)
+    bug_iterator_typed = _record_typer(bug_iterator, 'bug')
+    processed_bug_iterator = record_processor_inst.process(
+        bug_iterator_typed)
+    runtime_storage_inst.set_records(processed_bug_iterator,
+                                     utils.merge_records)
+
     vcs_inst = vcs.get_vcs(repo, cfg.CONF.sources_root)
     vcs_inst.fetch()
 
@@ -158,8 +168,12 @@ def update_members(runtime_storage_inst, record_processor_inst):
 def update_records(runtime_storage_inst, record_processor_inst):
     repos = utils.load_repos(runtime_storage_inst)
 
+    current_date = utils.timestamp_to_utc_date(int(time.time()))
+    last_bug_date = runtime_storage_inst.get_by_key('last_bug_date')
     for repo in repos:
-        process_repo(repo, runtime_storage_inst, record_processor_inst)
+        process_repo(repo, runtime_storage_inst, record_processor_inst,
+                     last_bug_date)
+    runtime_storage_inst.set_by_key('last_bug_date', current_date)
 
     mail_lists = runtime_storage_inst.get_by_key('mail_lists') or []
     for mail_list in mail_lists:

stackalytics/processor/record_processor.py

@@ -417,6 +417,30 @@ class RecordProcessor(object):
 
             yield bpc
 
+    def _process_bug(self, record):
+
+        bug_created = record.copy()
+        bug_created['primary_key'] = 'bugf:' + record['id']
+        bug_created['record_type'] = 'bugf'
+        bug_created['launchpad_id'] = record.get('owner')
+        bug_created['date'] = record['date_created']
+
+        self._update_record_and_user(bug_created)
+
+        yield bug_created
+
+        FIXED_BUGS = ['Fix Committed', 'Fix Released']
+        if 'date_fix_committed' in record and record['status'] in FIXED_BUGS:
+            bug_fixed = record.copy()
+            bug_fixed['primary_key'] = 'bugr:' + record['id']
+            bug_fixed['record_type'] = 'bugr'
+            bug_fixed['launchpad_id'] = record.get('assignee') or '*unassigned'
+            bug_fixed['date'] = record['date_fix_committed']
+
+            self._update_record_and_user(bug_fixed)
+
+            yield bug_fixed
+
     def _process_member(self, record):
         user_id = "member:" + record['member_id']
         record['primary_key'] = user_id
@@ -465,6 +489,9 @@ class RecordProcessor(object):
         elif record['record_type'] == 'member':
             for r in self._process_member(record):
                 yield r
+        elif record['record_type'] == 'bug':
+            for r in self._process_bug(record):
+                yield r
 
     def _renew_record_date(self, record):
         record['week'] = utils.timestamp_to_week(record['date'])

stackalytics/processor/utils.py

@@ -72,6 +72,11 @@ def timestamp_to_day(timestamp):
     return timestamp // (24 * 3600)
 
 
+def timestamp_to_utc_date(timestamp):
+    return (datetime.datetime.fromtimestamp(timestamp).
+            strftime('%Y-%m-%d'))
+
+
 def round_timestamp_to_day(timestamp):
     return (int(timestamp) // (24 * 3600)) * (24 * 3600)
 
@@ -173,6 +178,10 @@ def get_blueprint_id(module, name):
     return module + ':' + name
 
 
+def get_bug_id(module, bug_id):
+    return module + '/' + bug_id
+
+
 def get_patch_id(review_id, patch_number):
     return review_id + ':' + patch_number
 

tests/unit/test_record_processor.py

@@ -320,6 +320,90 @@ class TestRecordProcessor(testtools.TestCase):
         self.assertEqual('IBM', user['companies'][0]['company_name'])
         self.assertEqual(None, user['launchpad_id'])
 
+    def generate_bugs(self, assignee=None, date_fix_committed=None,
+                      status='Confirmed'):
+        yield {
+            'record_type': 'bug',
+            'id': 'bug_id',
+            'owner': 'owner',
+            'assignee': assignee,
+            'date_created': 1234567890,
+            'date_fix_committed': date_fix_committed,
+            'module': 'nova',
+            'status': status
+        }
+
+    def test_process_bug_not_fixed(self):
+        record = self.generate_bugs()
+        record_processor_inst = self.make_record_processor()
+        bugs = list(record_processor_inst.process(record))
+        self.assertEqual(len(bugs), 1)
+        self.assertRecordsMatch({
+            'primary_key': 'bugf:bug_id',
+            'record_type': 'bugf',
+            'launchpad_id': 'owner',
+            'date': 1234567890,
+        }, bugs[0])
+
+    def test_process_bug_fix_committed(self):
+        record = self.generate_bugs(status='Fix Committed',
+                                    date_fix_committed=1234567891,
+                                    assignee='assignee')
+        record_processor_inst = self.make_record_processor()
+        bugs = list(record_processor_inst.process(record))
+        self.assertEqual(len(bugs), 2)
+        self.assertRecordsMatch({
+            'primary_key': 'bugf:bug_id',
+            'record_type': 'bugf',
+            'launchpad_id': 'owner',
+            'date': 1234567890,
+        }, bugs[0])
+        self.assertRecordsMatch({
+            'primary_key': 'bugr:bug_id',
+            'record_type': 'bugr',
+            'launchpad_id': 'assignee',
+            'date': 1234567891,
+        }, bugs[1])
+
+    def test_process_bug_fix_released(self):
+        record = self.generate_bugs(status='Fix Released',
+                                    date_fix_committed=1234567891,
+                                    assignee='assignee')
+        record_processor_inst = self.make_record_processor()
+        bugs = list(record_processor_inst.process(record))
+        self.assertEqual(len(bugs), 2)
+        self.assertRecordsMatch({
+            'primary_key': 'bugf:bug_id',
+            'record_type': 'bugf',
+            'launchpad_id': 'owner',
+            'date': 1234567890,
+        }, bugs[0])
+        self.assertRecordsMatch({
+            'primary_key': 'bugr:bug_id',
+            'record_type': 'bugr',
+            'launchpad_id': 'assignee',
+            'date': 1234567891,
+        }, bugs[1])
+
+    def test_process_bug_fix_committed_without_assignee(self):
+        record = self.generate_bugs(status='Fix Committed',
+                                    date_fix_committed=1234567891)
+        record_processor_inst = self.make_record_processor()
+        bugs = list(record_processor_inst.process(record))
+        self.assertEqual(len(bugs), 2)
+        self.assertRecordsMatch({
+            'primary_key': 'bugf:bug_id',
+            'record_type': 'bugf',
+            'launchpad_id': 'owner',
+            'date': 1234567890,
+        }, bugs[0])
+        self.assertRecordsMatch({
+            'primary_key': 'bugr:bug_id',
+            'record_type': 'bugr',
+            'launchpad_id': '*unassigned',
+            'date': 1234567891,
+        }, bugs[1])
+
     # process records complex scenarios
 
     def test_process_blueprint_one_draft_spawned_lp_doesnt_know_user(self):