Added new metrics "bug_filed" and "bug_resolved"

implements bp metric-bugs Added new metrics 'bugs' by companies and users Bugs are retrieved using launchpad-api Bugs are shown in UI as other metrics as commits, emails, etc. Change-Id: Ia9d9d8ca2fcbbe0fa257d90585ab1c56403f2419
2014-04-29 16:32:59 +04:00
parent 6b247ce177
commit ceca8a18d3
12 changed files with 252 additions and 9 deletions
--- a/dashboard/decorators.py
+++ b/dashboard/decorators.py
@@ -310,6 +310,8 @@ def aggregate_filter():
                'emails': (incremental_filter, None),
                'bpd': (incremental_filter, None),
                'bpc': (incremental_filter, None),
+                'filed-bugs': (incremental_filter, None),
+                'resolved-bugs': (incremental_filter, None),
                'members': (incremental_filter, None),
                'person-day': (person_day_filter, None),
            }
--- a/dashboard/helpers.py
+++ b/dashboard/helpers.py
@@ -136,6 +136,8 @@ def get_contribution_summary(records):
    drafted_blueprint_count = 0
    completed_blueprint_count = 0
    email_count = 0
+    filed_bug_count = 0
+    resolved_bug_count = 0

    for record in records:
        record_type = record['record_type']
@@ -153,6 +155,10 @@ def get_contribution_summary(records):
            drafted_blueprint_count += 1
        elif record['record_type'] == 'bpc':
            completed_blueprint_count += 1
+        elif record['record_type'] == 'bugf':
+            filed_bug_count += 1
+        elif record['record_type'] == 'bugr':
+            resolved_bug_count += 1

    result = {
        'drafted_blueprint_count': drafted_blueprint_count,
@@ -161,6 +167,8 @@ def get_contribution_summary(records):
        'email_count': email_count,
        'loc': loc,
        'marks': marks,
+        'filed_bug_count': filed_bug_count,
+        'resolved_bug_count': resolved_bug_count,
    }
    return result

--- a/dashboard/parameters.py
+++ b/dashboard/parameters.py
@@ -35,6 +35,8 @@ METRIC_LABELS = {
    'emails': 'Emails',
    'bpd': 'Drafted Blueprints',
    'bpc': 'Completed Blueprints',
+    'filed-bugs': 'Filed Bugs',
+    'resolved-bugs': 'Resolved Bugs',
    # 'person-day': "Person-day effort"
 }

@@ -45,6 +47,8 @@ METRIC_TO_RECORD_TYPE = {
    'emails': 'email',
    'bpd': 'bpd',
    'bpc': 'bpc',
+    'filed-bugs': 'bugf',
+    'resolved-bugs': 'bugr',
    'members': 'member',
 }

--- a/dashboard/templates/_macros/activity_log.html
+++ b/dashboard/templates/_macros/activity_log.html
@@ -145,6 +145,10 @@ show_record_type=True, show_user_gravatar=True, gravatar_size=32, show_all=True)
            {%if mention_count %}
            <div><b>Mention count: ${mention_count}, last mention on ${mention_date_str}</b></div>
            {%/if%}
+        {%elif ((record_type == "bugf") || (record_type == "bugr")) %}
+            <div class="header">&ldquo;${title}&rdquo;</div>
+            <div>Status: <span class="status${status}">${status}</span></div>
+            <div>Importance: ${importance}</div>
        {%/if%}
    </div>
    </div>
--- a/dashboard/templates/_macros/contribution_summary.html
+++ b/dashboard/templates/_macros/contribution_summary.html
@@ -34,6 +34,8 @@
 <div>Review stat (-2, -1, +1, +2, A): <b>${marks["-2"]}, ${marks["-1"]}, ${marks["1"]}, ${marks["2"]}, ${marks["A"]}</b></div>
 <div>Draft Blueprints: <b>${drafted_blueprint_count}</b></div>
 <div>Completed Blueprints: <b>${completed_blueprint_count}</b></div>
+<div>Filed Bugs: <b>${filed_bug_count}</b></div>
+<div>Resolved Bugs: <b>${resolved_bug_count}</b></div>
 <div>Emails: <b>${email_count}</b></div>
 {% endraw %}
 </script>
--- a/stackalytics/processor/bps.py
+++ b/stackalytics/processor/bps.py
@@ -0,0 +1,63 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from stackalytics.openstack.common import log as logging
+from stackalytics.processor import launchpad_utils
+from stackalytics.processor import utils
+
+
+LOG = logging.getLogger(__name__)
+
+LINK_FIELDS = ['owner', 'assignee']
+BUG_FIELDS = ['web_link', 'status', 'title', 'importance']
+DATE_FIELDS = ['date_created', 'date_fix_committed']
+
+
+def _get_bug_id(web_link):
+    return web_link[web_link.rfind('/') + 1:]
+
+
+def log(repo, last_bug_date):
+    module = repo['module']
+    LOG.debug('Retrieving list of bugs for module: %s', module)
+
+    if not launchpad_utils.lp_module_exists(module):
+        LOG.debug('Module %s does not exist at Launchpad', module)
+        return
+
+    for record_draft in launchpad_utils.lp_bug_generator(module,
+                                                         last_bug_date):
+
+        record = {}
+
+        for field in LINK_FIELDS:
+            link = record_draft[field + '_link']
+            if link:
+                record[field] = launchpad_utils.link_to_launchpad_id(link)
+
+        for field in BUG_FIELDS:
+            record[field] = record_draft[field]
+
+        for field in DATE_FIELDS:
+            date = record_draft[field]
+            if date:
+                record[field] = utils.iso8601_to_timestamp(date)
+
+        bug_id = _get_bug_id(record_draft['web_link'])
+        record['module'] = module
+        record['id'] = utils.get_bug_id(module, bug_id)
+
+        LOG.debug('New bug: %s', record)
+        yield record
--- a/stackalytics/processor/launchpad_utils.py
+++ b/stackalytics/processor/launchpad_utils.py
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+import six
 from six.moves import http_client
 from six.moves.urllib import parse

@@ -22,11 +23,19 @@ from stackalytics.processor import utils

 LOG = logging.getLogger(__name__)

-
+BUG_STATUSES = ['New', 'Incomplete', 'Opinion', 'Invalid', 'Won\'t Fix',
+                'Expired', 'Confirmed', 'Triaged', 'In Progress',
+                'Fix Committed', 'Fix Released',
+                'Incomplete (with response)',
+                'Incomplete (without response)']
 LP_URI_V1 = 'https://api.launchpad.net/1.0/%s'
 LP_URI_DEVEL = 'https://api.launchpad.net/devel/%s'


+def link_to_launchpad_id(link):
+    return link[link.find('~') + 1:]
+
+
 def lp_profile_by_launchpad_id(launchpad_id):
    LOG.debug('Lookup user id %s at Launchpad', launchpad_id)
    uri = LP_URI_V1 % ('~' + launchpad_id)
@@ -65,3 +74,24 @@ def lp_blueprint_generator(module):
            yield record

        uri = chunk.get('next_collection_link')
+
+
+def lp_bug_generator(module, last_bug_date):
+    uri = LP_URI_DEVEL % (module + '?ws.op=searchTasks')
+    for status in BUG_STATUSES:
+        uri += '&status=' + six.moves.urllib.parse.quote_plus(status)
+    if last_bug_date:
+        uri += '&modified_since=' + last_bug_date
+
+    while uri:
+        LOG.debug('Reading chunk from uri %s', uri)
+        chunk = utils.read_json_from_uri(uri)
+
+        if not chunk:
+            LOG.warn('No data was read from uri %s', uri)
+            break
+
+        for record in chunk['entries']:
+            yield record
+
+        uri = chunk.get('next_collection_link')
--- a/stackalytics/processor/lp.py
+++ b/stackalytics/processor/lp.py
@@ -25,23 +25,19 @@ LINK_FIELDS = ['owner', 'drafter', 'starter', 'completer',
 DATE_FIELDS = ['date_created', 'date_completed', 'date_started']


-def _link_to_launchpad_id(link):
-    return link[link.find('~') + 1:]
-
-
 def log(repo):
    module = repo['module']
    LOG.debug('Retrieving list of blueprints for module: %s', module)

    if not launchpad_utils.lp_module_exists(module):
-        LOG.debug('Module %s not exist at Launchpad', module)
+        LOG.debug('Module %s does not exist at Launchpad', module)
        return

    for record in launchpad_utils.lp_blueprint_generator(module):
        for field in LINK_FIELDS:
            link = record[field + '_link']
            if link:
-                record[field] = _link_to_launchpad_id(link)
+                record[field] = launchpad_utils.link_to_launchpad_id(link)
                del record[field + '_link']
        for field in DATE_FIELDS:
            date = record[field]
--- a/stackalytics/processor/main.py
+++ b/stackalytics/processor/main.py
@@ -19,9 +19,11 @@ from oslo.config import cfg
 import psutil
 import six
 from six.moves.urllib import parse
+import time
 import yaml

 from stackalytics.openstack.common import log as logging
+from stackalytics.processor import bps
 from stackalytics.processor import config
 from stackalytics.processor import default_data_processor
 from stackalytics.processor import lp
@@ -76,7 +78,8 @@ def _record_typer(record_iterator, record_type):
        yield record


-def process_repo(repo, runtime_storage_inst, record_processor_inst):
+def process_repo(repo, runtime_storage_inst, record_processor_inst,
+                 last_bug_date):
    uri = repo['uri']
    LOG.debug('Processing repo uri %s' % uri)

@@ -87,6 +90,13 @@ def process_repo(repo, runtime_storage_inst, record_processor_inst):
    runtime_storage_inst.set_records(processed_bp_iterator,
                                     utils.merge_records)

+    bug_iterator = bps.log(repo, last_bug_date)
+    bug_iterator_typed = _record_typer(bug_iterator, 'bug')
+    processed_bug_iterator = record_processor_inst.process(
+        bug_iterator_typed)
+    runtime_storage_inst.set_records(processed_bug_iterator,
+                                     utils.merge_records)
+
    vcs_inst = vcs.get_vcs(repo, cfg.CONF.sources_root)
    vcs_inst.fetch()

@@ -158,8 +168,12 @@ def update_members(runtime_storage_inst, record_processor_inst):
 def update_records(runtime_storage_inst, record_processor_inst):
    repos = utils.load_repos(runtime_storage_inst)

+    current_date = utils.timestamp_to_utc_date(int(time.time()))
+    last_bug_date = runtime_storage_inst.get_by_key('last_bug_date')
    for repo in repos:
-        process_repo(repo, runtime_storage_inst, record_processor_inst)
+        process_repo(repo, runtime_storage_inst, record_processor_inst,
+                     last_bug_date)
+    runtime_storage_inst.set_by_key('last_bug_date', current_date)

    mail_lists = runtime_storage_inst.get_by_key('mail_lists') or []
    for mail_list in mail_lists:
--- a/stackalytics/processor/record_processor.py
+++ b/stackalytics/processor/record_processor.py
@@ -417,6 +417,30 @@ class RecordProcessor(object):

            yield bpc

+    def _process_bug(self, record):
+
+        bug_created = record.copy()
+        bug_created['primary_key'] = 'bugf:' + record['id']
+        bug_created['record_type'] = 'bugf'
+        bug_created['launchpad_id'] = record.get('owner')
+        bug_created['date'] = record['date_created']
+
+        self._update_record_and_user(bug_created)
+
+        yield bug_created
+
+        FIXED_BUGS = ['Fix Committed', 'Fix Released']
+        if 'date_fix_committed' in record and record['status'] in FIXED_BUGS:
+            bug_fixed = record.copy()
+            bug_fixed['primary_key'] = 'bugr:' + record['id']
+            bug_fixed['record_type'] = 'bugr'
+            bug_fixed['launchpad_id'] = record.get('assignee') or '*unassigned'
+            bug_fixed['date'] = record['date_fix_committed']
+
+            self._update_record_and_user(bug_fixed)
+
+            yield bug_fixed
+
    def _process_member(self, record):
        user_id = "member:" + record['member_id']
        record['primary_key'] = user_id
@@ -465,6 +489,9 @@ class RecordProcessor(object):
        elif record['record_type'] == 'member':
            for r in self._process_member(record):
                yield r
+        elif record['record_type'] == 'bug':
+            for r in self._process_bug(record):
+                yield r

    def _renew_record_date(self, record):
        record['week'] = utils.timestamp_to_week(record['date'])
--- a/stackalytics/processor/utils.py
+++ b/stackalytics/processor/utils.py
@@ -72,6 +72,11 @@ def timestamp_to_day(timestamp):
    return timestamp // (24 * 3600)


+def timestamp_to_utc_date(timestamp):
+    return (datetime.datetime.fromtimestamp(timestamp).
+            strftime('%Y-%m-%d'))
+
+
 def round_timestamp_to_day(timestamp):
    return (int(timestamp) // (24 * 3600)) * (24 * 3600)

@@ -173,6 +178,10 @@ def get_blueprint_id(module, name):
    return module + ':' + name


+def get_bug_id(module, bug_id):
+    return module + '/' + bug_id
+
+
 def get_patch_id(review_id, patch_number):
    return review_id + ':' + patch_number

--- a/tests/unit/test_record_processor.py
+++ b/tests/unit/test_record_processor.py
@@ -320,6 +320,90 @@ class TestRecordProcessor(testtools.TestCase):
        self.assertEqual('IBM', user['companies'][0]['company_name'])
        self.assertEqual(None, user['launchpad_id'])

+    def generate_bugs(self, assignee=None, date_fix_committed=None,
+                      status='Confirmed'):
+        yield {
+            'record_type': 'bug',
+            'id': 'bug_id',
+            'owner': 'owner',
+            'assignee': assignee,
+            'date_created': 1234567890,
+            'date_fix_committed': date_fix_committed,
+            'module': 'nova',
+            'status': status
+        }
+
+    def test_process_bug_not_fixed(self):
+        record = self.generate_bugs()
+        record_processor_inst = self.make_record_processor()
+        bugs = list(record_processor_inst.process(record))
+        self.assertEqual(len(bugs), 1)
+        self.assertRecordsMatch({
+            'primary_key': 'bugf:bug_id',
+            'record_type': 'bugf',
+            'launchpad_id': 'owner',
+            'date': 1234567890,
+        }, bugs[0])
+
+    def test_process_bug_fix_committed(self):
+        record = self.generate_bugs(status='Fix Committed',
+                                    date_fix_committed=1234567891,
+                                    assignee='assignee')
+        record_processor_inst = self.make_record_processor()
+        bugs = list(record_processor_inst.process(record))
+        self.assertEqual(len(bugs), 2)
+        self.assertRecordsMatch({
+            'primary_key': 'bugf:bug_id',
+            'record_type': 'bugf',
+            'launchpad_id': 'owner',
+            'date': 1234567890,
+        }, bugs[0])
+        self.assertRecordsMatch({
+            'primary_key': 'bugr:bug_id',
+            'record_type': 'bugr',
+            'launchpad_id': 'assignee',
+            'date': 1234567891,
+        }, bugs[1])
+
+    def test_process_bug_fix_released(self):
+        record = self.generate_bugs(status='Fix Released',
+                                    date_fix_committed=1234567891,
+                                    assignee='assignee')
+        record_processor_inst = self.make_record_processor()
+        bugs = list(record_processor_inst.process(record))
+        self.assertEqual(len(bugs), 2)
+        self.assertRecordsMatch({
+            'primary_key': 'bugf:bug_id',
+            'record_type': 'bugf',
+            'launchpad_id': 'owner',
+            'date': 1234567890,
+        }, bugs[0])
+        self.assertRecordsMatch({
+            'primary_key': 'bugr:bug_id',
+            'record_type': 'bugr',
+            'launchpad_id': 'assignee',
+            'date': 1234567891,
+        }, bugs[1])
+
+    def test_process_bug_fix_committed_without_assignee(self):
+        record = self.generate_bugs(status='Fix Committed',
+                                    date_fix_committed=1234567891)
+        record_processor_inst = self.make_record_processor()
+        bugs = list(record_processor_inst.process(record))
+        self.assertEqual(len(bugs), 2)
+        self.assertRecordsMatch({
+            'primary_key': 'bugf:bug_id',
+            'record_type': 'bugf',
+            'launchpad_id': 'owner',
+            'date': 1234567890,
+        }, bugs[0])
+        self.assertRecordsMatch({
+            'primary_key': 'bugr:bug_id',
+            'record_type': 'bugr',
+            'launchpad_id': '*unassigned',
+            'date': 1234567891,
+        }, bugs[1])
+
    # process records complex scenarios

    def test_process_blueprint_one_draft_spawned_lp_doesnt_know_user(self):