diff --git a/dashboard/decorators.py b/dashboard/decorators.py index b8e0b4780..9062d8af6 100644 --- a/dashboard/decorators.py +++ b/dashboard/decorators.py @@ -310,6 +310,8 @@ def aggregate_filter(): 'emails': (incremental_filter, None), 'bpd': (incremental_filter, None), 'bpc': (incremental_filter, None), + 'filed-bugs': (incremental_filter, None), + 'resolved-bugs': (incremental_filter, None), 'members': (incremental_filter, None), 'person-day': (person_day_filter, None), } diff --git a/dashboard/helpers.py b/dashboard/helpers.py index 114a0ef7a..7e67f640c 100644 --- a/dashboard/helpers.py +++ b/dashboard/helpers.py @@ -136,6 +136,8 @@ def get_contribution_summary(records): drafted_blueprint_count = 0 completed_blueprint_count = 0 email_count = 0 + filed_bug_count = 0 + resolved_bug_count = 0 for record in records: record_type = record['record_type'] @@ -153,6 +155,10 @@ def get_contribution_summary(records): drafted_blueprint_count += 1 elif record['record_type'] == 'bpc': completed_blueprint_count += 1 + elif record['record_type'] == 'bugf': + filed_bug_count += 1 + elif record['record_type'] == 'bugr': + resolved_bug_count += 1 result = { 'drafted_blueprint_count': drafted_blueprint_count, @@ -161,6 +167,8 @@ def get_contribution_summary(records): 'email_count': email_count, 'loc': loc, 'marks': marks, + 'filed_bug_count': filed_bug_count, + 'resolved_bug_count': resolved_bug_count, } return result diff --git a/dashboard/parameters.py b/dashboard/parameters.py index 706547e28..89787a25e 100644 --- a/dashboard/parameters.py +++ b/dashboard/parameters.py @@ -35,6 +35,8 @@ METRIC_LABELS = { 'emails': 'Emails', 'bpd': 'Drafted Blueprints', 'bpc': 'Completed Blueprints', + 'filed-bugs': 'Filed Bugs', + 'resolved-bugs': 'Resolved Bugs', # 'person-day': "Person-day effort" } @@ -45,6 +47,8 @@ METRIC_TO_RECORD_TYPE = { 'emails': 'email', 'bpd': 'bpd', 'bpc': 'bpc', + 'filed-bugs': 'bugf', + 'resolved-bugs': 'bugr', 'members': 'member', } diff --git a/dashboard/templates/_macros/activity_log.html b/dashboard/templates/_macros/activity_log.html index dd406884f..25db73731 100644 --- a/dashboard/templates/_macros/activity_log.html +++ b/dashboard/templates/_macros/activity_log.html @@ -145,6 +145,10 @@ show_record_type=True, show_user_gravatar=True, gravatar_size=32, show_all=True) {%if mention_count %}
Mention count: ${mention_count}, last mention on ${mention_date_str}
{%/if%} + {%elif ((record_type == "bugf") || (record_type == "bugr")) %} +
“${title}”
+
Status: ${status}
+
Importance: ${importance}
{%/if%} diff --git a/dashboard/templates/_macros/contribution_summary.html b/dashboard/templates/_macros/contribution_summary.html index 21eb67679..beae4d8da 100644 --- a/dashboard/templates/_macros/contribution_summary.html +++ b/dashboard/templates/_macros/contribution_summary.html @@ -34,6 +34,8 @@
Review stat (-2, -1, +1, +2, A): ${marks["-2"]}, ${marks["-1"]}, ${marks["1"]}, ${marks["2"]}, ${marks["A"]}
Draft Blueprints: ${drafted_blueprint_count}
Completed Blueprints: ${completed_blueprint_count}
+
Filed Bugs: ${filed_bug_count}
+
Resolved Bugs: ${resolved_bug_count}
Emails: ${email_count}
{% endraw %} diff --git a/stackalytics/processor/bps.py b/stackalytics/processor/bps.py new file mode 100644 index 000000000..667346bb2 --- /dev/null +++ b/stackalytics/processor/bps.py @@ -0,0 +1,63 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from stackalytics.openstack.common import log as logging +from stackalytics.processor import launchpad_utils +from stackalytics.processor import utils + + +LOG = logging.getLogger(__name__) + +LINK_FIELDS = ['owner', 'assignee'] +BUG_FIELDS = ['web_link', 'status', 'title', 'importance'] +DATE_FIELDS = ['date_created', 'date_fix_committed'] + + +def _get_bug_id(web_link): + return web_link[web_link.rfind('/') + 1:] + + +def log(repo, last_bug_date): + module = repo['module'] + LOG.debug('Retrieving list of bugs for module: %s', module) + + if not launchpad_utils.lp_module_exists(module): + LOG.debug('Module %s does not exist at Launchpad', module) + return + + for record_draft in launchpad_utils.lp_bug_generator(module, + last_bug_date): + + record = {} + + for field in LINK_FIELDS: + link = record_draft[field + '_link'] + if link: + record[field] = launchpad_utils.link_to_launchpad_id(link) + + for field in BUG_FIELDS: + record[field] = record_draft[field] + + for field in DATE_FIELDS: + date = record_draft[field] + if date: + record[field] = utils.iso8601_to_timestamp(date) + + bug_id = _get_bug_id(record_draft['web_link']) + record['module'] = module + record['id'] = utils.get_bug_id(module, bug_id) + + LOG.debug('New bug: %s', record) + yield record diff --git a/stackalytics/processor/launchpad_utils.py b/stackalytics/processor/launchpad_utils.py index 82fd1d7b4..d633412c2 100644 --- a/stackalytics/processor/launchpad_utils.py +++ b/stackalytics/processor/launchpad_utils.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import six from six.moves import http_client from six.moves.urllib import parse @@ -22,11 +23,19 @@ from stackalytics.processor import utils LOG = logging.getLogger(__name__) - +BUG_STATUSES = ['New', 'Incomplete', 'Opinion', 'Invalid', 'Won\'t Fix', + 'Expired', 'Confirmed', 'Triaged', 'In Progress', + 'Fix Committed', 'Fix Released', + 'Incomplete (with response)', + 'Incomplete (without response)'] LP_URI_V1 = 'https://api.launchpad.net/1.0/%s' LP_URI_DEVEL = 'https://api.launchpad.net/devel/%s' +def link_to_launchpad_id(link): + return link[link.find('~') + 1:] + + def lp_profile_by_launchpad_id(launchpad_id): LOG.debug('Lookup user id %s at Launchpad', launchpad_id) uri = LP_URI_V1 % ('~' + launchpad_id) @@ -65,3 +74,24 @@ def lp_blueprint_generator(module): yield record uri = chunk.get('next_collection_link') + + +def lp_bug_generator(module, last_bug_date): + uri = LP_URI_DEVEL % (module + '?ws.op=searchTasks') + for status in BUG_STATUSES: + uri += '&status=' + six.moves.urllib.parse.quote_plus(status) + if last_bug_date: + uri += '&modified_since=' + last_bug_date + + while uri: + LOG.debug('Reading chunk from uri %s', uri) + chunk = utils.read_json_from_uri(uri) + + if not chunk: + LOG.warn('No data was read from uri %s', uri) + break + + for record in chunk['entries']: + yield record + + uri = chunk.get('next_collection_link') diff --git a/stackalytics/processor/lp.py b/stackalytics/processor/lp.py index 436e17733..d3f38d601 100644 --- a/stackalytics/processor/lp.py +++ b/stackalytics/processor/lp.py @@ -25,23 +25,19 @@ LINK_FIELDS = ['owner', 'drafter', 'starter', 'completer', DATE_FIELDS = ['date_created', 'date_completed', 'date_started'] -def _link_to_launchpad_id(link): - return link[link.find('~') + 1:] - - def log(repo): module = repo['module'] LOG.debug('Retrieving list of blueprints for module: %s', module) if not launchpad_utils.lp_module_exists(module): - LOG.debug('Module %s not exist at Launchpad', module) + LOG.debug('Module %s does not exist at Launchpad', module) return for record in launchpad_utils.lp_blueprint_generator(module): for field in LINK_FIELDS: link = record[field + '_link'] if link: - record[field] = _link_to_launchpad_id(link) + record[field] = launchpad_utils.link_to_launchpad_id(link) del record[field + '_link'] for field in DATE_FIELDS: date = record[field] diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py index a980fbf71..f141c23fd 100644 --- a/stackalytics/processor/main.py +++ b/stackalytics/processor/main.py @@ -19,9 +19,11 @@ from oslo.config import cfg import psutil import six from six.moves.urllib import parse +import time import yaml from stackalytics.openstack.common import log as logging +from stackalytics.processor import bps from stackalytics.processor import config from stackalytics.processor import default_data_processor from stackalytics.processor import lp @@ -76,7 +78,8 @@ def _record_typer(record_iterator, record_type): yield record -def process_repo(repo, runtime_storage_inst, record_processor_inst): +def process_repo(repo, runtime_storage_inst, record_processor_inst, + last_bug_date): uri = repo['uri'] LOG.debug('Processing repo uri %s' % uri) @@ -87,6 +90,13 @@ def process_repo(repo, runtime_storage_inst, record_processor_inst): runtime_storage_inst.set_records(processed_bp_iterator, utils.merge_records) + bug_iterator = bps.log(repo, last_bug_date) + bug_iterator_typed = _record_typer(bug_iterator, 'bug') + processed_bug_iterator = record_processor_inst.process( + bug_iterator_typed) + runtime_storage_inst.set_records(processed_bug_iterator, + utils.merge_records) + vcs_inst = vcs.get_vcs(repo, cfg.CONF.sources_root) vcs_inst.fetch() @@ -158,8 +168,12 @@ def update_members(runtime_storage_inst, record_processor_inst): def update_records(runtime_storage_inst, record_processor_inst): repos = utils.load_repos(runtime_storage_inst) + current_date = utils.timestamp_to_utc_date(int(time.time())) + last_bug_date = runtime_storage_inst.get_by_key('last_bug_date') for repo in repos: - process_repo(repo, runtime_storage_inst, record_processor_inst) + process_repo(repo, runtime_storage_inst, record_processor_inst, + last_bug_date) + runtime_storage_inst.set_by_key('last_bug_date', current_date) mail_lists = runtime_storage_inst.get_by_key('mail_lists') or [] for mail_list in mail_lists: diff --git a/stackalytics/processor/record_processor.py b/stackalytics/processor/record_processor.py index 231b6b9aa..23a0cbdad 100644 --- a/stackalytics/processor/record_processor.py +++ b/stackalytics/processor/record_processor.py @@ -417,6 +417,30 @@ class RecordProcessor(object): yield bpc + def _process_bug(self, record): + + bug_created = record.copy() + bug_created['primary_key'] = 'bugf:' + record['id'] + bug_created['record_type'] = 'bugf' + bug_created['launchpad_id'] = record.get('owner') + bug_created['date'] = record['date_created'] + + self._update_record_and_user(bug_created) + + yield bug_created + + FIXED_BUGS = ['Fix Committed', 'Fix Released'] + if 'date_fix_committed' in record and record['status'] in FIXED_BUGS: + bug_fixed = record.copy() + bug_fixed['primary_key'] = 'bugr:' + record['id'] + bug_fixed['record_type'] = 'bugr' + bug_fixed['launchpad_id'] = record.get('assignee') or '*unassigned' + bug_fixed['date'] = record['date_fix_committed'] + + self._update_record_and_user(bug_fixed) + + yield bug_fixed + def _process_member(self, record): user_id = "member:" + record['member_id'] record['primary_key'] = user_id @@ -465,6 +489,9 @@ class RecordProcessor(object): elif record['record_type'] == 'member': for r in self._process_member(record): yield r + elif record['record_type'] == 'bug': + for r in self._process_bug(record): + yield r def _renew_record_date(self, record): record['week'] = utils.timestamp_to_week(record['date']) diff --git a/stackalytics/processor/utils.py b/stackalytics/processor/utils.py index d748fbadf..ec7584cb2 100644 --- a/stackalytics/processor/utils.py +++ b/stackalytics/processor/utils.py @@ -72,6 +72,11 @@ def timestamp_to_day(timestamp): return timestamp // (24 * 3600) +def timestamp_to_utc_date(timestamp): + return (datetime.datetime.fromtimestamp(timestamp). + strftime('%Y-%m-%d')) + + def round_timestamp_to_day(timestamp): return (int(timestamp) // (24 * 3600)) * (24 * 3600) @@ -175,6 +180,10 @@ def get_blueprint_id(module, name): return module + ':' + name +def get_bug_id(module, bug_id): + return module + '/' + bug_id + + def get_patch_id(review_id, patch_number): return review_id + ':' + patch_number diff --git a/tests/unit/test_record_processor.py b/tests/unit/test_record_processor.py index 39902c8f2..bcdc98fb2 100644 --- a/tests/unit/test_record_processor.py +++ b/tests/unit/test_record_processor.py @@ -320,6 +320,90 @@ class TestRecordProcessor(testtools.TestCase): self.assertEqual('IBM', user['companies'][0]['company_name']) self.assertEqual(None, user['launchpad_id']) + def generate_bugs(self, assignee=None, date_fix_committed=None, + status='Confirmed'): + yield { + 'record_type': 'bug', + 'id': 'bug_id', + 'owner': 'owner', + 'assignee': assignee, + 'date_created': 1234567890, + 'date_fix_committed': date_fix_committed, + 'module': 'nova', + 'status': status + } + + def test_process_bug_not_fixed(self): + record = self.generate_bugs() + record_processor_inst = self.make_record_processor() + bugs = list(record_processor_inst.process(record)) + self.assertEqual(len(bugs), 1) + self.assertRecordsMatch({ + 'primary_key': 'bugf:bug_id', + 'record_type': 'bugf', + 'launchpad_id': 'owner', + 'date': 1234567890, + }, bugs[0]) + + def test_process_bug_fix_committed(self): + record = self.generate_bugs(status='Fix Committed', + date_fix_committed=1234567891, + assignee='assignee') + record_processor_inst = self.make_record_processor() + bugs = list(record_processor_inst.process(record)) + self.assertEqual(len(bugs), 2) + self.assertRecordsMatch({ + 'primary_key': 'bugf:bug_id', + 'record_type': 'bugf', + 'launchpad_id': 'owner', + 'date': 1234567890, + }, bugs[0]) + self.assertRecordsMatch({ + 'primary_key': 'bugr:bug_id', + 'record_type': 'bugr', + 'launchpad_id': 'assignee', + 'date': 1234567891, + }, bugs[1]) + + def test_process_bug_fix_released(self): + record = self.generate_bugs(status='Fix Released', + date_fix_committed=1234567891, + assignee='assignee') + record_processor_inst = self.make_record_processor() + bugs = list(record_processor_inst.process(record)) + self.assertEqual(len(bugs), 2) + self.assertRecordsMatch({ + 'primary_key': 'bugf:bug_id', + 'record_type': 'bugf', + 'launchpad_id': 'owner', + 'date': 1234567890, + }, bugs[0]) + self.assertRecordsMatch({ + 'primary_key': 'bugr:bug_id', + 'record_type': 'bugr', + 'launchpad_id': 'assignee', + 'date': 1234567891, + }, bugs[1]) + + def test_process_bug_fix_committed_without_assignee(self): + record = self.generate_bugs(status='Fix Committed', + date_fix_committed=1234567891) + record_processor_inst = self.make_record_processor() + bugs = list(record_processor_inst.process(record)) + self.assertEqual(len(bugs), 2) + self.assertRecordsMatch({ + 'primary_key': 'bugf:bug_id', + 'record_type': 'bugf', + 'launchpad_id': 'owner', + 'date': 1234567890, + }, bugs[0]) + self.assertRecordsMatch({ + 'primary_key': 'bugr:bug_id', + 'record_type': 'bugr', + 'launchpad_id': '*unassigned', + 'date': 1234567891, + }, bugs[1]) + # process records complex scenarios def test_process_blueprint_one_draft_spawned_lp_doesnt_know_user(self):