x/stackalytics
Code Issues Proposed changes

Implemented processing of blueprints data

Browse Source 
* Grab blueprints from Launchpad
* Add new record types for new blueprints and completed blueprints
* Add new metrics

Part of bp metric-by-bugs-blueprints

Change-Id: I4656fc4d7c4626f7589c9551949d12d1f3b195f7
This commit is contained in:
Ilya Shakhat
2013-09-15 23:47:47 +04:00
committed by Gerrit Code Review
parent 71d631ae5b
commit 193d285019
7 changed files with 150 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
dashboard/templates/overview.html
View File

@@ -42,13 +42,7 @@
if ((start_record == 0) && (data["activity"].length == 0)) {
$('#activity_header').hide();
}
{% if metric == 'marks' %}
$("#review_activity_template").tmpl(data["activity"]).appendTo("#activity_container");
{% elif metric == 'emails' %}
$("#email_activity_template").tmpl(data["activity"]).appendTo("#activity_container");
{% else %}
$("#commit_activity_template").tmpl(data["activity"]).appendTo("#activity_container");
{% endif %}
$("#activity_template").tmpl(data["activity"]).appendTo("#activity_container");
}
});
}
@@ -90,7 +84,7 @@
</script>
{# Templates #}
<script id="commit_activity_template" type="text/x-jquery-tmpl">
<script id="activity_template" type="text/x-jquery-tmpl">
{% raw %}
<div style="margin-bottom: 1em;">
<div style='float: left; '><img src="${gravatar}" style="width: 32px; height: 32px;"></div>
@@ -99,6 +93,7 @@
<div style="font-weight: bold;">${date_str} to <a href="https://launchpad.net/${module}">${module}</a></div>
</div>
<div style="margin-left: 40px;">
{%if record_type == "commit" %}
{%if correction_comment != "" %}
<div style='font-weight: bold; color: red;'>Commit corrected:
<span>${correction_comment}</span></div>
@@ -107,38 +102,12 @@
<div style='white-space: pre-wrap; '>{%html message %}</div>
<div><span style="color: green">+<span>${lines_added}</span></span>
<span style="color: blue">- <span>${lines_deleted}</span></span></div>
</div>
</div>
{% endraw %}
</script>
<script id="review_activity_template" type="text/x-jquery-tmpl">
{% raw %}
<div style="margin-bottom: 1em;">
<div style='float: left; '><img src="${gravatar}" style="width: 32px; height: 32px;"></div>
<div style="margin-left: 40px;">
<div style="font-weight: bold;">{%html author_link %} ({%html company_link %})</div>
<div style="font-weight: bold;">${date_str} in <a href="https://launchpad.net/${module}">${module}</a></div>
</div>
<div style="margin-left: 40px;">
{%elif record_type == "mark" %}
<div>Patch submitted by {%html parent_author_link %}</div>
<div style='font-weight: bold;'>${subject}</div>
<div>Change Id: <a href="${url}">${review_id}</a></div>
<div style="color: {%if value > 0 %} green {%else%} blue {%/if%}">${description}: <span class="review_mark">${value}</span></div>
</div>
</div>
{% endraw %}
</script>
<script id="email_activity_template" type="text/x-jquery-tmpl">
{% raw %}
<div style="margin-bottom: 1em;">
<div style='float: left; '><img src="${gravatar}" style="width: 32px; height: 32px;"></div>
<div style="margin-left: 40px;">
<div style="font-weight: bold;">{%html author_link %} ({%html company_link %})</div>
<div style="font-weight: bold;">${date_str} to <a href="https://launchpad.net/${module}">${module}</a></div>
</div>
<div style="margin-left: 40px;">
{%elif record_type == "email" %}
<div style='font-weight: bold;'>
{%if email_link != "" %}
<a href='${email_link}'>
@@ -148,12 +117,16 @@
</a>
{%/if%}
</div>
{%elif ((record_type == "bp_draft") || (record_type == "bp_implementation")) %}
<div style='font-weight: bold;'>${title} (<a href='${web_link}'>${name}</a>)</div>
<div style='white-space: pre-wrap;'>${summary}</div>
{%/if%}
</div>
</div>
{% endraw %}
</script>
<script id="user_profile_template" type="text/x-jquery-tmpl">
{% raw %}
<div>

13
dashboard/web.py
View File

@@ -49,13 +49,17 @@ METRIC_LABELS = {
'commits': 'Commits',
'marks': 'Reviews',
'emails': 'Emails',
'bp_draft': 'New Blueprints',
'bp_implementation': 'Completed Blueprints',
}
METRIC_TO_RECORD_TYPE = {
'loc': 'commit',
'commits': 'commit',
'marks': 'mark',
'emails': 'email'
'emails': 'email',
'bp_draft': 'bp_draft',
'bp_implementation': 'bp_implementation',
}
DEFAULT_RECORDS_LIMIT = 10
@@ -402,6 +406,8 @@ def aggregate_filter():
'loc': (loc_filter, None),
'marks': (mark_filter, mark_finalize),
'emails': (incremental_filter, None),
'bp_draft': (incremental_filter, None),
'bp_implementation': (incremental_filter, None),
}
if metric not in metric_to_filters_map:
raise Exception('Invalid metric %s' % metric)
@@ -687,6 +693,11 @@ def get_activity_json(records):
_extend_record(email)
email['email_link'] = email.get('email_link') or ''
result.append(email)
elif ((record['record_type'] == 'bp_draft') or
(record['record_type'] == 'bp_implementation')):
blueprint = record.copy()
_extend_record(blueprint)
result.append(blueprint)
result.sort(key=lambda x: x['date'], reverse=True)
return result[start_record:start_record + page_size]

76
stackalytics/processor/lp.py Normal file
View File

@@ -0,0 +1,76 @@
# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import httplib
import urlparse
from stackalytics.openstack.common import log as logging
from stackalytics.processor import utils
LOG = logging.getLogger(__name__)
LINK_FIELDS = ['owner', 'drafter', 'starter', 'completer',
'assignee', 'approver']
DATE_FIELDS = ['date_created', 'date_completed', 'date_started']
def _module_exists(module):
uri = 'https://api.launchpad.net/devel/%s' % module
parsed_uri = urlparse.urlparse(uri)
conn = httplib.HTTPConnection(parsed_uri.netloc)
conn.request('GET', parsed_uri.path)
res = conn.getresponse()
LOG.debug('Checked uri: %(uri)s, status: %(status)s',
{'uri': uri, 'status': res.status})
conn.close()
return res.status != 404
def _link_to_launchpad_id(link):
return link[link.find('~') + 1:]
def log(repo):
module = repo['module']
LOG.debug('Retrieving list of blueprints for module: %s', module)
if not _module_exists(module):
LOG.debug('Module %s not exist at Launchpad', module)
return
uri = 'https://api.launchpad.net/devel/%s/all_specifications' % module
while True:
LOG.debug('Reading chunk from uri %s', uri)
chunk = utils.read_json_from_uri(uri)
if 'next_collection_link' not in chunk:
break
uri = chunk['next_collection_link']
for record in chunk['entries']:
for field in LINK_FIELDS:
link = record[field + '_link']
if link:
record[field] = _link_to_launchpad_id(link)
del record[field + '_link']
for field in DATE_FIELDS:
date = record[field]
if date:
record[field] = utils.iso8601_to_timestamp(date)
record['module'] = module
LOG.debug('New blueprint: %s', record)
yield record

7
stackalytics/processor/main.py
View File

@@ -22,6 +22,7 @@ from psutil import _error
from stackalytics.openstack.common import log as logging
from stackalytics.processor import config
from stackalytics.processor import default_data_processor
from stackalytics.processor import lp
from stackalytics.processor import mls
from stackalytics.processor import rcs
from stackalytics.processor import record_processor
@@ -78,6 +79,12 @@ def process_repo(repo, runtime_storage_inst, record_processor_inst):
uri = repo['uri']
LOG.debug('Processing repo uri %s' % uri)
bp_iterator = lp.log(repo)
bp_iterator_typed = _record_typer(bp_iterator, 'bp')
processed_mail_iterator = record_processor_inst.process(
bp_iterator_typed)
runtime_storage_inst.set_records(processed_mail_iterator)
vcs_inst = vcs.get_vcs(repo, cfg.CONF.sources_root)
vcs_inst.fetch()

1
stackalytics/processor/mls.py
View File

@@ -68,6 +68,7 @@ def _link_content_changed(link, runtime_storage_inst):
conn.request('HEAD', parsed_uri.path)
res = conn.getresponse()
last_modified = res.getheader('last-modified')
conn.close()
if last_modified != runtime_storage_inst.get_by_key('mail_link:' + link):
LOG.debug('Mail archive changed, last modified at: %s', last_modified)

39
stackalytics/processor/record_processor.py
View File

@@ -264,6 +264,42 @@ class RecordProcessor(object):
yield record
def _process_blueprint(self, record):
if record.get('drafter'):
bp_draft = dict([(k, v) for k, v in record.iteritems()])
bp_draft['primary_key'] = 'bpd:' + record['self_link']
drafter = utils.load_user(self.runtime_storage_inst,
record['drafter'])
if drafter and record['date_created']:
bp_draft['record_type'] = 'bp_draft'
bp_draft['author_name'] = drafter['user_name']
bp_draft['author_email'] = drafter['emails'][0]
bp_draft['launchpad_id'] = record['drafter']
bp_draft['date'] = record['date_created']
self._update_record_and_user(bp_draft)
yield bp_draft
if record.get('assignee'):
bp_implementation = dict([(k, v) for k, v in record.iteritems()])
bp_implementation['primary_key'] = 'bpi:' + record['self_link']
assignee = utils.load_user(self.runtime_storage_inst,
record['assignee'])
if assignee and record['date_completed']:
bp_implementation['record_type'] = 'bp_implementation'
bp_implementation['author_name'] = assignee['user_name']
bp_implementation['author_email'] = assignee['emails'][0]
bp_implementation['launchpad_id'] = record['assignee']
bp_implementation['date'] = record['date_completed']
if bp_implementation['author_email']:
self._update_record_and_user(bp_implementation)
yield bp_implementation
def _apply_type_based_processing(self, record):
if record['record_type'] == 'commit':
for r in self._process_commit(record):
@@ -274,6 +310,9 @@ class RecordProcessor(object):
elif record['record_type'] == 'email':
for r in self._process_email(record):
yield r
elif record['record_type'] == 'bp':
for r in self._process_blueprint(record):
yield r
def process(self, record_iterator):
for record in record_iterator:

5
stackalytics/processor/utils.py
View File

@@ -14,6 +14,7 @@
# limitations under the License.
import datetime
import iso8601
import json
import re
import time
@@ -32,6 +33,10 @@ def date_to_timestamp(d):
datetime.datetime.strptime(d, '%Y-%b-%d').timetuple()))
def iso8601_to_timestamp(s):
return int(time.mktime(iso8601.parse_date(s).timetuple()))
def timestamp_to_week(timestamp):
# Jan 4th 1970 is the first Sunday in the Epoch
return (timestamp - 3 * 24 * 3600) // (7 * 24 * 3600)