Implemented processing of blueprints data

* Grab blueprints from Launchpad
* Add new record types for new blueprints and completed blueprints
* Add new metrics

Part of bp metric-by-bugs-blueprints

Change-Id: I4656fc4d7c4626f7589c9551949d12d1f3b195f7

dashboard/templates/overview.html

@@ -42,13 +42,7 @@
                     if ((start_record == 0) && (data["activity"].length == 0)) {
                         $('#activity_header').hide();
                     }
-                    {% if metric == 'marks' %}
+                    $("#activity_template").tmpl(data["activity"]).appendTo("#activity_container");
-                    $("#review_activity_template").tmpl(data["activity"]).appendTo("#activity_container");
-                    {% elif metric == 'emails' %}
-                    $("#email_activity_template").tmpl(data["activity"]).appendTo("#activity_container");
-                    {% else %}
-                    $("#commit_activity_template").tmpl(data["activity"]).appendTo("#activity_container");
-                    {% endif %}
                 }
             });
         }
@@ -90,7 +84,7 @@
 </script>
 
 {# Templates #}
-<script id="commit_activity_template" type="text/x-jquery-tmpl">
+<script id="activity_template" type="text/x-jquery-tmpl">
     {% raw %}
     <div style="margin-bottom: 1em;">
     <div style='float: left; '><img src="${gravatar}" style="width: 32px; height: 32px;"></div>
@@ -99,6 +93,7 @@
         <div style="font-weight: bold;">${date_str} to <a href="https://launchpad.net/${module}">${module}</a></div>
     </div>
     <div style="margin-left: 40px;">
+        {%if record_type == "commit" %}
         {%if correction_comment != "" %}
         <div style='font-weight: bold; color: red;'>Commit corrected:
             <span>${correction_comment}</span></div>
@@ -107,38 +102,12 @@
         <div style='white-space: pre-wrap; '>{%html message %}</div>
         <div><span style="color: green">+<span>${lines_added}</span></span>
             <span style="color: blue">- <span>${lines_deleted}</span></span></div>
-    </div>
+        {%elif record_type == "mark" %}
-    </div>
-    {% endraw %}
-</script>
-
-<script id="review_activity_template" type="text/x-jquery-tmpl">
-    {% raw %}
-    <div style="margin-bottom: 1em;">
-    <div style='float: left; '><img src="${gravatar}" style="width: 32px; height: 32px;"></div>
-    <div style="margin-left: 40px;">
-        <div style="font-weight: bold;">{%html author_link %} ({%html company_link %})</div>
-        <div style="font-weight: bold;">${date_str} in <a href="https://launchpad.net/${module}">${module}</a></div>
-    </div>
-    <div style="margin-left: 40px;">
         <div>Patch submitted by {%html parent_author_link %}</div>
         <div style='font-weight: bold;'>${subject}</div>
         <div>Change Id: <a href="${url}">${review_id}</a></div>
         <div style="color: {%if value > 0 %} green {%else%} blue {%/if%}">${description}: <span class="review_mark">${value}</span></div>
-    </div>
+        {%elif record_type == "email" %}
-    </div>
-    {% endraw %}
-</script>
-
-<script id="email_activity_template" type="text/x-jquery-tmpl">
-    {% raw %}
-    <div style="margin-bottom: 1em;">
-    <div style='float: left; '><img src="${gravatar}" style="width: 32px; height: 32px;"></div>
-    <div style="margin-left: 40px;">
-        <div style="font-weight: bold;">{%html author_link %} ({%html company_link %})</div>
-        <div style="font-weight: bold;">${date_str} to <a href="https://launchpad.net/${module}">${module}</a></div>
-    </div>
-    <div style="margin-left: 40px;">
         <div style='font-weight: bold;'>
         {%if email_link != "" %}
         <a href='${email_link}'>
@@ -148,12 +117,16 @@
         </a>
         {%/if%}
         </div>
+        {%elif ((record_type == "bp_draft") || (record_type == "bp_implementation")) %}
+        <div style='font-weight: bold;'>${title} (<a href='${web_link}'>${name}</a>)</div>
+        <div style='white-space: pre-wrap;'>${summary}</div>
+
+        {%/if%}
     </div>
     </div>
     {% endraw %}
 </script>
 
-
 <script id="user_profile_template" type="text/x-jquery-tmpl">
     {% raw %}
     <div>

dashboard/web.py

@@ -49,13 +49,17 @@ METRIC_LABELS = {
     'commits': 'Commits',
     'marks': 'Reviews',
     'emails': 'Emails',
+    'bp_draft': 'New Blueprints',
+    'bp_implementation': 'Completed Blueprints',
 }
 
 METRIC_TO_RECORD_TYPE = {
     'loc': 'commit',
     'commits': 'commit',
     'marks': 'mark',
-    'emails': 'email'
+    'emails': 'email',
+    'bp_draft': 'bp_draft',
+    'bp_implementation': 'bp_implementation',
 }
 
 DEFAULT_RECORDS_LIMIT = 10
@@ -402,6 +406,8 @@ def aggregate_filter():
                 'loc': (loc_filter, None),
                 'marks': (mark_filter, mark_finalize),
                 'emails': (incremental_filter, None),
+                'bp_draft': (incremental_filter, None),
+                'bp_implementation': (incremental_filter, None),
             }
             if metric not in metric_to_filters_map:
                 raise Exception('Invalid metric %s' % metric)
@@ -687,6 +693,11 @@ def get_activity_json(records):
             _extend_record(email)
             email['email_link'] = email.get('email_link') or ''
             result.append(email)
+        elif ((record['record_type'] == 'bp_draft') or
+             (record['record_type'] == 'bp_implementation')):
+            blueprint = record.copy()
+            _extend_record(blueprint)
+            result.append(blueprint)
 
     result.sort(key=lambda x: x['date'], reverse=True)
     return result[start_record:start_record + page_size]

stackalytics/processor/lp.py

@@ -0,0 +1,76 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import httplib
+import urlparse
+
+from stackalytics.openstack.common import log as logging
+from stackalytics.processor import utils
+
+
+LOG = logging.getLogger(__name__)
+
+LINK_FIELDS = ['owner', 'drafter', 'starter', 'completer',
+               'assignee', 'approver']
+DATE_FIELDS = ['date_created', 'date_completed', 'date_started']
+
+
+def _module_exists(module):
+    uri = 'https://api.launchpad.net/devel/%s' % module
+    parsed_uri = urlparse.urlparse(uri)
+    conn = httplib.HTTPConnection(parsed_uri.netloc)
+    conn.request('GET', parsed_uri.path)
+    res = conn.getresponse()
+    LOG.debug('Checked uri: %(uri)s, status: %(status)s',
+              {'uri': uri, 'status': res.status})
+    conn.close()
+    return res.status != 404
+
+
+def _link_to_launchpad_id(link):
+    return link[link.find('~') + 1:]
+
+
+def log(repo):
+    module = repo['module']
+    LOG.debug('Retrieving list of blueprints for module: %s', module)
+
+    if not _module_exists(module):
+        LOG.debug('Module %s not exist at Launchpad', module)
+        return
+
+    uri = 'https://api.launchpad.net/devel/%s/all_specifications' % module
+    while True:
+        LOG.debug('Reading chunk from uri %s', uri)
+        chunk = utils.read_json_from_uri(uri)
+        if 'next_collection_link' not in chunk:
+            break
+        uri = chunk['next_collection_link']
+
+        for record in chunk['entries']:
+            for field in LINK_FIELDS:
+                link = record[field + '_link']
+                if link:
+                    record[field] = _link_to_launchpad_id(link)
+                    del record[field + '_link']
+            for field in DATE_FIELDS:
+                date = record[field]
+                if date:
+                    record[field] = utils.iso8601_to_timestamp(date)
+
+            record['module'] = module
+
+            LOG.debug('New blueprint: %s', record)
+            yield record

stackalytics/processor/main.py

@@ -22,6 +22,7 @@ from psutil import _error
 from stackalytics.openstack.common import log as logging
 from stackalytics.processor import config
 from stackalytics.processor import default_data_processor
+from stackalytics.processor import lp
 from stackalytics.processor import mls
 from stackalytics.processor import rcs
 from stackalytics.processor import record_processor
@@ -78,6 +79,12 @@ def process_repo(repo, runtime_storage_inst, record_processor_inst):
     uri = repo['uri']
     LOG.debug('Processing repo uri %s' % uri)
 
+    bp_iterator = lp.log(repo)
+    bp_iterator_typed = _record_typer(bp_iterator, 'bp')
+    processed_mail_iterator = record_processor_inst.process(
+        bp_iterator_typed)
+    runtime_storage_inst.set_records(processed_mail_iterator)
+
     vcs_inst = vcs.get_vcs(repo, cfg.CONF.sources_root)
     vcs_inst.fetch()
 

stackalytics/processor/mls.py

@@ -68,6 +68,7 @@ def _link_content_changed(link, runtime_storage_inst):
     conn.request('HEAD', parsed_uri.path)
     res = conn.getresponse()
     last_modified = res.getheader('last-modified')
+    conn.close()
 
     if last_modified != runtime_storage_inst.get_by_key('mail_link:' + link):
         LOG.debug('Mail archive changed, last modified at: %s', last_modified)

stackalytics/processor/record_processor.py

@@ -264,6 +264,42 @@ class RecordProcessor(object):
 
         yield record
 
+    def _process_blueprint(self, record):
+        if record.get('drafter'):
+            bp_draft = dict([(k, v) for k, v in record.iteritems()])
+            bp_draft['primary_key'] = 'bpd:' + record['self_link']
+
+            drafter = utils.load_user(self.runtime_storage_inst,
+                                      record['drafter'])
+            if drafter and record['date_created']:
+                bp_draft['record_type'] = 'bp_draft'
+                bp_draft['author_name'] = drafter['user_name']
+                bp_draft['author_email'] = drafter['emails'][0]
+                bp_draft['launchpad_id'] = record['drafter']
+                bp_draft['date'] = record['date_created']
+
+                self._update_record_and_user(bp_draft)
+
+                yield bp_draft
+
+        if record.get('assignee'):
+            bp_implementation = dict([(k, v) for k, v in record.iteritems()])
+            bp_implementation['primary_key'] = 'bpi:' + record['self_link']
+
+            assignee = utils.load_user(self.runtime_storage_inst,
+                                       record['assignee'])
+            if assignee and record['date_completed']:
+                bp_implementation['record_type'] = 'bp_implementation'
+                bp_implementation['author_name'] = assignee['user_name']
+                bp_implementation['author_email'] = assignee['emails'][0]
+                bp_implementation['launchpad_id'] = record['assignee']
+                bp_implementation['date'] = record['date_completed']
+
+                if bp_implementation['author_email']:
+                    self._update_record_and_user(bp_implementation)
+
+                yield bp_implementation
+
     def _apply_type_based_processing(self, record):
         if record['record_type'] == 'commit':
             for r in self._process_commit(record):
@@ -274,6 +310,9 @@ class RecordProcessor(object):
         elif record['record_type'] == 'email':
             for r in self._process_email(record):
                 yield r
+        elif record['record_type'] == 'bp':
+            for r in self._process_blueprint(record):
+                yield r
 
     def process(self, record_iterator):
         for record in record_iterator:

stackalytics/processor/utils.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import datetime
+import iso8601
 import json
 import re
 import time
@@ -32,6 +33,10 @@ def date_to_timestamp(d):
         datetime.datetime.strptime(d, '%Y-%b-%d').timetuple()))
 
 
+def iso8601_to_timestamp(s):
+    return int(time.mktime(iso8601.parse_date(s).timetuple()))
+
+
 def timestamp_to_week(timestamp):
     # Jan 4th 1970 is the first Sunday in the Epoch
     return (timestamp - 3 * 24 * 3600) // (7 * 24 * 3600)