Add crm114 logstash queries to uncategorized page

Adding code to add in links to crm114 logstash queries
for those entries on the uncategorized page.

We only add a link if results were found during page
production.  This is to not waste human time on empty
queries.  However, this is a nice-to-have and can
be removed should any objection exist.

We also add some verbiage at the top of the uncategorized
page that describes what the crm114 links are as well
as including links to more in-depth docs, which are added
with change I086b2dfa68ca2239d70cec6db0b22951e6786ed7.

Change-Id: Ie572ef90a3393b43d28608b95a8a9b24bd014962

elastic_recheck/cmd/uncategorized_fails.py

@@ -15,8 +15,10 @@
 #    under the License.
 
 import argparse
+import base64
 import collections
 import datetime
+import json
 import operator
 import re
 
@@ -72,7 +74,8 @@ def all_fails(classifier):
                 log = result.log_url.split("console.html")[0]
                 all_fails["%s.%s" % (build, name)] = {
                     'log': log,
-                    'timestamp': timestamp
+                    'timestamp': timestamp,
+                    'build_uuid': result.build_uuid
                 }
     return all_fails
 
@@ -85,7 +88,7 @@ def num_fails_per_build_name(all_jobs):
     return counts
 
 
-def classifying_rate(fails, data, engine):
+def classifying_rate(fails, data, engine, classifier):
     """Builds and prints the classification rate.
 
     It's important to know how good a job we are doing, so this
@@ -121,8 +124,22 @@ def classifying_rate(fails, data, engine):
                                    key=lambda v: v['timestamp'], reverse=True)
         # Convert timestamp into string
         for url in bad_job_urls[job]:
+            urlq = {}
             url['timestamp'] = url['timestamp'].strftime(
                 "%Y-%m-%dT%H:%M")
+            # setup crm114 query for build_uuid
+            query = ('build_uuid: "%s" '
+                     'AND error_pr:["-1000.0" TO "-10.0"] '
+                     % url['build_uuid'])
+            urlq = dict(search=query,
+                        fields=[],
+                        offset=0,
+                        timeframe=str(864000))
+            logstash_query = base64.urlsafe_b64encode(json.dumps(urlq))
+            logstash_url = 'http://logstash.openstack.org/#%s' % logstash_query
+            results = classifier.hits_by_query(query, size=1)
+            if results:
+                url['crm114'] = logstash_url
 
     classifying_rate = collections.defaultdict(int)
     classifying_rate['overall'] = "%.1f" % (
@@ -231,7 +248,7 @@ def main():
     fails = all_fails(classifier)
     data = collect_metrics(classifier, fails)
     engine = setup_template_engine(opts.templatedir)
-    html = classifying_rate(fails, data, engine)
+    html = classifying_rate(fails, data, engine, classifier)
     if opts.output:
         with open(opts.output, "w") as f:
             f.write(html)

web/share/templates/uncategorized.html

@@ -62,6 +62,10 @@
 <li><a href="#{{job[0]}}">{{job[0]}} ({{job[1]}})</a></li>
 {% endfor %}
 </div>
+<div class='crm114-verbiage'>
+The crm114 links are logstash queries showing log messages that have been flagged as potential errors.<br>
+More information on the system can be found <a href="http://ci.openstack.org/logstash.html#crm114">here</a>
+</div>
 <div class="jobs">
 <h1>Unclassified failed jobs</h1>
 Overall Categorization Rate: {{ rate['overall'] }}%
@@ -81,8 +85,11 @@ Overall Categorization Rate: {{ rate['overall'] }}%
 <h2>{{ job[0] }} : {{ job[1] }} Uncategorized Fails. {{rate[job[0]]}}% Classification Rate ({{total_job_failures[job[0]]}} Total Fails)</h2>
 <ul>
   {% for url in urls[job[0]] %}
-
+    {% if url['crm114'] %}
-  <li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a></li>
+      <li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a> : <a href="{{ url['crm114'] }}">crm114</a></li>
+    {% else %}
+      <li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a></li>
+    {% endif %}
   {% endfor %}
 </ul>
 {% endfor %}