From c7b42bb88d81b31562bcc5611017cb96c89d2719 Mon Sep 17 00:00:00 2001
From: pcrews <gleebix@gmail.com>
Date: Tue, 12 Aug 2014 22:00:02 -0700
Subject: [PATCH] Add crm114 logstash queries to uncategorized page

Adding code to add in links to crm114 logstash queries
for those entries on the uncategorized page.

We only add a link if results were found during page
production.  This is to not waste human time on empty
queries.  However, this is a nice-to-have and can
be removed should any objection exist.

We also add some verbiage at the top of the uncategorized
page that describes what the crm114 links are as well
as including links to more in-depth docs, which are added
with change I086b2dfa68ca2239d70cec6db0b22951e6786ed7.

Change-Id: Ie572ef90a3393b43d28608b95a8a9b24bd014962
---
 elastic_recheck/cmd/uncategorized_fails.py | 23 +++++++++++++++++++---
 web/share/templates/uncategorized.html     | 11 +++++++++--
 2 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/elastic_recheck/cmd/uncategorized_fails.py b/elastic_recheck/cmd/uncategorized_fails.py
index 64afde1e..3ef30d6b 100755
--- a/elastic_recheck/cmd/uncategorized_fails.py
+++ b/elastic_recheck/cmd/uncategorized_fails.py
@@ -15,8 +15,10 @@
 #    under the License.
 
 import argparse
+import base64
 import collections
 import datetime
+import json
 import operator
 import re
 
@@ -72,7 +74,8 @@ def all_fails(classifier):
                 log = result.log_url.split("console.html")[0]
                 all_fails["%s.%s" % (build, name)] = {
                     'log': log,
-                    'timestamp': timestamp
+                    'timestamp': timestamp,
+                    'build_uuid': result.build_uuid
                 }
     return all_fails
 
@@ -85,7 +88,7 @@ def num_fails_per_build_name(all_jobs):
     return counts
 
 
-def classifying_rate(fails, data, engine):
+def classifying_rate(fails, data, engine, classifier):
     """Builds and prints the classification rate.
 
     It's important to know how good a job we are doing, so this
@@ -121,8 +124,22 @@ def classifying_rate(fails, data, engine):
                                    key=lambda v: v['timestamp'], reverse=True)
         # Convert timestamp into string
         for url in bad_job_urls[job]:
+            urlq = {}
             url['timestamp'] = url['timestamp'].strftime(
                 "%Y-%m-%dT%H:%M")
+            # setup crm114 query for build_uuid
+            query = ('build_uuid: "%s" '
+                     'AND error_pr:["-1000.0" TO "-10.0"] '
+                     % url['build_uuid'])
+            urlq = dict(search=query,
+                        fields=[],
+                        offset=0,
+                        timeframe=str(864000))
+            logstash_query = base64.urlsafe_b64encode(json.dumps(urlq))
+            logstash_url = 'http://logstash.openstack.org/#%s' % logstash_query
+            results = classifier.hits_by_query(query, size=1)
+            if results:
+                url['crm114'] = logstash_url
 
     classifying_rate = collections.defaultdict(int)
     classifying_rate['overall'] = "%.1f" % (
@@ -231,7 +248,7 @@ def main():
     fails = all_fails(classifier)
     data = collect_metrics(classifier, fails)
     engine = setup_template_engine(opts.templatedir)
-    html = classifying_rate(fails, data, engine)
+    html = classifying_rate(fails, data, engine, classifier)
     if opts.output:
         with open(opts.output, "w") as f:
             f.write(html)
diff --git a/web/share/templates/uncategorized.html b/web/share/templates/uncategorized.html
index 82f1478a..04060000 100644
--- a/web/share/templates/uncategorized.html
+++ b/web/share/templates/uncategorized.html
@@ -62,6 +62,10 @@
 <li><a href="#{{job[0]}}">{{job[0]}} ({{job[1]}})</a></li>
 {% endfor %}
 </div>
+<div class='crm114-verbiage'>
+The crm114 links are logstash queries showing log messages that have been flagged as potential errors.<br>
+More information on the system can be found <a href="http://ci.openstack.org/logstash.html#crm114">here</a>
+</div>
 <div class="jobs">
 <h1>Unclassified failed jobs</h1>
 Overall Categorization Rate: {{ rate['overall'] }}%
@@ -81,8 +85,11 @@ Overall Categorization Rate: {{ rate['overall'] }}%
 <h2>{{ job[0] }} : {{ job[1] }} Uncategorized Fails. {{rate[job[0]]}}% Classification Rate ({{total_job_failures[job[0]]}} Total Fails)</h2>
 <ul>
   {% for url in urls[job[0]] %}
-
-  <li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a></li>
+    {% if url['crm114'] %}
+      <li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a> : <a href="{{ url['crm114'] }}">crm114</a></li>
+    {% else %}
+      <li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a></li>
+    {% endif %}
   {% endfor %}
 </ul>
 {% endfor %}