Fix the integrated gate filtering on uncategorized

When stackforge was compacted into the openstack/ git namespace this
broke our filtering on just integrated gate projects. This commit does
2 things to handle this. First it takes a list of the projects using
the integrated gate job template from zuul's layout.yaml and uses that
to be the filter. This way we actual have a view of the integrated gate
again. It then makes a second page for all the other projects failures
which can be used by those to track uncategorized failures. A future
next step will be to add a config file to make the functional split of
these views configurable so that we can more easily adjust the split
as the interactions between projects change.

Change-Id: I41c8ae1e75e8a3d8893f6af5af7c283b5f5c1bcf

elastic_recheck/cmd/uncategorized_fails.py

@@ -20,6 +20,7 @@ import ConfigParser
 import datetime
 import logging
 import operator
+import os
 import re
 import requests
 
@@ -57,7 +58,12 @@ def get_options():
     parser.add_argument('--dir', '-d', help="Queries Directory",
                         default="queries")
     parser.add_argument('-t', '--templatedir', help="Template Directory")
-    parser.add_argument('-o', '--output', help="Output File")
+    parser.add_argument('-o', '--output',
+                        help="The path for the directory to store the "
+                             "html output. 2 files will be created: "
+                             "integrated_gate.html and other.html. "
+                             "If this option is not specified these files "
+                             "will be written to the cwd.")
     parser.add_argument('-c', '--conf', help="Elastic Recheck Configuration "
                         "file to use for data_source options such as "
                         "elastic search url, logstash url, and database "
@@ -65,14 +71,20 @@ def get_options():
     return parser.parse_args()
 
 
-def setup_template_engine(directory):
+def setup_template_engine(directory, group='integrated_gate'):
     path = ["web/share/templates"]
     if directory:
         path.append(directory)
 
     loader = jinja2.FileSystemLoader(path)
     env = jinja2.Environment(loader=loader)
-    return env.get_template("uncategorized.html")
+    if group == 'integrated_gate':
+        filename = 'integrated_gate.html'
+    elif group == 'other':
+        filename = 'others.html'
+    else:
+        raise TypeError('Unknown job name %s' % group)
+    return env.get_template(filename)
 
 
 def all_fails(classifier):
@@ -81,6 +93,8 @@ def all_fails(classifier):
     This attempts to find all the build jobs in the integrated gate
     so we can figure out how good we are doing on total classification.
     """
+    integrated_fails = {}
+    other_fails = {}
     all_fails = {}
     query = ('filename:"console.html" '
              'AND (message:"Finished: FAILURE" '
@@ -96,17 +110,39 @@ def all_fails(classifier):
             if re.search(EXCLUDED_JOBS_REGEX, result.build_name):
                 continue
 
-            # not perfect, but basically an attempt to show the integrated
+            integrated_gate_projects = [
-            # gate. Would be nice if there was a zuul attr for this in es.
+                'openstack/cinder',
-            if re.search("(^openstack/|devstack|grenade)", result.project):
+                'openstack/glance',
+                'openstack/keystone',
+                'openstack/neutron',
+                'openstack/nova',
+                'openstack/requirements',
+                'openstack/tempest',
+                'openstack-dev/devstack',
+                'openstack-infra/devstack-gate',
+            ]
+            if result.project in integrated_gate_projects:
                 name = result.build_name
                 timestamp = dp.parse(result.timestamp)
                 log = result.log_url.split("console.html")[0]
-                all_fails["%s.%s" % (build, name)] = {
+                integrated_fails["%s.%s" % (build, name)] = {
                     'log': log,
                     'timestamp': timestamp,
                     'build_uuid': result.build_uuid
                 }
+            else:
+                name = result.build_name
+                timestamp = dp.parse(result.timestamp)
+                log = result.log_url.split("console.html")[0]
+                other_fails["%s.%s" % (build, name)] = {
+                    'log': log,
+                    'timestamp': timestamp,
+                    'build_uuid': result.build_uuid
+                }
+    all_fails = {
+        'integrated_gate': integrated_fails,
+        'other': other_fails
+    }
     return all_fails
 
 
@@ -294,15 +330,20 @@ def main():
             db_uri = config.get('data_source', 'db_uri')
 
     classifier = er.Classifier(opts.dir, es_url=es_url, db_uri=db_uri)
-    fails = all_fails(classifier)
+    all_gate_fails = all_fails(classifier)
+    for group in all_gate_fails:
+        fails = all_gate_fails[group]
+        if not fails:
+            continue
         data = collect_metrics(classifier, fails)
-    engine = setup_template_engine(opts.templatedir)
+        engine = setup_template_engine(opts.templatedir, group=group)
         html = classifying_rate(fails, data, engine, classifier, ls_url)
         if opts.output:
-        with open(opts.output, "w") as f:
+            out_dir = opts.output
-            f.write(html)
         else:
-        print html
+            out_dir = os.getcwd()
+        with open(os.path.join(out_dir, group + '.html'), "w") as f:
+            f.write(html)
 
 
 if __name__ == "__main__":

elastic_recheck/tests/unit/cmd/test_uncategorized_fails.py

@@ -54,5 +54,8 @@ class TestUncategorizedFails(testtools.TestCase):
         all_fails = fails.all_fails(classifier)
         # assert that we only have the single result
         self.assertThat(all_fails,
+                        testtools.matchers.HasLength(2))
+        self.assertThat(all_fails['integrated_gate'],
                         testtools.matchers.HasLength(1))
-        self.assertIn('gate-tempest-dsvm-full', all_fails.keys()[0])
+        self.assertIn('gate-tempest-dsvm-full',
+                      all_fails['integrated_gate'].keys()[0])

web/share/templates/integrated_gate.html

@@ -0,0 +1,120 @@
+{% extends "base.html" %}
+{% block body %}
+    {{ super() }}
+
+<style>
+  .menu {
+     float: right;
+     padding-top: 1em;
+  }
+  .jobs {
+     padding-top: 1em;
+  }
+</style>
+<script type="text/javascript">
+    function filter_log_age(ev, days) {
+        ev.preventDefault();
+        var generated = $('#generated-date').text();
+        var gen_date = Date.parse(generated);
+
+        $("div.job").each(function () {
+            $(this).show();
+        });
+        $( "li.log-link" ).each(function() {
+            if (! $( this ).hasClass("dated") ) {
+                var timestamp = $( this ).text().substr(0,16);
+                var item_date = Date.parse(timestamp);
+                var date_delta = (gen_date - item_date) / 86400000;
+                $( this ).addClass("dated");
+                $( this ).attr("age", date_delta);
+            }
+            if ($( this ).attr("age") > days ) {
+                $( this ).hide();
+            } else {
+                $( this ).show();
+            }
+        });
+        $("div.job").each(function () {
+            var visible = $('ul li:visible', $(this)).size()
+            if (visible == 0) $(this).hide();
+            else $(this).show();
+        });
+        $("#menu li").each(function () {
+            var h = $("a", this).attr('href').substring(1);
+            if ($('a[name="' + h + '"]').closest("div").is(":visible")){
+               $(this).show();
+            }else{
+               $(this).hide();
+            }
+
+        });
+
+    }
+
+    $(function() {
+        $("#24hours").click(function(e) {
+          filter_log_age(e, 1);
+        });
+        $("#2days").click(function(e) {
+          filter_log_age(e, 2);
+        });
+        $("#7days").click(function(e) {
+          filter_log_age(e, 7);
+        });
+        $("#10days").click(function(e) {
+          filter_log_age(e, 10);
+        });
+    });
+</script>
+<div class="container">
+  <ul class="nav nav-tabs">
+    <li><a href="../index.html">All Pipelines</a></li>
+    <li><a href="../gate.html">Gate Pipeline</a></li>
+    <li class="active"><a href="integrated_gate.html">Integrated Gate Uncategorized</a></li>
+    <li class="active"><a href="other.html">Other Projects Uncategorized</a></li>
+  </ul>
+</div>
+
+
+<div class="menu" id="menu">
+<a name="top"></a>
+{% for job in jobs %}
+<li><a href="#{{job[0]}}">{{job[0]}} ({{job[1]}})</a></li>
+{% endfor %}
+</div>
+<div class='crm114-verbiage'>
+Failures on this page are collected from all voting gate failures on projects running the integrated-gate job templates that don't match current elastic-recheck bug fingerprints.<br>
+The crm114 links are logstash queries showing log messages that have been flagged as potential errors.<br>
+More information on the system can be found <a href="http://docs.openstack.org/infra/system-config/logstash.html#crm114">here</a>
+</div>
+<div class="jobs">
+<h1>Unclassified failed integrated gate jobs</h1>
+Overall Categorization Rate: {{ rate['overall'] }}%
+<p>
+  Total: {{ total }} - Found: {{ count }} = Unclassified: {{ uncounted }}
+</p>
+<p>
+  Generated at: <span id="generated-date">{{ generated_at }}</span>
+  (View: <a id="24hours" href="#">24 hours</a>,
+         <a id="2days" href="#">2 days</a>,
+         <a id="7days" href="#">7 days</a>,
+         <a id="10days" href="#">10 days</a>)
+</p>
+{% for job in jobs %}
+<div class="job">
+  <a name="{{job[0]}}"></a>
+  <a href="#top"><i>back to top</i></a>
+  <h2>{{ job[0] }} : {{ job[1] }} Uncategorized Fails. {{rate[job[0]]}}% Classification Rate ({{total_job_failures[job[0]]}} Total Fails)</h2>
+  <ul>
+    {% for url in urls[job[0]] %}
+      {% if url['crm114'] %}
+        <li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a> : <a href="{{ url['crm114'] }}">crm114</a></li>
+      {% else %}
+        <li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a></li>
+      {% endif %}
+    {% endfor %}
+  </ul>
+</div>
+{% endfor %}
+{% endblock %}
+</div>

web/share/templates/other.html

@@ -70,7 +70,8 @@
   <ul class="nav nav-tabs">
     <li><a href="../index.html">All Pipelines</a></li>
     <li><a href="../gate.html">Gate Pipeline</a></li>
-    <li class="active"><a href="uncategorized.html">Uncategorized</a></li>
+    <li class="active"><a href="integrated_gate.html">Integrated Gate Uncategorized</a></li>
+    <li class="active"><a href="other.html">Other Projects Uncategorized</a></li>
   </ul>
 </div>
 
@@ -82,7 +83,7 @@
 {% endfor %}
 </div>
 <div class='crm114-verbiage'>
-Failures on this page are collected from all voting gate failures that don't match current elastic-recheck bug fingerprints.<br>
+Failures on this page are collected from all voting gate failures which do not use the integrated-gate job templates that don't match current elastic-recheck bug fingerprints.<br>
 The crm114 links are logstash queries showing log messages that have been flagged as potential errors.<br>
 More information on the system can be found <a href="http://docs.openstack.org/infra/system-config/logstash.html#crm114">here</a>
 </div>