Added timestamp field, changed duration field type; improve doc

The timestamp field helps to prepare visualization in
Kibana.
The duration field was set to integer in Kibana object, but
the value was string, so the Kibana was doing incorrect visualizations.
Also this commit provides fix for rst block in elasticserarch driver
document.

Change-Id: I92a034d78f9193476eccecc7efb4a818d4b4a658

doc/source/reference/drivers/elasticsearch.rst

@@ -88,9 +88,9 @@ Example of driver configuration:
 
 .. code-block:: text
 
-[connection elasticsearch]
-driver=elasticsearch
-uri=https://managesf.sftests.com:9200
+    [connection elasticsearch]
+    driver=elasticsearch
+    uri=https://managesf.sftests.com:9200
 
 
 Additional parameters to authenticate to the Elasticsearch server you

tests/unit/test_connection.py

@@ -14,6 +14,7 @@
 
 import textwrap
 import time
+import types
 
 import sqlalchemy as sa
 
@@ -703,6 +704,13 @@ class TestElasticsearchConnection(AnsibleZuulTestCase):
             build_doc['job_returned_vars'], {'foo': 'bar'})
 
         self.assertEqual(self.history[0].uuid, build_doc['uuid'])
+        self.assertIn('duration', build_doc)
+        self.assertTrue(type(build_doc['duration']) is int)
+
+        doc_gen = self.scheds.first.connections.connections[
+            'elasticsearch'].gen(indexed_docs, index)
+        self.assertIsInstance(doc_gen, types.GeneratorType)
+        self.assertTrue('@timestamp' in list(doc_gen)[0]['_source'])
 
     def test_elasticsearch_secret_leak(self):
         expected_secret = [{

zuul/driver/elasticsearch/connection.py

@@ -15,6 +15,7 @@
 import yaml
 import logging
 
+from datetime import datetime
 from elasticsearch import Elasticsearch
 from elasticsearch.client import IndicesClient
 from elasticsearch.helpers import bulk
@@ -32,6 +33,7 @@ class ElasticsearchConnection(BaseConnection):
         "build_type": {"type": "keyword"},
         "result": {"type": "keyword"},
         "duration": {"type": "integer"},
+        "@timestamp": {"type": "date"},
         # BuildSet type specific attributes
         "zuul_ref": {"type": "keyword"},
         "pipeline": {"type": "keyword"},
@@ -108,6 +110,8 @@ class ElasticsearchConnection(BaseConnection):
     def gen(self, it, index):
         for source in it:
             d = {}
+            source['@timestamp'] = datetime.fromtimestamp(
+                int(source['start_time'])).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
             d['_index'] = index
             d['_type'] = 'zuul'
             d['_op_type'] = 'index'

zuul/driver/elasticsearch/reporter.py

@@ -86,7 +86,7 @@ class ElasticsearchReporter(BaseReporter):
                 "result": result,
                 "start_time": str(start_time),
                 "end_time": str(end_time),
-                "duration": str(end_time - start_time),
+                "duration": end_time - start_time,
                 "voting": build.job.voting,
                 "log_url": url,
             }