Add option to include returned data in MQTT reporter

This adds an option to include result data from a job in the MQTT
reporter.  It is off by default since it may be quite large for
some jobs.

Change-Id: I802adee834b60256abd054eda2db834f8db82650

doc/source/drivers/mqtt.rst

@@ -316,3 +316,9 @@ reporter. Each pipeline must provide a topic name. For example:
 
       The quality of service level to use, it can be 0, 1 or 2. Read more in this
       `guide <https://www.hivemq.com/blog/mqtt-essentials-part-6-mqtt-quality-of-service-levels>`_
+
+   .. attr:: include-returned-data
+      :default: false
+
+      If set to ``true``, Zuul will include any data returned from the
+      job via :ref:`return_values`.

releasenotes/notes/mqtt-include-returned-data-c5836db472907c42.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    The MQTT driver now supports including data returned from a job in
+    its reports.  See
+    :attr:`pipeline.<reporter>.<mqtt>.include-returned-data`.

tests/fixtures/config/mqtt-driver/git/common-config/zuul.d/config.yaml

@@ -7,11 +7,15 @@
     start:
       mqtt:
         topic: "{tenant}/zuul_start/{pipeline}/{project}/{branch}"
+        # This doesn't make sense here -- there should be no return
+        # data yet, which is why we include it in this test.
+        include-returned-data: True
     success:
       gerrit:
         Verified: 1
       mqtt:
         topic: "{tenant}/zuul_buildset/{pipeline}/{project}/{branch}"
+        include-returned-data: True
     failure:
       gerrit:
         Verified: -1

tests/unit/test_connection.py

@@ -681,9 +681,13 @@ class TestMQTTConnection(ZuulTestCase):
                         'type': 'container_image'
                     }}
         self.executor_server.returnData(
-            "test", A, {"zuul": {"log_url": "some-log-url",
-                                 'artifacts': [artifact],
-                                 }}
+            "test", A, {
+                "zuul": {
+                    "log_url": "some-log-url",
+                    'artifacts': [artifact],
+                },
+                'foo': 'bar',
+            }
         )
         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
         self.waitUntilSettled()
@@ -703,6 +707,9 @@ class TestMQTTConnection(ZuulTestCase):
                           'test')
         self.assertNotIn('result', mqtt_payload['buildset']['builds'][0])
         self.assertNotIn('artifacts', mqtt_payload['buildset']['builds'][0])
+        builds = mqtt_payload['buildset']['builds']
+        test_job = [b for b in builds if b['job_name'] == 'test'][0]
+        self.assertNotIn('returned_data', test_job)
 
         self.assertEquals(success_event.get('topic'),
                           'tenant-one/zuul_buildset/check/org/project/master')
@@ -720,6 +727,7 @@ class TestMQTTConnection(ZuulTestCase):
         self.assertEquals(test_job['dependencies'], [])
         self.assertEquals(test_job['artifacts'], [artifact])
         self.assertEquals(test_job['log_url'], 'some-log-url/')
+        self.assertEquals(test_job['returned_data'], {'foo': 'bar'})
         build_id = test_job["uuid"]
         self.assertEquals(
             test_job["web_url"],

zuul/driver/mqtt/mqttreporter.py

@@ -30,6 +30,7 @@ class MQTTReporter(BaseReporter):
     def report(self, item, phase1=True, phase2=True):
         if not phase1:
             return
+        include_returned_data = self.config.get('include-returned-data')
         log = get_annotated_logger(self.log, item.event)
         log.debug("Report change %s, params %s", item.change, self.config)
         message = {
@@ -80,6 +81,10 @@ class MQTTReporter(BaseReporter):
                     'artifacts': get_artifacts_from_result_data(
                         build.result_data, logger=log)
                 })
+                if include_returned_data:
+                    rdata = build.result_data.copy()
+                    rdata.pop('zuul', None)
+                    job_informations['returned_data'] = rdata
 
                 # Report build data of retried builds if available
                 retry_builds = item.current_build_set.getRetryBuildsForJob(
@@ -144,4 +149,8 @@ def qosValue(value):
 
 
 def getSchema():
-    return v.Schema({v.Required('topic'): topicValue, 'qos': qosValue})
+    return v.Schema({
+        v.Required('topic'): topicValue,
+        'qos': qosValue,
+        'include-returned-data': bool,
+    })