Revamped results schema

Results are now formatted in the schema as defined here:
https://gist.github.com/cneill/a511451284a0c5f33295477150bd94d4

Furthermore, the json formatter is no longer responsible for the aggregation
of issues. Instead, this logic has been moved to the IssueTestResult class

Change-Id: Id39e122b2b4c1c9cafab09fdbc5d172dec012d22

syntribos/formatters/json_formatter.py

@@ -13,123 +13,14 @@
 # limitations under the License.
 import json
 
-import syntribos
-
 
 class JSONFormatter(object):
 
     def __init__(self, results):
         self.results = results
 
-    def report(self, min_severity, min_confidence, exclude_results):
+    def report(self, output):
-        min_sev = syntribos.RANKING_VALUES[min_severity]
+        output = json.dumps(output, sort_keys=True, cls=SetEncoder,
-        min_conf = syntribos.RANKING_VALUES[min_confidence]
-        machine_output = dict({'failures': {}, 'errors': [], 'stats': {}})
-        machine_output['stats']['severity'] = {
-            'UNDEFINED': 0, 'LOW': 0, 'MEDIUM': 0, 'HIGH': 0
-        }
-
-        severity_counter_dict = {}
-
-        # reports errors
-        for test, error in self.results.errors:
-            machine_output['errors'].append(
-                {
-                    'test': self.results.getDescription(test),
-                    'error': error
-                })
-
-        # reports failures
-        # Gets list of [issues] by flattening list of [(test, [issues])]
-        issues = [issue for test, failures in self.results.failures
-                  for issue in failures]
-
-        for issue in issues:
-            target = issue.target
-            path = issue.path
-            url = "{0}{1}".format(target, path)
-            defect_type = issue.defect_type
-            sev_rating = syntribos.RANKING[issue.severity]
-            conf_rating = syntribos.RANKING[issue.confidence]
-
-            if any([True for x in exclude_results if x and x in defect_type]):
-                continue
-
-            defect_obj = {
-                'description': issue.description,
-                'severity': sev_rating,
-                'signals': {
-                    'init_signals': [s.slug for s in issue.init_signals],
-                    'test_signals': [s.slug for s in issue.test_signals],
-                    'diff_signals': [s.slug for s in issue.diff_signals]
-                }
-
-            }
-
-            if defect_type not in severity_counter_dict:
-                severity_counter_dict[defect_type] = defect_obj
-                machine_output['stats']['severity'][sev_rating] += 1
-
-            if url not in machine_output['failures']:
-                if issue.severity >= min_sev and issue.confidence >= min_conf:
-                    machine_output['failures'][url] = {}
-                else:
-                    continue
-
-            issues_by_url = machine_output['failures'][url]
-            if defect_type not in issues_by_url:
-                if issue.severity >= min_sev and issue.confidence >= min_conf:
-                    issues_by_url[defect_type] = defect_obj
-                else:
-                    continue
-
-            issues_by_defect = issues_by_url[defect_type]
-            if issue.impacted_parameter:
-                # Only fuzz tests have an ImpactedParameter
-                method = issue.impacted_parameter.method
-                loc = issue.impacted_parameter.location
-                name = issue.impacted_parameter.name
-                content_type = issue.content_type
-                payload_string = issue.impacted_parameter.trunc_fuzz_string
-
-                param = {
-                    'method': method,
-                    'location': loc,
-                    'variables': [name],
-                }
-                if loc == "data":
-                    param['type'] = content_type
-
-                payload_obj = {
-                    'strings': [payload_string],
-                    'param': param,
-                    'confidence': conf_rating
-                }
-                if 'payloads' not in issues_by_defect:
-                    issues_by_defect['payloads'] = [payload_obj]
-                else:
-                    is_not_duplicate_payload = True
-
-                    for p in issues_by_defect['payloads']:
-
-                        if (p['param']['method'] == method and
-                                p['param']['location'] == loc):
-
-                            if payload_string not in p['strings']:
-                                p['strings'].append(payload_string)
-
-                            if name not in p['param']['variables']:
-                                p['param']['variables'].append(name)
-
-                            is_not_duplicate_payload = False
-                            break
-                    if is_not_duplicate_payload:
-                        issues_by_defect['payloads'].append(payload_obj)
-
-            else:
-                issues_by_defect['confidence'] = conf_rating
-
-        output = json.dumps(machine_output, sort_keys=True,
                             indent=2, separators=(',', ': '))
 
         self.results.stream.write(output)

syntribos/result.py

@@ -30,21 +30,168 @@ class IssueTestResult(unittest.TextTestResult):
     This class aggregates :class:`syntribos.issue.Issue` objects from all the
     tests as they run
     """
+    output = {"failures": {}, "errors": [], "stats": {}}
+    output["stats"]["severity"] = {
+        "UNDEFINED": 0, "LOW": 0, "MEDIUM": 0, "HIGH": 0
+    }
     stats = {"errors": 0, "failures": 0, "successes": 0}
+    severity_counter_dict = {}
     testsRunSinceLastPrint = 0
+    failure_id = 0
 
     def addFailure(self, test, err):
         """Adds issues to data structures
 
-        Appends issues to the result's list of failures, as well as
+        Appends issues to the result's list of failures, as well as updates the
-        to a dict of {url: {method: {test_name: issue}}} structure.
+        stats for the result. Each failure in the list of failures takes the
+        form:
+
+        .. code-block:: json
+
+            {
+              "url": "host.com/blah",
+              "type": "500_error",
+              "description": "500 errors r bad, mkay?",
+              "failure_id": 1234,
+              "instances": [
+                {
+                  "confidence": "HIGH",
+                  "param": {
+                    "location": "headers",
+                    "method": "POST",
+                    "variables": [
+                      "Content-Type"
+                    ]
+                  },
+                  "strings": [
+                    "derp"
+                  ],
+                  "severity": "LOW",
+                  "signals": {
+                    "diff_signals": [],
+                    "init_signals": [],
+                    "test_signals": []
+                  }
+                }
+              ]
+            }
 
         :param test: The test that has failed
         :type test: :class:`syntribos.tests.base.BaseTestCase`
         :param tuple err: Tuple of format ``(type, value, traceback)``
         """
-        self.failures.append((test, test.failures))
+        for issue in test.failures:
-        self.stats["failures"] += len(test.failures)
+            defect_type = issue.defect_type
+            if any([True for x in CONF.syntribos.exclude_results
+                    if x and x in defect_type]):
+                continue
+
+            min_sev = syntribos.RANKING_VALUES[CONF.min_severity]
+            min_conf = syntribos.RANKING_VALUES[CONF.min_confidence]
+            if issue.severity < min_sev or issue.confidence < min_conf:
+                continue
+
+            target = issue.target
+            path = issue.path
+            url = "{0}{1}".format(target, path)
+            description = issue.description
+            failure_obj = None
+
+            for f in self.failures:
+                if (f["url"] == url and f["defect_type"] == defect_type and
+                        f["description"] == description):
+                    failure_obj = f
+                    break
+            if not failure_obj:
+                failure_obj = {
+                    "url": url,
+                    "defect_type": defect_type,
+                    "description": description,
+                    "failure_id": self.failure_id,
+                    "instances": []
+                }
+                self.failures.append(failure_obj)
+                self.failure_id += 1
+
+            signals = {}
+            if issue.init_signals:
+                signals["init_signals"] = set(
+                    [s.slug for s in issue.init_signals])
+            if issue.test_signals:
+                signals["test_signals"] = set(
+                    [s.slug for s in issue.test_signals])
+            if issue.diff_signals:
+                signals["diff_signals"] = set(
+                    [s.slug for s in issue.diff_signals])
+            sev_rating = syntribos.RANKING[issue.severity]
+            conf_rating = syntribos.RANKING[issue.confidence]
+
+            if issue.impacted_parameter:
+                method = issue.impacted_parameter.method
+                loc = issue.impacted_parameter.location
+                name = issue.impacted_parameter.name
+                content_type = issue.content_type
+                payload_string = issue.impacted_parameter.trunc_fuzz_string
+
+                param = {
+                    "method": method,
+                    "location": loc,
+                }
+                if loc == "data":
+                    param["type"] = content_type
+
+                instance_obj = None
+                for i in failure_obj["instances"]:
+                    if(i["confidence"] == conf_rating and
+                            i["severity"] == sev_rating and
+                            i["param"]["method"] == method and
+                            i["param"]["location"] == loc):
+
+                        i["param"]["variables"].add(name)
+                        for sig_type in signals:
+                            if sig_type in i["signals"]:
+                                i["signals"][sig_type].update(
+                                    signals[sig_type])
+                            else:
+                                i["signals"][sig_type] = signals[sig_type]
+                        i["strings"].add(payload_string)
+                        instance_obj = i
+                        break
+
+                if not instance_obj:
+                    param["variables"] = set([name])
+                    instance_obj = {
+                        "confidence": conf_rating,
+                        "severity": sev_rating,
+                        "param": param,
+                        "strings": set([payload_string]),
+                        "signals": signals
+                    }
+                    failure_obj["instances"].append(instance_obj)
+                    self.stats["failures"] += 1
+                    self.output["stats"]["severity"][sev_rating] += 1
+            else:
+                instance_obj = None
+                for i in failure_obj["instances"]:
+                    if(i["confidence"] == conf_rating and
+                            i["severity"] == sev_rating):
+                        for sig_type in signals:
+                            if sig_type in i["signals"]:
+                                i["signals"][sig_type].update(
+                                    signals[sig_type])
+                            else:
+                                i["signals"][sig_type] = signals[sig_type]
+                        instance_obj = i
+                        break
+                if not instance_obj:
+                    instance_obj = {
+                        "confidence": conf_rating,
+                        "severity": sev_rating,
+                        "signals": signals
+                    }
+                    failure_obj["instances"].append(instance_obj)
+                    self.stats["failures"] += 1
+                    self.output["stats"]["severity"][sev_rating] += 1
 
     def addError(self, test, err):
         """Duplicates parent class addError functionality.
@@ -54,7 +201,11 @@ class IssueTestResult(unittest.TextTestResult):
         :param err:
         :type tuple: Tuple of format ``(type, value, traceback)``
         """
-        self.errors.append((test, self._exc_info_to_string(err, test)))
+        self.errors.append(
+            {
+                "test": self.getDescription(test),
+                "error": self._exc_info_to_string(err, test)
+            })
         self.stats["errors"] += 1
 
     def addSuccess(self, test):
@@ -65,36 +216,37 @@ class IssueTestResult(unittest.TextTestResult):
         """
         self.stats["successes"] += 1
 
-    def printErrors(self, output_format, min_severity, min_confidence,
+    def printErrors(self, output_format):
-                    exclude_results):
         """Print out each :class:`syntribos.issue.Issue` that was encountered
 
         :param str output_format: Either "json" or "xml"
         """
+        self.output["errors"] = self.errors
+        self.output["failures"] = self.failures
         formatter_types = {
             "json": JSONFormatter(self)
         }
         formatter = formatter_types[output_format]
-        formatter.report(min_severity, min_confidence, exclude_results)
+        formatter.report(self.output)
 
     def print_result(self, start_time):
         """Prints test summary/stats (e.g. # failures) to stdout."""
-        self.printErrors(
+        self.printErrors(CONF.output_format)
-            CONF.output_format, CONF.min_severity, CONF.min_confidence,
-            CONF.syntribos.exclude_results)
         self.print_log_path_and_stats(start_time)
 
     def print_log_path_and_stats(self, start_time):
         """Print the path to the log folder for this run."""
         test_log = Runner.get_log_file_name()
         run_time = time.time() - start_time
+        num_fail = self.stats["failures"]
+        num_err = self.stats["errors"]
         print("\n{sep}\nTotal: Ran {num} test{suff} in {time:.3f}s".format(
             sep=syntribos.SEP, num=self.testsRun,
             suff="s" * bool(self.testsRun - 1), time=run_time))
         print("Total: {f} failure{fsuff} and {e} error{esuff}".format(
-            f=len(self.failures), e=len(self.errors),
+            f=num_fail, e=num_err,
-            fsuff="s" * bool(len(self.failures) - 1),
+            fsuff="s" * bool(num_fail - 1),
-            esuff="s" * bool(len(self.errors) - 1)))
+            esuff="s" * bool(num_err - 1)))
         if test_log:
             print(syntribos.SEP)
             print("LOG PATH...: {path}".format(path=test_log))

syntribos/runner.py

@@ -233,8 +233,8 @@ class Runner(object):
                             LOG.debug(test_time)
                             bar.increment(1)
                         bar.print_bar()
-                        failures = len(result.failures) - last_failures
+                        failures = result.stats["failures"] - last_failures
-                        errors = len(result.errors) - last_errors
+                        errors = result.stats["errors"] - last_errors
                         total_tests = len(test_cases)
                         if failures > total_tests * 0.90:
                             # More than 90 percent failure
@@ -246,8 +246,8 @@ class Runner(object):
                             # More than 15 percent failure
                             failures = cli.colorize(failures, "blue")
                     if errors:
-                        last_failures = len(result.failures)
+                        last_failures = result.stats["failures"]
-                        last_errors = len(result.errors)
+                        last_errors = result.stats["errors"]
                         errors = cli.colorize(errors, "red")
                         print ("  :  {0} Failure(s), {1} Error(s)\r".format(
                             failures, errors))

syntribos/tests/fuzz/base_fuzz.py

@@ -208,10 +208,10 @@ class ImpactedParameter(object):
     def __init__(self, method, location, name, value):
         self.method = method
         self.location = location
-        if len(value) >= 512:
+        if len(value) >= 128:
             self.trunc_fuzz_string = "{0}...({1} chars)...{2}".format(
-                value[:256], len(value),
+                value[:64], len(value),
-                value[-256:])
+                value[-64:])
         else:
             self.trunc_fuzz_string = value
         self.fuzz_string = value

tests/unit/test_results.py

@@ -11,21 +11,35 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import sys
-
 import testtools
 
+import syntribos
+from syntribos.issue import Issue
 from syntribos.result import IssueTestResult
 
 
 class FakeTest(object):
     def __init__(self, name):
-        self.failures = [1, 2]
         self.errors = [3, 4]
         self.successes = [5, 6]
         self.name = name
         self.failureException = Exception
 
+        issue1 = Issue(defect_type="fake",
+                       severity=syntribos.LOW,
+                       description="x",
+                       confidence=syntribos.LOW)
+        issue1.target = "example.com"
+        issue1.path = "/test"
+
+        issue2 = Issue(defect_type="fake2",
+                       severity=syntribos.MEDIUM,
+                       description="x",
+                       confidence=syntribos.LOW)
+        issue2.target = "example.com"
+        issue2.path = "/test2"
+        self.failures = [issue1, issue2]
+
     def __str__(self):
         return self.name
 
@@ -40,11 +54,6 @@ class TestIssueTestResult(testtools.TestCase):
         self.issue_result.addFailure(test, ())
         self.assertEqual(self.issue_result.stats["failures"], 2)
 
-    def test_addError(self):
-        test = FakeTest("error")
-        self.issue_result.addError(test, sys.exc_info())
-        self.assertEqual(self.issue_result.stats["errors"], 1)
-
     def test_addSuccess(self):
         test = FakeTest("success")
         self.issue_result.addSuccess(test)