From 00348a4d0df48eda45035b78cb2a3e33b708553a Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Fri, 19 Jul 2019 09:06:30 -0700
Subject: [PATCH] Add tool to analyze check and gate success rates

This tool scans gerrit changes for comments from zuul over the last 30
days to build out success rates for check and gate pipelines. This only
looks at changes that have merged to avoid those that never can merge
because they only fail or are expected to fail.

This tool emits information like:

  Changes: 4475
  Check Failures: 5317.0
  Check Successes: 9173.0
  Check Rate of failure: 0.3669427191166322
  Gate Failures: 687.0
  Gate Successes: 4450.0
  Gate Rate of failure: 0.13373564337161767
  Total Failures: 6004.0
  Total Successes: 13623.0
  Total Rate of failure: 0.3059051306873185

Change-Id: I759ba670c6b81f4425ce618c412db9cbd0e51401
---
 tools/gerrit-success-rate.py | 75 ++++++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 tools/gerrit-success-rate.py

diff --git a/tools/gerrit-success-rate.py b/tools/gerrit-success-rate.py
new file mode 100644
index 0000000000..c61133de90
--- /dev/null
+++ b/tools/gerrit-success-rate.py
@@ -0,0 +1,75 @@
+import datetime
+import json
+import requests
+
+
+today = datetime.date.today()
+thirty_days = datetime.timedelta(days=30)
+last_month = today - thirty_days
+str_today = today.strftime('%Y-%m-%d')
+str_last_month = last_month.strftime('%Y-%m-%d')
+
+
+more_changes = True
+change_count = 0
+check_successes = 0.0
+check_failures = 0.0
+gate_successes = 0.0
+gate_failures = 0.0
+while(more_changes):
+    more_changes = False
+    # We look only at merged changes to ignore Do Not Merge changes that are
+    # expected to fail intentionally.
+    r = requests.get('https://review.opendev.org/changes/?q=status:merged+'
+                     'after:{after}+before:{before}&n=100&S={offset}'.format(
+                         after=str_last_month,
+                         before=str_today,
+                         offset=(change_count + 1)))
+
+    content = r.content[4:]
+    changes = json.loads(content)
+
+    for change in changes:
+        r = requests.get('https://review.opendev.org/changes/%s/detail' % change['id'])
+        content = r.content[4:]
+        details = json.loads(content)
+        change_count += 1
+        for message in details['messages']:
+            if 'author' not in message or 'message' not in message:
+                continue
+            if message['author']['_account_id'] != 22348:
+                # Don't care about non zuul comments
+                continue
+            if '(check pipeline)' in message['message']:
+                if 'Build succeeded' in message['message']:
+                    check_successes += 1
+                elif 'Build failed' in message['message']:
+                    # explicitly check for failed builds here to avoid merge
+                    # conflict messages
+                    check_failures += 1
+            if '(gate pipeline)' in message['message']:
+                if 'Build succeeded' in message['message']:
+                    gate_successes += 1
+                elif 'Build failed' in message['message']:
+                    # explicitly check for failed builds here to avoid merge
+                    # conflict messages
+                    gate_failures += 1
+            if '_more_changes' in change:
+                more_changes = True
+
+print('Changes: %s' % change_count)
+
+print('Check Failures: %s' % check_failures)
+print('Check Successes: %s' % check_successes)
+check_total = check_failures + check_successes
+print('Check Rate of failure: %s' % (check_failures/check_total))
+
+print('Gate Failures: %s' % gate_failures)
+print('Gate Successes: %s' % gate_successes)
+gate_total = gate_failures + gate_successes
+print('Gate Rate of failure: %s' % (gate_failures/gate_total))
+
+print('Total Failures: %s' % (check_failures + gate_failures))
+print('Total Successes: %s' % (check_successes + gate_successes))
+total = check_total + gate_total
+print('Total Rate of failure: %s' % ((check_failures + gate_failures)/total))