From 411f4bbc5c1a52acebf1e5ec13d5f98943e99b19 Mon Sep 17 00:00:00 2001
From: Masayuki Igawa <masayuki@igawa.io>
Date: Thu, 8 Aug 2019 16:15:11 +0900
Subject: [PATCH] Retry http requests to generate plugins list

This commit retries http requests to generate a plugins list. We often
see internal server errors(500) in the gate. It makes sanity jobs fail.
So, it would be better to retry internally instead of 'recheck'. To
accomplish it, this uses urllib3 module.
NOTE: The default total number of retries[0] is 10. It should be enough
for us.

[0] https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.retry.Retry

Change-Id: Id3da3e2895fc2324d5a2db91f82b12ec5bb610f8
---
 tools/generate-tempest-plugins-list.py | 32 ++++++++++----------------
 1 file changed, 12 insertions(+), 20 deletions(-)

diff --git a/tools/generate-tempest-plugins-list.py b/tools/generate-tempest-plugins-list.py
index f1003a0fb3..165af450c2 100644
--- a/tools/generate-tempest-plugins-list.py
+++ b/tools/generate-tempest-plugins-list.py
@@ -27,14 +27,8 @@ import json
 import re
 import sys
 
-try:
-    # For Python 3.0 and later
-    from urllib.error import HTTPError
-    import urllib.request as urllib
-except ImportError:
-    # Fall back to Python 2's urllib2
-    import urllib2 as urllib
-    from urllib2 import HTTPError
+import urllib3
+from urllib3.util import retry
 
 # List of projects having tempest plugin stale or unmaintained for a long time
 # (6 months or more)
@@ -69,23 +63,21 @@ url = 'https://review.opendev.org/projects/'
   },
 '''
 
+http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED')
+retries = retry.Retry(status_forcelist=[500], backoff_factor=1.0)
+
 
-# Rather than returning a 404 for a nonexistent file, cgit delivers a
-# 0-byte response to a GET request.  It also does not provide a
-# Content-Length in a HEAD response, so the way we tell if a file exists
-# is to check the length of the entire GET response body.
 def has_tempest_plugin(proj):
     try:
-        r = urllib.urlopen(
-            "https://opendev.org/%s/raw/branch/"
-            "master/setup.cfg" % proj)
-    except HTTPError as err:
-        if err.code == 404:
+        r = http.request('GET', "https://opendev.org/%s/raw/branch/"
+                         "master/setup.cfg" % proj, retries=retries)
+        if r.status == 404:
             return False
+    except urllib3.exceptions.MaxRetryError as err:
         # We should not ignore non 404 errors.
         raise err
     p = re.compile(r'^tempest\.test_plugins', re.M)
-    if p.findall(r.read().decode('utf-8')):
+    if p.findall(r.data.decode('utf-8')):
         return True
     else:
         False
@@ -98,11 +90,11 @@ if len(sys.argv) > 1 and sys.argv[1] == 'blacklist':
     # So, this exits here.
     sys.exit()
 
-r = urllib.urlopen(url)
+r = http.request('GET', url, retries=retries)
 # Gerrit prepends 4 garbage octets to the JSON, in order to counter
 # cross-site scripting attacks.  Therefore we must discard it so the
 # json library won't choke.
-content = r.read().decode('utf-8')[4:]
+content = r.data.decode('utf-8')[4:]
 projects = sorted(json.loads(content))
 
 # Retrieve projects having no deployment tool repo (such as deb,