Retry http requests to generate plugins list

This commit retries http requests to generate a plugins list. We often
see internal server errors(500) in the gate. It makes sanity jobs fail.
So, it would be better to retry internally instead of 'recheck'. To
accomplish it, this uses urllib3 module.
NOTE: The default total number of retries[0] is 10. It should be enough
for us.

[0] https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.retry.Retry

Change-Id: Id3da3e2895fc2324d5a2db91f82b12ec5bb610f8
This commit is contained in:
Masayuki Igawa 2019-08-08 16:15:11 +09:00
parent ecc33b29c8
commit 411f4bbc5c
No known key found for this signature in database
GPG Key ID: 290F53EDC899BF89
1 changed files with 12 additions and 20 deletions

View File

@ -27,14 +27,8 @@ import json
import re
import sys
try:
# For Python 3.0 and later
from urllib.error import HTTPError
import urllib.request as urllib
except ImportError:
# Fall back to Python 2's urllib2
import urllib2 as urllib
from urllib2 import HTTPError
import urllib3
from urllib3.util import retry
# List of projects having tempest plugin stale or unmaintained for a long time
# (6 months or more)
@ -69,23 +63,21 @@ url = 'https://review.opendev.org/projects/'
},
'''
http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED')
retries = retry.Retry(status_forcelist=[500], backoff_factor=1.0)
# Rather than returning a 404 for a nonexistent file, cgit delivers a
# 0-byte response to a GET request. It also does not provide a
# Content-Length in a HEAD response, so the way we tell if a file exists
# is to check the length of the entire GET response body.
def has_tempest_plugin(proj):
try:
r = urllib.urlopen(
"https://opendev.org/%s/raw/branch/"
"master/setup.cfg" % proj)
except HTTPError as err:
if err.code == 404:
r = http.request('GET', "https://opendev.org/%s/raw/branch/"
"master/setup.cfg" % proj, retries=retries)
if r.status == 404:
return False
except urllib3.exceptions.MaxRetryError as err:
# We should not ignore non 404 errors.
raise err
p = re.compile(r'^tempest\.test_plugins', re.M)
if p.findall(r.read().decode('utf-8')):
if p.findall(r.data.decode('utf-8')):
return True
else:
False
@ -98,11 +90,11 @@ if len(sys.argv) > 1 and sys.argv[1] == 'blacklist':
# So, this exits here.
sys.exit()
r = urllib.urlopen(url)
r = http.request('GET', url, retries=retries)
# Gerrit prepends 4 garbage octets to the JSON, in order to counter
# cross-site scripting attacks. Therefore we must discard it so the
# json library won't choke.
content = r.read().decode('utf-8')[4:]
content = r.data.decode('utf-8')[4:]
projects = sorted(json.loads(content))
# Retrieve projects having no deployment tool repo (such as deb,