openstack/fuel-qa
Code Issues Proposed changes

Fix encoding errors while working with APIs

Browse Source 
Use 'requests' module for HTTP requests to TestRail
and Jenkins API instead of 'urllib'. It provides a
method which decode response data using proper
character encoding (charset from headers) and returns
unicode string.

Also add one more environment variable to settings,
because it's needed by proboscis for test plan
generation (since I9b9d40a59d24f579502a38dfc9b8c142bc219a06
was merged).

Closes-bug: #1584401
Change-Id: I3d6cde2c8066bd58e735142fe26d56e83d1c90de
(cherry picked from commit 5072b8426c)
This commit is contained in:
Artem Panchenko 2016-05-22 02:02:24 +03:00
parent 9eb506b2ae
commit 21931edd79
5 changed files with 45 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
fuelweb_test/testrail/builds.py
View File

@ -14,27 +14,24 @@
from __future__ import unicode_literals
import json
import re
# pylint: disable=import-error
from six.moves.urllib import request
# pylint: enable=import-error
import requests
from requests.packages.urllib3 import disable_warnings
from fuelweb_test.testrail.settings import JENKINS
from fuelweb_test.testrail.settings import logger
disable_warnings()
def get_jobs_for_view(view):
"""Return list of jobs from specified view
"""
view_url = "/".join([JENKINS["url"], 'view', view, 'api/json'])
logger.debug("Request view data from {}".format(view_url))
req = request.Request(view_url)
opener = request.build_opener(request.HTTPHandler)
s = opener.open(req).read()
opener.close()
view_data = json.loads(s)
view_data = requests.get(view_url).json()
jobs = [job["name"] for job in view_data["jobs"]]
return jobs
@ -44,13 +41,10 @@ def get_downstream_builds_from_html(url):
"""
url = "/".join([url, 'downstreambuildview/'])
logger.debug("Request downstream builds data from {}".format(url))
req = request.Request(url)
opener = request.build_opener(request.HTTPHandler)
s = opener.open(req).read()
opener.close()
response = requests.get(url).text
jobs = []
raw_downstream_builds = re.findall(
'.*downstream-buildview.*href="(/job/\S+/[0-9]+/).*', s)
'.*downstream-buildview.*href="(/job/\S+/[0-9]+/).*', response)
for raw_build in raw_downstream_builds:
sub_job_name = raw_build.split('/')[2]
sub_job_build = raw_build.split('/')[3]
@ -71,11 +65,7 @@ def get_build_artifact(url, artifact):
"""
url = "/".join([url, 'artifact', artifact])
logger.debug("Request artifact content from {}".format(url))
req = request.Request(url)
opener = request.build_opener(request.HTTPHandler)
s = opener.open(req).read()
opener.close()
return s
return requests.get(url).text
class Build(object):
@ -104,13 +94,13 @@ class Build(object):
job_url = "/".join([JENKINS["url"], 'job', self.name,
'api/json?depth={depth}'.format(depth=depth)])
logger.debug("Request job info from {}".format(job_url))
return json.load(request.urlopen(job_url))
return requests.get(job_url).json()
def get_job_console(self):
job_url = "/".join([JENKINS["url"], 'job', self.name,
str(self.number), 'consoleText'])
logger.debug("Request job console from {}".format(job_url))
return request.urlopen(job_url)
return requests.get(job_url).text.split('\n')
def get_build_data(self, depth=1):
build_url = "/".join([JENKINS["url"], 'job',
@ -118,7 +108,7 @@ class Build(object):
str(self.number),
'api/json?depth={depth}'.format(depth=depth)])
logger.debug("Request build data from {}".format(build_url))
return json.load(request.urlopen(build_url))
return requests.get(build_url).json()
@staticmethod
def get_test_data(url, result_path=None):
@ -129,8 +119,7 @@ class Build(object):
test_url = "/".join([url.rstrip("/"), 'testReport', 'api/json'])
logger.debug("Request test data from {}".format(test_url))
response = request.urlopen(test_url)
return json.load(response)
return requests.get(test_url).json()
def test_data(self, result_path=None):
try:

7
fuelweb_test/testrail/generate_statistics.py
View File

@ -27,8 +27,6 @@ from collections import OrderedDict
from logging import CRITICAL
from logging import DEBUG
import six
from fuelweb_test.testrail.builds import Build
from fuelweb_test.testrail.launchpad_client import LaunchpadBug
from fuelweb_test.testrail.report import get_version
@ -427,10 +425,7 @@ def save_stats_to_file(stats, file_name, html=''):
html_file_path = '{}.html'.format(file_name)
warn_file_exists(html_file_path)
with open(html_file_path, 'w+') as f:
if isinstance(html, six.binary_type):
f.write(html)
else:
f.write(html.encode('utf-8', errors='xmlcharrefreplace'))
f.write(html)
def main():

9
fuelweb_test/testrail/report_pi.py
View File

@ -16,13 +16,10 @@
from __future__ import unicode_literals
import json
from logging import DEBUG
from optparse import OptionParser
# pylint: disable=import-error
from six.moves.urllib.request import urlopen
# pylint: enable=import-error
import requests
from fuelweb_test.testrail.builds import Build
from fuelweb_test.testrail.report import get_tests_results
@ -45,7 +42,7 @@ def find_run_by_name(test_plan, run_name):
def get_job_info(url):
job_url = "/".join([url, 'api/json'])
logger.debug("Request job info from %s", job_url)
return json.load(urlopen(job_url))
return requests.get(job_url).json()
def main():

1
fuelweb_test/testrail/settings.py
View File

@ -28,6 +28,7 @@ LOGS_DIR = os.environ.get('LOGS_DIR', os.getcwd())
os.environ["ENV_NAME"] = "some_environment"
os.environ["ISO_PATH"] = "./fuel.iso"
os.environ["CENTOS_CLOUD_IMAGE_PATH"] = "./centos-cloud-image.img"
JENKINS = {
'url': os.environ.get('JENKINS_URL', 'http://localhost/'),

86
fuelweb_test/testrail/testrail.py
View File

@ -26,18 +26,18 @@
from __future__ import unicode_literals
import base64
import json
import time
# pylint: disable=import-error
from six.moves.urllib.request import urlopen
from six.moves.urllib.request import Request
from six.moves.urllib.error import HTTPError
# pylint: enable=import-error
import requests
from requests.exceptions import HTTPError
from requests.packages.urllib3 import disable_warnings
from fuelweb_test.testrail.settings import logger
disable_warnings()
def request_retry(codes):
log_msg = "Got {0} Error! Waiting {1} seconds and trying again..."
@ -47,19 +47,21 @@ def request_retry(codes):
while True:
try:
response = func(*args, **kwargs)
response.raise_for_status()
except HTTPError as e:
if e.code in codes:
if iter_number < codes[e.code]:
error_code = e.response.status_code
if error_code in codes:
if iter_number < codes[error_code]:
wait = 5
if 'Retry-After' in e.hdrs:
wait = int(e.hdrs['Retry-after'])
logger.debug(log_msg.format(e.code, wait))
if 'Retry-After' in e.response.headers:
wait = int(e.response.headers['Retry-after'])
logger.debug(log_msg.format(error_code, wait))
time.sleep(wait)
iter_number += 1
continue
raise
else:
return response
return response.json()
return wrapper
return retry_request
@ -74,33 +76,9 @@ class APIClient(object):
base_url += '/'
self.__url = base_url + 'index.php?/api/v2/'
#
# Send Get
#
# Issues a GET request (read) against the API and returns the result
# (as Python dict).
#
# Arguments:
#
# uri The API method to call including parameters
# (e.g. get_case/1)
#
def send_get(self, uri):
return self.__send_request('GET', uri, None)
#
# Send POST
#
# Issues a POST request (write) against the API and returns the result
# (as Python dict).
#
# Arguments:
#
# uri The API method to call including parameters
# (e.g. add_case/1)
# data The data to submit as part of the request (as
# Python dict, strings must be UTF-8 encoded)
#
def send_post(self, uri, data):
return self.__send_request('POST', uri, data)
@ -108,38 +86,28 @@ class APIClient(object):
retry_codes = {429: 3}
@request_retry(codes=retry_codes)
def __get_response(_request):
return urlopen(_request).read()
def __get_response(_url, _headers, _data):
if method == 'POST':
return requests.post(_url, json=_data, headers=_headers)
return requests.get(_url, headers=_headers)
url = self.__url + uri
request = Request(url)
if method == 'POST':
request.add_data(json.dumps(data))
auth = base64.encodestring(
'{0}:{1}'.format(self.user, self.password)).strip()
request.add_header('Authorization', 'Basic {}'.format(auth))
request.add_header('Content-Type', 'application/json')
e = None
headers = {'Authorization': 'Basic {}'.format(auth),
'Content-Type': 'application/json'}
try:
response = __get_response(request)
return __get_response(url, headers, data)
except HTTPError as e:
response = e.read()
if response:
result = json.loads(response)
else:
result = {}
if e is not None:
if result and 'error' in result:
error = '"' + result['error'] + '"'
if e.message:
error = e.message
else:
error = 'No additional error message received'
raise APIError('TestRail API returned HTTP %s (%s)' %
(e.code, error))
return result
raise APIError('TestRail API returned HTTP {0}: "{1}"'.format(
e.response.status_code, error))
class APIError(Exception):