openstack/fuel-qa
Code Issues Proposed changes

Merge "Fix encoding errors while working with APIs" into stable/mitaka

Browse Source
This commit is contained in:
Jenkins 2016-05-23 10:17:39 +00:00 committed by Gerrit Code Review
commit 9ecd0e541c
5 changed files with 45 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
fuelweb_test/testrail/builds.py
View File

@ -14,27 +14,24 @@
from __future__ import unicode_literals
import json
import re
# pylint: disable=import-error
from six.moves.urllib import request
# pylint: enable=import-error
import requests
from requests.packages.urllib3 import disable_warnings
from fuelweb_test.testrail.settings import JENKINS
from fuelweb_test.testrail.settings import logger
disable_warnings()
def get_jobs_for_view(view):
"""Return list of jobs from specified view
"""
view_url = "/".join([JENKINS["url"], 'view', view, 'api/json'])
logger.debug("Request view data from {}".format(view_url))
req = request.Request(view_url)
opener = request.build_opener(request.HTTPHandler)
s = opener.open(req).read()
opener.close()
view_data = json.loads(s)
view_data = requests.get(view_url).json()
jobs = [job["name"] for job in view_data["jobs"]]
return jobs
@ -44,13 +41,10 @@ def get_downstream_builds_from_html(url):
"""
url = "/".join([url, 'downstreambuildview/'])
logger.debug("Request downstream builds data from {}".format(url))
req = request.Request(url)
opener = request.build_opener(request.HTTPHandler)
s = opener.open(req).read()
opener.close()
response = requests.get(url).text
jobs = []
raw_downstream_builds = re.findall(
'.*downstream-buildview.*href="(/job/\S+/[0-9]+/).*', s)
'.*downstream-buildview.*href="(/job/\S+/[0-9]+/).*', response)
for raw_build in raw_downstream_builds:
sub_job_name = raw_build.split('/')[2]
sub_job_build = raw_build.split('/')[3]
@ -71,11 +65,7 @@ def get_build_artifact(url, artifact):
"""
url = "/".join([url, 'artifact', artifact])
logger.debug("Request artifact content from {}".format(url))
req = request.Request(url)
opener = request.build_opener(request.HTTPHandler)
s = opener.open(req).read()
opener.close()
return s
return requests.get(url).text
class Build(object):
@ -104,13 +94,13 @@ class Build(object):
job_url = "/".join([JENKINS["url"], 'job', self.name,
'api/json?depth={depth}'.format(depth=depth)])
logger.debug("Request job info from {}".format(job_url))
return json.load(request.urlopen(job_url))
return requests.get(job_url).json()
def get_job_console(self):
job_url = "/".join([JENKINS["url"], 'job', self.name,
str(self.number), 'consoleText'])
logger.debug("Request job console from {}".format(job_url))
return request.urlopen(job_url)
return requests.get(job_url).text.split('\n')
def get_build_data(self, depth=1):
build_url = "/".join([JENKINS["url"], 'job',
@ -118,7 +108,7 @@ class Build(object):
str(self.number),
'api/json?depth={depth}'.format(depth=depth)])
logger.debug("Request build data from {}".format(build_url))
return json.load(request.urlopen(build_url))
return requests.get(build_url).json()
@staticmethod
def get_test_data(url, result_path=None):
@ -129,8 +119,7 @@ class Build(object):
test_url = "/".join([url.rstrip("/"), 'testReport', 'api/json'])
logger.debug("Request test data from {}".format(test_url))
response = request.urlopen(test_url)
return json.load(response)
return requests.get(test_url).json()
def test_data(self, result_path=None):
try:

7
fuelweb_test/testrail/generate_statistics.py
View File

@ -27,8 +27,6 @@ from collections import OrderedDict
from logging import CRITICAL
from logging import DEBUG
import six
from fuelweb_test.testrail.builds import Build
from fuelweb_test.testrail.launchpad_client import LaunchpadBug
from fuelweb_test.testrail.report import get_version
@ -427,10 +425,7 @@ def save_stats_to_file(stats, file_name, html=''):
html_file_path = '{}.html'.format(file_name)
warn_file_exists(html_file_path)
with open(html_file_path, 'w+') as f:
if isinstance(html, six.binary_type):
f.write(html)
else:
f.write(html.encode('utf-8', errors='xmlcharrefreplace'))
f.write(html)
def main():

9
fuelweb_test/testrail/report_pi.py
View File

@ -16,13 +16,10 @@
from __future__ import unicode_literals
import json
from logging import DEBUG
from optparse import OptionParser
# pylint: disable=import-error
from six.moves.urllib.request import urlopen
# pylint: enable=import-error
import requests
from fuelweb_test.testrail.builds import Build
from fuelweb_test.testrail.report import get_tests_results
@ -45,7 +42,7 @@ def find_run_by_name(test_plan, run_name):
def get_job_info(url):
job_url = "/".join([url, 'api/json'])
logger.debug("Request job info from %s", job_url)
return json.load(urlopen(job_url))
return requests.get(job_url).json()
def main():

1
fuelweb_test/testrail/settings.py
View File

@ -28,6 +28,7 @@ LOGS_DIR = os.environ.get('LOGS_DIR', os.getcwd())
os.environ["ENV_NAME"] = "some_environment"
os.environ["ISO_PATH"] = "./fuel.iso"
os.environ["CENTOS_CLOUD_IMAGE_PATH"] = "./centos-cloud-image.img"
JENKINS = {
'url': os.environ.get('JENKINS_URL', 'http://localhost/'),

86
fuelweb_test/testrail/testrail.py
View File

@ -26,18 +26,18 @@
from __future__ import unicode_literals
import base64
import json
import time
# pylint: disable=import-error
from six.moves.urllib.request import urlopen
from six.moves.urllib.request import Request
from six.moves.urllib.error import HTTPError
# pylint: enable=import-error
import requests
from requests.exceptions import HTTPError
from requests.packages.urllib3 import disable_warnings
from fuelweb_test.testrail.settings import logger
disable_warnings()
def request_retry(codes):
log_msg = "Got {0} Error! Waiting {1} seconds and trying again..."
@ -47,19 +47,21 @@ def request_retry(codes):
while True:
try:
response = func(*args, **kwargs)
response.raise_for_status()
except HTTPError as e:
if e.code in codes:
if iter_number < codes[e.code]:
error_code = e.response.status_code
if error_code in codes:
if iter_number < codes[error_code]:
wait = 5
if 'Retry-After' in e.hdrs:
wait = int(e.hdrs['Retry-after'])
logger.debug(log_msg.format(e.code, wait))
if 'Retry-After' in e.response.headers:
wait = int(e.response.headers['Retry-after'])
logger.debug(log_msg.format(error_code, wait))
time.sleep(wait)
iter_number += 1
continue
raise
else:
return response
return response.json()
return wrapper
return retry_request
@ -74,33 +76,9 @@ class APIClient(object):
base_url += '/'
self.__url = base_url + 'index.php?/api/v2/'
#
# Send Get
#
# Issues a GET request (read) against the API and returns the result
# (as Python dict).
#
# Arguments:
#
# uri The API method to call including parameters
# (e.g. get_case/1)
#
def send_get(self, uri):
return self.__send_request('GET', uri, None)
#
# Send POST
#
# Issues a POST request (write) against the API and returns the result
# (as Python dict).
#
# Arguments:
#
# uri The API method to call including parameters
# (e.g. add_case/1)
# data The data to submit as part of the request (as
# Python dict, strings must be UTF-8 encoded)
#
def send_post(self, uri, data):
return self.__send_request('POST', uri, data)
@ -108,38 +86,28 @@ class APIClient(object):
retry_codes = {429: 3}
@request_retry(codes=retry_codes)
def __get_response(_request):
return urlopen(_request).read()
def __get_response(_url, _headers, _data):
if method == 'POST':
return requests.post(_url, json=_data, headers=_headers)
return requests.get(_url, headers=_headers)
url = self.__url + uri
request = Request(url)
if method == 'POST':
request.add_data(json.dumps(data))
auth = base64.encodestring(
'{0}:{1}'.format(self.user, self.password)).strip()
request.add_header('Authorization', 'Basic {}'.format(auth))
request.add_header('Content-Type', 'application/json')
e = None
headers = {'Authorization': 'Basic {}'.format(auth),
'Content-Type': 'application/json'}
try:
response = __get_response(request)
return __get_response(url, headers, data)
except HTTPError as e:
response = e.read()
if response:
result = json.loads(response)
else:
result = {}
if e is not None:
if result and 'error' in result:
error = '"' + result['error'] + '"'
if e.message:
error = e.message
else:
error = 'No additional error message received'
raise APIError('TestRail API returned HTTP %s (%s)' %
(e.code, error))
return result
raise APIError('TestRail API returned HTTP {0}: "{1}"'.format(
e.response.status_code, error))
class APIError(Exception):