From 21931edd793e4f890cdc82190c2943589343af2e Mon Sep 17 00:00:00 2001 From: Artem Panchenko Date: Sun, 22 May 2016 02:02:24 +0300 Subject: [PATCH] Fix encoding errors while working with APIs Use 'requests' module for HTTP requests to TestRail and Jenkins API instead of 'urllib'. It provides a method which decode response data using proper character encoding (charset from headers) and returns unicode string. Also add one more environment variable to settings, because it's needed by proboscis for test plan generation (since I9b9d40a59d24f579502a38dfc9b8c142bc219a06 was merged). Closes-bug: #1584401 Change-Id: I3d6cde2c8066bd58e735142fe26d56e83d1c90de (cherry picked from commit 5072b8426c380a6b3bf6d81c958c7b0bfe3483da) --- fuelweb_test/testrail/builds.py | 37 +++------ fuelweb_test/testrail/generate_statistics.py | 7 +- fuelweb_test/testrail/report_pi.py | 9 +- fuelweb_test/testrail/settings.py | 1 + fuelweb_test/testrail/testrail.py | 86 ++++++-------------- 5 files changed, 45 insertions(+), 95 deletions(-) diff --git a/fuelweb_test/testrail/builds.py b/fuelweb_test/testrail/builds.py index 5c2598042..9d8d547a4 100644 --- a/fuelweb_test/testrail/builds.py +++ b/fuelweb_test/testrail/builds.py @@ -14,27 +14,24 @@ from __future__ import unicode_literals -import json import re -# pylint: disable=import-error -from six.moves.urllib import request -# pylint: enable=import-error +import requests +from requests.packages.urllib3 import disable_warnings from fuelweb_test.testrail.settings import JENKINS from fuelweb_test.testrail.settings import logger +disable_warnings() + + def get_jobs_for_view(view): """Return list of jobs from specified view """ view_url = "/".join([JENKINS["url"], 'view', view, 'api/json']) logger.debug("Request view data from {}".format(view_url)) - req = request.Request(view_url) - opener = request.build_opener(request.HTTPHandler) - s = opener.open(req).read() - opener.close() - view_data = json.loads(s) + view_data = requests.get(view_url).json() jobs = [job["name"] for job in view_data["jobs"]] return jobs @@ -44,13 +41,10 @@ def get_downstream_builds_from_html(url): """ url = "/".join([url, 'downstreambuildview/']) logger.debug("Request downstream builds data from {}".format(url)) - req = request.Request(url) - opener = request.build_opener(request.HTTPHandler) - s = opener.open(req).read() - opener.close() + response = requests.get(url).text jobs = [] raw_downstream_builds = re.findall( - '.*downstream-buildview.*href="(/job/\S+/[0-9]+/).*', s) + '.*downstream-buildview.*href="(/job/\S+/[0-9]+/).*', response) for raw_build in raw_downstream_builds: sub_job_name = raw_build.split('/')[2] sub_job_build = raw_build.split('/')[3] @@ -71,11 +65,7 @@ def get_build_artifact(url, artifact): """ url = "/".join([url, 'artifact', artifact]) logger.debug("Request artifact content from {}".format(url)) - req = request.Request(url) - opener = request.build_opener(request.HTTPHandler) - s = opener.open(req).read() - opener.close() - return s + return requests.get(url).text class Build(object): @@ -104,13 +94,13 @@ class Build(object): job_url = "/".join([JENKINS["url"], 'job', self.name, 'api/json?depth={depth}'.format(depth=depth)]) logger.debug("Request job info from {}".format(job_url)) - return json.load(request.urlopen(job_url)) + return requests.get(job_url).json() def get_job_console(self): job_url = "/".join([JENKINS["url"], 'job', self.name, str(self.number), 'consoleText']) logger.debug("Request job console from {}".format(job_url)) - return request.urlopen(job_url) + return requests.get(job_url).text.split('\n') def get_build_data(self, depth=1): build_url = "/".join([JENKINS["url"], 'job', @@ -118,7 +108,7 @@ class Build(object): str(self.number), 'api/json?depth={depth}'.format(depth=depth)]) logger.debug("Request build data from {}".format(build_url)) - return json.load(request.urlopen(build_url)) + return requests.get(build_url).json() @staticmethod def get_test_data(url, result_path=None): @@ -129,8 +119,7 @@ class Build(object): test_url = "/".join([url.rstrip("/"), 'testReport', 'api/json']) logger.debug("Request test data from {}".format(test_url)) - response = request.urlopen(test_url) - return json.load(response) + return requests.get(test_url).json() def test_data(self, result_path=None): try: diff --git a/fuelweb_test/testrail/generate_statistics.py b/fuelweb_test/testrail/generate_statistics.py index 844280b78..6542e6886 100644 --- a/fuelweb_test/testrail/generate_statistics.py +++ b/fuelweb_test/testrail/generate_statistics.py @@ -27,8 +27,6 @@ from collections import OrderedDict from logging import CRITICAL from logging import DEBUG -import six - from fuelweb_test.testrail.builds import Build from fuelweb_test.testrail.launchpad_client import LaunchpadBug from fuelweb_test.testrail.report import get_version @@ -427,10 +425,7 @@ def save_stats_to_file(stats, file_name, html=''): html_file_path = '{}.html'.format(file_name) warn_file_exists(html_file_path) with open(html_file_path, 'w+') as f: - if isinstance(html, six.binary_type): - f.write(html) - else: - f.write(html.encode('utf-8', errors='xmlcharrefreplace')) + f.write(html) def main(): diff --git a/fuelweb_test/testrail/report_pi.py b/fuelweb_test/testrail/report_pi.py index 7b82d03c3..ef7d44ca1 100644 --- a/fuelweb_test/testrail/report_pi.py +++ b/fuelweb_test/testrail/report_pi.py @@ -16,13 +16,10 @@ from __future__ import unicode_literals -import json - from logging import DEBUG from optparse import OptionParser -# pylint: disable=import-error -from six.moves.urllib.request import urlopen -# pylint: enable=import-error + +import requests from fuelweb_test.testrail.builds import Build from fuelweb_test.testrail.report import get_tests_results @@ -45,7 +42,7 @@ def find_run_by_name(test_plan, run_name): def get_job_info(url): job_url = "/".join([url, 'api/json']) logger.debug("Request job info from %s", job_url) - return json.load(urlopen(job_url)) + return requests.get(job_url).json() def main(): diff --git a/fuelweb_test/testrail/settings.py b/fuelweb_test/testrail/settings.py index 0644e43ff..d9531f4a7 100644 --- a/fuelweb_test/testrail/settings.py +++ b/fuelweb_test/testrail/settings.py @@ -28,6 +28,7 @@ LOGS_DIR = os.environ.get('LOGS_DIR', os.getcwd()) os.environ["ENV_NAME"] = "some_environment" os.environ["ISO_PATH"] = "./fuel.iso" +os.environ["CENTOS_CLOUD_IMAGE_PATH"] = "./centos-cloud-image.img" JENKINS = { 'url': os.environ.get('JENKINS_URL', 'http://localhost/'), diff --git a/fuelweb_test/testrail/testrail.py b/fuelweb_test/testrail/testrail.py index 4e00e8293..54874222a 100644 --- a/fuelweb_test/testrail/testrail.py +++ b/fuelweb_test/testrail/testrail.py @@ -26,18 +26,18 @@ from __future__ import unicode_literals import base64 -import json import time -# pylint: disable=import-error -from six.moves.urllib.request import urlopen -from six.moves.urllib.request import Request -from six.moves.urllib.error import HTTPError -# pylint: enable=import-error +import requests +from requests.exceptions import HTTPError +from requests.packages.urllib3 import disable_warnings from fuelweb_test.testrail.settings import logger +disable_warnings() + + def request_retry(codes): log_msg = "Got {0} Error! Waiting {1} seconds and trying again..." @@ -47,19 +47,21 @@ def request_retry(codes): while True: try: response = func(*args, **kwargs) + response.raise_for_status() except HTTPError as e: - if e.code in codes: - if iter_number < codes[e.code]: + error_code = e.response.status_code + if error_code in codes: + if iter_number < codes[error_code]: wait = 5 - if 'Retry-After' in e.hdrs: - wait = int(e.hdrs['Retry-after']) - logger.debug(log_msg.format(e.code, wait)) + if 'Retry-After' in e.response.headers: + wait = int(e.response.headers['Retry-after']) + logger.debug(log_msg.format(error_code, wait)) time.sleep(wait) iter_number += 1 continue raise else: - return response + return response.json() return wrapper return retry_request @@ -74,33 +76,9 @@ class APIClient(object): base_url += '/' self.__url = base_url + 'index.php?/api/v2/' - # - # Send Get - # - # Issues a GET request (read) against the API and returns the result - # (as Python dict). - # - # Arguments: - # - # uri The API method to call including parameters - # (e.g. get_case/1) - # def send_get(self, uri): return self.__send_request('GET', uri, None) - # - # Send POST - # - # Issues a POST request (write) against the API and returns the result - # (as Python dict). - # - # Arguments: - # - # uri The API method to call including parameters - # (e.g. add_case/1) - # data The data to submit as part of the request (as - # Python dict, strings must be UTF-8 encoded) - # def send_post(self, uri, data): return self.__send_request('POST', uri, data) @@ -108,38 +86,28 @@ class APIClient(object): retry_codes = {429: 3} @request_retry(codes=retry_codes) - def __get_response(_request): - return urlopen(_request).read() + def __get_response(_url, _headers, _data): + if method == 'POST': + return requests.post(_url, json=_data, headers=_headers) + return requests.get(_url, headers=_headers) url = self.__url + uri - request = Request(url) - if method == 'POST': - request.add_data(json.dumps(data)) + auth = base64.encodestring( '{0}:{1}'.format(self.user, self.password)).strip() - request.add_header('Authorization', 'Basic {}'.format(auth)) - request.add_header('Content-Type', 'application/json') - e = None + headers = {'Authorization': 'Basic {}'.format(auth), + 'Content-Type': 'application/json'} + try: - response = __get_response(request) + return __get_response(url, headers, data) except HTTPError as e: - response = e.read() - - if response: - result = json.loads(response) - else: - result = {} - - if e is not None: - if result and 'error' in result: - error = '"' + result['error'] + '"' + if e.message: + error = e.message else: error = 'No additional error message received' - raise APIError('TestRail API returned HTTP %s (%s)' % - (e.code, error)) - - return result + raise APIError('TestRail API returned HTTP {0}: "{1}"'.format( + e.response.status_code, error)) class APIError(Exception):