Fix encoding errors while working with APIs

Use 'requests' module for HTTP requests to TestRail
and Jenkins API instead of 'urllib'. It provides a
method which decode response data using proper
character encoding (charset from headers) and returns
unicode string.

Also add one more environment variable to settings,
because it's needed by proboscis for test plan
generation (since I9b9d40a59d24f579502a38dfc9b8c142bc219a06
was merged).

Closes-bug: #1584401
Change-Id: I3d6cde2c8066bd58e735142fe26d56e83d1c90de

fuelweb_test/testrail/builds.py

@@ -14,28 +14,24 @@
 
 from __future__ import unicode_literals
 
-import json
 import re
 
-# pylint: disable=import-error
+import requests
-# noinspection PyUnresolvedReferences
+from requests.packages.urllib3 import disable_warnings
-from six.moves.urllib import request
-# pylint: enable=import-error
 
 from fuelweb_test.testrail.settings import JENKINS
 from fuelweb_test.testrail.settings import logger
 
 
+disable_warnings()
+
+
 def get_jobs_for_view(view):
     """Return list of jobs from specified view
     """
     view_url = "/".join([JENKINS["url"], 'view', view, 'api/json'])
     logger.debug("Request view data from {}".format(view_url))
-    req = request.Request(view_url)
+    view_data = requests.get(view_url).json()
-    opener = request.build_opener(request.HTTPHandler)
-    s = opener.open(req).read()
-    opener.close()
-    view_data = json.loads(s)
     jobs = [job["name"] for job in view_data["jobs"]]
     return jobs
 
@@ -45,13 +41,10 @@ def get_downstream_builds_from_html(url):
     """
     url = "/".join([url, 'downstreambuildview/'])
     logger.debug("Request downstream builds data from {}".format(url))
-    req = request.Request(url)
+    response = requests.get(url).text
-    opener = request.build_opener(request.HTTPHandler)
-    s = opener.open(req).read()
-    opener.close()
     jobs = []
     raw_downstream_builds = re.findall(
-        '.*downstream-buildview.*href="(/job/\S+/[0-9]+/).*', s)
+        '.*downstream-buildview.*href="(/job/\S+/[0-9]+/).*', response)
     for raw_build in raw_downstream_builds:
         sub_job_name = raw_build.split('/')[2]
         sub_job_build = raw_build.split('/')[3]
@@ -72,11 +65,7 @@ def get_build_artifact(url, artifact):
     """
     url = "/".join([url, 'artifact', artifact])
     logger.debug("Request artifact content from {}".format(url))
-    req = request.Request(url)
+    return requests.get(url).text
-    opener = request.build_opener(request.HTTPHandler)
-    s = opener.open(req).read()
-    opener.close()
-    return s
 
 
 class Build(object):
@@ -105,13 +94,13 @@ class Build(object):
         job_url = "/".join([JENKINS["url"], 'job', self.name,
                             'api/json?depth={depth}'.format(depth=depth)])
         logger.debug("Request job info from {}".format(job_url))
-        return json.load(request.urlopen(job_url))
+        return requests.get(job_url).json()
 
     def get_job_console(self):
         job_url = "/".join([JENKINS["url"], 'job', self.name,
                             str(self.number), 'consoleText'])
         logger.debug("Request job console from {}".format(job_url))
-        return request.urlopen(job_url)
+        return requests.get(job_url).text.split('\n')
 
     def get_build_data(self, depth=1):
         build_url = "/".join([JENKINS["url"], 'job',
@@ -119,7 +108,7 @@ class Build(object):
                               str(self.number),
                               'api/json?depth={depth}'.format(depth=depth)])
         logger.debug("Request build data from {}".format(build_url))
-        return json.load(request.urlopen(build_url))
+        return requests.get(build_url).json()
 
     @staticmethod
     def get_test_data(url, result_path=None):
@@ -130,8 +119,7 @@ class Build(object):
             test_url = "/".join([url.rstrip("/"), 'testReport', 'api/json'])
 
         logger.debug("Request test data from {}".format(test_url))
-        response = request.urlopen(test_url)
+        return requests.get(test_url).json()
-        return json.load(response)
 
     def test_data(self, result_path=None):
         try:

fuelweb_test/testrail/generate_statistics.py

@@ -27,8 +27,6 @@ from collections import OrderedDict
 from logging import CRITICAL
 from logging import DEBUG
 
-import six
-
 from fuelweb_test.testrail.builds import Build
 from fuelweb_test.testrail.launchpad_client import LaunchpadBug
 from fuelweb_test.testrail.report import get_version
@@ -427,10 +425,7 @@ def save_stats_to_file(stats, file_name, html=''):
         html_file_path = '{}.html'.format(file_name)
         warn_file_exists(html_file_path)
         with open(html_file_path, 'w+') as f:
-            if isinstance(html, six.binary_type):
+            f.write(html)
-                f.write(html)
-            else:
-                f.write(html.encode('utf-8', errors='xmlcharrefreplace'))
 
 
 def main():

fuelweb_test/testrail/report_pi.py

@@ -16,14 +16,10 @@
 
 from __future__ import unicode_literals
 
-import json
-
 from logging import DEBUG
 from optparse import OptionParser
-# pylint: disable=import-error
+
-# noinspection PyUnresolvedReferences
+import requests
-from six.moves.urllib.request import urlopen
-# pylint: enable=import-error
 
 from fuelweb_test.testrail.builds import Build
 from fuelweb_test.testrail.report import get_tests_results
@@ -46,7 +42,7 @@ def find_run_by_name(test_plan, run_name):
 def get_job_info(url):
     job_url = "/".join([url, 'api/json'])
     logger.debug("Request job info from %s", job_url)
-    return json.load(urlopen(job_url))
+    return requests.get(job_url).json()
 
 
 def main():

fuelweb_test/testrail/settings.py

@@ -28,6 +28,7 @@ LOGS_DIR = os.environ.get('LOGS_DIR', os.getcwd())
 
 os.environ["ENV_NAME"] = "some_environment"
 os.environ["ISO_PATH"] = "./fuel.iso"
+os.environ["CENTOS_CLOUD_IMAGE_PATH"] = "./centos-cloud-image.img"
 
 JENKINS = {
     'url': os.environ.get('JENKINS_URL', 'http://localhost/'),

fuelweb_test/testrail/testrail.py

@@ -26,21 +26,18 @@
 from __future__ import unicode_literals
 
 import base64
-import json
 import time
 
-# pylint: disable=import-error
+import requests
-# noinspection PyUnresolvedReferences
+from requests.exceptions import HTTPError
-from six.moves.urllib.request import urlopen
+from requests.packages.urllib3 import disable_warnings
-# noinspection PyUnresolvedReferences
-from six.moves.urllib.request import Request
-# noinspection PyUnresolvedReferences
-from six.moves.urllib.error import HTTPError
-# pylint: enable=import-error
 
 from fuelweb_test.testrail.settings import logger
 
 
+disable_warnings()
+
+
 def request_retry(codes):
     log_msg = "Got {0} Error! Waiting {1} seconds and trying again..."
 
@@ -50,19 +47,21 @@ def request_retry(codes):
             while True:
                 try:
                     response = func(*args, **kwargs)
+                    response.raise_for_status()
                 except HTTPError as e:
-                    if e.code in codes:
+                    error_code = e.response.status_code
-                        if iter_number < codes[e.code]:
+                    if error_code in codes:
+                        if iter_number < codes[error_code]:
                             wait = 5
-                            if 'Retry-After' in e.hdrs:
+                            if 'Retry-After' in e.response.headers:
-                                wait = int(e.hdrs['Retry-after'])
+                                wait = int(e.response.headers['Retry-after'])
-                            logger.debug(log_msg.format(e.code, wait))
+                            logger.debug(log_msg.format(error_code, wait))
                             time.sleep(wait)
                             iter_number += 1
                             continue
                     raise
                 else:
-                    return response
+                    return response.json()
         return wrapper
     return retry_request
 
@@ -77,33 +76,9 @@ class APIClient(object):
             base_url += '/'
         self.__url = base_url + 'index.php?/api/v2/'
 
-    #
-    # Send Get
-    #
-    # Issues a GET request (read) against the API and returns the result
-    # (as Python dict).
-    #
-    # Arguments:
-    #
-    # uri                 The API method to call including parameters
-    #                     (e.g. get_case/1)
-    #
     def send_get(self, uri):
         return self.__send_request('GET', uri, None)
 
-    #
-    # Send POST
-    #
-    # Issues a POST request (write) against the API and returns the result
-    # (as Python dict).
-    #
-    # Arguments:
-    #
-    # uri                 The API method to call including parameters
-    #                     (e.g. add_case/1)
-    # data                The data to submit as part of the request (as
-    #                     Python dict, strings must be UTF-8 encoded)
-    #
     def send_post(self, uri, data):
         return self.__send_request('POST', uri, data)
 
@@ -111,38 +86,28 @@ class APIClient(object):
         retry_codes = {429: 3}
 
         @request_retry(codes=retry_codes)
-        def __get_response(_request):
+        def __get_response(_url, _headers, _data):
-            return urlopen(_request).read()
+            if method == 'POST':
+                return requests.post(_url, json=_data, headers=_headers)
+            return requests.get(_url, headers=_headers)
 
         url = self.__url + uri
-        request = Request(url)
+
-        if method == 'POST':
-            request.add_data(json.dumps(data))
         auth = base64.encodestring(
             '{0}:{1}'.format(self.user, self.password)).strip()
-        request.add_header('Authorization', 'Basic {}'.format(auth))
-        request.add_header('Content-Type', 'application/json')
 
-        e = None
+        headers = {'Authorization': 'Basic {}'.format(auth),
+                   'Content-Type': 'application/json'}
+
         try:
-            response = __get_response(request)
+            return __get_response(url, headers, data)
         except HTTPError as e:
-            response = e.read()
+            if e.message:
-
+                error = e.message
-        if response:
-            result = json.loads(response)
-        else:
-            result = {}
-
-        if e is not None:
-            if result and 'error' in result:
-                error = '"' + result['error'] + '"'
             else:
                 error = 'No additional error message received'
-            raise APIError('TestRail API returned HTTP %s (%s)' %
+            raise APIError('TestRail API returned HTTP {0}: "{1}"'.format(
-                           (e.code, error))
+                e.response.status_code, error))
-
-        return result
 
 
 class APIError(Exception):