Use requests lib in HTTPClient
This change proposes switch to requests lib instead using HTTPConnection and VerifiedHTTPSConnection, requests lib is recommended for higher-level http client interface. Also HTTPConnection can't work with API server behind proxy. Change-Id: I0393aaf38e59bdfdefb0e00aaa514a3246bf639b Closes-Bug: 1500929
This commit is contained in:
Anton Arefiev
parent
26df6c3c37
commit
b6b1981be4
|
|
@ -27,6 +27,7 @@ import time
|
|||
|
||||
from keystoneclient import adapter
|
||||
from oslo_utils import strutils
|
||||
import requests
|
||||
import six
|
||||
from six.moves import http_client
|
||||
import six.moves.urllib.parse as urlparse
|
||||
|
|
@ -59,6 +60,9 @@ DEFAULT_RETRY_INTERVAL = 2
|
|||
SENSITIVE_HEADERS = ('X-Auth-Token',)
|
||||
|
||||
|
||||
SUPPORTED_ENDPOINT_SCHEME = ('http', 'https')
|
||||
|
||||
|
||||
def _trim_endpoint_api_version(url):
|
||||
"""Trim API version and trailing slash from endpoint."""
|
||||
return url.rstrip('/').rstrip(API_VERSION)
|
||||
|
|
@ -214,39 +218,20 @@ class HTTPClient(VersionNegotiationMixin):
|
|||
DEFAULT_MAX_RETRIES)
|
||||
self.conflict_retry_interval = kwargs.pop('retry_interval',
|
||||
DEFAULT_RETRY_INTERVAL)
|
||||
self.connection_params = self.get_connection_params(endpoint, **kwargs)
|
||||
self.session = requests.Session()
|
||||
|
||||
@staticmethod
|
||||
def get_connection_params(endpoint, **kwargs):
|
||||
parts = urlparse.urlparse(endpoint)
|
||||
|
||||
path = _trim_endpoint_api_version(parts.path)
|
||||
|
||||
_args = (parts.hostname, parts.port, path)
|
||||
_kwargs = {'timeout': (float(kwargs.get('timeout'))
|
||||
if kwargs.get('timeout') else 600)}
|
||||
|
||||
if parts.scheme == 'https':
|
||||
_class = VerifiedHTTPSConnection
|
||||
_kwargs['ca_file'] = kwargs.get('ca_file', None)
|
||||
_kwargs['cert_file'] = kwargs.get('cert_file', None)
|
||||
_kwargs['key_file'] = kwargs.get('key_file', None)
|
||||
_kwargs['insecure'] = kwargs.get('insecure', False)
|
||||
elif parts.scheme == 'http':
|
||||
_class = six.moves.http_client.HTTPConnection
|
||||
else:
|
||||
if parts.scheme not in SUPPORTED_ENDPOINT_SCHEME:
|
||||
msg = _('Unsupported scheme: %s') % parts.scheme
|
||||
raise exc.EndpointException(msg)
|
||||
|
||||
return (_class, _args, _kwargs)
|
||||
|
||||
def get_connection(self):
|
||||
_class = self.connection_params[0]
|
||||
try:
|
||||
return _class(*self.connection_params[1][0:2],
|
||||
**self.connection_params[2])
|
||||
except six.moves.http_client.InvalidURL:
|
||||
raise exc.EndpointException()
|
||||
if parts.scheme == 'https':
|
||||
if kwargs.get('insecure') is True:
|
||||
self.session.verify = False
|
||||
elif kwargs.get('ca_file'):
|
||||
self.session.verify = kwargs['ca_file']
|
||||
self.session.cert = (kwargs.get('cert_file'),
|
||||
kwargs.get('key_file'))
|
||||
|
||||
def _process_header(self, name, value):
|
||||
"""Redacts any sensitive header
|
||||
|
|
@ -277,18 +262,14 @@ class HTTPClient(VersionNegotiationMixin):
|
|||
header = '-H \'%s: %s\'' % self._process_header(key, value)
|
||||
curl.append(header)
|
||||
|
||||
conn_params_fmt = [
|
||||
('key_file', '--key %s'),
|
||||
('cert_file', '--cert %s'),
|
||||
('ca_file', '--cacert %s'),
|
||||
]
|
||||
for (key, fmt) in conn_params_fmt:
|
||||
value = self.connection_params[2].get(key)
|
||||
if value:
|
||||
curl.append(fmt % value)
|
||||
|
||||
if self.connection_params[2].get('insecure'):
|
||||
if not self.session.verify:
|
||||
curl.append('-k')
|
||||
elif isinstance(self.session.verify, six.string_types):
|
||||
curl.append('--cacert %s' % self.session.verify)
|
||||
|
||||
if self.session.cert:
|
||||
curl.append('--cert %s' % self.session.cert[0])
|
||||
curl.append('--key %s' % self.session.cert[1])
|
||||
|
||||
if 'body' in kwargs:
|
||||
body = strutils.mask_password(kwargs['body'])
|
||||
|
|
@ -299,9 +280,12 @@ class HTTPClient(VersionNegotiationMixin):
|
|||
|
||||
@staticmethod
|
||||
def log_http_response(resp, body=None):
|
||||
status = (resp.version / 10.0, resp.status, resp.reason)
|
||||
# NOTE(aarefiev): resp.raw is urllib3 response object, it's used
|
||||
# only to get 'version', response from request with 'stream = True'
|
||||
# should be used for raw reading.
|
||||
status = (resp.raw.version / 10.0, resp.status_code, resp.reason)
|
||||
dump = ['\nHTTP/%.1f %s %s' % status]
|
||||
dump.extend(['%s: %s' % (k, v) for k, v in resp.getheaders()])
|
||||
dump.extend(['%s: %s' % (k, v) for k, v in resp.headers.items()])
|
||||
dump.append('')
|
||||
if body:
|
||||
body = strutils.mask_password(body)
|
||||
|
|
@ -309,22 +293,19 @@ class HTTPClient(VersionNegotiationMixin):
|
|||
LOG.debug('\n'.join(dump))
|
||||
|
||||
def _make_connection_url(self, url):
|
||||
(_class, _args, _kwargs) = self.connection_params
|
||||
base_url = _args[2]
|
||||
return '%s/%s' % (base_url, url.lstrip('/'))
|
||||
return urlparse.urljoin(self.endpoint_trimmed, url)
|
||||
|
||||
def _parse_version_headers(self, resp):
|
||||
return self._generic_parse_version_headers(resp.getheader)
|
||||
return self._generic_parse_version_headers(resp.headers.get)
|
||||
|
||||
def _make_simple_request(self, conn, method, url):
|
||||
conn.request(method, self._make_connection_url(url))
|
||||
return conn.getresponse()
|
||||
return conn.request(method, self._make_connection_url(url))
|
||||
|
||||
@with_retries
|
||||
def _http_request(self, url, method, **kwargs):
|
||||
"""Send an http request with the specified characteristics.
|
||||
|
||||
Wrapper around httplib.HTTP(S)Connection.request to handle tasks such
|
||||
Wrapper around request.Session.request to handle tasks such
|
||||
as setting headers and error handling.
|
||||
"""
|
||||
# Copy the kwargs so we can reuse the original in case of redirects
|
||||
|
|
@ -337,54 +318,63 @@ class HTTPClient(VersionNegotiationMixin):
|
|||
kwargs['headers'].setdefault('X-Auth-Token', self.auth_token)
|
||||
|
||||
self.log_curl_request(method, url, kwargs)
|
||||
conn = self.get_connection()
|
||||
|
||||
# NOTE(aarefiev): This is for backwards compatibility, request
|
||||
# expected body in 'data' field, previously we used httplib,
|
||||
# which expected 'body' field.
|
||||
body = kwargs.pop('body', None)
|
||||
if body:
|
||||
kwargs['data'] = body
|
||||
|
||||
conn_url = self._make_connection_url(url)
|
||||
try:
|
||||
conn_url = self._make_connection_url(url)
|
||||
conn.request(method, conn_url, **kwargs)
|
||||
resp = conn.getresponse()
|
||||
resp = self.session.request(method,
|
||||
conn_url,
|
||||
**kwargs)
|
||||
|
||||
# TODO(deva): implement graceful client downgrade when connecting
|
||||
# to servers that did not support microversions. Details here:
|
||||
# http://specs.openstack.org/openstack/ironic-specs/specs/kilo/api-microversions.html#use-case-3b-new-client-communicating-with-a-old-ironic-user-specified # noqa
|
||||
|
||||
if resp.status == http_client.NOT_ACCEPTABLE:
|
||||
negotiated_ver = self.negotiate_version(conn, resp)
|
||||
if resp.status_code == http_client.NOT_ACCEPTABLE:
|
||||
negotiated_ver = self.negotiate_version(self.session, resp)
|
||||
kwargs['headers']['X-OpenStack-Ironic-API-Version'] = (
|
||||
negotiated_ver)
|
||||
return self._http_request(url, method, **kwargs)
|
||||
|
||||
except socket.gaierror as e:
|
||||
message = (_("Error finding address for %(url)s: %(e)s")
|
||||
% dict(url=url, e=e))
|
||||
raise exc.EndpointNotFound(message)
|
||||
except (socket.error, socket.timeout) as e:
|
||||
endpoint = self.endpoint
|
||||
message = (_("Error communicating with %(endpoint)s %(e)s")
|
||||
% dict(endpoint=endpoint, e=e))
|
||||
except requests.exceptions.RequestException as e:
|
||||
message = (_("Error has occurred while handling "
|
||||
"request for %(url)s: %(e)s") %
|
||||
dict(url=conn_url, e=e))
|
||||
# NOTE(aarefiev): not valid request(invalid url, missing schema,
|
||||
# and so on), retrying is not needed.
|
||||
if isinstance(e, ValueError):
|
||||
raise exc.ValidationError(message)
|
||||
|
||||
raise exc.ConnectionRefused(message)
|
||||
|
||||
body_iter = ResponseBodyIterator(resp)
|
||||
body_iter = resp.iter_content(chunk_size=CHUNKSIZE)
|
||||
|
||||
# Read body into string if it isn't obviously image data
|
||||
body_str = None
|
||||
if resp.getheader('content-type', None) != 'application/octet-stream':
|
||||
if resp.headers.get('Content-Type') != 'application/octet-stream':
|
||||
body_str = ''.join([chunk for chunk in body_iter])
|
||||
self.log_http_response(resp, body_str)
|
||||
body_iter = six.StringIO(body_str)
|
||||
else:
|
||||
self.log_http_response(resp)
|
||||
|
||||
if resp.status >= http_client.BAD_REQUEST:
|
||||
if resp.status_code >= http_client.BAD_REQUEST:
|
||||
error_json = _extract_error_json(body_str)
|
||||
raise exc.from_response(
|
||||
resp, error_json.get('faultstring'),
|
||||
error_json.get('debuginfo'), method, url)
|
||||
elif resp.status in (http_client.MOVED_PERMANENTLY, http_client.FOUND,
|
||||
http_client.USE_PROXY):
|
||||
elif resp.status_code in (http_client.MOVED_PERMANENTLY,
|
||||
http_client.FOUND,
|
||||
http_client.USE_PROXY):
|
||||
# Redirected. Reissue the request to the new location.
|
||||
return self._http_request(resp['location'], method, **kwargs)
|
||||
elif resp.status == http_client.MULTIPLE_CHOICES:
|
||||
elif resp.status_code == http_client.MULTIPLE_CHOICES:
|
||||
raise exc.from_response(resp, method=method, url=url)
|
||||
|
||||
return resp, body_iter
|
||||
|
|
@ -398,9 +388,10 @@ class HTTPClient(VersionNegotiationMixin):
|
|||
kwargs['body'] = json.dumps(kwargs['body'])
|
||||
|
||||
resp, body_iter = self._http_request(url, method, **kwargs)
|
||||
content_type = resp.getheader('content-type', None)
|
||||
content_type = resp.headers.get('Content-Type')
|
||||
|
||||
if (resp.status in (http_client.NO_CONTENT, http_client.RESET_CONTENT)
|
||||
if (resp.status_code in (http_client.NO_CONTENT,
|
||||
http_client.RESET_CONTENT)
|
||||
or content_type is None):
|
||||
return resp, list()
|
||||
|
||||
|
|
@ -576,24 +567,6 @@ class SessionClient(VersionNegotiationMixin, adapter.LegacyJsonAdapter):
|
|||
return self._http_request(url, method, **kwargs)
|
||||
|
||||
|
||||
class ResponseBodyIterator(object):
|
||||
"""A class that acts as an iterator over an HTTP response."""
|
||||
|
||||
def __init__(self, resp):
|
||||
self.resp = resp
|
||||
|
||||
def __iter__(self):
|
||||
while True:
|
||||
yield self.next()
|
||||
|
||||
def next(self):
|
||||
chunk = self.resp.read(CHUNKSIZE)
|
||||
if chunk:
|
||||
return chunk
|
||||
else:
|
||||
raise StopIteration()
|
||||
|
||||
|
||||
def _construct_http_client(endpoint=None,
|
||||
session=None,
|
||||
token=None,
|
||||
|
|
|
|||
|
|
@ -26,8 +26,6 @@ from ironicclient import exc
|
|||
from ironicclient.tests.unit import utils
|
||||
|
||||
|
||||
HTTP_CLASS = six.moves.http_client.HTTPConnection
|
||||
HTTPS_CLASS = http.VerifiedHTTPSConnection
|
||||
DEFAULT_TIMEOUT = 600
|
||||
|
||||
DEFAULT_HOST = 'localhost'
|
||||
|
|
@ -192,32 +190,26 @@ class HttpClientTest(utils.BaseTestCase):
|
|||
def test_url_generation_trailing_slash_in_base(self):
|
||||
client = http.HTTPClient('http://localhost/')
|
||||
url = client._make_connection_url('/v1/resources')
|
||||
self.assertEqual('/v1/resources', url)
|
||||
self.assertEqual('http://localhost/v1/resources', url)
|
||||
|
||||
def test_url_generation_without_trailing_slash_in_base(self):
|
||||
client = http.HTTPClient('http://localhost')
|
||||
url = client._make_connection_url('/v1/resources')
|
||||
self.assertEqual('/v1/resources', url)
|
||||
|
||||
def test_url_generation_prefix_slash_in_path(self):
|
||||
client = http.HTTPClient('http://localhost/')
|
||||
url = client._make_connection_url('/v1/resources')
|
||||
self.assertEqual('/v1/resources', url)
|
||||
self.assertEqual('http://localhost/v1/resources', url)
|
||||
|
||||
def test_url_generation_without_prefix_slash_in_path(self):
|
||||
client = http.HTTPClient('http://localhost')
|
||||
url = client._make_connection_url('v1/resources')
|
||||
self.assertEqual('/v1/resources', url)
|
||||
self.assertEqual('http://localhost/v1/resources', url)
|
||||
|
||||
def test_server_exception_empty_body(self):
|
||||
error_body = _get_error_body()
|
||||
fake_resp = utils.FakeResponse(
|
||||
{'content-type': 'application/json'},
|
||||
six.StringIO(error_body), version=1,
|
||||
status=http_client.INTERNAL_SERVER_ERROR)
|
||||
client = http.HTTPClient('http://localhost/')
|
||||
client.get_connection = (
|
||||
lambda *a, **kw: utils.FakeConnection(fake_resp))
|
||||
client.session = utils.FakeSession(
|
||||
{'Content-Type': 'application/json'},
|
||||
six.StringIO(error_body),
|
||||
version=1,
|
||||
status_code=http_client.INTERNAL_SERVER_ERROR)
|
||||
|
||||
error = self.assertRaises(exc.InternalServerError,
|
||||
client.json_request,
|
||||
|
|
@ -227,13 +219,12 @@ class HttpClientTest(utils.BaseTestCase):
|
|||
def test_server_exception_msg_only(self):
|
||||
error_msg = 'test error msg'
|
||||
error_body = _get_error_body(error_msg)
|
||||
fake_resp = utils.FakeResponse(
|
||||
{'content-type': 'application/json'},
|
||||
six.StringIO(error_body), version=1,
|
||||
status=http_client.INTERNAL_SERVER_ERROR)
|
||||
client = http.HTTPClient('http://localhost/')
|
||||
client.get_connection = (
|
||||
lambda *a, **kw: utils.FakeConnection(fake_resp))
|
||||
client.session = utils.FakeSession(
|
||||
{'Content-Type': 'application/json'},
|
||||
six.StringIO(error_body),
|
||||
version=1,
|
||||
status_code=http_client.INTERNAL_SERVER_ERROR)
|
||||
|
||||
error = self.assertRaises(exc.InternalServerError,
|
||||
client.json_request,
|
||||
|
|
@ -245,13 +236,12 @@ class HttpClientTest(utils.BaseTestCase):
|
|||
error_trace = ("\"Traceback (most recent call last):\\n\\n "
|
||||
"File \\\"/usr/local/lib/python2.7/...")
|
||||
error_body = _get_error_body(error_msg, error_trace)
|
||||
fake_resp = utils.FakeResponse(
|
||||
{'content-type': 'application/json'},
|
||||
six.StringIO(error_body), version=1,
|
||||
status=http_client.INTERNAL_SERVER_ERROR)
|
||||
client = http.HTTPClient('http://localhost/')
|
||||
client.get_connection = (
|
||||
lambda *a, **kw: utils.FakeConnection(fake_resp))
|
||||
client.session = utils.FakeSession(
|
||||
{'Content-Type': 'application/json'},
|
||||
six.StringIO(error_body),
|
||||
version=1,
|
||||
status_code=http_client.INTERNAL_SERVER_ERROR)
|
||||
|
||||
error = self.assertRaises(exc.InternalServerError,
|
||||
client.json_request,
|
||||
|
|
@ -263,155 +253,83 @@ class HttpClientTest(utils.BaseTestCase):
|
|||
"%(error)s\n%(details)s" % {'error': str(error),
|
||||
'details': str(error.details)})
|
||||
|
||||
def test_get_connection_params(self):
|
||||
endpoint = 'http://ironic-host:6385'
|
||||
expected = (HTTP_CLASS,
|
||||
('ironic-host', 6385, ''),
|
||||
{'timeout': DEFAULT_TIMEOUT})
|
||||
params = http.HTTPClient.get_connection_params(endpoint)
|
||||
self.assertEqual(expected, params)
|
||||
def test_server_https_request_ok(self):
|
||||
client = http.HTTPClient('https://localhost/')
|
||||
client.session = utils.FakeSession(
|
||||
{'Content-Type': 'application/json'},
|
||||
six.StringIO("Body"),
|
||||
version=1,
|
||||
status_code=http_client.OK)
|
||||
|
||||
def test_get_connection_params_with_trailing_slash(self):
|
||||
endpoint = 'http://ironic-host:6385/'
|
||||
expected = (HTTP_CLASS,
|
||||
('ironic-host', 6385, ''),
|
||||
{'timeout': DEFAULT_TIMEOUT})
|
||||
params = http.HTTPClient.get_connection_params(endpoint)
|
||||
self.assertEqual(expected, params)
|
||||
client.json_request('GET', '/v1/resources')
|
||||
|
||||
def test_get_connection_params_with_ssl(self):
|
||||
endpoint = 'https://ironic-host:6385'
|
||||
expected = (HTTPS_CLASS,
|
||||
('ironic-host', 6385, ''),
|
||||
{
|
||||
'timeout': DEFAULT_TIMEOUT,
|
||||
'ca_file': None,
|
||||
'cert_file': None,
|
||||
'key_file': None,
|
||||
'insecure': False,
|
||||
})
|
||||
params = http.HTTPClient.get_connection_params(endpoint)
|
||||
self.assertEqual(expected, params)
|
||||
def test_server_https_empty_body(self):
|
||||
error_body = _get_error_body()
|
||||
|
||||
def test_get_connection_params_with_ssl_params(self):
|
||||
endpoint = 'https://ironic-host:6385'
|
||||
ssl_args = {
|
||||
'ca_file': '/path/to/ca_file',
|
||||
'cert_file': '/path/to/cert_file',
|
||||
'key_file': '/path/to/key_file',
|
||||
'insecure': True,
|
||||
}
|
||||
client = http.HTTPClient('https://localhost/')
|
||||
client.session = utils.FakeSession(
|
||||
{'Content-Type': 'application/json'},
|
||||
six.StringIO(error_body),
|
||||
version=1,
|
||||
status_code=http_client.INTERNAL_SERVER_ERROR)
|
||||
|
||||
expected_kwargs = {'timeout': DEFAULT_TIMEOUT}
|
||||
expected_kwargs.update(ssl_args)
|
||||
expected = (HTTPS_CLASS,
|
||||
('ironic-host', 6385, ''),
|
||||
expected_kwargs)
|
||||
params = http.HTTPClient.get_connection_params(endpoint, **ssl_args)
|
||||
self.assertEqual(expected, params)
|
||||
|
||||
def test_get_connection_params_with_timeout(self):
|
||||
endpoint = 'http://ironic-host:6385'
|
||||
expected = (HTTP_CLASS,
|
||||
('ironic-host', 6385, ''),
|
||||
{'timeout': 300.0})
|
||||
params = http.HTTPClient.get_connection_params(endpoint, timeout=300)
|
||||
self.assertEqual(expected, params)
|
||||
|
||||
def test_get_connection_params_with_version(self):
|
||||
endpoint = 'http://ironic-host:6385/v1'
|
||||
expected = (HTTP_CLASS,
|
||||
('ironic-host', 6385, ''),
|
||||
{'timeout': DEFAULT_TIMEOUT})
|
||||
params = http.HTTPClient.get_connection_params(endpoint)
|
||||
self.assertEqual(expected, params)
|
||||
|
||||
def test_get_connection_params_with_version_trailing_slash(self):
|
||||
endpoint = 'http://ironic-host:6385/v1/'
|
||||
expected = (HTTP_CLASS,
|
||||
('ironic-host', 6385, ''),
|
||||
{'timeout': DEFAULT_TIMEOUT})
|
||||
params = http.HTTPClient.get_connection_params(endpoint)
|
||||
self.assertEqual(expected, params)
|
||||
|
||||
def test_get_connection_params_with_subpath(self):
|
||||
endpoint = 'http://ironic-host:6385/ironic'
|
||||
expected = (HTTP_CLASS,
|
||||
('ironic-host', 6385, '/ironic'),
|
||||
{'timeout': DEFAULT_TIMEOUT})
|
||||
params = http.HTTPClient.get_connection_params(endpoint)
|
||||
self.assertEqual(expected, params)
|
||||
|
||||
def test_get_connection_params_with_subpath_trailing_slash(self):
|
||||
endpoint = 'http://ironic-host:6385/ironic/'
|
||||
expected = (HTTP_CLASS,
|
||||
('ironic-host', 6385, '/ironic'),
|
||||
{'timeout': DEFAULT_TIMEOUT})
|
||||
params = http.HTTPClient.get_connection_params(endpoint)
|
||||
self.assertEqual(expected, params)
|
||||
|
||||
def test_get_connection_params_with_subpath_version(self):
|
||||
endpoint = 'http://ironic-host:6385/ironic/v1'
|
||||
expected = (HTTP_CLASS,
|
||||
('ironic-host', 6385, '/ironic'),
|
||||
{'timeout': DEFAULT_TIMEOUT})
|
||||
params = http.HTTPClient.get_connection_params(endpoint)
|
||||
self.assertEqual(expected, params)
|
||||
|
||||
def test_get_connection_params_with_subpath_version_trailing_slash(self):
|
||||
endpoint = 'http://ironic-host:6385/ironic/v1/'
|
||||
expected = (HTTP_CLASS,
|
||||
('ironic-host', 6385, '/ironic'),
|
||||
{'timeout': DEFAULT_TIMEOUT})
|
||||
params = http.HTTPClient.get_connection_params(endpoint)
|
||||
self.assertEqual(expected, params)
|
||||
error = self.assertRaises(exc.InternalServerError,
|
||||
client.json_request,
|
||||
'GET', '/v1/resources')
|
||||
self.assertEqual('Internal Server Error (HTTP 500)', str(error))
|
||||
|
||||
def test_401_unauthorized_exception(self):
|
||||
error_body = _get_error_body()
|
||||
fake_resp = utils.FakeResponse({'content-type': 'text/plain'},
|
||||
six.StringIO(error_body),
|
||||
version=1,
|
||||
status=http_client.UNAUTHORIZED)
|
||||
client = http.HTTPClient('http://localhost/')
|
||||
client.get_connection = (
|
||||
lambda *a, **kw: utils.FakeConnection(fake_resp))
|
||||
client.session = utils.FakeSession(
|
||||
{'Content-Type': 'text/plain'},
|
||||
six.StringIO(error_body),
|
||||
version=1,
|
||||
status_code=http_client.UNAUTHORIZED)
|
||||
|
||||
self.assertRaises(exc.Unauthorized, client.json_request,
|
||||
'GET', '/v1/resources')
|
||||
|
||||
def test_http_request_not_valid_request(self):
|
||||
client = http.HTTPClient('http://localhost/')
|
||||
client.session.request = mock.Mock(
|
||||
side_effect=http.requests.exceptions.InvalidSchema)
|
||||
|
||||
self.assertRaises(exc.ValidationError, client._http_request,
|
||||
'http://localhost/', 'GET')
|
||||
|
||||
def test__parse_version_headers(self):
|
||||
# Test parsing of version headers from HTTPClient
|
||||
error_body = _get_error_body()
|
||||
fake_resp = utils.FakeResponse(
|
||||
expected_result = ('1.1', '1.6')
|
||||
|
||||
client = http.HTTPClient('http://localhost/')
|
||||
fake_resp = utils.FakeSessionResponse(
|
||||
{'X-OpenStack-Ironic-API-Minimum-Version': '1.1',
|
||||
'X-OpenStack-Ironic-API-Maximum-Version': '1.6',
|
||||
'content-type': 'text/plain',
|
||||
'Content-Type': 'text/plain',
|
||||
},
|
||||
six.StringIO(error_body),
|
||||
version=1,
|
||||
status=http_client.NOT_ACCEPTABLE)
|
||||
expected_result = ('1.1', '1.6')
|
||||
client = http.HTTPClient('http://localhost/')
|
||||
status_code=http_client.NOT_ACCEPTABLE)
|
||||
result = client._parse_version_headers(fake_resp)
|
||||
self.assertEqual(expected_result, result)
|
||||
|
||||
@mock.patch.object(filecache, 'save_data', autospec=True)
|
||||
@mock.patch.object(http.HTTPClient, 'get_connection', autospec=True)
|
||||
def test__http_request_client_fallback_fail(self, mock_getcon,
|
||||
mock_save_data):
|
||||
def test__http_request_client_fallback_fail(self, mock_save_data):
|
||||
# Test when fallback to a supported version fails
|
||||
host, port, latest_ver = 'localhost', '1234', '1.6'
|
||||
error_body = _get_error_body()
|
||||
fake_resp = utils.FakeResponse(
|
||||
|
||||
client = http.HTTPClient('http://%s:%s/' % (host, port))
|
||||
client.session = utils.FakeSession(
|
||||
{'X-OpenStack-Ironic-API-Minimum-Version': '1.1',
|
||||
'X-OpenStack-Ironic-API-Maximum-Version': latest_ver,
|
||||
'content-type': 'text/plain',
|
||||
},
|
||||
six.StringIO(error_body),
|
||||
version=1,
|
||||
status=http_client.NOT_ACCEPTABLE)
|
||||
client = http.HTTPClient('http://%s:%s/' % (host, port))
|
||||
mock_getcon.return_value = utils.FakeConnection(fake_resp)
|
||||
status_code=http_client.NOT_ACCEPTABLE)
|
||||
self.assertRaises(
|
||||
exc.UnsupportedVersion,
|
||||
client._http_request,
|
||||
|
|
@ -422,33 +340,35 @@ class HttpClientTest(utils.BaseTestCase):
|
|||
|
||||
@mock.patch.object(http.VersionNegotiationMixin, 'negotiate_version',
|
||||
autospec=False)
|
||||
@mock.patch.object(http.HTTPClient, 'get_connection', autospec=True)
|
||||
def test__http_request_client_fallback_success(
|
||||
self, mock_getcon, mock_negotiate):
|
||||
def test__http_request_client_fallback_success(self, mock_negotiate):
|
||||
# Test when fallback to a supported version succeeds
|
||||
mock_negotiate.return_value = '1.6'
|
||||
error_body = _get_error_body()
|
||||
bad_resp = utils.FakeResponse(
|
||||
bad_resp = utils.FakeSessionResponse(
|
||||
{'X-OpenStack-Ironic-API-Minimum-Version': '1.1',
|
||||
'X-OpenStack-Ironic-API-Maximum-Version': '1.6',
|
||||
'content-type': 'text/plain',
|
||||
},
|
||||
six.StringIO(error_body),
|
||||
version=1,
|
||||
status=http_client.NOT_ACCEPTABLE)
|
||||
good_resp = utils.FakeResponse(
|
||||
status_code=http_client.NOT_ACCEPTABLE)
|
||||
good_resp = utils.FakeSessionResponse(
|
||||
{'X-OpenStack-Ironic-API-Minimum-Version': '1.1',
|
||||
'X-OpenStack-Ironic-API-Maximum-Version': '1.6',
|
||||
'content-type': 'text/plain',
|
||||
},
|
||||
six.StringIO("We got some text"),
|
||||
version=1,
|
||||
status=http_client.OK)
|
||||
status_code=http_client.OK)
|
||||
client = http.HTTPClient('http://localhost/')
|
||||
mock_getcon.side_effect = iter([utils.FakeConnection(bad_resp),
|
||||
utils.FakeConnection(good_resp)])
|
||||
response, body_iter = client._http_request('/v1/resources', 'GET')
|
||||
self.assertEqual(http_client.OK, response.status)
|
||||
|
||||
with mock.patch.object(client, 'session',
|
||||
autospec=True) as mock_session:
|
||||
|
||||
mock_session.request.side_effect = iter([bad_resp, good_resp])
|
||||
response, body_iter = client._http_request('/v1/resources', 'GET')
|
||||
|
||||
self.assertEqual(http_client.OK, response.status_code)
|
||||
self.assertEqual(1, mock_negotiate.call_count)
|
||||
|
||||
@mock.patch.object(http.LOG, 'debug', autospec=True)
|
||||
|
|
@ -471,6 +391,64 @@ class HttpClientTest(utils.BaseTestCase):
|
|||
expected_log = ("\nHTTP/0.1 200 foo\n\n{\"password\": \"***\"}\n")
|
||||
mock_log.assert_called_once_with(expected_log)
|
||||
|
||||
def test__https_init_ssl_args_insecure(self):
|
||||
client = http.HTTPClient('https://localhost/', insecure=True)
|
||||
|
||||
self.assertEqual(False, client.session.verify)
|
||||
|
||||
def test__https_init_ssl_args_secure(self):
|
||||
client = http.HTTPClient('https://localhost/', ca_file='test_ca',
|
||||
key_file='test_key', cert_file='test_cert')
|
||||
|
||||
self.assertEqual('test_ca', client.session.verify)
|
||||
self.assertEqual(('test_cert', 'test_key'), client.session.cert)
|
||||
|
||||
@mock.patch. object(http.LOG, 'debug', autospec=True)
|
||||
def test_log_curl_request_with_body_and_header(self, mock_log):
|
||||
client = http.HTTPClient('http://test')
|
||||
headers = {'header1': 'value1'}
|
||||
body = 'example body'
|
||||
|
||||
client.log_curl_request('GET', '/v1/nodes',
|
||||
{'headers': headers, 'body': body})
|
||||
|
||||
self.assertTrue(mock_log.called)
|
||||
self.assertTrue(mock_log.call_args[0])
|
||||
self.assertEqual("curl -i -X GET -H 'header1: value1'"
|
||||
" -d 'example body' http://test/v1/nodes",
|
||||
mock_log.call_args[0][0])
|
||||
|
||||
@mock.patch. object(http.LOG, 'debug', autospec=True)
|
||||
def test_log_curl_request_with_certs(self, mock_log):
|
||||
headers = {'header1': 'value1'}
|
||||
client = http.HTTPClient('https://test', key_file='key',
|
||||
cert_file='cert', cacert='cacert',
|
||||
token='fake-token')
|
||||
|
||||
client.log_curl_request('GET', '/v1/test', {'headers': headers})
|
||||
|
||||
self.assertTrue(mock_log.called)
|
||||
self.assertTrue(mock_log.call_args[0])
|
||||
|
||||
self.assertEqual("curl -i -X GET -H 'header1: value1' "
|
||||
"--cert cert --key key https://test/v1/test",
|
||||
mock_log.call_args[0][0])
|
||||
|
||||
@mock.patch. object(http.LOG, 'debug', autospec=True)
|
||||
def test_log_curl_request_with_insecure_param(self, mock_log):
|
||||
headers = {'header1': 'value1'}
|
||||
http_client_object = http.HTTPClient('https://test', insecure=True,
|
||||
token='fake-token')
|
||||
|
||||
http_client_object.log_curl_request('GET', '/v1/test',
|
||||
{'headers': headers})
|
||||
|
||||
self.assertTrue(mock_log.called)
|
||||
self.assertTrue(mock_log.call_args[0])
|
||||
self.assertEqual("curl -i -X GET -H 'header1: value1' -k "
|
||||
"--cert None --key None https://test/v1/test",
|
||||
mock_log.call_args[0][0])
|
||||
|
||||
|
||||
class SessionClientTest(utils.BaseTestCase):
|
||||
|
||||
|
|
@ -529,116 +507,135 @@ class SessionClientTest(utils.BaseTestCase):
|
|||
@mock.patch.object(time, 'sleep', lambda *_: None)
|
||||
class RetriesTestCase(utils.BaseTestCase):
|
||||
|
||||
@mock.patch.object(http.HTTPClient, 'get_connection', autospec=True)
|
||||
def test_http_no_retry(self, mock_getcon):
|
||||
def test_http_no_retry(self):
|
||||
error_body = _get_error_body()
|
||||
bad_resp = utils.FakeResponse(
|
||||
{'content-type': 'text/plain'},
|
||||
bad_resp = utils.FakeSessionResponse(
|
||||
{'Content-Type': 'text/plain'},
|
||||
six.StringIO(error_body),
|
||||
version=1,
|
||||
status=http_client.CONFLICT)
|
||||
status_code=http_client.CONFLICT)
|
||||
client = http.HTTPClient('http://localhost/', max_retries=0)
|
||||
mock_getcon.return_value = utils.FakeConnection(bad_resp)
|
||||
self.assertRaises(exc.Conflict, client._http_request,
|
||||
'/v1/resources', 'GET')
|
||||
self.assertEqual(1, mock_getcon.call_count)
|
||||
|
||||
@mock.patch.object(http.HTTPClient, 'get_connection', autospec=True)
|
||||
def test_http_retry(self, mock_getcon):
|
||||
with mock.patch.object(client.session, 'request', autospec=True,
|
||||
return_value=bad_resp) as mock_request:
|
||||
|
||||
self.assertRaises(exc.Conflict, client._http_request,
|
||||
'/v1/resources', 'GET')
|
||||
self.assertEqual(1, mock_request.call_count)
|
||||
|
||||
def test_http_retry(self):
|
||||
error_body = _get_error_body()
|
||||
bad_resp = utils.FakeResponse(
|
||||
{'content-type': 'text/plain'},
|
||||
bad_resp = utils.FakeSessionResponse(
|
||||
{'Content-Type': 'text/plain'},
|
||||
six.StringIO(error_body),
|
||||
version=1,
|
||||
status=http_client.CONFLICT)
|
||||
good_resp = utils.FakeResponse(
|
||||
status_code=http_client.CONFLICT)
|
||||
good_resp = utils.FakeSessionResponse(
|
||||
{'Content-Type': 'text/plain'},
|
||||
six.StringIO("meow"),
|
||||
version=1,
|
||||
status_code=http_client.OK)
|
||||
client = http.HTTPClient('http://localhost/')
|
||||
|
||||
with mock.patch.object(client, 'session',
|
||||
autospec=True) as mock_session:
|
||||
|
||||
mock_session.request.side_effect = iter([bad_resp, good_resp])
|
||||
response, body_iter = client._http_request('/v1/resources', 'GET')
|
||||
|
||||
self.assertEqual(http_client.OK, response.status_code)
|
||||
self.assertEqual(2, mock_session.request.call_count)
|
||||
|
||||
def test_http_retry_503(self):
|
||||
error_body = _get_error_body()
|
||||
bad_resp = utils.FakeSessionResponse(
|
||||
{'Content-Type': 'text/plain'},
|
||||
six.StringIO(error_body),
|
||||
version=1,
|
||||
status_code=http_client.SERVICE_UNAVAILABLE)
|
||||
good_resp = utils.FakeSessionResponse(
|
||||
{'Content-Type': 'text/plain'},
|
||||
six.StringIO("meow"),
|
||||
version=1,
|
||||
status_code=http_client.OK)
|
||||
client = http.HTTPClient('http://localhost/')
|
||||
|
||||
with mock.patch.object(client, 'session',
|
||||
autospec=True) as mock_session:
|
||||
mock_session.request.side_effect = iter([bad_resp, good_resp])
|
||||
response, body_iter = client._http_request('/v1/resources', 'GET')
|
||||
|
||||
self.assertEqual(http_client.OK, response.status_code)
|
||||
self.assertEqual(2, mock_session.request.call_count)
|
||||
|
||||
def test_http_retry_connection_refused(self):
|
||||
good_resp = utils.FakeSessionResponse(
|
||||
{'content-type': 'text/plain'},
|
||||
six.StringIO("meow"),
|
||||
version=1,
|
||||
status=http_client.OK)
|
||||
status_code=http_client.OK)
|
||||
client = http.HTTPClient('http://localhost/')
|
||||
mock_getcon.side_effect = iter((utils.FakeConnection(bad_resp),
|
||||
utils.FakeConnection(good_resp)))
|
||||
response, body_iter = client._http_request('/v1/resources', 'GET')
|
||||
self.assertEqual(http_client.OK, response.status)
|
||||
self.assertEqual(2, mock_getcon.call_count)
|
||||
|
||||
@mock.patch.object(http.HTTPClient, 'get_connection', autospec=True)
|
||||
def test_http_retry_503(self, mock_getcon):
|
||||
with mock.patch.object(client, 'session',
|
||||
autospec=True) as mock_session:
|
||||
mock_session.request.side_effect = iter([exc.ConnectionRefused(),
|
||||
good_resp])
|
||||
response, body_iter = client._http_request('/v1/resources', 'GET')
|
||||
|
||||
self.assertEqual(http_client.OK, response.status_code)
|
||||
self.assertEqual(2, mock_session.request.call_count)
|
||||
|
||||
def test_http_failed_retry(self):
|
||||
error_body = _get_error_body()
|
||||
bad_resp = utils.FakeResponse(
|
||||
bad_resp = utils.FakeSessionResponse(
|
||||
{'content-type': 'text/plain'},
|
||||
six.StringIO(error_body),
|
||||
version=1,
|
||||
status=http_client.SERVICE_UNAVAILABLE)
|
||||
good_resp = utils.FakeResponse(
|
||||
{'content-type': 'text/plain'},
|
||||
six.StringIO("meow"),
|
||||
version=1,
|
||||
status=http_client.OK)
|
||||
status_code=http_client.CONFLICT)
|
||||
client = http.HTTPClient('http://localhost/')
|
||||
mock_getcon.side_effect = iter((utils.FakeConnection(bad_resp),
|
||||
utils.FakeConnection(good_resp)))
|
||||
response, body_iter = client._http_request('/v1/resources', 'GET')
|
||||
self.assertEqual(http_client.OK, response.status)
|
||||
self.assertEqual(2, mock_getcon.call_count)
|
||||
|
||||
@mock.patch.object(http.HTTPClient, 'get_connection', autospec=True)
|
||||
def test_http_retry_connection_refused(self, mock_getcon):
|
||||
good_resp = utils.FakeResponse(
|
||||
{'content-type': 'text/plain'},
|
||||
six.StringIO("meow"),
|
||||
version=1,
|
||||
status=http_client.OK)
|
||||
client = http.HTTPClient('http://localhost/')
|
||||
mock_getcon.side_effect = iter((exc.ConnectionRefused(),
|
||||
utils.FakeConnection(good_resp)))
|
||||
response, body_iter = client._http_request('/v1/resources', 'GET')
|
||||
self.assertEqual(http_client.OK, response.status)
|
||||
self.assertEqual(2, mock_getcon.call_count)
|
||||
with mock.patch.object(client, 'session',
|
||||
autospec=True) as mock_session:
|
||||
mock_session.request.return_value = bad_resp
|
||||
self.assertRaises(exc.Conflict, client._http_request,
|
||||
'/v1/resources', 'GET')
|
||||
self.assertEqual(http.DEFAULT_MAX_RETRIES + 1,
|
||||
mock_session.request.call_count)
|
||||
|
||||
@mock.patch.object(http.HTTPClient, 'get_connection', autospec=True)
|
||||
def test_http_failed_retry(self, mock_getcon):
|
||||
def test_http_max_retries_none(self):
|
||||
error_body = _get_error_body()
|
||||
bad_resp = utils.FakeResponse(
|
||||
bad_resp = utils.FakeSessionResponse(
|
||||
{'content-type': 'text/plain'},
|
||||
six.StringIO(error_body),
|
||||
version=1,
|
||||
status=http_client.CONFLICT)
|
||||
client = http.HTTPClient('http://localhost/')
|
||||
mock_getcon.return_value = utils.FakeConnection(bad_resp)
|
||||
self.assertRaises(exc.Conflict, client._http_request,
|
||||
'/v1/resources', 'GET')
|
||||
self.assertEqual(http.DEFAULT_MAX_RETRIES + 1, mock_getcon.call_count)
|
||||
|
||||
@mock.patch.object(http.HTTPClient, 'get_connection', autospec=True)
|
||||
def test_http_max_retries_none(self, mock_getcon):
|
||||
error_body = _get_error_body()
|
||||
bad_resp = utils.FakeResponse(
|
||||
{'content-type': 'text/plain'},
|
||||
six.StringIO(error_body),
|
||||
version=1,
|
||||
status=http_client.CONFLICT)
|
||||
status_code=http_client.CONFLICT)
|
||||
client = http.HTTPClient('http://localhost/', max_retries=None)
|
||||
mock_getcon.return_value = utils.FakeConnection(bad_resp)
|
||||
self.assertRaises(exc.Conflict, client._http_request,
|
||||
'/v1/resources', 'GET')
|
||||
self.assertEqual(http.DEFAULT_MAX_RETRIES + 1, mock_getcon.call_count)
|
||||
|
||||
@mock.patch.object(http.HTTPClient, 'get_connection', autospec=True)
|
||||
def test_http_change_max_retries(self, mock_getcon):
|
||||
with mock.patch.object(client, 'session',
|
||||
autospec=True) as mock_session:
|
||||
mock_session.request.return_value = bad_resp
|
||||
self.assertRaises(exc.Conflict, client._http_request,
|
||||
'/v1/resources', 'GET')
|
||||
self.assertEqual(http.DEFAULT_MAX_RETRIES + 1,
|
||||
mock_session.request.call_count)
|
||||
|
||||
def test_http_change_max_retries(self):
|
||||
error_body = _get_error_body()
|
||||
bad_resp = utils.FakeResponse(
|
||||
bad_resp = utils.FakeSessionResponse(
|
||||
{'content-type': 'text/plain'},
|
||||
six.StringIO(error_body),
|
||||
version=1,
|
||||
status=http_client.CONFLICT)
|
||||
status_code=http_client.CONFLICT)
|
||||
client = http.HTTPClient('http://localhost/',
|
||||
max_retries=http.DEFAULT_MAX_RETRIES + 1)
|
||||
mock_getcon.return_value = utils.FakeConnection(bad_resp)
|
||||
self.assertRaises(exc.Conflict, client._http_request,
|
||||
'/v1/resources', 'GET')
|
||||
self.assertEqual(http.DEFAULT_MAX_RETRIES + 2, mock_getcon.call_count)
|
||||
|
||||
with mock.patch.object(client, 'session',
|
||||
autospec=True) as mock_session:
|
||||
mock_session.request.return_value = bad_resp
|
||||
self.assertRaises(exc.Conflict, client._http_request,
|
||||
'/v1/resources', 'GET')
|
||||
self.assertEqual(http.DEFAULT_MAX_RETRIES + 2,
|
||||
mock_session.request.call_count)
|
||||
|
||||
def test_session_retry(self):
|
||||
error_body = _get_error_body()
|
||||
|
|
|
|||
|
|
@ -18,12 +18,11 @@ import datetime
|
|||
import os
|
||||
|
||||
import fixtures
|
||||
import mock
|
||||
from oslo_utils import strutils
|
||||
import six
|
||||
import testtools
|
||||
|
||||
from ironicclient.common import http
|
||||
|
||||
|
||||
DEFAULT_TEST_HOST = 'localhost'
|
||||
DEFAULT_TEST_REGION = 'regionhost'
|
||||
|
|
@ -58,7 +57,7 @@ class FakeAPI(object):
|
|||
|
||||
def raw_request(self, *args, **kwargs):
|
||||
response = self._request(*args, **kwargs)
|
||||
body_iter = http.ResponseBodyIterator(six.StringIO(response[1]))
|
||||
body_iter = iter(six.StringIO(response[1]))
|
||||
return FakeResponse(response[0]), body_iter
|
||||
|
||||
def json_request(self, *args, **kwargs):
|
||||
|
|
@ -94,8 +93,9 @@ class FakeResponse(object):
|
|||
"""
|
||||
self.headers = headers
|
||||
self.body = body
|
||||
self.version = version
|
||||
self.status = status
|
||||
self.raw = mock.Mock()
|
||||
self.raw.version = version
|
||||
self.status_code = status
|
||||
self.reason = reason
|
||||
|
||||
def getheaders(self):
|
||||
|
|
@ -139,19 +139,29 @@ class FakeKeystone(object):
|
|||
|
||||
class FakeSessionResponse(object):
|
||||
|
||||
def __init__(self, headers, content=None, status_code=None):
|
||||
def __init__(self, headers, content=None, status_code=None, version=None):
|
||||
self.headers = headers
|
||||
self.content = content
|
||||
self.status_code = status_code
|
||||
self.raw = mock.Mock()
|
||||
self.raw.version = version
|
||||
self.reason = ''
|
||||
|
||||
def iter_content(self, chunk_size):
|
||||
return iter(self.content)
|
||||
|
||||
|
||||
class FakeSession(object):
|
||||
|
||||
def __init__(self, headers, content=None, status_code=None):
|
||||
def __init__(self, headers, content=None, status_code=None, version=None):
|
||||
self.headers = headers
|
||||
self.content = content
|
||||
self.status_code = status_code
|
||||
self.version = version
|
||||
self.verify = False
|
||||
self.cert = ('test_cert', 'test_key')
|
||||
|
||||
def request(self, url, method, **kwargs):
|
||||
return FakeSessionResponse(self.headers, self.content,
|
||||
self.status_code)
|
||||
request = FakeSessionResponse(
|
||||
self.headers, self.content, self.status_code, self.version)
|
||||
return request
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
features:
|
||||
- Switch HTTP client to requests lib. It allows client to work with API
|
||||
behind the proxy, configure proxies by setting the environment
|
||||
variables HTTP_PROXY and HTTPS_PROXY.
|
||||
|
|
@ -11,4 +11,5 @@ oslo.utils>=3.5.0 # Apache-2.0
|
|||
PrettyTable<0.8,>=0.7 # BSD
|
||||
python-keystoneclient!=1.8.0,!=2.1.0,>=1.6.0 # Apache-2.0
|
||||
python-openstackclient>=2.1.0 # Apache-2.0
|
||||
requests>=2.8.1,!=2.9.0 # Apache-2.0
|
||||
six>=1.9.0 # MIT
|
||||
|
|
|
|||
Loading…
Reference in New Issue