Add requests.Session wrapper class

Wrapper class for requests.Session adds some common OpenStack functionality
* log all requests and responses at debug level
* json-encode request body passed in to request() in json keyword arg
* set default user_agent at Session creation
* set default verify at Session creation

This sets the base for some additional functionality to follow:
* handle redirects properly for a REST API client
* handle a default auth token to set the X-Auth-Token header for simple use cases

Partially implements blueprint: rest-client
Change-Id: Ice3f2dae4a99fe59c7cc384a857900e3ae45a7d2
This commit is contained in:
Dean Troyer 2014-03-30 15:29:40 -05:00
parent c1c6f87a71
commit aa8d82e5d0
4 changed files with 583 additions and 0 deletions

150
openstack/session.py Normal file
View File

@ -0,0 +1,150 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Wrapper class for requests.Session adds some common OpenStack functionality
- log all requests and responses at debug level
- json-encode request body passed in to request() in json keyword arg
- set default user_agent at Session creation; set to None to skip the header
- set default verify at Session creation
"""
import json
import logging
import requests
from six.moves import urllib
import openstack
DEFAULT_USER_AGENT = 'python-OpenStackSDK/' + openstack.__version__
_logger = logging.getLogger(__name__)
class Session(requests.Session):
_user_agent = DEFAULT_USER_AGENT
def __init__(
self,
user_agent=None,
verify=True,
):
"""Wraps requests.Session to add some OpenStack-specific features
:param string user_agent: Set the default ``User-Agent`` header;
Header is omitted if ``None`` and no value
is supplied in the ``request()`` call.
:param boolean/string verify: If ``True``, the SSL cert will be
verified. A CA_BUNDLE path can also be
provided.
User agent handling is as follows:
* if user_agent arg is included in the request() call, use it
* else if 'User-Agent' is set in the headers dict, use it
* else if user_agent arg is included in the __init__() call, use it
* else use DEFAULT_USER_AGENT
"""
super(Session, self).__init__()
if user_agent:
self._user_agent = user_agent
self.verify = verify
def request(self, method, url, **kwargs):
"""Send a request
:param string method: Request HTTP method
:param string url: Request URL
The following additional kw args are supported:
:param object json: Request body to be encoded as JSON
Overwrites ``data`` argument if present
:param string user_agent: Set the ``User-Agent`` header; overwrites
any value that may be in the headers dict.
Header is omitted if ``None``.
Remaining kw args from requests.Session.request() supported
"""
headers = kwargs.setdefault('headers', {})
# JSON-encode the data in json arg if present
# Overwrites any existing 'data' value
json_data = kwargs.pop('json', None)
if json_data is not None:
kwargs['data'] = json.dumps(json_data)
headers['Content-Type'] = 'application/json'
# Set User-Agent header if user_agent arg included, or
# fall through the default chain as described above
if 'user_agent' in kwargs:
headers['User-Agent'] = kwargs.pop('user_agent')
elif self._user_agent:
headers.setdefault('User-Agent', self._user_agent)
else:
headers.setdefault('User-Agent', DEFAULT_USER_AGENT)
self._log_request(method, url, **kwargs)
resp = super(Session, self).request(method, url, **kwargs)
self._log_response(resp)
return resp
def _log_request(self, method, url, **kwargs):
if 'params' in kwargs and kwargs['params']:
url += '?' + urllib.parse.urlencode(kwargs['params'])
string_parts = [
"curl -i",
"-X '%s'" % method,
"'%s'" % url,
]
# kwargs overrides the default
if (('verify' in kwargs and kwargs['verify'] is False) or
not self.verify):
string_parts.append('--insecure')
for element in kwargs['headers'].items():
header = " -H '%s: %s'" % element
string_parts.append(header)
_logger.debug("REQ: %s" % " ".join(string_parts))
if 'data' in kwargs:
_logger.debug("REQ BODY: %r\n" % (kwargs['data']))
def _log_response(self, response):
_logger.debug(
"RESP: [%s] %r" % (
response.status_code,
response.headers,
),
)
if response._content_consumed:
_logger.debug(
"RESP BODY: %s",
response.text,
)
_logger.debug(
"encoding: %s",
response.encoding,
)

View File

@ -0,0 +1,431 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
import logging
import six
import fixtures
import httpretty
from openstack import session
from openstack.tests import base
fake_url = 'http://www.root.url'
fake_request = 'Now is the time...'
fake_response = 'for the quick brown fox...'
fake_record1 = {
'key1': {
'id': '123',
'name': 'OneTwoThree',
'random': 'qwertyuiop',
},
}
fake_record2 = {
'hello': 'world',
}
class TestSessionBase(base.TestCase):
def stub_url(self, method, base_url=None, **kwargs):
if not base_url:
base_url = fake_url
if 'json' in kwargs:
json_data = kwargs.pop('json')
if json_data is not None:
kwargs['body'] = json.dumps(json_data)
kwargs['Content-Type'] = 'application/json'
httpretty.register_uri(method, base_url, **kwargs)
def assertRequestHeaderEqual(self, name, val):
"""Verify that the last request made contains a header and its value
The request must have already been made and httpretty must have been
activated for the request.
"""
headers = httpretty.last_request().headers
self.assertEqual(val, headers.get(name))
def assertResponseOK(self, resp, status=200, body=fake_response):
"""Verify the Response object contains expected values
Tests our defaults for a successful request.
"""
self.assertTrue(resp.ok)
self.assertEqual(status, resp.status_code)
self.assertEqual(body, resp.text)
class TestSession(TestSessionBase):
@httpretty.activate
def test_request(self):
self.stub_url(httpretty.GET, body=fake_response)
sess = session.Session()
resp = sess.request('GET', fake_url)
self.assertEqual(httpretty.GET, httpretty.last_request().method)
self.assertResponseOK(resp, body=fake_response)
@httpretty.activate
def test_request_json(self):
self.stub_url(httpretty.GET, json=fake_record1)
sess = session.Session()
resp = sess.request('GET', fake_url)
self.assertEqual(httpretty.GET, httpretty.last_request().method)
self.assertResponseOK(resp, body=json.dumps(fake_record1))
self.assertEqual(fake_record1, resp.json())
@httpretty.activate
def test_delete(self):
self.stub_url(httpretty.DELETE, body=fake_response)
sess = session.Session()
resp = sess.delete(fake_url)
self.assertEqual(httpretty.DELETE, httpretty.last_request().method)
self.assertResponseOK(resp, body=fake_response)
@httpretty.activate
def test_get(self):
self.stub_url(httpretty.GET, body=fake_response)
sess = session.Session()
resp = sess.get(fake_url)
self.assertEqual(httpretty.GET, httpretty.last_request().method)
self.assertResponseOK(resp, body=fake_response)
@httpretty.activate
def test_head(self):
self.stub_url(httpretty.HEAD, body=fake_response)
sess = session.Session()
resp = sess.head(fake_url)
self.assertEqual(httpretty.HEAD, httpretty.last_request().method)
self.assertResponseOK(resp, body='')
@httpretty.activate
def test_options(self):
self.stub_url(httpretty.OPTIONS, body=fake_response)
sess = session.Session()
resp = sess.options(fake_url)
self.assertEqual(httpretty.OPTIONS, httpretty.last_request().method)
self.assertResponseOK(resp, body=fake_response)
@httpretty.activate
def test_patch(self):
self.stub_url(httpretty.PATCH, body=fake_response)
sess = session.Session()
resp = sess.patch(fake_url, json=fake_record2)
self.assertEqual(httpretty.PATCH, httpretty.last_request().method)
self.assertEqual(
json.dumps(fake_record2),
httpretty.last_request().body.decode('utf-8'),
)
self.assertResponseOK(resp, body=fake_response)
@httpretty.activate
def test_post(self):
self.stub_url(httpretty.POST, body=fake_response)
sess = session.Session()
resp = sess.post(fake_url, json=fake_record2)
self.assertEqual(httpretty.POST, httpretty.last_request().method)
self.assertEqual(
json.dumps(fake_record2),
httpretty.last_request().body.decode('utf-8'),
)
self.assertResponseOK(resp, body=fake_response)
@httpretty.activate
def test_put(self):
self.stub_url(httpretty.PUT, body=fake_response)
sess = session.Session()
resp = sess.put(fake_url, data=fake_request)
self.assertEqual(httpretty.PUT, httpretty.last_request().method)
self.assertEqual(
fake_request,
httpretty.last_request().body.decode('utf-8'),
)
self.assertResponseOK(resp, body=fake_response)
resp = sess.put(fake_url, json=fake_record2)
self.assertEqual(httpretty.PUT, httpretty.last_request().method)
self.assertEqual(
json.dumps(fake_record2),
httpretty.last_request().body.decode('utf-8'),
)
self.assertResponseOK(resp, body=fake_response)
@httpretty.activate
def test_user_agent_no_arg(self):
self.stub_url(httpretty.GET, body=fake_response)
sess = session.Session()
resp = sess.get(fake_url)
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', session.DEFAULT_USER_AGENT)
resp = sess.get(fake_url, headers={'User-Agent': None})
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', None)
resp = sess.get(fake_url, user_agent=None)
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', None)
resp = sess.get(fake_url, headers={'User-Agent': ''})
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', '')
resp = sess.get(fake_url, user_agent='')
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', '')
resp = sess.get(fake_url, headers={'User-Agent': 'new-agent'})
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', 'new-agent')
resp = sess.get(fake_url, user_agent='new-agent')
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', 'new-agent')
resp = sess.get(
fake_url,
headers={'User-Agent': 'new-agent'},
user_agent=None,
)
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', None)
resp = sess.get(
fake_url,
headers={'User-Agent': None},
user_agent='overrides-agent',
)
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', 'overrides-agent')
resp = sess.get(
fake_url,
headers={'User-Agent': 'new-agent'},
user_agent='overrides-agent',
)
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', 'overrides-agent')
@httpretty.activate
def test_user_agent_arg_none(self):
self.stub_url(httpretty.GET, body=fake_response)
sess = session.Session(user_agent=None)
resp = sess.get(fake_url)
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', session.DEFAULT_USER_AGENT)
resp = sess.get(fake_url, headers={'User-Agent': None})
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', None)
resp = sess.get(fake_url, user_agent=None)
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', None)
resp = sess.get(fake_url, headers={'User-Agent': ''})
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', '')
resp = sess.get(fake_url, user_agent='')
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', '')
resp = sess.get(fake_url, headers={'User-Agent': 'new-agent'})
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', 'new-agent')
resp = sess.get(fake_url, user_agent='new-agent')
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', 'new-agent')
resp = sess.get(
fake_url,
headers={'User-Agent': 'new-agent'},
user_agent=None,
)
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', None)
resp = sess.get(
fake_url,
headers={'User-Agent': None},
user_agent='overrides-agent',
)
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', 'overrides-agent')
resp = sess.get(
fake_url,
headers={'User-Agent': 'new-agent'},
user_agent='overrides-agent',
)
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', 'overrides-agent')
@httpretty.activate
def test_user_agent_arg_default(self):
self.stub_url(httpretty.GET, body=fake_response)
sess = session.Session(user_agent='test-agent')
resp = sess.get(fake_url)
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', 'test-agent')
resp = sess.get(fake_url, headers={'User-Agent': None})
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', None)
resp = sess.get(fake_url, user_agent=None)
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', None)
resp = sess.get(fake_url, headers={'User-Agent': ''})
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', '')
resp = sess.get(fake_url, user_agent='')
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', '')
resp = sess.get(fake_url, headers={'User-Agent': 'new-agent'})
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', 'new-agent')
resp = sess.get(fake_url, user_agent='new-agent')
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', 'new-agent')
resp = sess.get(
fake_url,
headers={'User-Agent': 'new-agent'},
user_agent=None,
)
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', None)
resp = sess.get(
fake_url,
headers={'User-Agent': None},
user_agent='overrides-agent',
)
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', 'overrides-agent')
resp = sess.get(
fake_url,
headers={'User-Agent': 'new-agent'},
user_agent='overrides-agent',
)
self.assertTrue(resp.ok)
self.assertRequestHeaderEqual('User-Agent', 'overrides-agent')
def test_verify_no_arg(self):
sess = session.Session()
self.assertTrue(sess.verify)
def test_verify_arg_none(self):
sess = session.Session(verify=None)
self.assertIsNone(sess.verify)
def test_verify_arg_false(self):
sess = session.Session(verify=False)
self.assertFalse(sess.verify)
def test_verify_arg_true(self):
sess = session.Session(verify=True)
self.assertTrue(sess.verify)
def test_verify_arg_file(self):
sess = session.Session(verify='ca-file')
self.assertEqual('ca-file', sess.verify)
@httpretty.activate
def test_not_found(self):
sess = session.Session()
self.stub_url(httpretty.GET, status=404)
resp = sess.get(fake_url)
self.assertFalse(resp.ok)
self.assertEqual(404, resp.status_code)
@httpretty.activate
def test_server_error(self):
sess = session.Session()
self.stub_url(httpretty.GET, status=500)
resp = sess.get(fake_url)
self.assertFalse(resp.ok)
self.assertEqual(500, resp.status_code)
class TestSessionDebug(TestSessionBase):
def setUp(self):
super(TestSessionDebug, self).setUp()
self.log_fixture = self.useFixture(
fixtures.FakeLogger(level=logging.DEBUG),
)
@httpretty.activate
def test_debug_post(self):
self.stub_url(httpretty.POST, body=fake_response)
sess = session.Session()
headers = {
'User-Agent': 'fake-curl',
'X-Random-Header': 'x-random-value',
}
params = {
'detailed-arg-name': 'qaz11 wsx22+edc33',
'ssh_config_dir': '~/myusername/.ssh',
}
resp = sess.post(
fake_url,
headers=headers,
params=params,
json=fake_record2,
)
self.assertEqual(httpretty.POST, httpretty.last_request().method)
self.assertEqual(
json.dumps(fake_record2),
httpretty.last_request().body.decode('utf-8'),
)
self.assertResponseOK(resp, body=fake_response)
self.assertIn('curl', self.log_fixture.output)
self.assertIn('POST', self.log_fixture.output)
self.assertIn(
'detailed-arg-name=qaz11+wsx22%2Bedc33',
self.log_fixture.output,
)
self.assertIn(
'ssh_config_dir=%7E%2Fmyusername%2F.ssh',
self.log_fixture.output,
)
self.assertIn(json.dumps(fake_record2), self.log_fixture.output)
self.assertIn(fake_response, self.log_fixture.output)
for k, v in six.iteritems(headers):
self.assertIn(k, self.log_fixture.output)
self.assertIn(v, self.log_fixture.output)

View File

@ -1,3 +1,4 @@
pbr>=0.5.21,<1.0
Babel>=1.3
requests>=1.1

View File

@ -3,6 +3,7 @@ hacking>=0.0.8,<0.9
coverage>=3.6
discover
fixtures>=0.3.14
httpretty>=0.8.0
python-subunit
sphinx>=1.1.2
oslosphinx