openstack/python-swiftclient
Code Issues Proposed changes

Port to python-requests

Browse Source 
Currently, httplib implementation does not support SSL certificate
verification. This patch fixes this. Note that ssl compression parameter
and 100-continue thing is still missing from requests, though those are
lower priority.

Requests now takes care of:
* proxy configuration (get_environ_proxies),
* chunked encoding (with data generator),
* bulk uploading (with files dictionary),
* SSL certificate verification (with 'insecure' and 'cacert' parameter).

This patch have been tested with requests 1.1.0 (CentOS 6) and requests
2.2.1 (current version).

Change-Id: Ib5de962f4102d57c71ad85fd81a615362ef175dc
Closes-Bug: #1199783
DocImpact
SecurityImpact
This commit is contained in:
Tristan Cacqueray 2014-01-24 17:40:16 +01:00
parent 9b73547b7d
commit b182112719
8 changed files with 146 additions and 335 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
bin/swift
View File

@ -32,7 +32,7 @@ try:
except ImportError:
import json
from swiftclient import Connection, HTTPException
from swiftclient import Connection, RequestException
from swiftclient import command_helpers
from swiftclient.utils import config_true_value, prt_bytes
from swiftclient.multithreading import MultiThreadingManager
@ -1388,16 +1388,16 @@ Examples:
parser.add_option('--insecure',
action="store_true", dest="insecure",
default=default_val,
help='Allow swiftclient to access insecure keystone '
'server. The keystone\'s certificate will not '
'be verified. '
help='Allow swiftclient to access servers without '
'having to verify the SSL certificate. '
'Defaults to env[SWIFTCLIENT_INSECURE] '
'(set to \'true\' to enable).')
parser.add_option('--no-ssl-compression',
action='store_false', dest='ssl_compression',
default=True,
help='Disable SSL compression when using https. '
'This may increase performance.')
help='This option is deprecated and not used anymore. '
'SSL compression should be disabled by default '
'by the system SSL library')
parser.disable_interspersed_args()
(options, args) = parse_args(parser, argv[1:], enforce_requires=False)
parser.enable_interspersed_args()
@ -1425,7 +1425,7 @@ Examples:
parser.usage = globals()['st_%s_help' % args[0]]
try:
globals()['st_%s' % args[0]](parser, argv[1:], thread_manager)
except (ClientException, HTTPException, socket.error) as err:
except (ClientException, RequestException, socket.error) as err:
thread_manager.error(str(err))
had_error = thread_manager.error_count

1
requirements.txt
View File

@ -1 +1,2 @@
requests>=1.1
simplejson>=2.0.9

196
swiftclient/client.py
View File

@ -18,24 +18,18 @@ OpenStack Swift client library used internally
"""
import socket
import requests
import sys
import logging
import warnings
from functools import wraps
from distutils.version import StrictVersion
from requests.exceptions import RequestException, SSLError
from urllib import quote as _quote
from urlparse import urlparse, urlunparse
from httplib import HTTPException, HTTPConnection, HTTPSConnection
from time import sleep, time
from swiftclient.exceptions import ClientException, InvalidHeadersException
from swiftclient.utils import get_environ_proxies
try:
from swiftclient.https_connection import HTTPSConnectionNoSSLComp
except ImportError:
HTTPSConnectionNoSSLComp = HTTPSConnection
try:
from logging import NullHandler
@ -50,6 +44,18 @@ except ImportError:
def createLock(self):
self.lock = None
# requests version 1.2.3 try to encode headers in ascii, preventing
# utf-8 encoded header to be 'prepared'
if StrictVersion(requests.__version__) < StrictVersion('2.0.0'):
from requests.structures import CaseInsensitiveDict
def prepare_unicode_headers(self, headers):
if headers:
self.headers = CaseInsensitiveDict(headers)
else:
self.headers = CaseInsensitiveDict()
requests.models.PreparedRequest.prepare_headers = prepare_unicode_headers
logger = logging.getLogger("swiftclient")
logger.addHandler(NullHandler())
@ -124,68 +130,93 @@ except ImportError:
from json import loads as json_loads
def http_connection(url, proxy=None, ssl_compression=True):
"""
Make an HTTPConnection or HTTPSConnection
class HTTPConnection:
def __init__(self, url, proxy=None, cacert=None, insecure=False,
ssl_compression=False):
"""
Make an HTTPConnection or HTTPSConnection
:param url: url to connect to
:param proxy: proxy to connect through, if any; None by default; str of the
format 'http://127.0.0.1:8888' to set one
:param ssl_compression: Whether to enable compression at the SSL layer.
If set to 'False' and the pyOpenSSL library is
present an attempt to disable SSL compression
will be made. This may provide a performance
increase for https upload/download operations.
:returns: tuple of (parsed url, connection object)
:raises ClientException: Unable to handle protocol scheme
"""
url = encode_utf8(url)
parsed = urlparse(url)
if proxy:
proxy_parsed = urlparse(proxy)
else:
proxies = get_environ_proxies(parsed.netloc)
proxy = proxies.get(parsed.scheme, None)
proxy_parsed = urlparse(proxy) if proxy else None
host = proxy_parsed.netloc if proxy else parsed.netloc
if parsed.scheme == 'http':
conn = HTTPConnection(host)
elif parsed.scheme == 'https':
if ssl_compression is True:
conn = HTTPSConnection(host)
else:
conn = HTTPSConnectionNoSSLComp(host)
else:
raise ClientException('Cannot handle protocol scheme %s for url %s' %
(parsed.scheme, repr(url)))
:param url: url to connect to
:param proxy: proxy to connect through, if any; None by default; str
of the format 'http://127.0.0.1:8888' to set one
:param cacert: A CA bundle file to use in verifying a TLS server
certificate.
:param insecure: Allow to access servers without checking SSL certs.
The server's certificate will not be verified.
:param ssl_compression: SSL compression should be disabled by default
and this setting is not usable as of now. The
parameter is kept for backward compatibility.
:raises ClientException: Unable to handle protocol scheme
"""
self.url = url
self.parsed_url = urlparse(url)
self.host = self.parsed_url.netloc
self.port = self.parsed_url.port
self.requests_args = {}
if self.parsed_url.scheme not in ('http', 'https'):
raise ClientException("Unsupported scheme")
self.requests_args['verify'] = not insecure
if cacert:
# verify requests parameter is used to pass the CA_BUNDLE file
# see: http://docs.python-requests.org/en/latest/user/advanced/
self.requests_args['verify'] = cacert
if proxy:
proxy_parsed = urlparse(proxy)
if not proxy_parsed.scheme:
raise ClientException("Proxy's missing scheme")
self.requests_args['proxies'] = {
proxy_parsed.scheme: '%s://%s' % (
proxy_parsed.scheme, proxy_parsed.netloc
)
}
self.requests_args['stream'] = True
def putheader_wrapper(func):
def _request(self, *arg, **kwarg):
""" Final wrapper before requests call, to be patched in tests """
return requests.request(*arg, **kwarg)
@wraps(func)
def putheader_escaped(key, value):
func(encode_utf8(key), encode_utf8(value))
return putheader_escaped
conn.putheader = putheader_wrapper(conn.putheader)
def request(self, method, full_path, data=None, headers={}, files=None):
""" Encode url and header, then call requests.request """
headers = dict((encode_utf8(x), encode_utf8(y)) for x, y in
headers.iteritems())
url = encode_utf8("%s://%s%s" % (
self.parsed_url.scheme,
self.parsed_url.netloc,
full_path))
self.resp = self._request(method, url, headers=headers, data=data,
files=files, **self.requests_args)
return self.resp
def request_wrapper(func):
def putrequest(self, full_path, data=None, headers={}, files=None):
"""
Use python-requests files upload
@wraps(func)
def request_escaped(method, url, body=None, headers=None):
validate_headers(headers)
url = encode_utf8(url)
if body:
body = encode_utf8(body)
func(method, url, body=body, headers=headers or {})
return request_escaped
conn.request = request_wrapper(conn.request)
if proxy:
try:
# python 2.6 method
conn._set_tunnel(parsed.hostname, parsed.port)
except AttributeError:
# python 2.7 method
conn.set_tunnel(parsed.hostname, parsed.port)
return parsed, conn
:param data: Use data generator for chunked-transfer
:param files: Use files for default transfer
"""
return self.request('PUT', full_path, data, headers, files)
def getresponse(self):
""" Adapt requests response to httplib interface """
self.resp.status = self.resp.status_code
old_getheader = self.resp.raw.getheader
def getheaders():
return self.resp.headers.items()
def getheader(k, v=None):
return old_getheader(k.lower(), v)
self.resp.getheaders = getheaders
self.resp.getheader = getheader
self.resp.read = self.resp.raw.read
return self.resp
def http_connection(*arg, **kwarg):
""" :returns: tuple of (parsed url, connection object) """
conn = HTTPConnection(*arg, **kwarg)
return conn.parsed_url, conn
def get_auth_1_0(url, user, key, snet):
@ -890,27 +921,16 @@ def put_object(url, token=None, container=None, name=None, contents=None,
if hasattr(contents, 'read'):
if chunk_size is None:
chunk_size = 65536
conn.putrequest('PUT', path)
for header, value in headers.iteritems():
conn.putheader(header, value)
if content_length is None:
conn.putheader('Transfer-Encoding', 'chunked')
conn.endheaders()
chunk = contents.read(chunk_size)
while chunk:
conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
chunk = contents.read(chunk_size)
conn.send('0\r\n\r\n')
def chunk_reader():
while True:
data = contents.read(chunk_size)
if not data:
break
yield data
conn.putrequest(path, headers=headers, data=chunk_reader())
else:
conn.endheaders()
left = content_length
while left > 0:
size = chunk_size
if size > left:
size = left
chunk = contents.read(size)
conn.send(chunk)
left -= len(chunk)
conn.putrequest(path, headers=headers, files={"file": contents})
else:
if chunk_size is not None:
warn_msg = '%s object has no \"read\" method, ignoring chunk_size'\
@ -1129,6 +1149,8 @@ class Connection(object):
def http_connection(self):
return http_connection(self.url,
cacert=self.cacert,
insecure=self.insecure,
ssl_compression=self.ssl_compression)
def _add_response_dict(self, target_dict, kwargs):
@ -1160,7 +1182,9 @@ class Connection(object):
rv = func(self.url, self.token, *args, **kwargs)
self._add_response_dict(caller_response_dict, kwargs)
return rv
except (socket.error, HTTPException) as e:
except SSLError:
raise
except (socket.error, RequestException) as e:
self._add_response_dict(caller_response_dict, kwargs)
if self.attempts > self.retries:
logger.exception(e)

95
swiftclient/https_connection.py
View File

@ -1,95 +0,0 @@
# Copyright (c) 2013 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
HTTPS/SSL related functionality
"""
import socket
from httplib import HTTPSConnection
import OpenSSL
try:
from eventlet.green.OpenSSL.SSL import GreenConnection
from eventlet.greenio import GreenSocket
from eventlet.patcher import is_monkey_patched
def getsockopt(self, *args, **kwargs):
return self.fd.getsockopt(*args, **kwargs)
# The above is a workaround for an eventlet bug in getsockopt.
# TODO(mclaren): Workaround can be removed when this fix lands:
# https://bitbucket.org/eventlet/eventlet/commits/609f230
GreenSocket.getsockopt = getsockopt
except ImportError:
def is_monkey_patched(*args):
return False
class HTTPSConnectionNoSSLComp(HTTPSConnection):
"""
Extended HTTPSConnection which uses the OpenSSL library
for disabling SSL compression.
Note: This functionality can eventually be replaced
with native Python 3.3 code.
"""
def __init__(self, host):
HTTPSConnection.__init__(self, host)
self.setcontext()
def setcontext(self):
"""
Set up the OpenSSL context.
"""
self.context = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
# Disable SSL layer compression.
self.context.set_options(0x20000) # SSL_OP_NO_COMPRESSION
def connect(self):
"""
Connect to an SSL port using the OpenSSL library and apply
per-connection parameters.
"""
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock = OpenSSLConnectionDelegator(self.context, sock)
self.sock.connect((self.host, self.port))
class OpenSSLConnectionDelegator(object):
"""
An OpenSSL.SSL.Connection delegator.
Supplies an additional 'makefile' method which httplib requires
and is not present in OpenSSL.SSL.Connection.
Note: Since it is not possible to inherit from OpenSSL.SSL.Connection
a delegator must be used.
"""
def __init__(self, *args, **kwargs):
if is_monkey_patched('socket'):
# If we are running in a monkey patched environment
# use eventlet's GreenConnection -- it handles eventlet's
# non-blocking sockets correctly.
Connection = GreenConnection
else:
Connection = OpenSSL.SSL.Connection
self.connection = Connection(*args, **kwargs)
def __getattr__(self, name):
return getattr(self.connection, name)
def makefile(self, *args, **kwargs):
return socket._fileobject(self.connection, *args, **kwargs)

50
swiftclient/utils.py
View File

@ -14,23 +14,6 @@
# limitations under the License.
"""Miscellaneous utility functions for use with Swift."""
import sys
import os
_ver = sys.version_info
#: Python 2.x?
is_py2 = (_ver[0] == 2)
#: Python 3.x?
is_py3 = (_ver[0] == 3)
if is_py2:
from urllib import getproxies, proxy_bypass
elif is_py3:
from urllib.request import getproxies, proxy_bypass
TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))
@ -72,36 +55,3 @@ def prt_bytes(bytes, human_flag):
bytes = '%12s' % bytes
return(bytes)
# get_environ_proxies function, borrowed from python Requests
# (www.python-requests.org)
def get_environ_proxies(netloc):
"""Return a dict of environment proxies."""
get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper())
# First check whether no_proxy is defined. If it is, check that the URL
# we're getting isn't in the no_proxy list.
no_proxy = get_proxy('no_proxy')
if no_proxy:
# We need to check whether we match here. We need to see if we match
# the end of the netloc, both with and without the port.
no_proxy = no_proxy.replace(' ', '').split(',')
for host in no_proxy:
if netloc.endswith(host) or netloc.split(':')[0].endswith(host):
# The URL does match something in no_proxy, so we don't want
# to apply the proxies on this URL.
return {}
# If the system proxy settings indicate that this URL should be bypassed,
# don't proxy.
if proxy_bypass(netloc):
return {}
# If we get here, we either didn't have no_proxy set or we're not going
# anywhere that no_proxy applies to, and the system settings don't require
# bypassing the proxy for the current URL.
return getproxies()

39
tests/test_swiftclient.py
View File

@ -15,7 +15,6 @@
# TODO: More tests
import mock
import httplib
import logging
import socket
import StringIO
@ -107,7 +106,8 @@ class MockHttpTest(testtools.TestCase):
query_string = kwargs.get('query_string')
storage_url = kwargs.get('storage_url')
def wrapper(url, proxy=None, ssl_compression=True):
def wrapper(url, proxy=None, cacert=None, insecure=False,
ssl_compression=True):
if storage_url:
self.assertEqual(storage_url, url)
@ -138,11 +138,17 @@ class MockHttpTest(testtools.TestCase):
class MockHttpResponse():
def __init__(self):
self.status = 200
def __init__(self, status=0):
self.status = status
self.status_code = status
self.reason = "OK"
self.buffer = []
class Raw:
def read():
pass
self.raw = Raw()
def read(self):
return ""
@ -153,10 +159,15 @@ class MockHttpResponse():
return {"key1": "value1", "key2": "value2"}
def fake_response(self):
return MockHttpResponse()
return MockHttpResponse(self.status)
def fake_send(self, msg):
self.buffer.append(msg)
def _fake_request(self, *arg, **kwarg):
self.status = 200
# This simulate previous httplib implementation that would do a
# putrequest() and then use putheader() to send header.
for k, v in kwarg['headers'].iteritems():
self.buffer.append('%s: %s' % (k, v))
return self.fake_response()
class TestHttpHelpers(MockHttpTest):
@ -173,8 +184,7 @@ class TestHttpHelpers(MockHttpTest):
self.assertTrue(isinstance(conn, c.HTTPConnection))
url = 'https://www.test.com'
_junk, conn = c.http_connection(url)
self.assertTrue(isinstance(conn, httplib.HTTPSConnection) or
isinstance(conn, c.HTTPSConnectionNoSSLComp))
self.assertTrue(isinstance(conn, c.HTTPConnection))
url = 'ftp://www.test.com'
self.assertRaises(c.ClientException, c.http_connection, url)
@ -560,7 +570,7 @@ class TestPutObject(MockHttpTest):
resp = MockHttpResponse()
conn[1].getresponse = resp.fake_response
conn[1].send = resp.fake_send
conn[1]._request = resp._fake_request
value = c.put_object(*args, headers=headers, http_conn=conn)
self.assertTrue(isinstance(value, basestring))
# Test for RFC-2616 encoded symbols
@ -573,7 +583,7 @@ class TestPutObject(MockHttpTest):
args = ('asdf', 'asdf', 'asdf', 'asdf', mock_file)
resp = MockHttpResponse()
conn[1].getresponse = resp.fake_response
conn[1].send = resp.fake_send
conn[1]._request = resp._fake_request
with warnings.catch_warnings(record=True) as w:
c.put_object(*args, chunk_size=20, headers={}, http_conn=conn)
self.assertEqual(len(w), 0)
@ -621,7 +631,7 @@ class TestPostObject(MockHttpTest):
resp = MockHttpResponse()
conn[1].getresponse = resp.fake_response
conn[1].send = resp.fake_send
conn[1]._request = resp._fake_request
c.post_object(*args, headers=headers, http_conn=conn)
# Test for RFC-2616 encoded symbols
self.assertTrue("a-b: .x:yz mn:kl:qr" in resp.buffer[0],
@ -853,7 +863,7 @@ class TestConnection(MockHttpTest):
self.port = parsed_url.netloc
def putrequest(self, *args, **kwargs):
return
self.send()
def putheader(self, *args, **kwargs):
return
@ -880,7 +890,8 @@ class TestConnection(MockHttpTest):
def read(self, *args, **kwargs):
return ''
def local_http_connection(url, proxy=None, ssl_compression=True):
def local_http_connection(url, proxy=None, cacert=None,
insecure=False, ssl_compression=True):
parsed = urlparse(url)
return parsed, LocalConnection()

80
tests/test_utils.py
View File

@ -14,7 +14,6 @@
# limitations under the License.
import testtools
import os
from swiftclient import utils as u
@ -118,82 +117,3 @@ class TestPrtBytes(testtools.TestCase):
def test_overflow(self):
bytes_ = 2 ** 90
self.assertEqual('1024Y', u.prt_bytes(bytes_, True).lstrip())
class TestGetEnvironProxy(testtools.TestCase):
ENV_VARS = ('http_proxy', 'https_proxy', 'no_proxy',
'HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY')
def setUp(self):
self.proxy_dict = {}
super(TestGetEnvironProxy, self).setUp()
for proxy_s in TestGetEnvironProxy.ENV_VARS:
# Save old env value
self.proxy_dict[proxy_s] = os.environ.get(proxy_s, None)
def tearDown(self):
super(TestGetEnvironProxy, self).tearDown()
for proxy_s in TestGetEnvironProxy.ENV_VARS:
if self.proxy_dict[proxy_s]:
os.environ[proxy_s] = self.proxy_dict[proxy_s]
elif os.environ.get(proxy_s):
del os.environ[proxy_s]
def setup_env(self, new_env={}):
for proxy_s in TestGetEnvironProxy.ENV_VARS:
# Set new env value
if new_env.get(proxy_s):
os.environ[proxy_s] = new_env.get(proxy_s)
elif os.environ.get(proxy_s):
del os.environ[proxy_s]
def test_http_proxy(self):
self.setup_env({'http_proxy': 'http://proxy.tests.com:8080'})
proxy_dict = u.get_environ_proxies('www.tests.com:81')
self.assertEqual(proxy_dict['http'], 'http://proxy.tests.com:8080')
self.assertEqual(proxy_dict.get('https'), None)
self.assertEqual(len(proxy_dict), 1)
self.setup_env({'HTTP_PROXY': 'http://proxy.tests.com:8080'})
proxy_dict = u.get_environ_proxies('www.tests.com:81')
self.assertEqual(proxy_dict['http'], 'http://proxy.tests.com:8080')
self.assertEqual(proxy_dict.get('https'), None)
self.assertEqual(len(proxy_dict), 1)
def test_https_proxy(self):
self.setup_env({'https_proxy': 'http://proxy.tests.com:8080'})
proxy_dict = u.get_environ_proxies('www.tests.com:81')
self.assertEqual(proxy_dict['https'], 'http://proxy.tests.com:8080')
self.assertEqual(proxy_dict.get('http'), None)
self.assertEqual(len(proxy_dict), 1)
self.setup_env({'HTTPS_PROXY': 'http://proxy.tests.com:8080'})
proxy_dict = u.get_environ_proxies('www.tests.com:81')
self.assertEqual(proxy_dict['https'], 'http://proxy.tests.com:8080')
self.assertEqual(proxy_dict.get('http'), None)
self.assertEqual(len(proxy_dict), 1)
def test_http_https_proxy(self):
self.setup_env({'http_proxy': 'http://proxy1.tests.com:8081',
'https_proxy': 'http://proxy2.tests.com:8082'})
proxy_dict = u.get_environ_proxies('www.tests.com:81')
self.assertEqual(proxy_dict['http'], 'http://proxy1.tests.com:8081')
self.assertEqual(proxy_dict['https'], 'http://proxy2.tests.com:8082')
self.assertEqual(len(proxy_dict), 2)
self.setup_env({'http_proxy': 'http://proxy1.tests.com:8081',
'HTTPS_PROXY': 'http://proxy2.tests.com:8082'})
proxy_dict = u.get_environ_proxies('www.tests.com:81')
self.assertEqual(proxy_dict['http'], 'http://proxy1.tests.com:8081')
self.assertEqual(proxy_dict['https'], 'http://proxy2.tests.com:8082')
self.assertEqual(len(proxy_dict), 2)
def test_proxy_exclusion(self):
self.setup_env({'http_proxy': 'http://proxy1.tests.com:8081',
'https_proxy': 'http://proxy2.tests.com:8082',
'no_proxy': 'www.tests.com'})
proxy_dict = u.get_environ_proxies('www.tests.com:81')
self.assertEqual(len(proxy_dict), 0)
self.setup_env({'http_proxy': 'http://proxy1.tests.com:8081',
'HTTPS_PROXY': 'http://proxy2.tests.com:8082',
'NO_PROXY': 'www.tests.com'})
proxy_dict = u.get_environ_proxies('www.tests.com:81')
self.assertEqual(len(proxy_dict), 0)

6
tests/utils.py
View File

@ -12,7 +12,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from httplib import HTTPException
from requests import RequestException
from time import sleep
@ -74,7 +74,7 @@ def fake_http_connect(*code_iter, **kwargs):
def getexpect(self):
if self.status == -2:
raise HTTPException()
raise RequestException()
if self.status == -3:
return FakeConn(507)
return FakeConn(100)
@ -141,7 +141,7 @@ def fake_http_connect(*code_iter, **kwargs):
etag = etag_iter.next()
timestamp = timestamps_iter.next()
if status <= 0:
raise HTTPException()
raise RequestException()
fake_conn = FakeConn(status, etag, body=kwargs.get('body', ''),
timestamp=timestamp)
fake_conn.connect()