openstack/python-swiftclient
Code Issues Proposed changes

Add option to disable SSL compression

Browse Source 
Allows optionally disabling SSL compression. This can significantly
improve HTTPS upload/download performance in some cases -- in particular
when the object is not compressible and you have very high network
bandwidth.

Implements blueprint ssl-compression.

Change-Id: I1260055f9c2e83cdabfeb51aed11b3899bed4d55
This commit is contained in:
Stuart McLaren 2013-01-18 14:17:21 +00:00
parent 2d12f09c66
commit 790f087a67
4 changed files with 133 additions and 9 deletions

8
bin/swift

@ -50,7 +50,8 @@ def get_conn(options):
os_options=options.os_options,
snet=options.snet,
cacert=options.os_cacert,
insecure=options.insecure)
insecure=options.insecure,
ssl_compression=options.ssl_compression)
def mkdirs(path):
@ -1268,6 +1269,11 @@ Examples:
'be verified. '
'Defaults to env[SWIFTCLIENT_INSECURE] '
'(set to \'true\' to enable).')
parser.add_option('--no-ssl-compression',
action='store_false', dest='ssl_compression',
default=True,
help='Disable SSL compression when using https. '
'This may increase performance.')
parser.disable_interspersed_args()
(options, args) = parse_args(parser, argv[1:], enforce_requires=False)
parser.enable_interspersed_args()

31
swiftclient/client.py

@ -28,6 +28,10 @@ from urlparse import urlparse, urlunparse
from httplib import HTTPException, HTTPConnection, HTTPSConnection
from time import sleep
try:
from swiftclient.https_connection import HTTPSConnectionNoSSLComp
except ImportError:
HTTPSConnectionNoSSLComp = HTTPSConnection
logger = logging.getLogger("swiftclient")
@ -141,23 +145,32 @@ class ClientException(Exception):
return b and '%s: %s' % (a, b) or a
def http_connection(url, proxy=None):
def http_connection(url, proxy=None, ssl_compression=True):
"""
Make an HTTPConnection or HTTPSConnection
:param url: url to connect to
:param proxy: proxy to connect through, if any; None by default; str of the
format 'http://127.0.0.1:8888' to set one
:param ssl_compression: Whether to enable compression at the SSL layer.
If set to 'False' and the pyOpenSSL library is
present an attempt to disable SSL compression
will be made. This may provide a performance
increase for https upload/download operations.
:returns: tuple of (parsed url, connection object)
:raises ClientException: Unable to handle protocol scheme
"""
url = encode_utf8(url)
parsed = urlparse(url)
proxy_parsed = urlparse(proxy) if proxy else None
host = proxy_parsed if proxy else parsed.netloc
if parsed.scheme == 'http':
conn = HTTPConnection((proxy_parsed if proxy else parsed).netloc)
conn = HTTPConnection(host)
elif parsed.scheme == 'https':
conn = HTTPSConnection((proxy_parsed if proxy else parsed).netloc)
if ssl_compression is True:
conn = HTTPSConnection(host)
else:
conn = HTTPSConnectionNoSSLComp(host)
else:
raise ClientException('Cannot handle protocol scheme %s for url %s' %
(parsed.scheme, repr(url)))
@ -956,7 +969,8 @@ class Connection(object):
def __init__(self, authurl=None, user=None, key=None, retries=5,
preauthurl=None, preauthtoken=None, snet=False,
starting_backoff=1, tenant_name=None, os_options=None,
auth_version="1", cacert=None, insecure=False):
auth_version="1", cacert=None, insecure=False,
ssl_compression=True):
"""
:param authurl: authentication URL
:param user: user name to authenticate as
@ -975,6 +989,11 @@ class Connection(object):
tenant_name, object_storage_url, region_name
:param insecure: Allow to access insecure keystone server.
The keystone's certificate will not be verified.
:param ssl_compression: Whether to enable compression at the SSL layer.
If set to 'False' and the pyOpenSSL library is
present an attempt to disable SSL compression
will be made. This may provide a performance
increase for https upload/download operations.
"""
self.authurl = authurl
self.user = user
@ -992,6 +1011,7 @@ class Connection(object):
self.os_options['tenant_name'] = tenant_name
self.cacert = cacert
self.insecure = insecure
self.ssl_compression = ssl_compression
def get_auth(self):
return get_auth(self.authurl,
@ -1004,7 +1024,8 @@ class Connection(object):
insecure=self.insecure)
def http_connection(self):
return http_connection(self.url)
return http_connection(self.url,
ssl_compression=self.ssl_compression)
def _retry(self, reset_func, func, *args, **kwargs):
self.attempts = 0

95
swiftclient/https_connection.py Normal file

@ -0,0 +1,95 @@
# Copyright (c) 2013 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
HTTPS/SSL related functionality
"""
import socket
from httplib import HTTPSConnection
import OpenSSL
try:
from eventlet.green.OpenSSL.SSL import GreenConnection
from eventlet.greenio import GreenSocket
from eventlet.patcher import is_monkey_patched
def getsockopt(self, *args, **kwargs):
return self.fd.getsockopt(*args, **kwargs)
# The above is a workaround for an eventlet bug in getsockopt.
# TODO(mclaren): Workaround can be removed when this fix lands:
# https://bitbucket.org/eventlet/eventlet/commits/609f230
GreenSocket.getsockopt = getsockopt
except ImportError:
def is_monkey_patched(*args):
return False
class HTTPSConnectionNoSSLComp(HTTPSConnection):
"""
Extended HTTPSConnection which uses the OpenSSL library
for disabling SSL compression.
Note: This functionality can eventually be replaced
with native Python 3.3 code.
"""
def __init__(self, host):
HTTPSConnection.__init__(self, host)
self.setcontext()
def setcontext(self):
"""
Set up the OpenSSL context.
"""
self.context = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
# Disable SSL layer compression.
self.context.set_options(0x20000) # SSL_OP_NO_COMPRESSION
def connect(self):
"""
Connect to an SSL port using the OpenSSL library and apply
per-connection parameters.
"""
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock = OpenSSLConnectionDelegator(self.context, sock)
self.sock.connect((self.host, self.port))
class OpenSSLConnectionDelegator(object):
"""
An OpenSSL.SSL.Connection delegator.
Supplies an additional 'makefile' method which httplib requires
and is not present in OpenSSL.SSL.Connection.
Note: Since it is not possible to inherit from OpenSSL.SSL.Connection
a delegator must be used.
"""
def __init__(self, *args, **kwargs):
if is_monkey_patched('socket'):
# If we are running in a monkey patched environment
# use eventlet's GreenConnection -- it handles eventlet's
# non-blocking sockets correctly.
Connection = GreenConnection
else:
Connection = OpenSSL.SSL.Connection
self.connection = Connection(*args, **kwargs)
def __getattr__(self, name):
return getattr(self.connection, name)
def makefile(self, *args, **kwargs):
return socket._fileobject(self.connection, *args, **kwargs)

8
tests/test_swiftclient.py

@ -14,6 +14,7 @@
# limitations under the License.
# TODO: More tests
import httplib
import socket
import StringIO
import testtools
@ -123,7 +124,7 @@ class MockHttpTest(testtools.TestCase):
return_read = kwargs.get('return_read')
query_string = kwargs.get('query_string')
def wrapper(url, proxy=None):
def wrapper(url, proxy=None, ssl_compression=True):
parsed, _conn = _orig_http_connection(url, proxy=proxy)
conn = fake_http_connect(*args, **kwargs)()
@ -182,7 +183,8 @@ class TestHttpHelpers(MockHttpTest):
self.assertTrue(isinstance(conn, c.HTTPConnection))
url = 'https://www.test.com'
_junk, conn = c.http_connection(url)
self.assertTrue(isinstance(conn, c.HTTPSConnection))
self.assertTrue(isinstance(conn, httplib.HTTPSConnection) or
isinstance(conn, c.HTTPSConnectionNoSSLComp))
url = 'ftp://www.test.com'
self.assertRaises(c.ClientException, c.http_connection, url)
@ -775,7 +777,7 @@ class TestConnection(MockHttpTest):
def read(self, *args, **kwargs):
return ''
def local_http_connection(url, proxy=None):
def local_http_connection(url, proxy=None, ssl_compression=True):
parsed = urlparse(url)
return parsed, LocalConnection()