diff --git a/bin/swift b/bin/swift index 6459a436..c15431f3 100755 --- a/bin/swift +++ b/bin/swift @@ -32,7 +32,7 @@ try: except ImportError: import json -from swiftclient import Connection, HTTPException +from swiftclient import Connection, RequestException from swiftclient import command_helpers from swiftclient.utils import config_true_value, prt_bytes from swiftclient.multithreading import MultiThreadingManager @@ -1388,16 +1388,16 @@ Examples: parser.add_option('--insecure', action="store_true", dest="insecure", default=default_val, - help='Allow swiftclient to access insecure keystone ' - 'server. The keystone\'s certificate will not ' - 'be verified. ' + help='Allow swiftclient to access servers without ' + 'having to verify the SSL certificate. ' 'Defaults to env[SWIFTCLIENT_INSECURE] ' '(set to \'true\' to enable).') parser.add_option('--no-ssl-compression', action='store_false', dest='ssl_compression', default=True, - help='Disable SSL compression when using https. ' - 'This may increase performance.') + help='This option is deprecated and not used anymore. ' + 'SSL compression should be disabled by default ' + 'by the system SSL library') parser.disable_interspersed_args() (options, args) = parse_args(parser, argv[1:], enforce_requires=False) parser.enable_interspersed_args() @@ -1425,7 +1425,7 @@ Examples: parser.usage = globals()['st_%s_help' % args[0]] try: globals()['st_%s' % args[0]](parser, argv[1:], thread_manager) - except (ClientException, HTTPException, socket.error) as err: + except (ClientException, RequestException, socket.error) as err: thread_manager.error(str(err)) had_error = thread_manager.error_count diff --git a/requirements.txt b/requirements.txt index 519b0481..f1cf59ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ +requests>=1.1 simplejson>=2.0.9 diff --git a/swiftclient/client.py b/swiftclient/client.py index afd11bec..a8587cb8 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -18,24 +18,18 @@ OpenStack Swift client library used internally """ import socket +import requests import sys import logging import warnings -from functools import wraps +from distutils.version import StrictVersion +from requests.exceptions import RequestException, SSLError from urllib import quote as _quote from urlparse import urlparse, urlunparse -from httplib import HTTPException, HTTPConnection, HTTPSConnection from time import sleep, time from swiftclient.exceptions import ClientException, InvalidHeadersException -from swiftclient.utils import get_environ_proxies - -try: - from swiftclient.https_connection import HTTPSConnectionNoSSLComp -except ImportError: - HTTPSConnectionNoSSLComp = HTTPSConnection - try: from logging import NullHandler @@ -50,6 +44,18 @@ except ImportError: def createLock(self): self.lock = None +# requests version 1.2.3 try to encode headers in ascii, preventing +# utf-8 encoded header to be 'prepared' +if StrictVersion(requests.__version__) < StrictVersion('2.0.0'): + from requests.structures import CaseInsensitiveDict + + def prepare_unicode_headers(self, headers): + if headers: + self.headers = CaseInsensitiveDict(headers) + else: + self.headers = CaseInsensitiveDict() + requests.models.PreparedRequest.prepare_headers = prepare_unicode_headers + logger = logging.getLogger("swiftclient") logger.addHandler(NullHandler()) @@ -124,68 +130,93 @@ except ImportError: from json import loads as json_loads -def http_connection(url, proxy=None, ssl_compression=True): - """ - Make an HTTPConnection or HTTPSConnection +class HTTPConnection: + def __init__(self, url, proxy=None, cacert=None, insecure=False, + ssl_compression=False): + """ + Make an HTTPConnection or HTTPSConnection - :param url: url to connect to - :param proxy: proxy to connect through, if any; None by default; str of the - format 'http://127.0.0.1:8888' to set one - :param ssl_compression: Whether to enable compression at the SSL layer. - If set to 'False' and the pyOpenSSL library is - present an attempt to disable SSL compression - will be made. This may provide a performance - increase for https upload/download operations. - :returns: tuple of (parsed url, connection object) - :raises ClientException: Unable to handle protocol scheme - """ - url = encode_utf8(url) - parsed = urlparse(url) - if proxy: - proxy_parsed = urlparse(proxy) - else: - proxies = get_environ_proxies(parsed.netloc) - proxy = proxies.get(parsed.scheme, None) - proxy_parsed = urlparse(proxy) if proxy else None - host = proxy_parsed.netloc if proxy else parsed.netloc - if parsed.scheme == 'http': - conn = HTTPConnection(host) - elif parsed.scheme == 'https': - if ssl_compression is True: - conn = HTTPSConnection(host) - else: - conn = HTTPSConnectionNoSSLComp(host) - else: - raise ClientException('Cannot handle protocol scheme %s for url %s' % - (parsed.scheme, repr(url))) + :param url: url to connect to + :param proxy: proxy to connect through, if any; None by default; str + of the format 'http://127.0.0.1:8888' to set one + :param cacert: A CA bundle file to use in verifying a TLS server + certificate. + :param insecure: Allow to access servers without checking SSL certs. + The server's certificate will not be verified. + :param ssl_compression: SSL compression should be disabled by default + and this setting is not usable as of now. The + parameter is kept for backward compatibility. + :raises ClientException: Unable to handle protocol scheme + """ + self.url = url + self.parsed_url = urlparse(url) + self.host = self.parsed_url.netloc + self.port = self.parsed_url.port + self.requests_args = {} + if self.parsed_url.scheme not in ('http', 'https'): + raise ClientException("Unsupported scheme") + self.requests_args['verify'] = not insecure + if cacert: + # verify requests parameter is used to pass the CA_BUNDLE file + # see: http://docs.python-requests.org/en/latest/user/advanced/ + self.requests_args['verify'] = cacert + if proxy: + proxy_parsed = urlparse(proxy) + if not proxy_parsed.scheme: + raise ClientException("Proxy's missing scheme") + self.requests_args['proxies'] = { + proxy_parsed.scheme: '%s://%s' % ( + proxy_parsed.scheme, proxy_parsed.netloc + ) + } + self.requests_args['stream'] = True - def putheader_wrapper(func): + def _request(self, *arg, **kwarg): + """ Final wrapper before requests call, to be patched in tests """ + return requests.request(*arg, **kwarg) - @wraps(func) - def putheader_escaped(key, value): - func(encode_utf8(key), encode_utf8(value)) - return putheader_escaped - conn.putheader = putheader_wrapper(conn.putheader) + def request(self, method, full_path, data=None, headers={}, files=None): + """ Encode url and header, then call requests.request """ + headers = dict((encode_utf8(x), encode_utf8(y)) for x, y in + headers.iteritems()) + url = encode_utf8("%s://%s%s" % ( + self.parsed_url.scheme, + self.parsed_url.netloc, + full_path)) + self.resp = self._request(method, url, headers=headers, data=data, + files=files, **self.requests_args) + return self.resp - def request_wrapper(func): + def putrequest(self, full_path, data=None, headers={}, files=None): + """ + Use python-requests files upload - @wraps(func) - def request_escaped(method, url, body=None, headers=None): - validate_headers(headers) - url = encode_utf8(url) - if body: - body = encode_utf8(body) - func(method, url, body=body, headers=headers or {}) - return request_escaped - conn.request = request_wrapper(conn.request) - if proxy: - try: - # python 2.6 method - conn._set_tunnel(parsed.hostname, parsed.port) - except AttributeError: - # python 2.7 method - conn.set_tunnel(parsed.hostname, parsed.port) - return parsed, conn + :param data: Use data generator for chunked-transfer + :param files: Use files for default transfer + """ + return self.request('PUT', full_path, data, headers, files) + + def getresponse(self): + """ Adapt requests response to httplib interface """ + self.resp.status = self.resp.status_code + old_getheader = self.resp.raw.getheader + + def getheaders(): + return self.resp.headers.items() + + def getheader(k, v=None): + return old_getheader(k.lower(), v) + + self.resp.getheaders = getheaders + self.resp.getheader = getheader + self.resp.read = self.resp.raw.read + return self.resp + + +def http_connection(*arg, **kwarg): + """ :returns: tuple of (parsed url, connection object) """ + conn = HTTPConnection(*arg, **kwarg) + return conn.parsed_url, conn def get_auth_1_0(url, user, key, snet): @@ -890,27 +921,16 @@ def put_object(url, token=None, container=None, name=None, contents=None, if hasattr(contents, 'read'): if chunk_size is None: chunk_size = 65536 - conn.putrequest('PUT', path) - for header, value in headers.iteritems(): - conn.putheader(header, value) if content_length is None: - conn.putheader('Transfer-Encoding', 'chunked') - conn.endheaders() - chunk = contents.read(chunk_size) - while chunk: - conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) - chunk = contents.read(chunk_size) - conn.send('0\r\n\r\n') + def chunk_reader(): + while True: + data = contents.read(chunk_size) + if not data: + break + yield data + conn.putrequest(path, headers=headers, data=chunk_reader()) else: - conn.endheaders() - left = content_length - while left > 0: - size = chunk_size - if size > left: - size = left - chunk = contents.read(size) - conn.send(chunk) - left -= len(chunk) + conn.putrequest(path, headers=headers, files={"file": contents}) else: if chunk_size is not None: warn_msg = '%s object has no \"read\" method, ignoring chunk_size'\ @@ -1129,6 +1149,8 @@ class Connection(object): def http_connection(self): return http_connection(self.url, + cacert=self.cacert, + insecure=self.insecure, ssl_compression=self.ssl_compression) def _add_response_dict(self, target_dict, kwargs): @@ -1160,7 +1182,9 @@ class Connection(object): rv = func(self.url, self.token, *args, **kwargs) self._add_response_dict(caller_response_dict, kwargs) return rv - except (socket.error, HTTPException) as e: + except SSLError: + raise + except (socket.error, RequestException) as e: self._add_response_dict(caller_response_dict, kwargs) if self.attempts > self.retries: logger.exception(e) diff --git a/swiftclient/https_connection.py b/swiftclient/https_connection.py deleted file mode 100644 index 2a2dc1f0..00000000 --- a/swiftclient/https_connection.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright (c) 2013 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -HTTPS/SSL related functionality -""" - -import socket - -from httplib import HTTPSConnection - -import OpenSSL - -try: - from eventlet.green.OpenSSL.SSL import GreenConnection - from eventlet.greenio import GreenSocket - from eventlet.patcher import is_monkey_patched - - def getsockopt(self, *args, **kwargs): - return self.fd.getsockopt(*args, **kwargs) - # The above is a workaround for an eventlet bug in getsockopt. - # TODO(mclaren): Workaround can be removed when this fix lands: - # https://bitbucket.org/eventlet/eventlet/commits/609f230 - GreenSocket.getsockopt = getsockopt -except ImportError: - def is_monkey_patched(*args): - return False - - -class HTTPSConnectionNoSSLComp(HTTPSConnection): - """ - Extended HTTPSConnection which uses the OpenSSL library - for disabling SSL compression. - Note: This functionality can eventually be replaced - with native Python 3.3 code. - """ - def __init__(self, host): - HTTPSConnection.__init__(self, host) - self.setcontext() - - def setcontext(self): - """ - Set up the OpenSSL context. - """ - self.context = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) - # Disable SSL layer compression. - self.context.set_options(0x20000) # SSL_OP_NO_COMPRESSION - - def connect(self): - """ - Connect to an SSL port using the OpenSSL library and apply - per-connection parameters. - """ - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.sock = OpenSSLConnectionDelegator(self.context, sock) - self.sock.connect((self.host, self.port)) - - -class OpenSSLConnectionDelegator(object): - """ - An OpenSSL.SSL.Connection delegator. - - Supplies an additional 'makefile' method which httplib requires - and is not present in OpenSSL.SSL.Connection. - - Note: Since it is not possible to inherit from OpenSSL.SSL.Connection - a delegator must be used. - """ - def __init__(self, *args, **kwargs): - if is_monkey_patched('socket'): - # If we are running in a monkey patched environment - # use eventlet's GreenConnection -- it handles eventlet's - # non-blocking sockets correctly. - Connection = GreenConnection - else: - Connection = OpenSSL.SSL.Connection - self.connection = Connection(*args, **kwargs) - - def __getattr__(self, name): - return getattr(self.connection, name) - - def makefile(self, *args, **kwargs): - return socket._fileobject(self.connection, *args, **kwargs) diff --git a/swiftclient/utils.py b/swiftclient/utils.py index a74eadab..a038dcc5 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -14,23 +14,6 @@ # limitations under the License. """Miscellaneous utility functions for use with Swift.""" -import sys -import os - -_ver = sys.version_info - -#: Python 2.x? -is_py2 = (_ver[0] == 2) - -#: Python 3.x? -is_py3 = (_ver[0] == 3) - -if is_py2: - from urllib import getproxies, proxy_bypass -elif is_py3: - from urllib.request import getproxies, proxy_bypass - - TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y')) @@ -72,36 +55,3 @@ def prt_bytes(bytes, human_flag): bytes = '%12s' % bytes return(bytes) - - -# get_environ_proxies function, borrowed from python Requests -# (www.python-requests.org) -def get_environ_proxies(netloc): - """Return a dict of environment proxies.""" - - get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper()) - - # First check whether no_proxy is defined. If it is, check that the URL - # we're getting isn't in the no_proxy list. - no_proxy = get_proxy('no_proxy') - - if no_proxy: - # We need to check whether we match here. We need to see if we match - # the end of the netloc, both with and without the port. - no_proxy = no_proxy.replace(' ', '').split(',') - - for host in no_proxy: - if netloc.endswith(host) or netloc.split(':')[0].endswith(host): - # The URL does match something in no_proxy, so we don't want - # to apply the proxies on this URL. - return {} - - # If the system proxy settings indicate that this URL should be bypassed, - # don't proxy. - if proxy_bypass(netloc): - return {} - - # If we get here, we either didn't have no_proxy set or we're not going - # anywhere that no_proxy applies to, and the system settings don't require - # bypassing the proxy for the current URL. - return getproxies() diff --git a/tests/test_swiftclient.py b/tests/test_swiftclient.py index 19764db2..7a9f1f00 100644 --- a/tests/test_swiftclient.py +++ b/tests/test_swiftclient.py @@ -15,7 +15,6 @@ # TODO: More tests import mock -import httplib import logging import socket import StringIO @@ -107,7 +106,8 @@ class MockHttpTest(testtools.TestCase): query_string = kwargs.get('query_string') storage_url = kwargs.get('storage_url') - def wrapper(url, proxy=None, ssl_compression=True): + def wrapper(url, proxy=None, cacert=None, insecure=False, + ssl_compression=True): if storage_url: self.assertEqual(storage_url, url) @@ -138,11 +138,17 @@ class MockHttpTest(testtools.TestCase): class MockHttpResponse(): - def __init__(self): - self.status = 200 + def __init__(self, status=0): + self.status = status + self.status_code = status self.reason = "OK" self.buffer = [] + class Raw: + def read(): + pass + self.raw = Raw() + def read(self): return "" @@ -153,10 +159,15 @@ class MockHttpResponse(): return {"key1": "value1", "key2": "value2"} def fake_response(self): - return MockHttpResponse() + return MockHttpResponse(self.status) - def fake_send(self, msg): - self.buffer.append(msg) + def _fake_request(self, *arg, **kwarg): + self.status = 200 + # This simulate previous httplib implementation that would do a + # putrequest() and then use putheader() to send header. + for k, v in kwarg['headers'].iteritems(): + self.buffer.append('%s: %s' % (k, v)) + return self.fake_response() class TestHttpHelpers(MockHttpTest): @@ -173,8 +184,7 @@ class TestHttpHelpers(MockHttpTest): self.assertTrue(isinstance(conn, c.HTTPConnection)) url = 'https://www.test.com' _junk, conn = c.http_connection(url) - self.assertTrue(isinstance(conn, httplib.HTTPSConnection) or - isinstance(conn, c.HTTPSConnectionNoSSLComp)) + self.assertTrue(isinstance(conn, c.HTTPConnection)) url = 'ftp://www.test.com' self.assertRaises(c.ClientException, c.http_connection, url) @@ -560,7 +570,7 @@ class TestPutObject(MockHttpTest): resp = MockHttpResponse() conn[1].getresponse = resp.fake_response - conn[1].send = resp.fake_send + conn[1]._request = resp._fake_request value = c.put_object(*args, headers=headers, http_conn=conn) self.assertTrue(isinstance(value, basestring)) # Test for RFC-2616 encoded symbols @@ -573,7 +583,7 @@ class TestPutObject(MockHttpTest): args = ('asdf', 'asdf', 'asdf', 'asdf', mock_file) resp = MockHttpResponse() conn[1].getresponse = resp.fake_response - conn[1].send = resp.fake_send + conn[1]._request = resp._fake_request with warnings.catch_warnings(record=True) as w: c.put_object(*args, chunk_size=20, headers={}, http_conn=conn) self.assertEqual(len(w), 0) @@ -621,7 +631,7 @@ class TestPostObject(MockHttpTest): resp = MockHttpResponse() conn[1].getresponse = resp.fake_response - conn[1].send = resp.fake_send + conn[1]._request = resp._fake_request c.post_object(*args, headers=headers, http_conn=conn) # Test for RFC-2616 encoded symbols self.assertTrue("a-b: .x:yz mn:kl:qr" in resp.buffer[0], @@ -853,7 +863,7 @@ class TestConnection(MockHttpTest): self.port = parsed_url.netloc def putrequest(self, *args, **kwargs): - return + self.send() def putheader(self, *args, **kwargs): return @@ -880,7 +890,8 @@ class TestConnection(MockHttpTest): def read(self, *args, **kwargs): return '' - def local_http_connection(url, proxy=None, ssl_compression=True): + def local_http_connection(url, proxy=None, cacert=None, + insecure=False, ssl_compression=True): parsed = urlparse(url) return parsed, LocalConnection() diff --git a/tests/test_utils.py b/tests/test_utils.py index 7b2ef76a..33d94670 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -14,7 +14,6 @@ # limitations under the License. import testtools -import os from swiftclient import utils as u @@ -118,82 +117,3 @@ class TestPrtBytes(testtools.TestCase): def test_overflow(self): bytes_ = 2 ** 90 self.assertEqual('1024Y', u.prt_bytes(bytes_, True).lstrip()) - - -class TestGetEnvironProxy(testtools.TestCase): - - ENV_VARS = ('http_proxy', 'https_proxy', 'no_proxy', - 'HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY') - - def setUp(self): - self.proxy_dict = {} - super(TestGetEnvironProxy, self).setUp() - for proxy_s in TestGetEnvironProxy.ENV_VARS: - # Save old env value - self.proxy_dict[proxy_s] = os.environ.get(proxy_s, None) - - def tearDown(self): - super(TestGetEnvironProxy, self).tearDown() - for proxy_s in TestGetEnvironProxy.ENV_VARS: - if self.proxy_dict[proxy_s]: - os.environ[proxy_s] = self.proxy_dict[proxy_s] - elif os.environ.get(proxy_s): - del os.environ[proxy_s] - - def setup_env(self, new_env={}): - for proxy_s in TestGetEnvironProxy.ENV_VARS: - # Set new env value - if new_env.get(proxy_s): - os.environ[proxy_s] = new_env.get(proxy_s) - elif os.environ.get(proxy_s): - del os.environ[proxy_s] - - def test_http_proxy(self): - self.setup_env({'http_proxy': 'http://proxy.tests.com:8080'}) - proxy_dict = u.get_environ_proxies('www.tests.com:81') - self.assertEqual(proxy_dict['http'], 'http://proxy.tests.com:8080') - self.assertEqual(proxy_dict.get('https'), None) - self.assertEqual(len(proxy_dict), 1) - self.setup_env({'HTTP_PROXY': 'http://proxy.tests.com:8080'}) - proxy_dict = u.get_environ_proxies('www.tests.com:81') - self.assertEqual(proxy_dict['http'], 'http://proxy.tests.com:8080') - self.assertEqual(proxy_dict.get('https'), None) - self.assertEqual(len(proxy_dict), 1) - - def test_https_proxy(self): - self.setup_env({'https_proxy': 'http://proxy.tests.com:8080'}) - proxy_dict = u.get_environ_proxies('www.tests.com:81') - self.assertEqual(proxy_dict['https'], 'http://proxy.tests.com:8080') - self.assertEqual(proxy_dict.get('http'), None) - self.assertEqual(len(proxy_dict), 1) - self.setup_env({'HTTPS_PROXY': 'http://proxy.tests.com:8080'}) - proxy_dict = u.get_environ_proxies('www.tests.com:81') - self.assertEqual(proxy_dict['https'], 'http://proxy.tests.com:8080') - self.assertEqual(proxy_dict.get('http'), None) - self.assertEqual(len(proxy_dict), 1) - - def test_http_https_proxy(self): - self.setup_env({'http_proxy': 'http://proxy1.tests.com:8081', - 'https_proxy': 'http://proxy2.tests.com:8082'}) - proxy_dict = u.get_environ_proxies('www.tests.com:81') - self.assertEqual(proxy_dict['http'], 'http://proxy1.tests.com:8081') - self.assertEqual(proxy_dict['https'], 'http://proxy2.tests.com:8082') - self.assertEqual(len(proxy_dict), 2) - self.setup_env({'http_proxy': 'http://proxy1.tests.com:8081', - 'HTTPS_PROXY': 'http://proxy2.tests.com:8082'}) - proxy_dict = u.get_environ_proxies('www.tests.com:81') - self.assertEqual(proxy_dict['http'], 'http://proxy1.tests.com:8081') - self.assertEqual(proxy_dict['https'], 'http://proxy2.tests.com:8082') - self.assertEqual(len(proxy_dict), 2) - - def test_proxy_exclusion(self): - self.setup_env({'http_proxy': 'http://proxy1.tests.com:8081', - 'https_proxy': 'http://proxy2.tests.com:8082', - 'no_proxy': 'www.tests.com'}) - proxy_dict = u.get_environ_proxies('www.tests.com:81') - self.assertEqual(len(proxy_dict), 0) - self.setup_env({'http_proxy': 'http://proxy1.tests.com:8081', - 'HTTPS_PROXY': 'http://proxy2.tests.com:8082', - 'NO_PROXY': 'www.tests.com'}) - proxy_dict = u.get_environ_proxies('www.tests.com:81') - self.assertEqual(len(proxy_dict), 0) diff --git a/tests/utils.py b/tests/utils.py index 93972e41..ff2834a2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -12,7 +12,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -from httplib import HTTPException +from requests import RequestException from time import sleep @@ -74,7 +74,7 @@ def fake_http_connect(*code_iter, **kwargs): def getexpect(self): if self.status == -2: - raise HTTPException() + raise RequestException() if self.status == -3: return FakeConn(507) return FakeConn(100) @@ -141,7 +141,7 @@ def fake_http_connect(*code_iter, **kwargs): etag = etag_iter.next() timestamp = timestamps_iter.next() if status <= 0: - raise HTTPException() + raise RequestException() fake_conn = FakeConn(status, etag, body=kwargs.get('body', ''), timestamp=timestamp) fake_conn.connect()