From 1f3ae6d8dadef838f1ea7fa571339932074fbe38 Mon Sep 17 00:00:00 2001 From: Christian Schwede Date: Thu, 9 Jan 2014 10:15:50 +0000 Subject: [PATCH] Remove swiftclient dependency Removes the requirement for swiftclient in swift-dispersion-report and swift-dispersion-populate. To prevent a dependency on keystoneclient and to avoid reinventing the wheel with an internal keystoneclient, authentication with keystone is only supported if swiftclient is available. If not, only auth v1 is supported. The dependency in swift/container/sync.py has also been removed. Implements: blueprint remove-swiftclient-dependency Change-Id: I6ec3b3c85a67b9ab6eb04b90ffc16daf1600e8a7 --- bin/swift-dispersion-populate | 14 +-- bin/swift-dispersion-report | 12 +-- requirements.txt | 1 - swift/common/internal_client.py | 107 +++++++++++++++++++++++ swift/container/sync.py | 4 +- test-requirements.txt | 1 + test/unit/common/test_internal_client.py | 107 +++++++++++++++++++++++ 7 files changed, 231 insertions(+), 15 deletions(-) diff --git a/bin/swift-dispersion-populate b/bin/swift-dispersion-populate index 351af6309b..a475e40e4c 100755 --- a/bin/swift-dispersion-populate +++ b/bin/swift-dispersion-populate @@ -24,7 +24,11 @@ from time import time from eventlet import GreenPool, patcher, sleep from eventlet.pools import Pool -from swiftclient import Connection, get_auth +try: + from swiftclient import get_auth +except ImportError: + from swift.common.internal_client import get_auth +from swift.common.internal_client import SimpleClient from swift.common.ring import Ring from swift.common.utils import compute_eta, get_time_units, config_true_value @@ -133,12 +137,8 @@ Usage: %%prog [options] [conf_file] insecure=insecure) account = url.rsplit('/', 1)[1] connpool = Pool(max_size=concurrency) - connpool.create = lambda: Connection(conf['auth_url'], - conf['auth_user'], conf['auth_key'], - retries=retries, - preauthurl=url, preauthtoken=token, - os_options=os_options, - insecure=insecure) + connpool.create = lambda: SimpleClient( + url=url, token=token, retries=retries) if container_populate: container_ring = Ring(swift_dir, ring_name='container') diff --git a/bin/swift-dispersion-report b/bin/swift-dispersion-report index aabea6c502..34f239c876 100755 --- a/bin/swift-dispersion-report +++ b/bin/swift-dispersion-report @@ -28,7 +28,11 @@ from eventlet import GreenPool, hubs, patcher, Timeout from eventlet.pools import Pool from swift.common import direct_client -from swiftclient import Connection, get_auth +try: + from swiftclient import get_auth +except ImportError: + from swift.common.internal_client import get_auth +from swift.common.internal_client import SimpleClient from swift.common.ring import Ring from swift.common.exceptions import ClientException from swift.common.utils import compute_eta, get_time_units, config_true_value @@ -356,10 +360,8 @@ Usage: %%prog [options] [conf_file] insecure=insecure) account = url.rsplit('/', 1)[1] connpool = Pool(max_size=concurrency) - connpool.create = lambda: Connection( - conf['auth_url'], conf['auth_user'], conf['auth_key'], retries=retries, - preauthurl=url, preauthtoken=token, os_options=os_options, - insecure=insecure) + connpool.create = lambda: SimpleClient( + url=url, token=token, retries=retries) container_ring = Ring(swift_dir, ring_name='container') object_ring = Ring(swift_dir, ring_name='object') diff --git a/requirements.txt b/requirements.txt index 113986693f..bbac51a612 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,3 @@ netifaces>=0.5 pastedeploy>=1.3.3 simplejson>=2.0.9 xattr>=0.4 -python-swiftclient diff --git a/swift/common/internal_client.py b/swift/common/internal_client.py index f6a22c818f..8394706a09 100644 --- a/swift/common/internal_client.py +++ b/swift/common/internal_client.py @@ -14,12 +14,14 @@ # limitations under the License. from eventlet import sleep, Timeout +from eventlet.green import httplib, socket, urllib2 import json from paste.deploy import loadapp import struct from sys import exc_info import zlib from swift import gettext_ as _ +import urlparse from zlib import compressobj from swift.common.utils import quote @@ -675,3 +677,108 @@ class InternalClient(object): headers['Transfer-Encoding'] = 'chunked' path = self.make_path(account, container, obj) self.make_request('PUT', path, headers, (2,), fobj) + + +def get_auth(url, user, key, auth_version='1.0', **kwargs): + if auth_version != '1.0': + exit('ERROR: swiftclient missing, only auth v1.0 supported') + req = urllib2.Request(url) + req.add_header('X-Auth-User', user) + req.add_header('X-Auth-Key', key) + conn = urllib2.urlopen(req) + headers = conn.info() + return ( + headers.getheader('X-Storage-Url'), + headers.getheader('X-Auth-Token')) + + +class SimpleClient(object): + """ + Simple client that is used in bin/swift-dispersion-* and container sync + """ + def __init__(self, url=None, token=None, starting_backoff=1, + max_backoff=5, retries=5): + self.url = url + self.token = token + self.attempts = 0 + self.starting_backoff = starting_backoff + self.max_backoff = max_backoff + self.retries = retries + + def base_request(self, method, container=None, name=None, prefix=None, + headers={}, proxy=None, contents=None, full_listing=None): + # Common request method + url = self.url + + if self.token: + headers['X-Auth-Token'] = self.token + + if container: + url = '%s/%s' % (url.rstrip('/'), quote(container)) + + if name: + url = '%s/%s' % (url.rstrip('/'), quote(name)) + + url += '?format=json' + + if prefix: + url += '&prefix=%s' % prefix + + if proxy: + proxy = urlparse.urlparse(proxy) + proxy = urllib2.ProxyHandler({proxy.scheme: proxy.netloc}) + opener = urllib2.build_opener(proxy) + urllib2.install_opener(opener) + + req = urllib2.Request(url, headers=headers, data=contents) + req.get_method = lambda: method + urllib2.urlopen(req) + conn = urllib2.urlopen(req) + body = conn.read() + try: + body_data = json.loads(body) + except ValueError: + body_data = None + return [None, body_data] + + def retry_request(self, method, **kwargs): + self.attempts = 0 + backoff = self.starting_backoff + while self.attempts <= self.retries: + self.attempts += 1 + try: + return self.base_request(method, **kwargs) + except (socket.error, httplib.HTTPException, urllib2.URLError): + if self.attempts > self.retries: + raise + sleep(backoff) + backoff = min(backoff * 2, self.max_backoff) + + def get_account(self, *args, **kwargs): + # Used in swift-dispertion-populate + return self.retry_request('GET', **kwargs) + + def put_container(self, container, **kwargs): + # Used in swift-dispertion-populate + return self.retry_request('PUT', container=container, **kwargs) + + def get_container(self, container, **kwargs): + # Used in swift-dispertion-populate + return self.retry_request('GET', container=container, **kwargs) + + def put_object(self, container, name, contents, **kwargs): + # Used in swift-dispertion-populate + return self.retry_request('PUT', container=container, name=name, + contents=contents.read(), **kwargs) + + +def put_object(url, **kwargs): + """For usage with container sync """ + client = SimpleClient(url=url) + client.retry_request('PUT', **kwargs) + + +def delete_object(url, **kwargs): + """For usage with container sync """ + client = SimpleClient(url=url) + client.retry_request('DELETE', **kwargs) diff --git a/swift/container/sync.py b/swift/container/sync.py index 41302bfb77..87ca387b76 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -24,15 +24,15 @@ from eventlet import sleep, Timeout import swift.common.db from swift.container import server as container_server -from swiftclient import delete_object, put_object, quote from swift.container.backend import ContainerBroker from swift.common.container_sync_realms import ContainerSyncRealms from swift.common.direct_client import direct_get_object +from swift.common.internal_client import delete_object, put_object from swift.common.exceptions import ClientException from swift.common.ring import Ring from swift.common.utils import audit_location_generator, get_logger, \ hash_path, config_true_value, validate_sync_to, whataremyips, \ - FileLikeIter, urlparse + FileLikeIter, urlparse, quote from swift.common.daemon import Daemon from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND diff --git a/test-requirements.txt b/test-requirements.txt index 8db26120a9..7036e8d192 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,3 +7,4 @@ openstack.nose_plugin nosehtmloutput sphinx>=1.1.2,<1.2 mock>=0.8.0 +python-swiftclient diff --git a/test/unit/common/test_internal_client.py b/test/unit/common/test_internal_client.py index 6c7e6272e4..4772688d62 100644 --- a/test/unit/common/test_internal_client.py +++ b/test/unit/common/test_internal_client.py @@ -14,11 +14,13 @@ # limitations under the License. import json +import mock from StringIO import StringIO import unittest from urllib import quote import zlib +from eventlet.green import urllib2 from swift.common import internal_client @@ -919,5 +921,110 @@ class TestInternalClient(unittest.TestCase): client.upload_object(fobj, account, container, obj, headers) self.assertEquals(1, client.make_request_called) + +class TestGetAuth(unittest.TestCase): + @mock.patch('eventlet.green.urllib2.urlopen') + @mock.patch('eventlet.green.urllib2.Request') + def test_ok(self, request, urlopen): + def getheader(name): + d = {'X-Storage-Url': 'url', 'X-Auth-Token': 'token'} + return d.get(name) + urlopen.return_value.info.return_value.getheader = getheader + + url, token = internal_client.get_auth( + 'http://127.0.0.1', 'user', 'key') + + self.assertEqual(url, "url") + self.assertEqual(token, "token") + request.assert_called_with('http://127.0.0.1') + request.return_value.add_header.assert_any_call('X-Auth-User', 'user') + request.return_value.add_header.assert_any_call('X-Auth-Key', 'key') + + def test_invalid_version(self): + self.assertRaises(SystemExit, internal_client.get_auth, + 'http://127.0.0.1', 'user', 'key', auth_version=2.0) + + +class TestSimpleClient(unittest.TestCase): + + @mock.patch('eventlet.green.urllib2.urlopen') + @mock.patch('eventlet.green.urllib2.Request') + def test_get(self, request, urlopen): + # basic GET request, only url as kwarg + request.return_value.get_type.return_value = "http" + urlopen.return_value.read.return_value = '' + sc = internal_client.SimpleClient(url='http://127.0.0.1') + retval = sc.retry_request('GET') + request.assert_called_with('http://127.0.0.1?format=json', + headers={}, + data=None) + self.assertEqual([None, None], retval) + self.assertEqual('GET', request.return_value.get_method()) + + # Check if JSON is decoded + urlopen.return_value.read.return_value = '{}' + retval = sc.retry_request('GET') + self.assertEqual([None, {}], retval) + + # same as above, now with token + sc = internal_client.SimpleClient(url='http://127.0.0.1', + token='token') + retval = sc.retry_request('GET') + request.assert_called_with('http://127.0.0.1?format=json', + headers={'X-Auth-Token': 'token'}, + data=None) + self.assertEqual([None, {}], retval) + + # same as above, now with prefix + sc = internal_client.SimpleClient(url='http://127.0.0.1', + token='token') + retval = sc.retry_request('GET', prefix="pre_") + request.assert_called_with('http://127.0.0.1?format=json&prefix=pre_', + headers={'X-Auth-Token': 'token'}, + data=None) + self.assertEqual([None, {}], retval) + + # same as above, now with container name + retval = sc.retry_request('GET', container='cont') + request.assert_called_with('http://127.0.0.1/cont?format=json', + headers={'X-Auth-Token': 'token'}, + data=None) + self.assertEqual([None, {}], retval) + + # same as above, now with object name + retval = sc.retry_request('GET', container='cont', name='obj') + request.assert_called_with('http://127.0.0.1/cont/obj?format=json', + headers={'X-Auth-Token': 'token'}, + data=None) + self.assertEqual([None, {}], retval) + + @mock.patch('eventlet.green.urllib2.urlopen') + @mock.patch('eventlet.green.urllib2.Request') + def test_get_with_retries_all_failed(self, request, urlopen): + # Simulate a failing request, ensure retries done + request.return_value.get_type.return_value = "http" + request.side_effect = urllib2.URLError('') + urlopen.return_value.read.return_value = '' + sc = internal_client.SimpleClient(url='http://127.0.0.1', retries=1) + self.assertRaises(urllib2.URLError, sc.retry_request, 'GET') + self.assertEqual(request.call_count, 2) + + @mock.patch('eventlet.green.urllib2.urlopen') + @mock.patch('eventlet.green.urllib2.Request') + def test_get_with_retries(self, request, urlopen): + # First request fails, retry successful + request.return_value.get_type.return_value = "http" + urlopen.return_value.read.return_value = '' + req = urllib2.Request('http://127.0.0.1', method='GET') + request.side_effect = [urllib2.URLError(''), req] + sc = internal_client.SimpleClient(url='http://127.0.0.1', retries=1) + + retval = sc.retry_request('GET') + self.assertEqual(request.call_count, 3) + request.assert_called_with('http://127.0.0.1?format=json', data=None, + headers={'X-Auth-Token': 'token'}) + self.assertEqual([None, None], retval) + + if __name__ == '__main__': unittest.main()