openstack/swift
Code Issues Proposed changes

Remove swiftclient dependency

Browse Source 
Removes the requirement for swiftclient in swift-dispersion-report
and swift-dispersion-populate. To prevent a dependency on
keystoneclient and to avoid reinventing the wheel with an internal
keystoneclient, authentication with keystone is only supported if
swiftclient is available. If not, only auth v1 is supported.

The dependency in swift/container/sync.py has also been removed.

Implements: blueprint remove-swiftclient-dependency

Change-Id: I6ec3b3c85a67b9ab6eb04b90ffc16daf1600e8a7
This commit is contained in:
Christian Schwede 2014-01-09 10:15:50 +00:00
parent 6fec0dd735
commit 1f3ae6d8da
7 changed files with 231 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
bin/swift-dispersion-populate
View File

@ -24,7 +24,11 @@ from time import time
from eventlet import GreenPool, patcher, sleep
from eventlet.pools import Pool
from swiftclient import Connection, get_auth
try:
from swiftclient import get_auth
except ImportError:
from swift.common.internal_client import get_auth
from swift.common.internal_client import SimpleClient
from swift.common.ring import Ring
from swift.common.utils import compute_eta, get_time_units, config_true_value
@ -133,12 +137,8 @@ Usage: %%prog [options] [conf_file]
insecure=insecure)
account = url.rsplit('/', 1)[1]
connpool = Pool(max_size=concurrency)
connpool.create = lambda: Connection(conf['auth_url'],
conf['auth_user'], conf['auth_key'],
retries=retries,
preauthurl=url, preauthtoken=token,
os_options=os_options,
insecure=insecure)
connpool.create = lambda: SimpleClient(
url=url, token=token, retries=retries)
if container_populate:
container_ring = Ring(swift_dir, ring_name='container')

12
bin/swift-dispersion-report
View File

@ -28,7 +28,11 @@ from eventlet import GreenPool, hubs, patcher, Timeout
from eventlet.pools import Pool
from swift.common import direct_client
from swiftclient import Connection, get_auth
try:
from swiftclient import get_auth
except ImportError:
from swift.common.internal_client import get_auth
from swift.common.internal_client import SimpleClient
from swift.common.ring import Ring
from swift.common.exceptions import ClientException
from swift.common.utils import compute_eta, get_time_units, config_true_value
@ -356,10 +360,8 @@ Usage: %%prog [options] [conf_file]
insecure=insecure)
account = url.rsplit('/', 1)[1]
connpool = Pool(max_size=concurrency)
connpool.create = lambda: Connection(
conf['auth_url'], conf['auth_user'], conf['auth_key'], retries=retries,
preauthurl=url, preauthtoken=token, os_options=os_options,
insecure=insecure)
connpool.create = lambda: SimpleClient(
url=url, token=token, retries=retries)
container_ring = Ring(swift_dir, ring_name='container')
object_ring = Ring(swift_dir, ring_name='object')

1
requirements.txt
View File

@ -5,4 +5,3 @@ netifaces>=0.5
pastedeploy>=1.3.3
simplejson>=2.0.9
xattr>=0.4
python-swiftclient

107
swift/common/internal_client.py
View File

@ -14,12 +14,14 @@
# limitations under the License.
from eventlet import sleep, Timeout
from eventlet.green import httplib, socket, urllib2
import json
from paste.deploy import loadapp
import struct
from sys import exc_info
import zlib
from swift import gettext_ as _
import urlparse
from zlib import compressobj
from swift.common.utils import quote
@ -675,3 +677,108 @@ class InternalClient(object):
headers['Transfer-Encoding'] = 'chunked'
path = self.make_path(account, container, obj)
self.make_request('PUT', path, headers, (2,), fobj)
def get_auth(url, user, key, auth_version='1.0', **kwargs):
if auth_version != '1.0':
exit('ERROR: swiftclient missing, only auth v1.0 supported')
req = urllib2.Request(url)
req.add_header('X-Auth-User', user)
req.add_header('X-Auth-Key', key)
conn = urllib2.urlopen(req)
headers = conn.info()
return (
headers.getheader('X-Storage-Url'),
headers.getheader('X-Auth-Token'))
class SimpleClient(object):
"""
Simple client that is used in bin/swift-dispersion-* and container sync
"""
def __init__(self, url=None, token=None, starting_backoff=1,
max_backoff=5, retries=5):
self.url = url
self.token = token
self.attempts = 0
self.starting_backoff = starting_backoff
self.max_backoff = max_backoff
self.retries = retries
def base_request(self, method, container=None, name=None, prefix=None,
headers={}, proxy=None, contents=None, full_listing=None):
# Common request method
url = self.url
if self.token:
headers['X-Auth-Token'] = self.token
if container:
url = '%s/%s' % (url.rstrip('/'), quote(container))
if name:
url = '%s/%s' % (url.rstrip('/'), quote(name))
url += '?format=json'
if prefix:
url += '&prefix=%s' % prefix
if proxy:
proxy = urlparse.urlparse(proxy)
proxy = urllib2.ProxyHandler({proxy.scheme: proxy.netloc})
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
req = urllib2.Request(url, headers=headers, data=contents)
req.get_method = lambda: method
urllib2.urlopen(req)
conn = urllib2.urlopen(req)
body = conn.read()
try:
body_data = json.loads(body)
except ValueError:
body_data = None
return [None, body_data]
def retry_request(self, method, **kwargs):
self.attempts = 0
backoff = self.starting_backoff
while self.attempts <= self.retries:
self.attempts += 1
try:
return self.base_request(method, **kwargs)
except (socket.error, httplib.HTTPException, urllib2.URLError):
if self.attempts > self.retries:
raise
sleep(backoff)
backoff = min(backoff * 2, self.max_backoff)
def get_account(self, *args, **kwargs):
# Used in swift-dispertion-populate
return self.retry_request('GET', **kwargs)
def put_container(self, container, **kwargs):
# Used in swift-dispertion-populate
return self.retry_request('PUT', container=container, **kwargs)
def get_container(self, container, **kwargs):
# Used in swift-dispertion-populate
return self.retry_request('GET', container=container, **kwargs)
def put_object(self, container, name, contents, **kwargs):
# Used in swift-dispertion-populate
return self.retry_request('PUT', container=container, name=name,
contents=contents.read(), **kwargs)
def put_object(url, **kwargs):
"""For usage with container sync """
client = SimpleClient(url=url)
client.retry_request('PUT', **kwargs)
def delete_object(url, **kwargs):
"""For usage with container sync """
client = SimpleClient(url=url)
client.retry_request('DELETE', **kwargs)

4
swift/container/sync.py
View File

@ -24,15 +24,15 @@ from eventlet import sleep, Timeout
import swift.common.db
from swift.container import server as container_server
from swiftclient import delete_object, put_object, quote
from swift.container.backend import ContainerBroker
from swift.common.container_sync_realms import ContainerSyncRealms
from swift.common.direct_client import direct_get_object
from swift.common.internal_client import delete_object, put_object
from swift.common.exceptions import ClientException
from swift.common.ring import Ring
from swift.common.utils import audit_location_generator, get_logger, \
hash_path, config_true_value, validate_sync_to, whataremyips, \
FileLikeIter, urlparse
FileLikeIter, urlparse, quote
from swift.common.daemon import Daemon
from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND

1
test-requirements.txt
View File

@ -7,3 +7,4 @@ openstack.nose_plugin
nosehtmloutput
sphinx>=1.1.2,<1.2
mock>=0.8.0
python-swiftclient

107
test/unit/common/test_internal_client.py
View File

@ -14,11 +14,13 @@
# limitations under the License.
import json
import mock
from StringIO import StringIO
import unittest
from urllib import quote
import zlib
from eventlet.green import urllib2
from swift.common import internal_client
@ -919,5 +921,110 @@ class TestInternalClient(unittest.TestCase):
client.upload_object(fobj, account, container, obj, headers)
self.assertEquals(1, client.make_request_called)
class TestGetAuth(unittest.TestCase):
@mock.patch('eventlet.green.urllib2.urlopen')
@mock.patch('eventlet.green.urllib2.Request')
def test_ok(self, request, urlopen):
def getheader(name):
d = {'X-Storage-Url': 'url', 'X-Auth-Token': 'token'}
return d.get(name)
urlopen.return_value.info.return_value.getheader = getheader
url, token = internal_client.get_auth(
'http://127.0.0.1', 'user', 'key')
self.assertEqual(url, "url")
self.assertEqual(token, "token")
request.assert_called_with('http://127.0.0.1')
request.return_value.add_header.assert_any_call('X-Auth-User', 'user')
request.return_value.add_header.assert_any_call('X-Auth-Key', 'key')
def test_invalid_version(self):
self.assertRaises(SystemExit, internal_client.get_auth,
'http://127.0.0.1', 'user', 'key', auth_version=2.0)
class TestSimpleClient(unittest.TestCase):
@mock.patch('eventlet.green.urllib2.urlopen')
@mock.patch('eventlet.green.urllib2.Request')
def test_get(self, request, urlopen):
# basic GET request, only url as kwarg
request.return_value.get_type.return_value = "http"
urlopen.return_value.read.return_value = ''
sc = internal_client.SimpleClient(url='http://127.0.0.1')
retval = sc.retry_request('GET')
request.assert_called_with('http://127.0.0.1?format=json',
headers={},
data=None)
self.assertEqual([None, None], retval)
self.assertEqual('GET', request.return_value.get_method())
# Check if JSON is decoded
urlopen.return_value.read.return_value = '{}'
retval = sc.retry_request('GET')
self.assertEqual([None, {}], retval)
# same as above, now with token
sc = internal_client.SimpleClient(url='http://127.0.0.1',
token='token')
retval = sc.retry_request('GET')
request.assert_called_with('http://127.0.0.1?format=json',
headers={'X-Auth-Token': 'token'},
data=None)
self.assertEqual([None, {}], retval)
# same as above, now with prefix
sc = internal_client.SimpleClient(url='http://127.0.0.1',
token='token')
retval = sc.retry_request('GET', prefix="pre_")
request.assert_called_with('http://127.0.0.1?format=json&prefix=pre_',
headers={'X-Auth-Token': 'token'},
data=None)
self.assertEqual([None, {}], retval)
# same as above, now with container name
retval = sc.retry_request('GET', container='cont')
request.assert_called_with('http://127.0.0.1/cont?format=json',
headers={'X-Auth-Token': 'token'},
data=None)
self.assertEqual([None, {}], retval)
# same as above, now with object name
retval = sc.retry_request('GET', container='cont', name='obj')
request.assert_called_with('http://127.0.0.1/cont/obj?format=json',
headers={'X-Auth-Token': 'token'},
data=None)
self.assertEqual([None, {}], retval)
@mock.patch('eventlet.green.urllib2.urlopen')
@mock.patch('eventlet.green.urllib2.Request')
def test_get_with_retries_all_failed(self, request, urlopen):
# Simulate a failing request, ensure retries done
request.return_value.get_type.return_value = "http"
request.side_effect = urllib2.URLError('')
urlopen.return_value.read.return_value = ''
sc = internal_client.SimpleClient(url='http://127.0.0.1', retries=1)
self.assertRaises(urllib2.URLError, sc.retry_request, 'GET')
self.assertEqual(request.call_count, 2)
@mock.patch('eventlet.green.urllib2.urlopen')
@mock.patch('eventlet.green.urllib2.Request')
def test_get_with_retries(self, request, urlopen):
# First request fails, retry successful
request.return_value.get_type.return_value = "http"
urlopen.return_value.read.return_value = ''
req = urllib2.Request('http://127.0.0.1', method='GET')
request.side_effect = [urllib2.URLError(''), req]
sc = internal_client.SimpleClient(url='http://127.0.0.1', retries=1)
retval = sc.retry_request('GET')
self.assertEqual(request.call_count, 3)
request.assert_called_with('http://127.0.0.1?format=json', data=None,
headers={'X-Auth-Token': 'token'})
self.assertEqual([None, None], retval)
if __name__ == '__main__':
unittest.main()