openstack/swift
Code Issues Proposed changes

Update swift.common.client with bin/swift changes.

Browse Source 
- Add auth version 2 to swift.common.client.
- Remove ununsed imports.
- Fix bug where auth_version should be a string.
- Add test for auth version 2.
- Allow to override the returns of http_connection for tests.
- Sync the passing of headers in bin/swift as well from client.
- Fixes bug 885011
- Previously it was review 3680 but abandoned.
- Address: Maru newby review.
- TODO: properly test auth_v1.

Change-Id: I579d8154828e892596fae9ab75f69d353f15e12c
This commit is contained in:
Chmouel Boudjnah 2012-03-04 15:46:55 +00:00
parent 2bdf0d32bc
commit 4f93c5d5e4
3 changed files with 188 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
bin/swift
View File

@ -30,9 +30,6 @@ from traceback import format_exception
# Inclusion of swift.common.client for convenience of single file distribution
import socket
from cStringIO import StringIO
from re import compile, DOTALL
from tokenize import generate_tokens, STRING, NAME, OP
from urllib import quote as _quote
from urlparse import urlparse, urlunparse, urljoin
@ -154,6 +151,31 @@ def http_connection(url, proxy=None):
return parsed, conn
def json_request(method, url, **kwargs):
"""Takes a request in json parse it and return in json"""
kwargs.setdefault('headers', {})
if 'body' in kwargs:
kwargs['headers']['Content-Type'] = 'application/json'
kwargs['body'] = json_dumps(kwargs['body'])
parsed, conn = http_connection(url)
conn.request(method, parsed.path, **kwargs)
resp = conn.getresponse()
body = resp.read()
if body:
try:
body = json_loads(body)
except ValueError:
body = None
if not body or resp.status < 200 or resp.status >= 300:
raise ClientException('Auth GET failed', http_scheme=parsed.scheme,
http_host=conn.host,
http_port=conn.port,
http_path=parsed.path,
http_status=resp.status,
http_reason=resp.reason)
return resp, body
def get_conn(options):
"""
Return a connection building it from the options.
@ -180,7 +202,8 @@ def _get_auth_v1_0(url, user, key, snet):
if snet:
parsed = list(urlparse(url))
# Second item in the list is the netloc
parsed[1] = 'snet-' + parsed[1]
netloc = parsed[1]
parsed[1] = 'snet-' + netloc
url = urlunparse(parsed)
return url, resp.getheader('x-storage-token',
resp.getheader('x-auth-token'))
@ -192,30 +215,6 @@ def _get_auth_v2_0(url, user, key, snet):
else:
tenant = user
def json_request(method, token_url, **kwargs):
kwargs.setdefault('headers', {})
if 'body' in kwargs:
kwargs['headers']['Content-Type'] = 'application/json'
kwargs['body'] = json_dumps(kwargs['body'])
parsed, conn = http_connection(token_url)
conn.request(method, parsed.path, **kwargs)
resp = conn.getresponse()
body = resp.read()
if body:
try:
body = json_loads(body)
except ValueError:
pass
else:
body = None
if resp.status < 200 or resp.status >= 300:
raise ClientException('Auth GET failed', http_scheme=parsed.scheme,
http_host=conn.host,
http_port=conn.port,
http_path=parsed.path,
http_status=resp.status,
http_reason=resp.reason)
return resp, body
body = {"auth": {"tenantName": tenant,
"passwordCredentials":
{"username": user, "password": key}}}
@ -260,11 +259,11 @@ def get_auth(url, user, key, snet=False, auth_version="1.0"):
:param snet: use SERVICENET internal network (see above), default is False
:param auth_version: OpenStack authentication version (default is 1.0)
:returns: tuple of (storage URL, auth token)
:raises ClientException: HTTP GET request to auth URL failed
:raises: ClientException: HTTP GET request to auth URL failed
"""
if auth_version == "1.0" or auth_version == "1":
if auth_version in ["1.0", "1"]:
return _get_auth_v1_0(url, user, key, snet)
elif auth_version == "2.0" or auth_version == "2":
elif auth_version in ["2.0", "2"]:
return _get_auth_v2_0(url, user, key, snet)
@ -450,7 +449,7 @@ def get_container(url, token, container, marker=None, limit=None,
return resp_headers, json_loads(resp.read())
def head_container(url, token, container, http_conn=None):
def head_container(url, token, container, http_conn=None, headers=None):
"""
Get container stats.
@ -468,7 +467,10 @@ def head_container(url, token, container, http_conn=None):
else:
parsed, conn = http_connection(url)
path = '%s/%s' % (parsed.path, quote(container))
conn.request('HEAD', path, '', {'X-Auth-Token': token})
req_headers = {'X-Auth-Token': token}
if headers:
req_headers.update(headers)
conn.request('HEAD', path, '', req_headers)
resp = conn.getresponse()
body = resp.read()
if resp.status < 200 or resp.status >= 300:
@ -816,7 +818,7 @@ class Connection(object):
def __init__(self, authurl, user, key, retries=5, preauthurl=None,
preauthtoken=None, snet=False, starting_backoff=1,
auth_version=1):
auth_version="1"):
"""
:param authurl: authenitcation URL
:param user: user name to authenticate as

122
swift/common/client.py
View File

@ -16,9 +16,10 @@
"""
Cloud Files client library used internally
"""
import socket
from urllib import quote as _quote
from urlparse import urlparse, urlunparse
from urlparse import urlparse, urlunparse, urljoin
try:
from eventlet.green.httplib import HTTPException, HTTPSConnection
@ -53,9 +54,11 @@ def quote(value, safe='/'):
try:
# simplejson is popular and pretty good
from simplejson import loads as json_loads
from simplejson import dumps as json_dumps
except ImportError:
# 2.6 will have a json module in the stdlib
from json import loads as json_loads
from json import dumps as json_dumps
class ClientException(Exception):
@ -136,23 +139,32 @@ def http_connection(url, proxy=None):
return parsed, conn
def get_auth(url, user, key, snet=False):
"""
Get authentication/authorization credentials.
def json_request(method, url, **kwargs):
"""Takes a request in json parse it and return in json"""
kwargs.setdefault('headers', {})
if 'body' in kwargs:
kwargs['headers']['Content-Type'] = 'application/json'
kwargs['body'] = json_dumps(kwargs['body'])
parsed, conn = http_connection(url)
conn.request(method, parsed.path, **kwargs)
resp = conn.getresponse()
body = resp.read()
if body:
try:
body = json_loads(body)
except ValueError:
body = None
if not body or resp.status < 200 or resp.status >= 300:
raise ClientException('Auth GET failed', http_scheme=parsed.scheme,
http_host=conn.host,
http_port=conn.port,
http_path=parsed.path,
http_status=resp.status,
http_reason=resp.reason)
return resp, body
The snet parameter is used for Rackspace's ServiceNet internal network
implementation. In this function, it simply adds *snet-* to the beginning
of the host name for the returned storage URL. With Rackspace Cloud Files,
use of this network path causes no bandwidth charges but requires the
client to be running on Rackspace's ServiceNet network.
:param url: authentication/authorization URL
:param user: user to authenticate as
:param key: key or password for authorization
:param snet: use SERVICENET internal network (see above), default is False
:returns: tuple of (storage URL, auth token)
:raises ClientException: HTTP GET request to auth URL failed
"""
def _get_auth_v1_0(url, user, key, snet):
parsed, conn = http_connection(url)
conn.request('GET', parsed.path, '',
{'X-Auth-User': user, 'X-Auth-Key': key})
@ -173,6 +185,64 @@ def get_auth(url, user, key, snet=False):
resp.getheader('x-auth-token'))
def _get_auth_v2_0(url, user, key, snet):
if ':' in user:
tenant, user = user.split(':')
else:
tenant = user
body = {"auth": {"tenantName": tenant,
"passwordCredentials":
{"username": user, "password": key}}}
token_url = urljoin(url, "tokens")
resp, body = json_request("POST", token_url, body=body)
token_id = None
try:
url = None
catalogs = body['access']['serviceCatalog']
for service in catalogs:
if service['type'] == 'object-store':
url = service['endpoints'][0]['publicURL']
token_id = body['access']['token']['id']
if not url:
raise ClientException("There is no object-store endpoint " \
"on this auth server.")
except(KeyError, IndexError):
raise ClientException("Error while getting answers from auth server")
if snet:
parsed = list(urlparse(url))
# Second item in the list is the netloc
parsed[1] = 'snet-' + parsed[1]
url = urlunparse(parsed)
return url, token_id
def get_auth(url, user, key, snet=False, auth_version="1.0"):
"""
Get authentication/authorization credentials.
The snet parameter is used for Rackspace's ServiceNet internal network
implementation. In this function, it simply adds *snet-* to the beginning
of the host name for the returned storage URL. With Rackspace Cloud Files,
use of this network path causes no bandwidth charges but requires the
client to be running on Rackspace's ServiceNet network.
:param url: authentication/authorization URL
:param user: user to authenticate as
:param key: key or password for authorization
:param snet: use SERVICENET internal network (see above), default is False
:param auth_version: OpenStack authentication version (default is 1.0)
:returns: tuple of (storage URL, auth token)
:raises: ClientException: HTTP GET request to auth URL failed
"""
if auth_version in ["1.0", "1"]:
return _get_auth_v1_0(url, user, key, snet)
elif auth_version in ["2.0", "2"]:
return _get_auth_v2_0(url, user, key, snet)
def get_account(url, token, marker=None, limit=None, prefix=None,
http_conn=None, full_listing=False):
"""
@ -280,11 +350,13 @@ def post_account(url, token, headers, http_conn=None):
body = resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Account POST failed',
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=parsed.path,
http_status=resp.status, http_reason=resp.reason,
http_response_content=body)
http_scheme=parsed.scheme,
http_host=conn.host,
http_port=conn.port,
http_path=parsed.path,
http_status=resp.status,
http_reason=resp.reason,
http_response_content=body)
def get_container(url, token, container, marker=None, limit=None,
prefix=None, delimiter=None, http_conn=None,
@ -720,7 +792,8 @@ class Connection(object):
"""Convenience class to make requests that will also retry the request"""
def __init__(self, authurl, user, key, retries=5, preauthurl=None,
preauthtoken=None, snet=False, starting_backoff=1):
preauthtoken=None, snet=False, starting_backoff=1,
auth_version="1"):
"""
:param authurl: authentication URL
:param user: user name to authenticate as
@ -730,6 +803,7 @@ class Connection(object):
:param preauthtoken: authentication token (if you have already
authenticated)
:param snet: use SERVICENET internal network default is False
:param auth_version: Openstack auth version.
"""
self.authurl = authurl
self.user = user
@ -741,9 +815,11 @@ class Connection(object):
self.attempts = 0
self.snet = snet
self.starting_backoff = starting_backoff
self.auth_version = auth_version
def get_auth(self):
return get_auth(self.authurl, self.user, self.key, snet=self.snet)
return get_auth(self.authurl, self.user, self.key, snet=self.snet,
auth_version=self.auth_version)
def http_connection(self):
return http_connection(self.url)

74
test/unit/common/test_client.py
View File

@ -16,7 +16,6 @@
# TODO: More tests
import socket
import unittest
from StringIO import StringIO
from urlparse import urlparse
# TODO: mock http connection class with more control over headers
@ -25,25 +24,6 @@ from test.unit.proxy.test_server import fake_http_connect
from swift.common import client as c
class TestHttpHelpers(unittest.TestCase):
def test_quote(self):
value = 'standard string'
self.assertEquals('standard%20string', c.quote(value))
value = u'\u0075nicode string'
self.assertEquals('unicode%20string', c.quote(value))
def test_http_connection(self):
url = 'http://www.test.com'
_junk, conn = c.http_connection(url)
self.assertTrue(isinstance(conn, c.HTTPConnection))
url = 'https://www.test.com'
_junk, conn = c.http_connection(url)
self.assertTrue(isinstance(conn, c.HTTPSConnection))
url = 'ftp://www.test.com'
self.assertRaises(c.ClientException, c.http_connection, url)
class TestClientException(unittest.TestCase):
def test_is_exception(self):
@ -115,6 +95,7 @@ class MockHttpTest(unittest.TestCase):
def setUp(self):
def fake_http_connection(*args, **kwargs):
_orig_http_connection = c.http_connection
return_read = kwargs.get('return_read')
def wrapper(url, proxy=None):
parsed, _conn = _orig_http_connection(url, proxy=proxy)
@ -130,7 +111,7 @@ class MockHttpTest(unittest.TestCase):
def read(*args, **kwargs):
conn.has_been_read = True
return _orig_read(*args, **kwargs)
conn.read = read
conn.read = return_read or read
return parsed, conn
return wrapper
@ -139,6 +120,36 @@ class MockHttpTest(unittest.TestCase):
def tearDown(self):
reload(c)
class TestHttpHelpers(MockHttpTest):
def test_quote(self):
value = 'standard string'
self.assertEquals('standard%20string', c.quote(value))
value = u'\u0075nicode string'
self.assertEquals('unicode%20string', c.quote(value))
def test_http_connection(self):
url = 'http://www.test.com'
_junk, conn = c.http_connection(url)
self.assertTrue(isinstance(conn, c.HTTPConnection))
url = 'https://www.test.com'
_junk, conn = c.http_connection(url)
self.assertTrue(isinstance(conn, c.HTTPSConnection))
url = 'ftp://www.test.com'
self.assertRaises(c.ClientException, c.http_connection, url)
def test_json_request(self):
def read(*args, **kwargs):
body = {'a': '1',
'b': '2'}
return c.json_dumps(body)
c.http_connection = self.fake_http_connection(200, return_read=read)
url = 'http://www.test.com'
_junk, conn = c.json_request('GET', url, body={'username': 'user1',
'password': 'secure'})
self.assertTrue(type(conn) is dict)
# TODO: following tests are placeholders, need more tests, better coverage
@ -150,6 +161,27 @@ class TestGetAuth(MockHttpTest):
self.assertEquals(url, None)
self.assertEquals(token, None)
def test_auth_v1(self):
c.http_connection = self.fake_http_connection(200)
url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf',
auth_version="1.0")
self.assertEquals(url, None)
self.assertEquals(token, None)
def test_auth_v2(self):
def read(*args, **kwargs):
acct_url = 'http://127.0.01/AUTH_FOO'
body = {'access': {'serviceCatalog':
[{u'endpoints': [{'publicURL': acct_url}],
'type': 'object-store'}],
'token': {'id': 'XXXXXXX'}}}
return c.json_dumps(body)
c.http_connection = self.fake_http_connection(200, return_read=read)
url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf',
auth_version="2.0")
self.assertTrue(url.startswith("http"))
self.assertTrue(token)
class TestGetAccount(MockHttpTest):