Merge "Update swift.common.client with bin/swift changes."

bin/swift

@@ -30,9 +30,6 @@ from traceback import format_exception
 # Inclusion of swift.common.client for convenience of single file distribution
 
 import socket
-from cStringIO import StringIO
-from re import compile, DOTALL
-from tokenize import generate_tokens, STRING, NAME, OP
 from urllib import quote as _quote
 from urlparse import urlparse, urlunparse, urljoin
 
@@ -154,6 +151,31 @@ def http_connection(url, proxy=None):
     return parsed, conn
 
 
+def json_request(method, url, **kwargs):
+    """Takes a request in json parse it and return in json"""
+    kwargs.setdefault('headers', {})
+    if 'body' in kwargs:
+        kwargs['headers']['Content-Type'] = 'application/json'
+        kwargs['body'] = json_dumps(kwargs['body'])
+    parsed, conn = http_connection(url)
+    conn.request(method, parsed.path, **kwargs)
+    resp = conn.getresponse()
+    body = resp.read()
+    if body:
+        try:
+            body = json_loads(body)
+        except ValueError:
+            body = None
+    if not body or resp.status < 200 or resp.status >= 300:
+        raise ClientException('Auth GET failed', http_scheme=parsed.scheme,
+                              http_host=conn.host,
+                              http_port=conn.port,
+                              http_path=parsed.path,
+                              http_status=resp.status,
+                              http_reason=resp.reason)
+    return resp, body
+
+
 def get_conn(options):
     """
     Return a connection building it from the options.
@@ -180,7 +202,8 @@ def _get_auth_v1_0(url, user, key, snet):
     if snet:
         parsed = list(urlparse(url))
         # Second item in the list is the netloc
-        parsed[1] = 'snet-' + parsed[1]
+        netloc = parsed[1]
+        parsed[1] = 'snet-' + netloc
         url = urlunparse(parsed)
     return url, resp.getheader('x-storage-token',
                                                 resp.getheader('x-auth-token'))
@@ -192,30 +215,6 @@ def _get_auth_v2_0(url, user, key, snet):
     else:
         tenant = user
 
-    def json_request(method, token_url, **kwargs):
-        kwargs.setdefault('headers', {})
-        if 'body' in kwargs:
-            kwargs['headers']['Content-Type'] = 'application/json'
-            kwargs['body'] = json_dumps(kwargs['body'])
-        parsed, conn = http_connection(token_url)
-        conn.request(method, parsed.path, **kwargs)
-        resp = conn.getresponse()
-        body = resp.read()
-        if body:
-            try:
-                body = json_loads(body)
-            except ValueError:
-                pass
-        else:
-            body = None
-        if resp.status < 200 or resp.status >= 300:
-            raise ClientException('Auth GET failed', http_scheme=parsed.scheme,
-                                  http_host=conn.host,
-                                  http_port=conn.port,
-                                  http_path=parsed.path,
-                                  http_status=resp.status,
-                                  http_reason=resp.reason)
-        return resp, body
     body = {"auth": {"tenantName": tenant,
                      "passwordCredentials":
                          {"username": user, "password": key}}}
@@ -260,11 +259,11 @@ def get_auth(url, user, key, snet=False, auth_version="1.0"):
     :param snet: use SERVICENET internal network (see above), default is False
     :param auth_version: OpenStack authentication version (default is 1.0)
     :returns: tuple of (storage URL, auth token)
-    :raises ClientException: HTTP GET request to auth URL failed
+    :raises: ClientException: HTTP GET request to auth URL failed
     """
-    if auth_version == "1.0" or auth_version == "1":
+    if auth_version in ["1.0", "1"]:
         return _get_auth_v1_0(url, user, key, snet)
-    elif auth_version == "2.0" or auth_version == "2":
+    elif auth_version in ["2.0", "2"]:
         return _get_auth_v2_0(url, user, key, snet)
 
 
@@ -450,7 +449,7 @@ def get_container(url, token, container, marker=None, limit=None,
     return resp_headers, json_loads(resp.read())
 
 
-def head_container(url, token, container, http_conn=None):
+def head_container(url, token, container, http_conn=None, headers=None):
     """
     Get container stats.
 
@@ -468,7 +467,10 @@ def head_container(url, token, container, http_conn=None):
     else:
         parsed, conn = http_connection(url)
     path = '%s/%s' % (parsed.path, quote(container))
-    conn.request('HEAD', path, '', {'X-Auth-Token': token})
+    req_headers = {'X-Auth-Token': token}
+    if headers:
+        req_headers.update(headers)
+    conn.request('HEAD', path, '', req_headers)
     resp = conn.getresponse()
     body = resp.read()
     if resp.status < 200 or resp.status >= 300:
@@ -816,7 +818,7 @@ class Connection(object):
 
     def __init__(self, authurl, user, key, retries=5, preauthurl=None,
                  preauthtoken=None, snet=False, starting_backoff=1,
-                 auth_version=1):
+                 auth_version="1"):
         """
         :param authurl: authenitcation URL
         :param user: user name to authenticate as

swift/common/client.py

@@ -16,9 +16,10 @@
 """
 Cloud Files client library used internally
 """
+
 import socket
 from urllib import quote as _quote
-from urlparse import urlparse, urlunparse
+from urlparse import urlparse, urlunparse, urljoin
 
 try:
     from eventlet.green.httplib import HTTPException, HTTPSConnection
@@ -53,9 +54,11 @@ def quote(value, safe='/'):
 try:
     # simplejson is popular and pretty good
     from simplejson import loads as json_loads
+    from simplejson import dumps as json_dumps
 except ImportError:
     # 2.6 will have a json module in the stdlib
     from json import loads as json_loads
+    from json import dumps as json_dumps
 
 
 class ClientException(Exception):
@@ -136,23 +139,32 @@ def http_connection(url, proxy=None):
     return parsed, conn
 
 
-def get_auth(url, user, key, snet=False):
+def json_request(method, url, **kwargs):
-    """
+    """Takes a request in json parse it and return in json"""
-    Get authentication/authorization credentials.
+    kwargs.setdefault('headers', {})
+    if 'body' in kwargs:
+        kwargs['headers']['Content-Type'] = 'application/json'
+        kwargs['body'] = json_dumps(kwargs['body'])
+    parsed, conn = http_connection(url)
+    conn.request(method, parsed.path, **kwargs)
+    resp = conn.getresponse()
+    body = resp.read()
+    if body:
+        try:
+            body = json_loads(body)
+        except ValueError:
+            body = None
+    if not body or resp.status < 200 or resp.status >= 300:
+        raise ClientException('Auth GET failed', http_scheme=parsed.scheme,
+                              http_host=conn.host,
+                              http_port=conn.port,
+                              http_path=parsed.path,
+                              http_status=resp.status,
+                              http_reason=resp.reason)
+    return resp, body
 
-    The snet parameter is used for Rackspace's ServiceNet internal network
-    implementation. In this function, it simply adds *snet-* to the beginning
-    of the host name for the returned storage URL. With Rackspace Cloud Files,
-    use of this network path causes no bandwidth charges but requires the
-    client to be running on Rackspace's ServiceNet network.
 
-    :param url: authentication/authorization URL
+def _get_auth_v1_0(url, user, key, snet):
-    :param user: user to authenticate as
-    :param key: key or password for authorization
-    :param snet: use SERVICENET internal network (see above), default is False
-    :returns: tuple of (storage URL, auth token)
-    :raises ClientException: HTTP GET request to auth URL failed
-    """
     parsed, conn = http_connection(url)
     conn.request('GET', parsed.path, '',
                  {'X-Auth-User': user, 'X-Auth-Key': key})
@@ -173,6 +185,64 @@ def get_auth(url, user, key, snet=False):
                                                 resp.getheader('x-auth-token'))
 
 
+def _get_auth_v2_0(url, user, key, snet):
+    if ':' in user:
+        tenant, user = user.split(':')
+    else:
+        tenant = user
+
+    body = {"auth": {"tenantName": tenant,
+                     "passwordCredentials":
+                         {"username": user, "password": key}}}
+    token_url = urljoin(url, "tokens")
+    resp, body = json_request("POST", token_url, body=body)
+    token_id = None
+    try:
+        url = None
+        catalogs = body['access']['serviceCatalog']
+        for service in catalogs:
+            if service['type'] == 'object-store':
+                url = service['endpoints'][0]['publicURL']
+        token_id = body['access']['token']['id']
+        if not url:
+            raise ClientException("There is no object-store endpoint " \
+                                  "on this auth server.")
+    except(KeyError, IndexError):
+        raise ClientException("Error while getting answers from auth server")
+
+    if snet:
+        parsed = list(urlparse(url))
+        # Second item in the list is the netloc
+        parsed[1] = 'snet-' + parsed[1]
+        url = urlunparse(parsed)
+
+    return url, token_id
+
+
+def get_auth(url, user, key, snet=False, auth_version="1.0"):
+    """
+    Get authentication/authorization credentials.
+
+    The snet parameter is used for Rackspace's ServiceNet internal network
+    implementation. In this function, it simply adds *snet-* to the beginning
+    of the host name for the returned storage URL. With Rackspace Cloud Files,
+    use of this network path causes no bandwidth charges but requires the
+    client to be running on Rackspace's ServiceNet network.
+
+    :param url: authentication/authorization URL
+    :param user: user to authenticate as
+    :param key: key or password for authorization
+    :param snet: use SERVICENET internal network (see above), default is False
+    :param auth_version: OpenStack authentication version (default is 1.0)
+    :returns: tuple of (storage URL, auth token)
+    :raises: ClientException: HTTP GET request to auth URL failed
+    """
+    if auth_version in ["1.0", "1"]:
+        return _get_auth_v1_0(url, user, key, snet)
+    elif auth_version in ["2.0", "2"]:
+        return _get_auth_v2_0(url, user, key, snet)
+
+
 def get_account(url, token, marker=None, limit=None, prefix=None,
                 http_conn=None, full_listing=False):
     """
@@ -280,11 +350,13 @@ def post_account(url, token, headers, http_conn=None):
     body = resp.read()
     if resp.status < 200 or resp.status >= 300:
         raise ClientException('Account POST failed',
-                http_scheme=parsed.scheme, http_host=conn.host,
+                              http_scheme=parsed.scheme,
-                http_port=conn.port, http_path=parsed.path,
+                              http_host=conn.host,
-                http_status=resp.status, http_reason=resp.reason,
+                              http_port=conn.port,
-                http_response_content=body)
+                              http_path=parsed.path,
-
+                              http_status=resp.status,
+                              http_reason=resp.reason,
+                              http_response_content=body)
 
 def get_container(url, token, container, marker=None, limit=None,
                   prefix=None, delimiter=None, http_conn=None,
@@ -720,7 +792,8 @@ class Connection(object):
     """Convenience class to make requests that will also retry the request"""
 
     def __init__(self, authurl, user, key, retries=5, preauthurl=None,
-                 preauthtoken=None, snet=False, starting_backoff=1):
+                 preauthtoken=None, snet=False, starting_backoff=1,
+                 auth_version="1"):
         """
         :param authurl: authentication URL
         :param user: user name to authenticate as
@@ -730,6 +803,7 @@ class Connection(object):
         :param preauthtoken: authentication token (if you have already
                              authenticated)
         :param snet: use SERVICENET internal network default is False
+        :param auth_version: Openstack auth version.
         """
         self.authurl = authurl
         self.user = user
@@ -741,9 +815,11 @@ class Connection(object):
         self.attempts = 0
         self.snet = snet
         self.starting_backoff = starting_backoff
+        self.auth_version = auth_version
 
     def get_auth(self):
-        return get_auth(self.authurl, self.user, self.key, snet=self.snet)
+        return get_auth(self.authurl, self.user, self.key, snet=self.snet,
+                        auth_version=self.auth_version)
 
     def http_connection(self):
         return http_connection(self.url)

test/unit/common/test_client.py

@@ -16,7 +16,6 @@
 # TODO: More tests
 import socket
 import unittest
-from StringIO import StringIO
 from urlparse import urlparse
 
 # TODO: mock http connection class with more control over headers
@@ -25,25 +24,6 @@ from test.unit.proxy.test_server import fake_http_connect
 from swift.common import client as c
 
 
-class TestHttpHelpers(unittest.TestCase):
-
-    def test_quote(self):
-        value = 'standard string'
-        self.assertEquals('standard%20string', c.quote(value))
-        value = u'\u0075nicode string'
-        self.assertEquals('unicode%20string', c.quote(value))
-
-    def test_http_connection(self):
-        url = 'http://www.test.com'
-        _junk, conn = c.http_connection(url)
-        self.assertTrue(isinstance(conn, c.HTTPConnection))
-        url = 'https://www.test.com'
-        _junk, conn = c.http_connection(url)
-        self.assertTrue(isinstance(conn, c.HTTPSConnection))
-        url = 'ftp://www.test.com'
-        self.assertRaises(c.ClientException, c.http_connection, url)
-
-
 class TestClientException(unittest.TestCase):
 
     def test_is_exception(self):
@@ -115,6 +95,7 @@ class MockHttpTest(unittest.TestCase):
     def setUp(self):
         def fake_http_connection(*args, **kwargs):
             _orig_http_connection = c.http_connection
+            return_read = kwargs.get('return_read')
 
             def wrapper(url, proxy=None):
                 parsed, _conn = _orig_http_connection(url, proxy=proxy)
@@ -130,7 +111,7 @@ class MockHttpTest(unittest.TestCase):
                 def read(*args, **kwargs):
                     conn.has_been_read = True
                     return _orig_read(*args, **kwargs)
-                conn.read = read
+                conn.read = return_read or read
 
                 return parsed, conn
             return wrapper
@@ -139,6 +120,36 @@ class MockHttpTest(unittest.TestCase):
     def tearDown(self):
         reload(c)
 
+
+class TestHttpHelpers(MockHttpTest):
+
+    def test_quote(self):
+        value = 'standard string'
+        self.assertEquals('standard%20string', c.quote(value))
+        value = u'\u0075nicode string'
+        self.assertEquals('unicode%20string', c.quote(value))
+
+    def test_http_connection(self):
+        url = 'http://www.test.com'
+        _junk, conn = c.http_connection(url)
+        self.assertTrue(isinstance(conn, c.HTTPConnection))
+        url = 'https://www.test.com'
+        _junk, conn = c.http_connection(url)
+        self.assertTrue(isinstance(conn, c.HTTPSConnection))
+        url = 'ftp://www.test.com'
+        self.assertRaises(c.ClientException, c.http_connection, url)
+
+    def test_json_request(self):
+        def read(*args, **kwargs):
+            body = {'a': '1',
+                    'b': '2'}
+            return c.json_dumps(body)
+        c.http_connection = self.fake_http_connection(200, return_read=read)
+        url = 'http://www.test.com'
+        _junk, conn = c.json_request('GET', url, body={'username': 'user1',
+                                                       'password': 'secure'})
+        self.assertTrue(type(conn) is dict)
+
 # TODO: following tests are placeholders, need more tests, better coverage
 
 
@@ -150,6 +161,27 @@ class TestGetAuth(MockHttpTest):
         self.assertEquals(url, None)
         self.assertEquals(token, None)
 
+    def test_auth_v1(self):
+        c.http_connection = self.fake_http_connection(200)
+        url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf',
+                                auth_version="1.0")
+        self.assertEquals(url, None)
+        self.assertEquals(token, None)
+
+    def test_auth_v2(self):
+        def read(*args, **kwargs):
+            acct_url = 'http://127.0.01/AUTH_FOO'
+            body = {'access': {'serviceCatalog':
+                                   [{u'endpoints': [{'publicURL': acct_url}],
+                                     'type': 'object-store'}],
+                               'token': {'id': 'XXXXXXX'}}}
+            return c.json_dumps(body)
+        c.http_connection = self.fake_http_connection(200, return_read=read)
+        url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf',
+                                auth_version="2.0")
+        self.assertTrue(url.startswith("http"))
+        self.assertTrue(token)
+
 
 class TestGetAccount(MockHttpTest):