Use keystoneclient for authentication.

- This allows us to delegate all 2.0 authentication directly to the
  library without reimplementing ourselves.
- Support reusing a token / storage-url without re-authenticating every
  time via the switch os_storage_url os_auth_token.
- Allow auth via tenant_id instead of just tenant_name via the switch
  os_tenant_id.
- Refactor a bit to make it easier in the future to add new OS features
  (i.e: region).
- Implements blueprint use-keystoneclient-for-swiftclient.
- Fixes bug 1016641.

Change-Id: I532f38a68af884de25326aaac05a2050f5ffa1c7

bin/swift

@@ -38,9 +38,9 @@ def get_conn(options):
     return Connection(options.auth,
                       options.user,
                       options.key,
-                      snet=options.snet,
+                      auth_version=options.auth_version,
-                      tenant_name=options.os_tenant_name,
+                      os_options=options.os_options,
-                      auth_version=options.auth_version)
+                      snet=options.snet)
 
 
 def mkdirs(path):
@@ -991,13 +991,6 @@ def parse_args(parser, args, enforce_requires=True):
         # Use 2.0 auth if none of the old args are present
         options.auth_version = '2.0'
 
-    if options.auth_version in ('2.0', '2') and not \
-            options.os_tenant_name and options.user and \
-            ':' in options.user:
-        (options.os_tenant_name,
-         options.os_username) = options.user.split(':')
-        options.user = options.os_username
-
     # Use new-style args if old ones not present
     if not options.auth and options.os_auth_url:
         options.auth = options.os_auth_url
@@ -1006,6 +999,15 @@ def parse_args(parser, args, enforce_requires=True):
     if not options.key and options.os_password:
         options.key = options.os_password
 
+    # Specific OpenStack options
+    options.os_options = {
+        'tenant_id': options.os_tenant_id,
+        'tenant_name': options.os_tenant_name,
+        'service_type': options.os_service_type,
+        'auth_token': options.os_auth_token,
+        'object_storage_url': options.os_storage_url,
+    }
+
     # Handle trailing '/' in URL
     if options.auth and not options.auth.endswith('/'):
         options.auth += '/'
@@ -1017,8 +1019,8 @@ Auth version 1.0 requires ST_AUTH, ST_USER, and ST_KEY environment variables
 to be set or overridden with -A, -U, or -K.
 
 Auth version 2.0 requires OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, and
-OS_TENANT_NAME to be set or overridden with --os_auth_url, --os_username,
+OS_TENANT_NAME OS_TENANT_ID to be set or overridden with --os-auth_url,
---os_password, or --os_tenant_name.'''.strip('\n'))
+--os_username, --os_password, --os_tenant_name or os_tenant_id.'''.strip('\n'))
     return options, args
 
 
@@ -1058,19 +1060,36 @@ Example:
     parser.add_option('-K', '--key', dest='key',
                       default=environ.get('ST_KEY'),
                       help='Key for obtaining an auth token')
-    parser.add_option('--os_auth_url', dest='os_auth_url',
-                      default=environ.get('OS_AUTH_URL'),
-                      help='Openstack auth URL. Defaults to env[OS_AUTH_URL].')
     parser.add_option('--os_username', dest='os_username',
                       default=environ.get('OS_USERNAME'),
                       help='Openstack username. Defaults to env[OS_USERNAME].')
+    parser.add_option('--os_password', dest='os_password',
+                      default=environ.get('OS_PASSWORD'),
+                      help='Openstack password. Defaults to env[OS_PASSWORD].')
+    parser.add_option('--os_tenant_id',
+                      default=environ.get('OS_TENANT_ID'),
+                      help='OpenStack tenant ID.' \
+                           'Defaults to env[OS_TENANT_ID]')
     parser.add_option('--os_tenant_name', dest='os_tenant_name',
                       default=environ.get('OS_TENANT_NAME'),
                       help='Openstack tenant name.' \
                            'Defaults to env[OS_TENANT_NAME].')
-    parser.add_option('--os_password', dest='os_password',
+    parser.add_option('--os_auth_url', dest='os_auth_url',
-                      default=environ.get('OS_PASSWORD'),
+                      default=environ.get('OS_AUTH_URL'),
-                      help='Openstack password. Defaults to env[OS_PASSWORD].')
+                      help='Openstack auth URL. Defaults to env[OS_AUTH_URL].')
+    parser.add_option('--os_auth_token', dest='os_auth_token',
+                      default=environ.get('OS_AUTH_TOKEN'),
+                      help='Openstack token. Defaults to env[OS_AUTH_TOKEN]')
+    parser.add_option('--os_storage_url',
+                      dest='os_storage_url',
+                      default=environ.get('OS_STORAGE_URL'),
+                      help='Openstack storage URL.' \
+                           'Defaults to env[OS_STORAGE_URL]')
+    parser.add_option('--os_service_type',
+                      dest='os_service_type',
+                      default=environ.get('OS_SERVICE_TYPE'),
+                      help='Openstack Service type.' \
+                           'Defaults to env[OS_SERVICE_TYPE]')
     parser.disable_interspersed_args()
     (options, args) = parse_args(parser, argv[1:], enforce_requires=False)
     parser.enable_interspersed_args()

swiftclient/client.py

@@ -20,10 +20,9 @@ Cloud Files client library used internally
 import socket
 import os
 import logging
-import httplib
 
 from urllib import quote as _quote
-from urlparse import urlparse, urlunparse, urljoin
+from urlparse import urlparse, urlunparse
 
 try:
     from eventlet.green.httplib import HTTPException, HTTPSConnection
@@ -202,7 +201,7 @@ def json_request(method, url, **kwargs):
     return resp, body
 
 
-def _get_auth_v1_0(url, user, key, snet):
+def get_auth_1_0(url, user, key, snet):
     parsed, conn = http_connection(url)
     method = 'GET'
     conn.request(method, parsed.path, '',
@@ -230,36 +229,26 @@ def _get_auth_v1_0(url, user, key, snet):
                                resp.getheader('x-auth-token'))
 
 
-def _get_auth_v2_0(url, user, tenant_name, key, snet):
+def get_keystoneclient_2_0(auth_url, user, key, os_options):
-    body = {'auth':
+    """
-            {'passwordCredentials': {'password': key, 'username': user},
+    Authenticate against a auth 2.0 server.
-             'tenantName': tenant_name}}
-    token_url = urljoin(url, "tokens")
-    resp, body = json_request("POST", token_url, body=body)
-    token_id = None
-    try:
-        url = None
-        catalogs = body['access']['serviceCatalog']
-        for service in catalogs:
-            if service['type'] == 'object-store':
-                url = service['endpoints'][0]['publicURL']
-        token_id = body['access']['token']['id']
-        if not url:
-            raise ClientException("There is no object-store endpoint "
-                                  "on this auth server.")
-    except(KeyError, IndexError):
-        raise ClientException("Error while getting answers from auth server")
 
-    if snet:
+    We are using the keystoneclient library for our 2.0 authentication.
-        parsed = list(urlparse(url))
+    """
-        # Second item in the list is the netloc
+    from keystoneclient.v2_0 import client as ksclient
-        parsed[1] = 'snet-' + parsed[1]
+    _ksclient = ksclient.Client(username=user,
-        url = urlunparse(parsed)
+                                password=key,
-
+                                tenant_name=os_options.get('tenant_name'),
-    return url, token_id
+                                tenant_id=os_options.get('tenant_id'),
+                                auth_url=auth_url)
+    service_type = os_options.get('service_type') or 'object-store'
+    endpoint = _ksclient.service_catalog.url_for(
+        service_type=service_type,
+        endpoint_type='publicURL')
+    return (endpoint, _ksclient.auth_token)
 
 
-def get_auth(url, user, key, snet=False, tenant_name=None, auth_version="1.0"):
+def get_auth(auth_url, user, key, **kwargs):
     """
     Get authentication/authorization credentials.
 
@@ -268,26 +257,43 @@ def get_auth(url, user, key, snet=False, tenant_name=None, auth_version="1.0"):
     of the host name for the returned storage URL. With Rackspace Cloud Files,
     use of this network path causes no bandwidth charges but requires the
     client to be running on Rackspace's ServiceNet network.
-
-    :param url: authentication/authorization URL
-    :param user: user to authenticate as
-    :param key: key or password for authorization
-    :param snet: use SERVICENET internal network (see above), default is False
-    :param auth_version: OpenStack auth version, default is 1.0
-    :param tenant_name: The tenant/account name, required when connecting
-                        to a auth 2.0 system.
-    :returns: tuple of (storage URL, auth token)
-    :raises: ClientException: HTTP GET request to auth URL failed
     """
-    if auth_version in ["1.0", "1"]:
+    auth_version = kwargs.get('auth_version', '1')
-        return _get_auth_v1_0(url, user, key, snet)
+
-    elif auth_version in ["2.0", "2"]:
+    if auth_version in ['1.0', '1', 1]:
-        if not tenant_name and ':' in user:
+        return get_auth_1_0(auth_url,
-            (tenant_name, user) = user.split(':')
+                            user,
-        if not tenant_name:
+                            key,
+                            kwargs.get('snet'))
+
+    if auth_version in ['2.0', '2', 2]:
+
+        # We are allowing to specify a token/storage-url to re-use
+        # without having to re-authenticate.
+        if (kwargs['os_options'].get('object_storage_url') and
+                kwargs['os_options'].get('auth_token')):
+            return(kwargs['os_options'].get('object_storage_url'),
+                   kwargs['os_options'].get('auth_token'))
+
+        # We are handling a special use case here when we were
+        # allowing specifying the account/tenant_name with the -U
+        # argument
+        if not kwargs.get('tenant_name') and ':' in user:
+            (kwargs['os_options']['tenant_name'],
+             user) = user.split(':')
+
+        # We are allowing to have an tenant_name argument in get_auth
+        # directly without having os_options
+        if kwargs.get('tenant_name'):
+            kwargs['os_options']['tenant_name'] = kwargs['tenant_name']
+
+        if (not 'tenant_name' in kwargs['os_options']):
             raise ClientException('No tenant specified')
-        return _get_auth_v2_0(url, user, tenant_name, key, snet)
+
-    else:
+        (auth_url, token) = get_keystoneclient_2_0(auth_url, user,
+                                                   key, kwargs['os_options'])
+        return (auth_url, token)
+
     raise ClientException('Unknown auth_version %s specified.'
                           % auth_version)
 
@@ -898,8 +904,7 @@ class Connection(object):
 
     def __init__(self, authurl, user, key, retries=5, preauthurl=None,
                  preauthtoken=None, snet=False, starting_backoff=1,
-                 tenant_name=None,
+                 tenant_name=None, os_options={}, auth_version="1"):
-                 auth_version="1"):
         """
         :param authurl: authentication URL
         :param user: user name to authenticate as
@@ -912,6 +917,9 @@ class Connection(object):
         :param auth_version: OpenStack auth version, default is 1.0
         :param tenant_name: The tenant/account name, required when connecting
                             to a auth 2.0 system.
+        :param os_options: The OpenStack options which can have tenant_id,
+                           auth_token, service_type, tenant_name,
+                           object_storage_url
         """
         self.authurl = authurl
         self.user = user
@@ -924,13 +932,17 @@ class Connection(object):
         self.snet = snet
         self.starting_backoff = starting_backoff
         self.auth_version = auth_version
-        self.tenant_name = tenant_name
+        if tenant_name:
+            os_options['tenant_name'] = tenant_name
+        self.os_options = os_options
 
     def get_auth(self):
-        return get_auth(self.authurl, self.user,
+        return get_auth(self.authurl,
-                        self.key, snet=self.snet,
+                        self.user,
-                        tenant_name=self.tenant_name,
+                        self.key,
-                        auth_version=self.auth_version)
+                        snet=self.snet,
+                        auth_version=self.auth_version,
+                        os_options=self.os_options)
 
     def http_connection(self):
         return http_connection(self.url)

tests/test_swiftclient.py

@@ -19,7 +19,7 @@ import unittest
 from urlparse import urlparse
 
 # TODO: mock http connection class with more control over headers
-from utils import fake_http_connect
+from utils import fake_http_connect, fake_get_keystoneclient_2_0
 
 from swiftclient import client as c
 
@@ -175,42 +175,33 @@ class TestGetAuth(MockHttpTest):
         self.assertEquals(token, None)
 
     def test_auth_v2(self):
-        def read(*args, **kwargs):
+        c.get_keystoneclient_2_0 = fake_get_keystoneclient_2_0
-            acct_url = 'http://127.0.01/AUTH_FOO'
-            body = {'access': {'serviceCatalog':
-                                   [{u'endpoints': [{'publicURL': acct_url}],
-                                     'type': 'object-store'}],
-                               'token': {'id': 'XXXXXXX'}}}
-            return c.json_dumps(body)
-        c.http_connection = self.fake_http_connection(200, return_read=read)
         url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf',
-                                tenant_name='asdf', auth_version="2.0")
+                                os_options={'tenant_name': 'asdf'},
+                                auth_version="2.0")
         self.assertTrue(url.startswith("http"))
         self.assertTrue(token)
 
     def test_auth_v2_no_tenant_name(self):
-        def read(*args, **kwargs):
+        c.get_keystoneclient_2_0 = fake_get_keystoneclient_2_0
-            acct_url = 'http://127.0.01/AUTH_FOO'
-            body = {'access': {'serviceCatalog':
-                                   [{u'endpoints': [{'publicURL': acct_url}],
-                                     'type': 'object-store'}],
-                               'token': {'id': 'XXXXXXX'}}}
-            return c.json_dumps(body)
-        c.http_connection = self.fake_http_connection(200, return_read=read)
         self.assertRaises(c.ClientException, c.get_auth,
                           'http://www.tests.com', 'asdf', 'asdf',
+                          os_options={},
                           auth_version='2.0')
 
     def test_auth_v2_with_tenant_user_in_user(self):
-        def read(*args, **kwargs):
+        c.get_keystoneclient_2_0 = fake_get_keystoneclient_2_0
-            acct_url = 'http://127.0.01/AUTH_FOO'
-            body = {'access': {'serviceCatalog':
-                                   [{u'endpoints': [{'publicURL': acct_url}],
-                                     'type': 'object-store'}],
-                               'token': {'id': 'XXXXXXX'}}}
-            return c.json_dumps(body)
-        c.http_connection = self.fake_http_connection(200, return_read=read)
         url, token = c.get_auth('http://www.test.com', 'foo:bar', 'asdf',
+                                os_options={},
+                                auth_version="2.0")
+        self.assertTrue(url.startswith("http"))
+        self.assertTrue(token)
+
+    def test_auth_v2_tenant_name_no_os_options(self):
+        c.get_keystoneclient_2_0 = fake_get_keystoneclient_2_0
+        url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf',
+                                tenant_name='asdf',
+                                os_options={},
                                 auth_version="2.0")
         self.assertTrue(url.startswith("http"))
         self.assertTrue(token)

tests/utils.py

@@ -17,6 +17,14 @@ from httplib import HTTPException
 from eventlet import Timeout, sleep
 
 
+def fake_get_keystoneclient_2_0(auth_url,
+                                username,
+                                tenant_name,
+                                password,
+                                service_type='object-store'):
+    return ("http://url/", "token")
+
+
 def fake_http_connect(*code_iter, **kwargs):
 
     class FakeConn(object):

tools/pip-requires

@@ -1 +1,2 @@
 simplejson
+python-keystoneclient