Migrate to keystoneauth from keystoneclient

As a stepping stone to the os-client-config patch, first switch to
using keystoneauth, its Session and its argparse registration and
plugin loading to sort out any issues with that level of plumbing.
The next patch will layer on the ability to use os-client-config
for argument processing and client construction.

Change-Id: Id681e5eb56b47d06000620f7c92c9b0c5f8d4408

doc/source/api.rst

@@ -19,22 +19,28 @@ If you prefer string value, you can use ``1.1`` (deprecated now), ``2`` or
 ``2.X`` (where X is a microversion).
 
 
-Alternatively, you can create a client instance using the keystoneclient
+Alternatively, you can create a client instance using the keystoneauth
 session API::
 
-    >>> from keystoneclient.auth.identity import v2
+    >>> from keystoneauth1 import loading
-    >>> from keystoneclient import session
+    >>> from keystoneauth1 import session
     >>> from novaclient import client
-    >>> auth = v2.Password(auth_url=AUTH_URL,
+    >>> loader = loading.get_plugin_loader('password')
-    ...                    username=USERNAME,
+    >>> auth = loader.Password(auth_url=AUTH_URL,
-    ...                    password=PASSWORD,
+    ...                        username=USERNAME,
-    ...                    tenant_name=PROJECT_ID)
+    ...                        password=PASSWORD,
+    ...                        project_id=PROJECT_ID)
     >>> sess = session.Session(auth=auth)
     >>> nova = client.Client(VERSION, session=sess)
 
-For more information on this keystoneclient API, see `Using Sessions`_.
+If you have PROJECT_NAME instead of a PROJECT_ID, use the project_name
+parameter. Similarly, if your cloud uses keystone v3 and you have a DOMAIN_NAME
+or DOMAIN_ID, provide it as `user_domain_(name|id)` and if you are using a
+PROJECT_NAME also provide the domain information as `project_domain_(name|id)`.
 
-.. _Using Sessions: http://docs.openstack.org/developer/python-keystoneclient/using-sessions.html
+For more information on this keystoneauth API, see `Using Sessions`_.
+
+.. _Using Sessions: http://docs.openstack.org/developer/keystoneauth/using-sessions.html
 
 It is also possible to use an instance as a context manager in which case
 there will be a session kept alive for the duration of the with statement::

novaclient/client.py

@@ -32,8 +32,8 @@ import pkgutil
 import re
 import warnings
 
-from keystoneclient import adapter
+from keystoneauth1 import adapter
-from keystoneclient import session
+from keystoneauth1 import session
 from oslo_utils import importutils
 from oslo_utils import netutils
 import pkg_resources
@@ -80,7 +80,7 @@ class SessionClient(adapter.LegacyJsonAdapter):
         kwargs.setdefault('headers', kwargs.get('headers', {}))
         api_versions.update_headers(kwargs["headers"], self.api_version)
         # NOTE(jamielennox): The standard call raises errors from
-        # keystoneclient, where we need to raise the novaclient errors.
+        # keystoneauth1, where we need to raise the novaclient errors.
         raise_exc = kwargs.pop('raise_exc', True)
         with utils.record_time(self.times, self.timings, method, url):
             resp, body = super(SessionClient, self).request(url,
@@ -680,6 +680,8 @@ def _construct_http_client(username=None, password=None, project_id=None,
                            user_id=None, connection_pool=False, session=None,
                            auth=None, user_agent='python-novaclient',
                            interface=None, api_version=None, **kwargs):
+    # TODO(mordred): If not session, just make a Session, then return
+    # SessionClient always
     if session:
         return SessionClient(session=session,
                              auth=auth,
@@ -806,7 +808,7 @@ def Client(version, *args, **kwargs):
     (where X is a microversion).
 
 
-    Alternatively, you can create a client instance using the keystoneclient
+    Alternatively, you can create a client instance using the keystoneauth
     session API. See "The novaclient Python API" page at
     python-novaclient's doc.
     """

novaclient/exceptions.py

@@ -231,7 +231,7 @@ _code_map = dict((c.http_status, c) for c in _error_classes)
 class InvalidUsage(RuntimeError):
     """This function call is invalid in the way you are using this client.
 
-    Due to the transition to using keystoneclient some function calls are no
+    Due to the transition to using keystoneauth some function calls are no
     longer available. You should make a similar call to the session object
     instead.
     """

novaclient/shell.py

@@ -23,11 +23,9 @@ import argparse
 import getpass
 import logging
 import sys
+import warnings
 
-from keystoneclient.auth.identity.generic import password
+from keystoneauth1 import loading
-from keystoneclient.auth.identity.generic import token
-from keystoneclient.auth.identity import v3 as identity
-from keystoneclient import session as ksession
 from oslo_utils import encodeutils
 from oslo_utils import importutils
 from oslo_utils import strutils
@@ -246,23 +244,33 @@ class NovaClientArgumentParser(argparse.ArgumentParser):
 class OpenStackComputeShell(object):
     times = []
 
-    def _append_global_identity_args(self, parser):
+    def _append_global_identity_args(self, parser, argv):
         # Register the CLI arguments that have moved to the session object.
-        ksession.Session.register_cli_options(parser)
+        loading.register_session_argparse_arguments(parser)
+        # Peek into argv to see if os-auth-token or os-token were given,
+        # in which case, the token auth plugin is what the user wants
+        # else, we'll default to password
+        default_auth_plugin = 'password'
+        if 'os-token' in argv:
+            default_auth_plugin = 'token'
+        loading.register_auth_argparse_arguments(
+            parser, argv, default=default_auth_plugin)
 
         parser.set_defaults(insecure=cliutils.env('NOVACLIENT_INSECURE',
                             default=False))
-
+        parser.set_defaults(os_auth_url=cliutils.env('OS_AUTH_URL',
-        identity.Password.register_argparse_arguments(parser)
+                                                     'NOVA_URL'))
 
         parser.set_defaults(os_username=cliutils.env('OS_USERNAME',
                                                      'NOVA_USERNAME'))
         parser.set_defaults(os_password=cliutils.env('OS_PASSWORD',
                                                      'NOVA_PASSWORD'))
-        parser.set_defaults(os_auth_url=cliutils.env('OS_AUTH_URL',
+        parser.set_defaults(os_project_name=cliutils.env(
-                                                     'NOVA_URL'))
+            'OS_PROJECT_NAME', 'OS_TENANT_NAME', 'NOVA_PROJECT_ID'))
+        parser.set_defaults(os_project_id=cliutils.env(
+            'OS_PROJECT_ID', 'OS_TENANT_ID'))
 
-    def get_base_parser(self):
+    def get_base_parser(self, argv):
         parser = NovaClientArgumentParser(
             prog='nova',
             description=__doc__.strip(),
@@ -305,8 +313,7 @@ class OpenStackComputeShell(object):
 
         parser.add_argument(
             '--os-auth-token',
-            default=cliutils.env('OS_AUTH_TOKEN'),
+            help=argparse.SUPPRESS)
-            help='Defaults to env[OS_AUTH_TOKEN].')
 
         parser.add_argument(
             '--os_username',
@@ -316,21 +323,10 @@ class OpenStackComputeShell(object):
             '--os_password',
             help=argparse.SUPPRESS)
 
-        parser.add_argument(
-            '--os-tenant-name',
-            metavar='<auth-tenant-name>',
-            default=cliutils.env('OS_TENANT_NAME', 'NOVA_PROJECT_ID'),
-            help=_('Defaults to env[OS_TENANT_NAME].'))
         parser.add_argument(
             '--os_tenant_name',
             help=argparse.SUPPRESS)
 
-        parser.add_argument(
-            '--os-tenant-id',
-            metavar='<auth-tenant-id>',
-            default=cliutils.env('OS_TENANT_ID'),
-            help=_('Defaults to env[OS_TENANT_ID].'))
-
         parser.add_argument(
             '--os_auth_url',
             help=argparse.SUPPRESS)
@@ -348,7 +344,7 @@ class OpenStackComputeShell(object):
             '--os-auth-system',
             metavar='<auth-system>',
             default=cliutils.env('OS_AUTH_SYSTEM'),
-            help='Defaults to env[OS_AUTH_SYSTEM].')
+            help=argparse.SUPPRESS)
         parser.add_argument(
             '--os_auth_system',
             help=argparse.SUPPRESS)
@@ -426,12 +422,12 @@ class OpenStackComputeShell(object):
         # The auth-system-plugins might require some extra options
         novaclient.auth_plugin.load_auth_system_opts(parser)
 
-        self._append_global_identity_args(parser)
+        self._append_global_identity_args(parser, argv)
 
         return parser
 
-    def get_subcommand_parser(self, version, do_help=False):
+    def get_subcommand_parser(self, version, do_help=False, argv=None):
-        parser = self.get_base_parser()
+        parser = self.get_base_parser(argv)
 
         self.subcommands = {}
         subparsers = parser.add_subparsers(metavar='<subcommand>')
@@ -529,23 +525,9 @@ class OpenStackComputeShell(object):
                             format=streamformat)
         logging.getLogger('iso8601').setLevel(logging.WARNING)
 
-    def _get_keystone_auth(self, session, auth_url, **kwargs):
-        auth_token = kwargs.pop('auth_token', None)
-        if auth_token:
-            return token.Token(auth_url, auth_token, **kwargs)
-        else:
-            return password.Password(
-                auth_url,
-                username=kwargs.pop('username'),
-                user_id=kwargs.pop('user_id'),
-                password=kwargs.pop('password'),
-                user_domain_id=kwargs.pop('user_domain_id'),
-                user_domain_name=kwargs.pop('user_domain_name'),
-                **kwargs)
-
     def main(self, argv):
         # Parse args once to find version and debug settings
-        parser = self.get_base_parser()
+        parser = self.get_base_parser(argv)
 
         # NOTE(dtroyer): Hackery to handle --endpoint_type due to argparse
         #                thinking usage-list --end is ambiguous; but it
@@ -554,6 +536,10 @@ class OpenStackComputeShell(object):
         if '--endpoint_type' in argv:
             spot = argv.index('--endpoint_type')
             argv[spot] = '--endpoint-type'
+        # For backwards compat with old os-auth-token parameter
+        if '--os-auth-token' in argv:
+            spot = argv.index('--os-auth-token')
+            argv[spot] = '--os-token'
 
         (args, args_list) = parser.parse_known_args(argv)
 
@@ -579,8 +565,10 @@ class OpenStackComputeShell(object):
         os_username = args.os_username
         os_user_id = args.os_user_id
         os_password = None  # Fetched and set later as needed
-        os_tenant_name = args.os_tenant_name
+        os_project_name = getattr(
-        os_tenant_id = args.os_tenant_id
+            args, 'os_project_name', getattr(args, 'os_tenant_name', None))
+        os_project_id = getattr(
+            args, 'os_project_id', getattr(args, 'os_tenant_id', None))
         os_auth_url = args.os_auth_url
         os_region_name = args.os_region_name
         os_auth_system = args.os_auth_system
@@ -603,10 +591,14 @@ class OpenStackComputeShell(object):
         # Finally, authenticate unless we have both.
         # Note if we don't auth we probably don't have a tenant ID so we can't
         # cache the token.
-        auth_token = args.os_auth_token if args.os_auth_token else None
+        auth_token = getattr(args, 'os_token', None)
         management_url = bypass_url if bypass_url else None
 
         if os_auth_system and os_auth_system != "keystone":
+            warnings.warn(_(
+                'novaclient auth plugins that are not keystone are deprecated.'
+                ' Auth plugins should now be done as plugins to keystoneauth'
+                ' and selected with --os-auth-type or OS_AUTH_TYPE'))
             auth_plugin = novaclient.auth_plugin.load_plugin(os_auth_system)
         else:
             auth_plugin = None
@@ -648,8 +640,7 @@ class OpenStackComputeShell(object):
                           "or user id via --os-username, --os-user-id, "
                           "env[OS_USERNAME] or env[OS_USER_ID]"))
 
-            if not any([args.os_tenant_name, args.os_tenant_id,
+            if not any([os_project_name, os_project_id]):
-                        args.os_project_id, args.os_project_name]):
                 raise exc.CommandError(_("You must provide a project name or"
                                          " project id via --os-project-name,"
                                          " --os-project-id, env[OS_PROJECT_ID]"
@@ -669,34 +660,20 @@ class OpenStackComputeShell(object):
                           "default url with --os-auth-system "
                           "or env[OS_AUTH_SYSTEM]"))
 
-            project_id = args.os_project_id or args.os_tenant_id
-            project_name = args.os_project_name or args.os_tenant_name
             if use_session:
                 # Not using Nova auth plugin, so use keystone
                 with utils.record_time(self.times, args.timings,
                                        'auth_url', args.os_auth_url):
-                    keystone_session = (ksession.Session
+                    keystone_session = (
-                                        .load_from_cli_options(args))
+                        loading.load_session_from_argparse_arguments(args))
-                    keystone_auth = self._get_keystone_auth(
+                    keystone_auth = (
-                        keystone_session,
+                        loading.load_auth_from_argparse_arguments(args))
-                        args.os_auth_url,
-                        username=args.os_username,
-                        user_id=args.os_user_id,
-                        user_domain_id=args.os_user_domain_id,
-                        user_domain_name=args.os_user_domain_name,
-                        password=args.os_password,
-                        auth_token=args.os_auth_token,
-                        project_id=project_id,
-                        project_name=project_name,
-                        project_domain_id=args.os_project_domain_id,
-                        project_domain_name=args.os_project_domain_name)
             else:
                 # set password for auth plugins
                 os_password = args.os_password
 
         if (not skip_auth and
-                not any([args.os_tenant_id, args.os_tenant_name,
+                not any([os_project_name, os_project_id])):
-                         args.os_project_id, args.os_project_name])):
             raise exc.CommandError(_("You must provide a project name or"
                                      " project id via --os-project-name,"
                                      " --os-project-id, env[OS_PROJECT_ID]"
@@ -713,8 +690,8 @@ class OpenStackComputeShell(object):
         # microversion, so we just pass version 2 at here.
         self.cs = client.Client(
             api_versions.APIVersion("2.0"),
-            os_username, os_password, os_tenant_name,
+            os_username, os_password, os_project_name,
-            tenant_id=os_tenant_id, user_id=os_user_id,
+            tenant_id=os_project_id, user_id=os_user_id,
             auth_url=os_auth_url, insecure=insecure,
             region_name=os_region_name, endpoint_type=endpoint_type,
             extensions=self.extensions, service_type=service_type,
@@ -745,7 +722,7 @@ class OpenStackComputeShell(object):
         self._run_extension_hooks('__pre_parse_args__')
 
         subcommand_parser = self.get_subcommand_parser(
-            api_version, do_help=do_help)
+            api_version, do_help=do_help, argv=argv)
         self.parser = subcommand_parser
 
         if args.help or not argv:
@@ -777,8 +754,8 @@ class OpenStackComputeShell(object):
         # Recreate client object with discovered version.
         self.cs = client.Client(
             api_version,
-            os_username, os_password, os_tenant_name,
+            os_username, os_password, os_project_name,
-            tenant_id=os_tenant_id, user_id=os_user_id,
+            tenant_id=os_project_id, user_id=os_user_id,
             auth_url=os_auth_url, insecure=insecure,
             region_name=os_region_name, endpoint_type=endpoint_type,
             extensions=self.extensions, service_type=service_type,

novaclient/tests/unit/fixture_data/client.py

@@ -11,9 +11,9 @@
 # under the License.
 
 import fixtures
-from keystoneclient.auth.identity import v2
+from keystoneauth1 import fixture
-from keystoneclient import fixture
+from keystoneauth1 import loading
-from keystoneclient import session
+from keystoneauth1 import session
 
 from novaclient.v2 import client as v2client
 
@@ -33,6 +33,7 @@ class V1(fixtures.Fixture):
 
         self.token = fixture.V2Token()
         self.token.set_scope()
+        self.discovery = fixture.V2Discovery(href=self.identity_url)
 
         s = self.token.add_service('compute')
         s.add_endpoint(self.compute_url)
@@ -48,6 +49,9 @@ class V1(fixtures.Fixture):
         self.requests.register_uri('POST', auth_url,
                                    json=self.token,
                                    headers=headers)
+        self.requests.register_uri('GET', self.identity_url,
+                                   json=self.discovery,
+                                   headers=headers)
         self.client = self.new_client()
 
     def new_client(self):
@@ -61,5 +65,7 @@ class SessionV1(V1):
 
     def new_client(self):
         self.session = session.Session()
-        self.session.auth = v2.Password(self.identity_url, 'xx', 'xx')
+        loader = loading.get_plugin_loader('password')
+        self.session.auth = loader.load_from_options(
+            auth_url=self.identity_url, username='xx', password='xx')
         return v2client.Client(session=self.session)

novaclient/tests/unit/test_auth_plugins.py

@@ -15,7 +15,7 @@
 
 import argparse
 
-from keystoneclient import fixture
+from keystoneauth1 import fixture
 import mock
 import pkg_resources
 import requests

novaclient/tests/unit/test_client.py

@@ -18,7 +18,7 @@ import json
 import logging
 
 import fixtures
-from keystoneclient import adapter
+from keystoneauth1 import adapter
 import mock
 import requests
 
@@ -30,7 +30,7 @@ import novaclient.v2.client
 
 class ClientConnectionPoolTest(utils.TestCase):
 
-    @mock.patch("keystoneclient.session.TCPKeepAliveAdapter")
+    @mock.patch("keystoneauth1.session.TCPKeepAliveAdapter")
     def test_get(self, mock_http_adapter):
         mock_http_adapter.side_effect = lambda: mock.Mock()
         pool = novaclient.client._ClientConnectionPool()

novaclient/tests/unit/test_service_catalog.py

@@ -11,7 +11,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from keystoneclient import fixture
+from keystoneauth1 import fixture
 
 from novaclient import exceptions
 from novaclient import service_catalog

novaclient/tests/unit/test_shell.py

@@ -16,7 +16,7 @@ import re
 import sys
 
 import fixtures
-from keystoneclient import fixture
+from keystoneauth1 import fixture
 import mock
 import prettytable
 import requests_mock

novaclient/tests/unit/v2/test_auth.py

@@ -14,7 +14,7 @@
 import copy
 import json
 
-from keystoneclient import fixture
+from keystoneauth1 import fixture
 import mock
 import requests
 

novaclient/tests/unit/v2/test_client.py

@@ -12,7 +12,7 @@
 
 import uuid
 
-from keystoneclient import session
+from keystoneauth1 import session
 
 from novaclient.tests.unit import utils
 from novaclient.v2 import client

novaclient/v2/shell.py

@@ -27,6 +27,7 @@ import logging
 import os
 import sys
 import time
+import warnings
 
 from oslo_utils import encodeutils
 from oslo_utils import strutils
@@ -3874,10 +3875,11 @@ def ensure_service_catalog_present(cs):
 
 def do_endpoints(cs, _args):
     """Discover endpoints that get returned from the authenticate services."""
+    warnings.warn(
+        "nova endpoints is deprecated, use openstack catalog list instead")
     if isinstance(cs.client, client.SessionClient):
-        auth = cs.client.auth
+        access = cs.client.auth.get_access(cs.client.session)
-        sc = auth.get_access(cs.client.session).service_catalog
+        for service in access.service_catalog.catalog:
-        for service in sc.get_data():
             _print_endpoints(service, cs.client.region_name)
     else:
         ensure_service_catalog_present(cs)
@@ -3926,12 +3928,14 @@ def _get_first_endpoint(endpoints, region):
     help=_('Wrap PKI tokens to a specified length, or 0 to disable.'))
 def do_credentials(cs, _args):
     """Show user credentials returned from auth."""
+    warnings.warn(
+        "nova credentials is deprecated, use openstack client instead")
     if isinstance(cs.client, client.SessionClient):
-        auth = cs.client.auth
+        access = cs.client.auth.get_access(cs.client.session)
-        sc = auth.get_access(cs.client.session).service_catalog
+        utils.print_dict(access._user, 'User Credentials',
-        utils.print_dict(sc.catalog['user'], 'User Credentials',
                          wrap=int(_args.wrap))
-        utils.print_dict(sc.get_token(), 'Token', wrap=int(_args.wrap))
+        if hasattr(access, '_token'):
+            utils.print_dict(access._token, 'Token', wrap=int(_args.wrap))
     else:
         ensure_service_catalog_present(cs)
         catalog = cs.client.service_catalog.catalog

releasenotes/notes/keystoneauth-8ec1e6be14cdbae3.yaml

@@ -0,0 +1,11 @@
+---
+features:
+- keystoneauth plugins are now supported.
+upgrade:
+- novaclient now requires the keystoneauth library.
+deprecations:
+- novaclient auth strategy plugins are deprecated. Please use
+  keystoneauth auth plugins instead.
+- nova credentials is deprecated. Please use openstack token issue
+- nova endpoints is deprecated. Please use openstack catalog list
+  instead.

requirements.txt

@@ -3,6 +3,7 @@
 # process, which may cause wedges in the gate later.
 pbr>=1.6
 argparse
+keystoneauth1>=2.1.0
 iso8601>=0.1.9
 oslo.i18n>=1.5.0 # Apache-2.0
 oslo.serialization>=1.10.0 # Apache-2.0
@@ -12,4 +13,3 @@ requests>=2.8.1
 simplejson>=2.2.0
 six>=1.9.0
 Babel>=1.3
-python-keystoneclient!=1.8.0,>=1.6.0

test-requirements.txt

@@ -8,6 +8,7 @@ discover
 fixtures>=1.3.1
 keyring>=5.5.1
 mock>=1.2
+python-keystoneclient!=1.8.0,>=1.6.0
 requests-mock>=0.7.0 # Apache-2.0
 sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2
 os-client-config!=1.6.2,>=1.4.0