Import common keystone configuration from ironic

This code is used by both ironic and ironic-inspector with barely
any difference and will be required to import the JSON RPC code.

To prevent IPA from depending on keystoneauth, the new requirement
is added as a new extra feature "keystone".

Change-Id: I8bc08ec9e081a67d1687033413fee63698e14e69

extra-requirements.txt

@@ -0,0 +1,3 @@
+# This file mirrors all extra requirements from setup.cfg and must be kept
+# in sync. It is used both in unit tests and when building docs.
+keystoneauth1>=4.2.0 # Apache-2.0

ironic_lib/exception.py

@@ -181,3 +181,16 @@ class Unauthorized(IronicException):
 
 class ConfigInvalid(IronicException):
     _msg_fmt = _("Invalid configuration file. %(error_msg)s")
+
+
+class CatalogNotFound(IronicException):
+    _msg_fmt = _("Service type %(service_type)s with endpoint type "
+                 "%(endpoint_type)s not found in keystone service catalog.")
+
+
+class KeystoneUnauthorized(IronicException):
+    _msg_fmt = _("Not authorized in Keystone.")
+
+
+class KeystoneFailure(IronicException):
+    pass

ironic_lib/keystone.py

@@ -0,0 +1,196 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Central place for handling Keystone authorization and service lookup."""
+
+import copy
+import functools
+
+from keystoneauth1 import exceptions as ks_exception
+from keystoneauth1 import loading as ks_loading
+from keystoneauth1 import service_token
+from keystoneauth1 import token_endpoint
+from oslo_config import cfg
+from oslo_log import log as logging
+
+from ironic_lib import exception
+
+
+CONF = cfg.CONF
+LOG = logging.getLogger(__name__)
+
+DEFAULT_VALID_INTERFACES = ['internal', 'public']
+
+
+def ks_exceptions(f):
+    """Wraps keystoneclient functions and centralizes exception handling."""
+    @functools.wraps(f)
+    def wrapper(group, *args, **kwargs):
+        try:
+            return f(group, *args, **kwargs)
+        except ks_exception.EndpointNotFound:
+            service_type = kwargs.get(
+                'service_type',
+                getattr(getattr(CONF, group), 'service_type', group))
+            endpoint_type = kwargs.get('endpoint_type', 'internal')
+            raise exception.CatalogNotFound(
+                service_type=service_type, endpoint_type=endpoint_type)
+        except (ks_exception.Unauthorized, ks_exception.AuthorizationFailure):
+            raise exception.KeystoneUnauthorized()
+        except (ks_exception.NoMatchingPlugin,
+                ks_exception.MissingRequiredOptions) as e:
+            raise exception.ConfigInvalid(str(e))
+        except Exception as e:
+            LOG.exception('Keystone request failed with unexpected exception')
+            raise exception.KeystoneFailure(str(e))
+    return wrapper
+
+
+@ks_exceptions
+def get_session(group, **session_kwargs):
+    """Loads session object from options in a configuration file section.
+
+    The session_kwargs will be passed directly to keystoneauth1 Session
+    and will override the values loaded from config.
+    Consult keystoneauth1 docs for available options.
+
+    :param group: name of the config section to load session options from
+
+    """
+    return ks_loading.load_session_from_conf_options(
+        CONF, group, **session_kwargs)
+
+
+@ks_exceptions
+def get_auth(group, **auth_kwargs):
+    """Loads auth plugin from options in a configuration file section.
+
+    The auth_kwargs will be passed directly to keystoneauth1 auth plugin
+    and will override the values loaded from config.
+    Note that the accepted kwargs will depend on auth plugin type as defined
+    by [group]auth_type option.
+    Consult keystoneauth1 docs for available auth plugins and their options.
+
+    :param group: name of the config section to load auth plugin options from
+
+    """
+    try:
+        auth = ks_loading.load_auth_from_conf_options(CONF, group,
+                                                      **auth_kwargs)
+    except ks_exception.MissingRequiredOptions:
+        LOG.error('Failed to load auth plugin from group %s', group)
+        raise
+    return auth
+
+
+@ks_exceptions
+def get_adapter(group, **adapter_kwargs):
+    """Loads adapter from options in a configuration file section.
+
+    The adapter_kwargs will be passed directly to keystoneauth1 Adapter
+    and will override the values loaded from config.
+    Consult keystoneauth1 docs for available adapter options.
+
+    :param group: name of the config section to load adapter options from
+
+    """
+    return ks_loading.load_adapter_from_conf_options(CONF, group,
+                                                     **adapter_kwargs)
+
+
+def get_endpoint(group, **adapter_kwargs):
+    """Get an endpoint from an adapter.
+
+    The adapter_kwargs will be passed directly to keystoneauth1 Adapter
+    and will override the values loaded from config.
+    Consult keystoneauth1 docs for available adapter options.
+
+    :param group: name of the config section to load adapter options from
+    :raises: CatalogNotFound if the endpoint is not found
+    """
+    result = get_adapter(group, **adapter_kwargs).get_endpoint()
+    if not result:
+        service_type = adapter_kwargs.get(
+            'service_type',
+            getattr(getattr(CONF, group), 'service_type', group))
+        endpoint_type = adapter_kwargs.get('endpoint_type', 'internal')
+        raise exception.CatalogNotFound(
+            service_type=service_type, endpoint_type=endpoint_type)
+    return result
+
+
+def get_service_auth(context, endpoint, service_auth):
+    """Create auth plugin wrapping both user and service auth.
+
+    When properly configured and using auth_token middleware,
+    requests with valid service auth will not fail
+    if the user token is expired.
+
+    Ideally we would use the plugin provided by auth_token middleware
+    however this plugin isn't serialized yet.
+    """
+    # TODO(pas-ha) use auth plugin from context when it is available
+    user_auth = token_endpoint.Token(endpoint, context.auth_token)
+    return service_token.ServiceTokenAuthWrapper(user_auth=user_auth,
+                                                 service_auth=service_auth)
+
+
+def register_auth_opts(conf, group, service_type=None):
+    """Register session- and auth-related options
+
+    Registers only basic auth options shared by all auth plugins.
+    The rest are registered at runtime depending on auth plugin used.
+    """
+    ks_loading.register_session_conf_options(conf, group)
+    ks_loading.register_auth_conf_options(conf, group)
+    CONF.set_default('auth_type', default='password', group=group)
+    ks_loading.register_adapter_conf_options(conf, group)
+    conf.set_default('valid_interfaces', DEFAULT_VALID_INTERFACES, group=group)
+    # TODO(pas-ha) use os-service-type to try find the service_type by the
+    # config group name assuming it is a project name (e.g. 'glance')
+    if service_type:
+        conf.set_default('service_type', service_type, group=group)
+
+
+def add_auth_opts(options, service_type=None):
+    """Add auth options to sample config
+
+    As these are dynamically registered at runtime,
+    this adds options for most used auth_plugins
+    when generating sample config.
+    """
+    def add_options(opts, opts_to_add):
+        for new_opt in opts_to_add:
+            for opt in opts:
+                if opt.name == new_opt.name:
+                    break
+            else:
+                opts.append(new_opt)
+
+    opts = copy.deepcopy(options)
+    opts.insert(0, ks_loading.get_auth_common_conf_options()[0])
+    # NOTE(dims): There are a lot of auth plugins, we just generate
+    # the config options for a few common ones
+    plugins = ['password', 'v2password', 'v3password']
+    for name in plugins:
+        plugin = ks_loading.get_plugin_loader(name)
+        add_options(opts, ks_loading.get_auth_plugin_conf_options(plugin))
+    add_options(opts, ks_loading.get_session_conf_options())
+    if service_type:
+        adapter_opts = ks_loading.get_adapter_conf_options(
+            include_deprecated=False)
+        # adding defaults for valid interfaces
+        cfg.set_defaults(adapter_opts, service_type=service_type,
+                         valid_interfaces=DEFAULT_VALID_INTERFACES)
+        add_options(opts, adapter_opts)
+    opts.sort(key=lambda x: x.name)
+    return opts

ironic_lib/tests/base.py

@@ -57,6 +57,10 @@ class IronicLibTestCase(test_base.BaseTestCase):
             # subprocess.Popen is a class
             self.patch(subprocess, 'Popen', DoNotCallPopen)
 
+    def config(self, **kw):
+        """Override config options for a test."""
+        self.cfg_fixture.config(**kw)
+
 
 def do_not_call(*args, **kwargs):
     """Helper function to raise an exception if it is called"""

ironic_lib/tests/test_keystone.py

@@ -0,0 +1,116 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from unittest import mock
+
+from keystoneauth1 import loading as ks_loading
+from oslo_config import cfg
+
+from ironic_lib import exception
+from ironic_lib import keystone
+from ironic_lib.tests import base
+
+
+class KeystoneTestCase(base.IronicLibTestCase):
+
+    def setUp(self):
+        super(KeystoneTestCase, self).setUp()
+        self.test_group = 'test_group'
+        self.cfg_fixture.conf.register_group(cfg.OptGroup(self.test_group))
+        keystone.register_auth_opts(self.cfg_fixture.conf, self.test_group,
+                                    service_type='vikings')
+        self.config(auth_type='password',
+                    group=self.test_group)
+        # NOTE(pas-ha) this is due to auth_plugin options
+        # being dynamically registered on first load,
+        # but we need to set the config before
+        plugin = ks_loading.get_plugin_loader('password')
+        opts = ks_loading.get_auth_plugin_conf_options(plugin)
+        self.cfg_fixture.register_opts(opts, group=self.test_group)
+        self.config(auth_url='http://127.0.0.1:9898',
+                    username='fake_user',
+                    password='fake_pass',
+                    project_name='fake_tenant',
+                    group=self.test_group)
+
+    def test_get_session(self):
+        self.config(timeout=10, group=self.test_group)
+        session = keystone.get_session(self.test_group, timeout=20)
+        self.assertEqual(20, session.timeout)
+
+    def test_get_auth(self):
+        auth = keystone.get_auth(self.test_group)
+        self.assertEqual('http://127.0.0.1:9898', auth.auth_url)
+
+    def test_get_auth_fail(self):
+        # NOTE(pas-ha) 'password' auth_plugin is used,
+        # so when we set the required auth_url to None,
+        # MissingOption is raised
+        self.config(auth_url=None, group=self.test_group)
+        self.assertRaises(exception.ConfigInvalid,
+                          keystone.get_auth,
+                          self.test_group)
+
+    def test_get_adapter_from_config(self):
+        self.config(valid_interfaces=['internal', 'public'],
+                    group=self.test_group)
+        session = keystone.get_session(self.test_group)
+        adapter = keystone.get_adapter(self.test_group, session=session,
+                                       interface='admin')
+        self.assertEqual('admin', adapter.interface)
+        self.assertEqual(session, adapter.session)
+
+    @mock.patch('keystoneauth1.service_token.ServiceTokenAuthWrapper',
+                autospec=True)
+    @mock.patch('keystoneauth1.token_endpoint.Token', autospec=True)
+    def test_get_service_auth(self, token_mock, service_auth_mock):
+        ctxt = mock.Mock(spec=['auth_token'], auth_token='spam')
+        mock_auth = mock.Mock()
+        self.assertEqual(service_auth_mock.return_value,
+                         keystone.get_service_auth(ctxt, 'ham', mock_auth))
+        token_mock.assert_called_once_with('ham', 'spam')
+        service_auth_mock.assert_called_once_with(
+            user_auth=token_mock.return_value, service_auth=mock_auth)
+
+
+class AuthConfTestCase(base.IronicLibTestCase):
+
+    def setUp(self):
+        super(AuthConfTestCase, self).setUp()
+        self.test_group = 'test_group'
+        self.cfg_fixture.conf.register_group(cfg.OptGroup(self.test_group))
+        keystone.register_auth_opts(self.cfg_fixture.conf, self.test_group)
+        self.config(auth_type='password',
+                    group=self.test_group)
+        # NOTE(pas-ha) this is due to auth_plugin options
+        # being dynamically registered on first load,
+        # but we need to set the config before
+        plugin = ks_loading.get_plugin_loader('password')
+        opts = ks_loading.get_auth_plugin_conf_options(plugin)
+        self.cfg_fixture.register_opts(opts, group=self.test_group)
+        self.config(auth_url='http://127.0.0.1:9898',
+                    username='fake_user',
+                    password='fake_pass',
+                    project_name='fake_tenant',
+                    group=self.test_group)
+
+    def test_add_auth_opts(self):
+        opts = keystone.add_auth_opts([])
+        # check that there is no duplicates
+        names = {o.dest for o in opts}
+        self.assertEqual(len(names), len(opts))
+        # NOTE(pas-ha) checking for most standard auth and session ones only
+        expected = {'timeout', 'insecure', 'cafile', 'certfile', 'keyfile',
+                    'auth_type', 'auth_url', 'username', 'password',
+                    'tenant_name', 'project_name', 'trust_id',
+                    'domain_id', 'user_domain_id', 'project_domain_id'}
+        self.assertTrue(expected.issubset(names))

setup.cfg

@@ -36,3 +36,7 @@ oslo.config.opts =
     ironic_lib.metrics = ironic_lib.metrics_utils:list_opts
     ironic_lib.metrics_statsd = ironic_lib.metrics_statsd:list_opts
     ironic_lib.utils = ironic_lib.utils:list_opts
+
+[extra]
+keystone =
+    keystoneauth1>=4.2.0 # Apache-2.0

test-requirements.txt

@@ -8,4 +8,3 @@ oslotest>=3.2.0 # Apache-2.0
 
 # doc requirements
 doc8>=0.6.0 # Apache-2.0
-

tox.ini

@@ -15,6 +15,7 @@ setenv = VIRTUAL_ENV={envdir}
 deps =
   -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
   -r{toxinidir}/test-requirements.txt
+  -r{toxinidir}/extra-requirements.txt
   -r{toxinidir}/requirements.txt
 commands = stestr run {posargs}
 
@@ -63,6 +64,7 @@ commands = {posargs}
 deps =
   -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
   -r{toxinidir}/doc/requirements.txt
+  -r{toxinidir}/extra-requirements.txt
 commands =
   sphinx-build -W -b html doc/source doc/build/html
 
@@ -77,4 +79,5 @@ commands = sphinx-build -b latex doc/source doc/build/pdf
 deps =
   -c{toxinidir}/lower-constraints.txt
   -r{toxinidir}/test-requirements.txt
+  -r{toxinidir}/extra-requirements.txt
   -r{toxinidir}/requirements.txt