Fix: TrustMiddleware unable to fetch trusts/credentials from identity service

When using token to init a new openstacksdk connection, the SDK try to fetch
another token from it, causing keystone raise exception and return 500 status
response. Refer: https://bugs.launchpad.net/keystone/+bug/1959674
Switch from `token` to `admin_token` to make the SDK session use the provided
token directly instead of fetching a new one.

Closes-bug: #2048452
Depends-On: https://review.opendev.org/c/openstack/senlin/+/905555
Change-Id: I8f9b2db3d4851cf54c2113b2fb0ae97ae38ac286

releasenotes/notes/bug-2048452-8a690353815601a0.yaml

@@ -0,0 +1,9 @@
+---
+fixes:
+  - |
+    [`bug 2048452 <https://bugs.launchpad.net/senlin/+bug/2048452>`_]
+    Fixed a bug where `TrustMiddleware` unable to fetch trusts/credentials
+    from Identity service, may be related to: 
+    https://bugs.launchpad.net/keystone/+bug/1959674
+    This bug is fixed by using `admin_token` instead of `token` auth method
+    to fetch trusts/credentials from Identity service.

senlin/api/middleware/trust.py

@@ -44,7 +44,6 @@ class TrustMiddleware(wsgi.Middleware):
         params = {
             'auth_url': ctx.auth_url,
             'token': ctx.auth_token,
-            'project_id': ctx.project_id,
             'user_id': ctx.user_id,
         }
         kc = driver_base.SenlinDriver().identity(params)

senlin/drivers/os/cinder_v2.py

@@ -19,7 +19,7 @@ class CinderClient(base.DriverBase):
 
     def __init__(self, params):
         super(CinderClient, self).__init__(params)
-        self.conn = sdk.create_connection(params)
+        self.conn = sdk.create_connection(params, service_type='block-storage')
         self.session = self.conn.session
 
     @sdk.translate_exception

senlin/drivers/os/glance_v2.py

@@ -19,7 +19,7 @@ class GlanceClient(base.DriverBase):
 
     def __init__(self, params):
         super(GlanceClient, self).__init__(params)
-        self.conn = sdk.create_connection(params)
+        self.conn = sdk.create_connection(params, service_type='image')
         self.session = self.conn.session
 
     @sdk.translate_exception

senlin/drivers/os/heat_v1.py

@@ -21,7 +21,7 @@ class HeatClient(base.DriverBase):
 
     def __init__(self, params):
         super(HeatClient, self).__init__(params)
-        self.conn = sdk.create_connection(params)
+        self.conn = sdk.create_connection(params, service_type='orchestration')
 
     @sdk.translate_exception
     def stack_create(self, **params):

senlin/drivers/os/mistral_v2.py

@@ -21,7 +21,7 @@ class MistralClient(base.DriverBase):
 
     def __init__(self, params):
         super(MistralClient, self).__init__(params)
-        self.conn = sdk.create_connection(params)
+        self.conn = sdk.create_connection(params, service_type='workflow')
         self.session = self.conn.session
 
     @sdk.translate_exception

senlin/drivers/os/neutron_v2.py

@@ -21,7 +21,7 @@ class NeutronClient(base.DriverBase):
 
     def __init__(self, params):
         super(NeutronClient, self).__init__(params)
-        self.conn = sdk.create_connection(params)
+        self.conn = sdk.create_connection(params, service_type='network')
 
     @sdk.translate_exception
     def network_get(self, name_or_id, ignore_missing=False):

senlin/drivers/os/nova_v2.py

@@ -26,7 +26,7 @@ class NovaClient(base.DriverBase):
 
     def __init__(self, params):
         super(NovaClient, self).__init__(params)
-        self.conn = sdk.create_connection(params)
+        self.conn = sdk.create_connection(params, service_type='compute')
         self.session = self.conn.session
 
     @sdk.translate_exception

senlin/drivers/os/octavia_v2.py

@@ -19,7 +19,7 @@ class OctaviaClient(base.DriverBase):
 
     def __init__(self, params):
         super(OctaviaClient, self).__init__(params)
-        self.conn = sdk.create_connection(params)
+        self.conn = sdk.create_connection(params, service_type='load-balancer')
 
     @sdk.translate_exception
     def loadbalancer_get(self, name_or_id, ignore_missing=False,

senlin/drivers/os/zaqar_v2.py

@@ -21,7 +21,7 @@ class ZaqarClient(base.DriverBase):
 
     def __init__(self, params):
         super(ZaqarClient, self).__init__(params)
-        self.conn = sdk.create_connection(params)
+        self.conn = sdk.create_connection(params, service_type='messaging')
         self.session = self.conn.session
 
     @sdk.translate_exception

senlin/drivers/sdk.py

@@ -17,13 +17,13 @@ import sys
 
 import functools
 import openstack
-from openstack import connection
 from openstack import exceptions as sdk_exc
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_serialization import jsonutils
 from requests import exceptions as req_exc
 
+from senlin.common import context
 from senlin.common import exception as senlin_exc
 from senlin import version
 
@@ -106,34 +106,61 @@ def translate_exception(func):
     return invoke_with_catch
 
 
-def create_connection(params=None):
+def create_connection(params=None, service_type='identity'):
-    if params is None:
+    """Create a connection to SDK service client."""
-        params = {}
+    params = params or {}
-
-    if 'token' in params:
-        params['auth_type'] = 'token'
-    params['app_name'] = USER_AGENT
-    params['app_version'] = version.version_info.version_string()
     params.setdefault('region_name', cfg.CONF.default_region_name)
     params.setdefault('identity_api_version', '3')
     params.setdefault('messaging_api_version', '2')
 
+    if 'token' in params:
+        # NOTE(daiplg): If existing token is provided, use admin_token plugin
+        # to authenticate to avoid fetching service catalog or determining
+        # scope info because of:
+        #   https://bugs.launchpad.net/keystone/+bug/1959674
+        # Refer: keystoneauth1.loading._plugins.admin_token.AdminToken
+        params['auth_type'] = 'admin_token'
+        if 'endpoint' not in params:
+            # NOTE(daiplg): Because there is no service catalog the endpoint
+            # that is supplied with initialization is used for all operations
+            # performed with this plugin so must be the full base URL to
+            # an actual service.
+            service_credentials = context.get_service_credentials() or {}
+            admin_connection = _create_connection(service_credentials)
+
+            region_name = params['region_name']
+            interface = service_credentials.get('interface', 'public')
+
+            temp_adapter = admin_connection.config.get_session_client(
+                service_type=service_type,
+                region_name=region_name,
+                allow_version_hack=True,
+            )
+            params['endpoint'] = temp_adapter.get_endpoint(
+                region_name=region_name,
+                interface=interface
+            )
+    return _create_connection(params)
+
+
+def _create_connection(params=None):
+    """Create a connection to SDK service client."""
+    params = params or {}
     try:
-        conn = connection.Connection(**params)
+        connection = openstack.connect(
+            load_envvars=False,
+            load_yaml_config=False,
+            insecure=not cfg.CONF.authentication.verify_ssl,
+            cafile=cfg.CONF.authentication.cafile,
+            cert=cfg.CONF.authentication.certfile,
+            key=cfg.CONF.authentication.keyfile,
+            app_name=USER_AGENT,
+            app_version=version.version_info.version_string(),
+            **params,
+        )
     except Exception as ex:
         raise parse_exception(ex)
-
+    return connection
-    if cfg.CONF.authentication.certfile and \
-            cfg.CONF.authentication.keyfile:
-        conn.session.cert = (cfg.CONF.authentication.certfile,
-                             cfg.CONF.authentication.keyfile)
-    if cfg.CONF.authentication.verify_ssl:
-        if cfg.CONF.authentication.cafile:
-            conn.session.verify = cfg.CONF.authentication.cafile
-        else:
-            conn.session.verify = cfg.CONF.authentication.verify_ssl
-
-    return conn
 
 
 def authenticate(**kwargs):
@@ -151,6 +178,7 @@ def authenticate(**kwargs):
 
 class FakeResourceObject(object):
     """Generate a fake SDK resource object based on given dictionary"""
+
     def __init__(self, params):
         for key in params:
             setattr(self, key, params[key])

senlin/tests/unit/api/middleware/test_trust.py

@@ -85,7 +85,6 @@ class TestTrustMiddleware(base.SenlinTestCase):
         mock_driver.assert_called_once_with()
         x_driver.identity.assert_called_once_with({
             'auth_url': self.context.auth_url,
-            'project_id': self.context.project_id,
             'user_id': self.context.user_id,
             'token': self.context.auth_token,
         })
@@ -123,7 +122,6 @@ class TestTrustMiddleware(base.SenlinTestCase):
         mock_driver.assert_called_once_with()
         x_driver.identity.assert_called_once_with({
             'auth_url': self.context.auth_url,
-            'project_id': self.context.project_id,
             'user_id': self.context.user_id,
             'token': self.context.auth_token,
         })
@@ -162,7 +160,6 @@ class TestTrustMiddleware(base.SenlinTestCase):
         mock_driver.assert_called_once_with()
         x_driver.identity.assert_called_once_with({
             'auth_url': self.context.auth_url,
-            'project_id': self.context.project_id,
             'user_id': self.context.user_id,
             'token': self.context.auth_token,
         })
@@ -205,7 +202,6 @@ class TestTrustMiddleware(base.SenlinTestCase):
         mock_driver.assert_called_once_with()
         x_driver.identity.assert_called_once_with({
             'auth_url': self.context.auth_url,
-            'project_id': self.context.project_id,
             'user_id': self.context.user_id,
             'token': self.context.auth_token,
         })

senlin/tests/unit/drivers/test_cinder_v2.py

@@ -32,7 +32,8 @@ class TestCinderV2(base.SenlinTestCase):
         self.vo = cinder_v2.CinderClient(self.conn_params)
 
     def test_init(self):
-        self.mock_create.assert_called_once_with(self.conn_params)
+        self.mock_create.assert_called_once_with(self.conn_params,
+                                                 service_type='block-storage')
         self.assertEqual(self.mock_conn, self.vo.conn)
 
     def test_volume_get(self):

senlin/tests/unit/drivers/test_glance_v2.py

@@ -35,7 +35,8 @@ class TestGlanceV2(base.SenlinTestCase):
         gc = glance_v2.GlanceClient(self.conn_params)
 
         self.assertEqual(self.fake_conn, gc.conn)
-        mock_create.assert_called_once_with(self.conn_params)
+        mock_create.assert_called_once_with(self.conn_params,
+                                            service_type='image')
 
     def test_image_find(self, mock_create):
         mock_create.return_value = self.fake_conn

senlin/tests/unit/drivers/test_heat_v1.py

@@ -34,7 +34,8 @@ class TestHeatV1(base.SenlinTestCase):
         self.hc = heat_v1.HeatClient(self.conn_params)
 
     def test_init(self):
-        self.mock_create.assert_called_once_with(self.conn_params)
+        self.mock_create.assert_called_once_with(self.conn_params,
+                                                 service_type='orchestration')
         self.assertEqual(self.mock_conn, self.hc.conn)
 
     def test_stack_create(self):

senlin/tests/unit/drivers/test_mistral_v2.py

@@ -34,7 +34,8 @@ class TestMistralV2(base.SenlinTestCase):
     def test_init(self):
         d = mistral_v2.MistralClient(self.conn_params)
 
-        self.mock_create.assert_called_once_with(self.conn_params)
+        self.mock_create.assert_called_once_with(self.conn_params,
+                                                 service_type='workflow')
         self.assertEqual(self.mock_conn, d.conn)
 
     def test_workflow_find(self):

senlin/tests/unit/drivers/test_neutron_v2.py

@@ -35,7 +35,8 @@ class TestNeutronV2Driver(base.SenlinTestCase):
     def test_init(self, mock_create_connection):
         params = self.conn_params
         neutron_v2.NeutronClient(params)
-        mock_create_connection.assert_called_once_with(params)
+        mock_create_connection.assert_called_once_with(params,
+                                                       service_type='network')
 
     def test_network_get_with_uuid(self):
         net_id = uuidutils.generate_uuid()

senlin/tests/unit/drivers/test_nova_v2.py

@@ -37,7 +37,8 @@ class TestNovaV2(base.SenlinTestCase):
 
     def test_init(self):
         d = nova_v2.NovaClient(self.conn_params)
-        self.mock_create.assert_called_once_with(self.conn_params)
+        self.mock_create.assert_called_once_with(self.conn_params,
+                                                 service_type='compute')
         self.assertEqual(self.mock_conn, d.conn)
 
     def test_flavor_find(self):

senlin/tests/unit/drivers/test_octavia_v2.py

@@ -33,7 +33,9 @@ class TestOctaviaV2Driver(base.SenlinTestCase):
     def test_init(self, mock_create_connection):
         params = self.conn_params
         octavia_v2.OctaviaClient(params)
-        mock_create_connection.assert_called_once_with(params)
+        mock_create_connection.assert_called_once_with(
+            params,
+            service_type='load-balancer')
 
     def test_loadbalancer_get(self):
         lb_id = 'loadbalancer_identifier'

senlin/tests/unit/drivers/test_sdk.py

@@ -13,7 +13,7 @@
 import types
 from unittest import mock
 
-from openstack import connection
+import openstack
 from oslo_serialization import jsonutils
 from requests import exceptions as req_exc
 
@@ -127,7 +127,6 @@ class OpenStackSDKTest(base.SenlinTestCase):
         self.assertEqual('Unknown Error', str(ex))
 
     def test_translate_exception_wrapper(self):
-
         @sdk.translate_exception
         def test_func(driver):
             return driver.__name__
@@ -136,7 +135,6 @@ class OpenStackSDKTest(base.SenlinTestCase):
         self.assertEqual(types.FunctionType, type(res))
 
     def test_translate_exception_with_exception(self):
-
         @sdk.translate_exception
         def test_func(driver):
             raise (Exception('test exception'))
@@ -149,24 +147,62 @@ class OpenStackSDKTest(base.SenlinTestCase):
         self.assertEqual(500, ex.code)
         self.assertEqual('BOOM', ex.message)
 
-    @mock.patch.object(connection, 'Connection')
+    @mock.patch.object(openstack, 'connect')
     def test_create_connection_token(self, mock_conn):
         x_conn = mock.Mock()
+        mock_session_client = mock.Mock()
+        x_conn.config.get_session_client.return_value = mock_session_client
         mock_conn.return_value = x_conn
+        mock_session_client.get_endpoint.return_value = 'https://FAKE_URL'
 
         res = sdk.create_connection({'token': 'TOKEN', 'foo': 'bar'})
 
         self.assertEqual(x_conn, res)
-        mock_conn.assert_called_once_with(
+        calls = [
-            app_name=sdk.USER_AGENT, app_version=self.app_version,
+            mock.call(
+                load_envvars=False,
+                load_yaml_config=False,
+                insecure=False,
+                cafile=None,
+                cert=None,
+                key=None,
+                app_name=sdk.USER_AGENT,
+                app_version=self.app_version,
+                auth_url='',
+                username='senlin',
+                password='',
+                project_name='service',
+                user_domain_name='Default',
+                project_domain_name='Default',
+                verify=True,
+                interface='public'
+            ),
+            mock.call().config.get_session_client(
+                service_type='identity',
+                region_name=None,
+                allow_version_hack=True
+            ),
+            mock.call().config.get_session_client().get_endpoint(
+                region_name=None, interface='public'),
+            mock.call(load_envvars=False,
+                      load_yaml_config=False,
+                      insecure=False,
+                      cafile=None,
+                      cert=None,
+                      key=None,
+                      app_name=sdk.USER_AGENT,
+                      app_version=self.app_version,
+                      token='TOKEN',
+                      foo='bar',
+                      region_name=None,
                       identity_api_version='3',
                       messaging_api_version='2',
-            region_name=None,
+                      auth_type='admin_token',
-            auth_type='token',
+                      endpoint='https://FAKE_URL'),
-            token='TOKEN',
+        ]
-            foo='bar')
+        mock_conn.assert_has_calls(calls)
 
-    @mock.patch.object(connection, 'Connection')
+    @mock.patch.object(openstack, 'connect')
     def test_create_connection_password(self, mock_conn):
         x_conn = mock.Mock()
         mock_conn.return_value = x_conn
@@ -176,7 +212,14 @@ class OpenStackSDKTest(base.SenlinTestCase):
 
         self.assertEqual(x_conn, res)
         mock_conn.assert_called_once_with(
-            app_name=sdk.USER_AGENT, app_version=self.app_version,
+            load_envvars=False,
+            load_yaml_config=False,
+            insecure=False,
+            cafile=None,
+            cert=None,
+            key=None,
+            app_name=sdk.USER_AGENT,
+            app_version=self.app_version,
             identity_api_version='3',
             messaging_api_version='2',
             region_name=None,
@@ -184,7 +227,7 @@ class OpenStackSDKTest(base.SenlinTestCase):
             password='abc',
             foo='bar')
 
-    @mock.patch.object(connection, 'Connection')
+    @mock.patch.object(openstack, 'connect')
     def test_create_connection_with_region(self, mock_conn):
         x_conn = mock.Mock()
         mock_conn.return_value = x_conn
@@ -193,12 +236,19 @@ class OpenStackSDKTest(base.SenlinTestCase):
 
         self.assertEqual(x_conn, res)
         mock_conn.assert_called_once_with(
-            app_name=sdk.USER_AGENT, app_version=self.app_version,
+            load_envvars=False,
+            load_yaml_config=False,
+            insecure=False,
+            cafile=None,
+            cert=None,
+            key=None,
+            app_name=sdk.USER_AGENT,
+            app_version=self.app_version,
             identity_api_version='3',
             messaging_api_version='2',
             region_name='REGION_ONE')
 
-    @mock.patch.object(connection, 'Connection')
+    @mock.patch.object(openstack, 'connect')
     @mock.patch.object(sdk, 'parse_exception')
     def test_create_connection_with_exception(self, mock_parse, mock_conn):
         ex_raw = Exception('Whatever')
@@ -210,7 +260,14 @@ class OpenStackSDKTest(base.SenlinTestCase):
                                sdk.create_connection)
 
         mock_conn.assert_called_once_with(
-            app_name=sdk.USER_AGENT, app_version=self.app_version,
+            load_envvars=False,
+            load_yaml_config=False,
+            insecure=False,
+            cafile=None,
+            cert=None,
+            key=None,
+            app_name=sdk.USER_AGENT,
+            app_version=self.app_version,
             identity_api_version='3',
             messaging_api_version='2',
             region_name=None)

senlin/tests/unit/drivers/test_zaqar_v2.py

@@ -35,7 +35,8 @@ class TestZaqarV2(base.SenlinTestCase):
 
     def test_init(self):
         zc = zaqar_v2.ZaqarClient(self.conn_params)
-        self.mock_create.assert_called_once_with(self.conn_params)
+        self.mock_create.assert_called_once_with(self.conn_params,
+                                                 service_type='messaging')
         self.assertEqual(self.mock_conn, zc.conn)
 
     def test_queue_create(self):