Start using keystoneauth for keystone sessions

This is being split out from keystoneclient. That's a happymaking.
In order to make this work, we also need to fix glance image data upload
for v2 put. It seems that the internal interface has changed. Pass the file
object and not a size. We could do the work to figure out how to do it
in two discreet steps, but I believe that would involved a chunk of
engineering on a dead interface and is not worth it.

Also, Change ignore-errors to ignore_errors

Change-Id: I816752f8f4d29e40d41622bd8a271eb5c8e5a9fb

.coveragerc

@@ -4,4 +4,4 @@ source = shade
 omit = shade/tests/*
 
 [report]
-ignore-errors = True
+ignore_errors = True

requirements.txt

@@ -3,9 +3,10 @@ pbr>=0.11,<2.0
 bunch
 decorator
 jsonpatch
-os-client-config>=1.6.2
+os-client-config>=1.7.4
 six
 
+keystoneauth1>=1.0.0
 python-novaclient>=2.21.0,!=2.27.0
 python-keystoneclient>=0.11.0
 python-glanceclient>=1.0.0

shade/__init__.py

@@ -17,7 +17,6 @@ import hashlib
 import inspect
 import logging
 import operator
-import os
 
 from cinderclient.v1 import client as cinder_client
 from designateclient.v1 import Client as designate_client
@@ -28,12 +27,13 @@ import glanceclient.exc
 from ironicclient import client as ironic_client
 from ironicclient import exceptions as ironic_exceptions
 import jsonpatch
-from keystoneclient import auth as ksc_auth
-from keystoneclient.auth import token_endpoint
+import keystoneauth1.exceptions
+from keystoneauth1 import loading
+from keystoneauth1 import plugin as ksc_plugin
+from keystoneauth1 import session as ksc_session
 from keystoneclient.v2_0 import client as k2_client
 from keystoneclient.v3 import client as k3_client
 from keystoneclient import exceptions as keystone_exceptions
-from keystoneclient import session as ksc_session
 from novaclient import client as nova_client
 from novaclient import exceptions as nova_exceptions
 from neutronclient.common import exceptions as neutron_exceptions
@@ -229,6 +229,9 @@ class OpenStackCloud(object):
         self.auth = cloud_config.get_auth_args()
         self.region_name = cloud_config.region_name
         self.auth_type = cloud_config.config['auth_type']
+        # provide backwards compat to the old name of this plugin
+        if self.auth_type == 'token_endpoint':
+            self.auth_type = 'admin_token'
         self.default_interface = cloud_config.get_interface()
         self.private = cloud_config.config.get('private', False)
         self.api_timeout = cloud_config.config['api_timeout']
@@ -344,18 +347,6 @@ class OpenStackCloud(object):
                 'compute', nova_client.Client)
         return self._nova_client
 
-    def _get_auth_plugin_class(self):
-        try:
-            if self.auth_type == 'token_endpoint':
-                return token_endpoint.Token
-            else:
-                return ksc_auth.get_plugin_class(self.auth_type)
-        except Exception as e:
-            self.log.debug("keystone auth plugin failure", exc_info=True)
-            raise OpenStackCloudException(
-                "Could not find auth plugin: {plugin} {error}".format(
-                    plugin=self.auth_type, error=str(e)))
-
     def _get_identity_client_class(self):
         if self.cloud_config.get_api_version('identity') == '3':
             return k3_client.Client
@@ -369,9 +360,18 @@ class OpenStackCloud(object):
     def keystone_session(self):
         if self._keystone_session is None:
 
-            auth_plugin = self._get_auth_plugin_class()
             try:
-                keystone_auth = auth_plugin(**self.auth)
+                loader = loading.get_plugin_loader(self.auth_type)
+            except keystoneauth1.exceptions.auth_plugins.NoMatchingPlugin:
+                self.log.debug(
+                    "keystoneauth could not find auth plugin {plugin}".format(
+                        plugin=self.auth_type), exc_info=True)
+                raise OpenStackCloudException(
+                    "No auth plugin named {plugin}".format(
+                        plugin=self.auth_type))
+
+            try:
+                keystone_auth = loader.plugin_class(**self.auth)
             except Exception as e:
                 self.log.debug(
                     "keystone couldn't construct plugin", exc_info=True)
@@ -401,7 +401,7 @@ class OpenStackCloud(object):
     @property
     def service_catalog(self):
         return self.keystone_session.auth.get_access(
-            self.keystone_session).service_catalog.get_data()
+            self.keystone_session).service_catalog.catalog
 
     @property
     def auth_token(self):
@@ -732,7 +732,7 @@ class OpenStackCloud(object):
             # keystone is a special case in keystone, because what?
             if service_key == 'identity':
                 endpoint = self.keystone_session.get_endpoint(
-                    interface=ksc_auth.AUTH_INTERFACE)
+                    interface=ksc_plugin.AUTH_INTERFACE)
             else:
                 endpoint = self.keystone_session.get_endpoint(
                     service_type=self.cloud_config.get_service_type(
@@ -1382,12 +1382,8 @@ class OpenStackCloud(object):
                 image_kwargs[k] = str(v)
         image = self.manager.submitTask(_tasks.ImageCreate(
             name=name, **image_kwargs))
-        curr = image_data.tell()
-        image_data.seek(0, os.SEEK_END)
-        data_size = image_data.tell()
-        image_data.seek(curr)
         self.manager.submitTask(_tasks.ImageUpload(
-            image_id=image.id, image_data=image_data, image_size=data_size))
+            image_id=image.id, image_data=image_data))
         return image
 
     def _upload_image_put_v1(self, name, image_data, **image_kwargs):

shade/tests/unit/test_caching.py

@@ -15,9 +15,11 @@ import tempfile
 
 import mock
 import os_client_config as occ
+import testtools
 import yaml
 
 import shade
+from shade import exc
 from shade import meta
 from shade.tests import fakes
 from shade.tests.unit import base
@@ -299,7 +301,7 @@ class TestMemoryCache(base.TestCase):
                 'owner_specified.shade.sha256': mock.ANY}
         glance_mock.images.create.assert_called_with(**args)
         glance_mock.images.upload.assert_called_with(
-            image_data=mock.ANY, image_id=fake_image.id, image_size=1)
+            image_data=mock.ANY, image_id=fake_image.id)
         fake_image_dict = meta.obj_to_dict(fake_image)
         self.assertEqual([fake_image_dict], self.cloud.list_images())
 
@@ -399,3 +401,8 @@ class TestMemoryCache(base.TestCase):
         self.cloud.list_images.invalidate(self.cloud)
         self.assertEqual(
             [fi, fi2], [dict(x) for x in self.cloud.list_images()])
+
+    def test_get_auth_bogus(self):
+        self.cloud.auth_type = 'bogus'
+        with testtools.ExpectedException(exc.OpenStackCloudException):
+            self.cloud.keystone_session

shade/tests/unit/test_create_server.py

@@ -32,7 +32,8 @@ class TestCreateServer(base.TestCase):
     def setUp(self):
         super(TestCreateServer, self).setUp()
         config = os_client_config.OpenStackConfig()
-        self.client = OpenStackCloud(cloud_config=config.get_one_cloud())
+        self.client = OpenStackCloud(
+            cloud_config=config.get_one_cloud(validate=False))
 
     def test_create_server_with_create_exception(self):
         """

shade/tests/unit/test_delete_server.py

@@ -42,7 +42,8 @@ class TestDeleteServer(base.TestCase):
     def setUp(self):
         super(TestDeleteServer, self).setUp()
         config = os_client_config.OpenStackConfig()
-        self.cloud = OpenStackCloud(cloud_config=config.get_one_cloud())
+        self.cloud = OpenStackCloud(
+            cloud_config=config.get_one_cloud(validate=False))
 
     @mock.patch('shade.OpenStackCloud.nova_client')
     def test_delete_server(self, nova_mock):

shade/tests/unit/test_domain_params.py

@@ -26,7 +26,7 @@ class TestDomainParams(base.TestCase):
 
     def setUp(self):
         super(TestDomainParams, self).setUp()
-        self.cloud = shade.openstack_cloud()
+        self.cloud = shade.openstack_cloud(validate=False)
 
     @mock.patch.object(occ.cloud_config.CloudConfig, 'get_api_version')
     @mock.patch.object(shade.OpenStackCloud, '_get_project')

shade/tests/unit/test_endpoints.py

@@ -40,7 +40,8 @@ class TestCloudEndpoints(base.TestCase):
     def setUp(self):
         super(TestCloudEndpoints, self).setUp()
         config = os_client_config.OpenStackConfig()
-        self.client = OperatorCloud(cloud_config=config.get_one_cloud())
+        self.client = OperatorCloud(
+            cloud_config=config.get_one_cloud(validate=False))
         self.mock_ks_endpoints = \
             [FakeEndpoint(**kwa) for kwa in self.mock_endpoints]
 

shade/tests/unit/test_flavors.py

@@ -24,7 +24,7 @@ class TestFlavors(base.TestCase):
 
     def setUp(self):
         super(TestFlavors, self).setUp()
-        self.op_cloud = shade.operator_cloud()
+        self.op_cloud = shade.operator_cloud(validate=False)
 
     @mock.patch.object(shade.OpenStackCloud, 'nova_client')
     def test_create_flavor(self, mock_nova):

shade/tests/unit/test_floating_ip_common.py

@@ -31,7 +31,8 @@ class TestFloatingIP(base.TestCase):
     def setUp(self):
         super(TestFloatingIP, self).setUp()
         config = os_client_config.OpenStackConfig()
-        self.client = OpenStackCloud(cloud_config=config.get_one_cloud())
+        self.client = OpenStackCloud(
+            cloud_config=config.get_one_cloud(validate=False))
 
     @patch.object(OpenStackCloud, 'get_floating_ip')
     @patch.object(OpenStackCloud, 'attach_ip_to_server')

shade/tests/unit/test_floating_ip_neutron.py

@@ -123,7 +123,8 @@ class TestFloatingIP(base.TestCase):
         super(TestFloatingIP, self).setUp()
         # floating_ip_source='neutron' is default for OpenStackCloud()
         config = os_client_config.OpenStackConfig()
-        self.client = OpenStackCloud(cloud_config=config.get_one_cloud())
+        self.client = OpenStackCloud(
+            cloud_config=config.get_one_cloud(validate=False))
 
     @patch.object(OpenStackCloud, 'neutron_client')
     @patch.object(OpenStackCloud, 'has_service')

shade/tests/unit/test_floating_ip_nova.py

@@ -71,7 +71,8 @@ class TestFloatingIP(base.TestCase):
     def setUp(self):
         super(TestFloatingIP, self).setUp()
         config = os_client_config.OpenStackConfig()
-        self.client = OpenStackCloud(cloud_config=config.get_one_cloud())
+        self.client = OpenStackCloud(
+            cloud_config=config.get_one_cloud(validate=False))
 
     @patch.object(OpenStackCloud, 'nova_client')
     @patch.object(OpenStackCloud, 'has_service')

shade/tests/unit/test_floating_ip_pool.py

@@ -35,7 +35,8 @@ class TestFloatingIPPool(base.TestCase):
     def setUp(self):
         super(TestFloatingIPPool, self).setUp()
         config = os_client_config.OpenStackConfig()
-        self.client = OpenStackCloud(cloud_config=config.get_one_cloud())
+        self.client = OpenStackCloud(
+            cloud_config=config.get_one_cloud(validate=False))
 
     @patch.object(OpenStackCloud, '_has_nova_extension')
     @patch.object(OpenStackCloud, 'nova_client')

shade/tests/unit/test_identity_domains.py

@@ -32,7 +32,7 @@ class TestIdentityDomains(base.TestCase):
 
     def setUp(self):
         super(TestIdentityDomains, self).setUp()
-        self.cloud = shade.operator_cloud()
+        self.cloud = shade.operator_cloud(validate=False)
 
     @mock.patch.object(shade.OpenStackCloud, 'keystone_client')
     def test_list_identity_domains(self, mock_keystone):

shade/tests/unit/test_keypair.py

@@ -27,7 +27,7 @@ class TestKeypair(base.TestCase):
 
     def setUp(self):
         super(TestKeypair, self).setUp()
-        self.cloud = shade.openstack_cloud()
+        self.cloud = shade.openstack_cloud(validate=False)
 
     @patch.object(shade.OpenStackCloud, 'nova_client')
     def test_create_keypair(self, mock_nova):

shade/tests/unit/test_meta.py

@@ -105,7 +105,7 @@ class TestMeta(testtools.TestCase):
 
     def test_get_server_ip(self):
         srv = meta.obj_to_dict(FakeServer())
-        cloud = shade.openstack_cloud()
+        cloud = shade.openstack_cloud(validate=False)
         self.assertEqual(PRIVATE_V4, meta.get_server_private_ip(srv))
         self.assertEqual(PUBLIC_V4, meta.get_server_external_ipv4(cloud, srv))
 
@@ -124,7 +124,7 @@ class TestMeta(testtools.TestCase):
 
         srv = meta.obj_to_dict(fakes.FakeServer(
             id='test-id', name='test-name', status='ACTIVE'))
-        cloud = shade.openstack_cloud()
+        cloud = shade.openstack_cloud(validate=False)
 
         self.assertEqual(PRIVATE_V4, meta.get_server_private_ip(srv, cloud))
         mock_has_service.assert_called_once_with('network')
@@ -154,7 +154,7 @@ class TestMeta(testtools.TestCase):
         srv = meta.obj_to_dict(fakes.FakeServer(
             id='test-id', name='test-name', status='ACTIVE'))
         ip = meta.get_server_external_ipv4(
-            cloud=shade.openstack_cloud(), server=srv)
+            cloud=shade.openstack_cloud(validate=False), server=srv)
 
         self.assertEqual(PUBLIC_V4, ip)
         self.assertFalse(mock_get_server_ip.called)
@@ -164,7 +164,7 @@ class TestMeta(testtools.TestCase):
             id='test-id', name='test-name', status='ACTIVE',
             accessIPv4=PUBLIC_V4))
         ip = meta.get_server_external_ipv4(
-            cloud=shade.openstack_cloud(), server=srv)
+            cloud=shade.openstack_cloud(validate=False), server=srv)
 
         self.assertEqual(PUBLIC_V4, ip)
 
@@ -189,7 +189,7 @@ class TestMeta(testtools.TestCase):
         srv = meta.obj_to_dict(fakes.FakeServer(
             id='test-id', name='test-name', status='ACTIVE'))
         ip = meta.get_server_external_ipv4(
-            cloud=shade.openstack_cloud(), server=srv)
+            cloud=shade.openstack_cloud(validate=False), server=srv)
 
         self.assertEqual(PUBLIC_V4, ip)
         self.assertTrue(mock_get_server_ip.called)
@@ -209,7 +209,7 @@ class TestMeta(testtools.TestCase):
             id='test-id', name='test-name', status='ACTIVE',
             addresses={'test-net': [{'addr': PUBLIC_V4}]}))
         ip = meta.get_server_external_ipv4(
-            cloud=shade.openstack_cloud(), server=srv)
+            cloud=shade.openstack_cloud(validate=False), server=srv)
 
         self.assertEqual(PUBLIC_V4, ip)
         self.assertTrue(mock_get_server_ip.called)
@@ -230,7 +230,7 @@ class TestMeta(testtools.TestCase):
             id='test-id', name='test-name', status='ACTIVE',
             addresses={'test-net': [{'addr': PRIVATE_V4}]}))
         ip = meta.get_server_external_ipv4(
-            cloud=shade.openstack_cloud(), server=srv)
+            cloud=shade.openstack_cloud(validate=False), server=srv)
 
         self.assertIsNone(ip)
         self.assertTrue(mock_get_server_ip.called)

shade/tests/unit/test_object.py

@@ -30,7 +30,8 @@ class TestObject(base.TestCase):
     def setUp(self):
         super(TestObject, self).setUp()
         config = os_client_config.OpenStackConfig()
-        self.cloud = OpenStackCloud(cloud_config=config.get_one_cloud())
+        self.cloud = OpenStackCloud(
+            cloud_config=config.get_one_cloud(validate=False))
 
     @mock.patch.object(swift_client, 'Connection')
     @mock.patch.object(shade.OpenStackCloud, 'auth_token',

shade/tests/unit/test_operator_noauth.py

@@ -27,11 +27,15 @@ class TestShadeOperatorNoAuth(base.TestCase):
         URL in the auth data.  This is permits testing of the basic
         mechanism that enables Ironic noauth mode to be utilized with
         Shade.
+
+        @todo(mordred): remove the token in the next patch - occ handles
+        this right.
         """
         super(TestShadeOperatorNoAuth, self).setUp()
         self.cloud_noauth = shade.operator_cloud(
-            auth_type='None',
-            auth=dict(endpoint="http://localhost:6385")
+            auth_type='admin_token',
+            auth=dict(endpoint="http://localhost:6385", token='foo'),
+            validate=False,
         )
 
     @mock.patch.object(shade.OperatorCloud, 'get_session_endpoint')
@@ -45,5 +49,5 @@ class TestShadeOperatorNoAuth(base.TestCase):
         was still called.
         """
         self.cloud_noauth.patch_machine('name', {})
-        self.assertFalse(mock_endpoint.called)
+        self.assertTrue(mock_endpoint.called)
         self.assertTrue(mock_client.called)

shade/tests/unit/test_port.py

@@ -145,7 +145,8 @@ class TestPort(base.TestCase):
     def setUp(self):
         super(TestPort, self).setUp()
         config = os_client_config.OpenStackConfig()
-        self.client = OpenStackCloud(cloud_config=config.get_one_cloud())
+        self.client = OpenStackCloud(
+            cloud_config=config.get_one_cloud(validate=False))
 
     @patch.object(OpenStackCloud, 'neutron_client')
     def test_create_port(self, mock_neutron_client):

shade/tests/unit/test_rebuild_server.py

@@ -32,7 +32,8 @@ class TestRebuildServer(base.TestCase):
     def setUp(self):
         super(TestRebuildServer, self).setUp()
         config = os_client_config.OpenStackConfig()
-        self.client = OpenStackCloud(cloud_config=config.get_one_cloud())
+        self.client = OpenStackCloud(
+            cloud_config=config.get_one_cloud(validate=False))
 
     def test_rebuild_server_rebuild_exception(self):
         """

shade/tests/unit/test_security_groups.py

@@ -57,7 +57,7 @@ class TestSecurityGroups(base.TestCase):
 
     def setUp(self):
         super(TestSecurityGroups, self).setUp()
-        self.cloud = shade.openstack_cloud()
+        self.cloud = shade.openstack_cloud(validate=False)
 
     @mock.patch.object(shade.OpenStackCloud, 'neutron_client')
     @mock.patch.object(shade.OpenStackCloud, 'nova_client')

shade/tests/unit/test_services.py

@@ -42,7 +42,8 @@ class CloudServices(base.TestCase):
     def setUp(self):
         super(CloudServices, self).setUp()
         config = os_client_config.OpenStackConfig()
-        self.client = OperatorCloud(cloud_config=config.get_one_cloud())
+        self.client = OperatorCloud(cloud_config=config.get_one_cloud(
+            validate=False))
         self.mock_ks_services = [FakeService(**kwa) for kwa in
                                  self.mock_services]
 

shade/tests/unit/test_shade.py

@@ -15,8 +15,6 @@
 import mock
 
 import glanceclient
-import keystoneclient.auth.identity.generic.password
-import keystoneclient.auth.token_endpoint
 from keystoneclient.v2_0 import client as k2_client
 from keystoneclient.v3 import client as k3_client
 from neutronclient.common import exceptions as n_exc
@@ -32,30 +30,11 @@ class TestShade(base.TestCase):
 
     def setUp(self):
         super(TestShade, self).setUp()
-        self.cloud = shade.openstack_cloud()
+        self.cloud = shade.openstack_cloud(validate=False)
 
     def test_openstack_cloud(self):
         self.assertIsInstance(self.cloud, shade.OpenStackCloud)
 
-    def test_get_auth_token_endpoint(self):
-        self.cloud.auth_type = 'token_endpoint'
-        plugin = self.cloud._get_auth_plugin_class()
-
-        self.assertIs(plugin, keystoneclient.auth.token_endpoint.Token)
-
-    def test_get_auth_bogus(self):
-        self.cloud.auth_type = 'bogus'
-        self.assertRaises(
-            exc.OpenStackCloudException,
-            self.cloud._get_auth_plugin_class)
-
-    def test_get_auth_password(self):
-        plugin = self.cloud._get_auth_plugin_class()
-
-        self.assertIs(
-            plugin,
-            keystoneclient.auth.identity.generic.password.Password)
-
     @mock.patch.object(
         os_client_config.cloud_config.CloudConfig, 'get_api_version')
     def test_get_client_v2(self, mock_api_version):
@@ -123,7 +102,7 @@ class TestShade(base.TestCase):
         mock_keystone_session.return_value = None
         self.cloud.glance_client
         mock_client.assert_called_with(
-            version='1', region_name='', service_name=None,
+            version='2', region_name='', service_name=None,
             interface='public',
             service_type='image', session=mock.ANY,
         )

shade/tests/unit/test_shade_operator.py

@@ -12,7 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from keystoneclient import auth as ksc_auth
+from keystoneauth1 import plugin as ksc_plugin
 
 import mock
 import testtools
@@ -29,7 +29,7 @@ class TestShadeOperator(base.TestCase):
 
     def setUp(self):
         super(TestShadeOperator, self).setUp()
-        self.cloud = shade.operator_cloud()
+        self.cloud = shade.operator_cloud(validate=False)
 
     def test_operator_cloud(self):
         self.assertIsInstance(self.cloud, shade.OperatorCloud)
@@ -768,7 +768,7 @@ class TestShadeOperator(base.TestCase):
     def test_get_session_endpoint_identity(self, session_mock):
         self.cloud.get_session_endpoint('identity')
         session_mock.get_endpoint.assert_called_with(
-            interface=ksc_auth.AUTH_INTERFACE)
+            interface=ksc_plugin.AUTH_INTERFACE)
 
     @mock.patch.object(shade.OpenStackCloud, 'keystone_session')
     def test_has_service_no(self, session_mock):