Remove RefStack dependency on Keystone client.

RefStack should use Tempest approach for retrieving Keystone service
Id. For RefStack purpose, raw API call to keystone should be enough
to get necessary information.

Closes-Bug: #1528357

Change-Id: I555308aabb59b638e22249ead882ca5b09043b7b

refstack_client/refstack_client.py

@@ -38,8 +38,6 @@ import time
 from Crypto.Hash import SHA256
 from Crypto.PublicKey import RSA
 from Crypto.Signature import PKCS1_v1_5
-from keystoneclient.v2_0 import client as ksclient2
-from keystoneclient.v3 import client as ksclient3
 import requests
 import requests.exceptions
 import six.moves
@@ -140,8 +138,9 @@ class RefstackClient:
 
         return os.path.join(tempest_dir, '.testrepository', subunit_file)
 
-    def _get_cpid_from_keystone(self, conf_file):
+    def _get_keystone_config(self, conf_file):
-        '''This will get the Keystone service ID which is used as the CPID.'''
+        '''This will get and return the keystone configs
+        from config file.'''
         try:
             # Prefer Keystone V3 API if it is enabled
             auth_version = (
@@ -151,7 +150,15 @@ class RefstackClient:
                                                   'api_v3')
                          and conf_file.has_option('identity', 'uri_v3'))
                 else 'v2')
-
+            if auth_version == 'v2':
+                auth_url = '%s/tokens' % (conf_file.get('identity', 'uri')
+                                          .rstrip('/'))
+            elif auth_version == 'v3':
+                auth_url = '%s/auth/tokens' % (conf_file.get('identity',
+                                               'uri_v3').rstrip('/'))
+            domain_name = 'Default'
+            if conf_file.has_option('identity', 'domain_name'):
+                domain_name = conf_file.get('identity', 'domain_name')
             if conf_file.has_option('auth', 'test_accounts_file'):
                 account_file = os.path.expanduser(
                     conf_file.get('auth', 'test_accounts_file'))
@@ -165,12 +172,17 @@ class RefstackClient:
                     self.logger.error('Accounts file %s found, '
                                       'but was empty.' % account_file)
                     exit(1)
-
                 account = accounts[0]
                 username = account.get('username')
                 password = account.get('password')
                 tenant_id = account.get('tenant_id')
                 tenant_name = account.get('tenant_name')
+                return {'auth_version': auth_version,
+                        'auth_url': auth_url,
+                        'domain_name': domain_name,
+                        'username': username, 'password': password,
+                        'tenant_id': tenant_id, 'tenant_name': tenant_name
+                        }
             else:
                 username = conf_file.get('identity', 'username')
                 password = conf_file.get('identity', 'password')
@@ -179,69 +191,107 @@ class RefstackClient:
                     tenant_id = conf_file.get('identity', 'tenant_id')
                 else:
                     tenant_id = None
-                    tenant_name = conf_file.get('identity', 'tenant_name')
+                tenant_name = conf_file.get('identity', 'tenant_name')
-
+                return {'auth_version': auth_version,
-            args = {
+                        'auth_url': auth_url,
-                'insecure': self.args.insecure,
+                        'domain_name': domain_name,
-                'username': username,
+                        'username': username, 'password': password,
-                'password': password
+                        'tenant_id': tenant_id, 'tenant_name': tenant_name}
-            }
-            if tenant_id:
-                args['tenant_id'] = tenant_id
-            else:
-                args['tenant_name'] = tenant_name
-
-            try:
-                if auth_version == 'v2':
-                    args['auth_url'] = conf_file.get('identity', 'uri')
-                    client = ksclient2.Client(**args)
-                    token = client.auth_ref
-                    for service in token['serviceCatalog']:
-                        if service['type'] == 'identity':
-                            if service['endpoints'][0]['id']:
-                                return service['endpoints'][0]['id']
-                    # Raise a key error if 'identity' was not found so that it
-                    # can be caught and have an appropriate error displayed.
-                    raise KeyError
-                elif auth_version == 'v3':
-                    args['auth_url'] = conf_file.get('identity', 'uri_v3')
-                    if conf_file.has_option('identity', 'domain_name'):
-                        args['project_domain_name'] = \
-                            conf_file.get('identity', 'domain_name')
-                        args['user_domain_name'] = conf_file.get('identity',
-                                                                 'domain_name')
-                    if conf_file.has_option('identity', 'region'):
-                        args['region_name'] = conf_file.get('identity',
-                                                            'region')
-                    client = ksclient3.Client(**args)
-                    token = client.auth_ref
-                    for service in token['catalog']:
-                        if service['type'] == 'identity' and service['id']:
-                            return service['id']
-                    # Raise a key error if 'identity' was not found. It will
-                    # be caught below as well.
-                    raise KeyError
-                else:
-                    raise ValueError('Auth_version %s is unsupported'
-                                     '' % auth_version)
-            # If a Key or Index Error was raised, one of the expected keys or
-            # indices for retrieving the identity service ID was not found.
-            except (KeyError, IndexError) as e:
-                self.logger.warning('Unable to retrieve CPID from Keystone %s '
-                                    'catalog. The catalog or the identity '
-                                    'service endpoint was not '
-                                    'found.' % auth_version)
-            except Exception as e:
-                self.logger.warning('Problems retrieving CPID from Keystone '
-                                    'using %s endpoint (%s)' % (auth_version,
-                                    args['auth_url']))
-
-            return self._generate_cpid_from_endpoint(args['auth_url'])
         except ConfigParser.Error as e:
             # Most likely a missing section or option in the config file.
             self.logger.error("Invalid Config File: %s" % e)
             exit(1)
 
+    def _generate_keystone_data(self, auth_config):
+        '''This will generate data for http post to keystone
+        API from auth_config.'''
+        auth_version = auth_config['auth_version']
+        auth_url = auth_config['auth_url']
+        if auth_version == 'v2':
+            password_credential = {'username': auth_config['username'],
+                                   'password': auth_config['password']}
+            if auth_config['tenant_id']:
+                data = {
+                    'auth': {
+                        'tenantId': auth_config['tenant_id'],
+                        'passwordCredentials': password_credential
+                    }
+                }
+            else:
+                data = {
+                    'auth': {
+                        'tenantName': auth_config['tenant_name'],
+                        'passwordCredentials': password_credential
+                    }
+                }
+            return auth_version, auth_url, data
+        elif auth_version == 'v3':
+            identity = {'methods': ['password'], 'password':
+                        {'user': {'name': auth_config['username'],
+                         'domain': {
+                             'name': auth_config['domain_name']
+                         },
+                        'password': auth_config['password']}}}
+            data = {
+                'auth': {
+                    'identity': identity,
+                    'scope': {
+                        'project': {
+                            'name': auth_config['username'],
+                            'domain': {'name': auth_config['domain_name']}
+                        }
+                    }
+                }
+            }
+            return auth_version, auth_url, data
+
+    def _get_cpid_from_keystone(self, auth_version, auth_url, content):
+        '''This will get the Keystone service ID which is used as the CPID.'''
+        try:
+            headers = {'content-type': 'application/json'}
+            response = requests.post(auth_url,
+                                     data=json.dumps(content),
+                                     headers=headers,
+                                     verify=not self.args.insecure)
+            rsp = response.json()
+            if response.status_code in (200, 203):
+                # keystone API v2 response.
+                access = rsp['access']
+                for service in access['serviceCatalog']:
+                    if service['type'] == 'identity':
+                        if service['endpoints'][0]['id']:
+                            return service['endpoints'][0]['id']
+                # Raise a key error if 'identity' was not found so that it
+                # can be caught and have an appropriate error displayed.
+                raise KeyError
+            elif response.status_code == 201:
+                # keystone API v3 response.
+                token = rsp['token']
+                for service in token['catalog']:
+                    if service['type'] == 'identity' and service['id']:
+                        return service['id']
+                # Raise a key error if 'identity' was not found.
+                # It will be caught below as well.
+                raise KeyError
+            else:
+                message = ('Invalid request with error '
+                           'code: %s. Error message: %s'
+                           '' % (rsp['error']['code'],
+                           rsp['error']['message']))
+                raise requests.exceptions.HTTPError(message)
+            # If a Key or Index Error was raised, one of the expected keys or
+            # indices for retrieving the identity service ID was not found.
+        except (KeyError, IndexError) as e:
+            self.logger.warning('Unable to retrieve CPID from Keystone %s '
+                                'catalog. The catalog or the identity '
+                                'service endpoint was not '
+                                'found.' % auth_version)
+        except Exception as e:
+            self.logger.warning('Problems retrieving CPID from Keystone '
+                                'using %s endpoint (%s) with error (%s)'
+                                % (auth_version, auth_url, e))
+        return self._generate_cpid_from_endpoint(auth_url)
+
     def _generate_cpid_from_endpoint(self, endpoint):
         '''This method will md5 hash the hostname of a Keystone endpoint to
            generate a CPID.'''
@@ -333,7 +383,10 @@ class RefstackClient:
         self._prep_test()
         results_file = self._get_next_stream_subunit_output_file(
             self.tempest_dir)
-        cpid = self._get_cpid_from_keystone(self.conf)
+        keystone_config = self._get_keystone_config(self.conf)
+        auth_version, auth_url, content = \
+            self._generate_keystone_data(keystone_config)
+        cpid = self._get_cpid_from_keystone(auth_version, auth_url, content)
 
         self.logger.info("Starting Tempest test...")
         start_time = time.time()

refstack_client/tests/unit/test_client.py

@@ -24,6 +24,7 @@ import httmock
 import mock
 from mock import MagicMock
 import unittest
+import requests
 
 
 import refstack_client.refstack_client as rc
@@ -68,7 +69,7 @@ class TestRefstackClient(unittest.TestCase):
                 argv.extend(('--', kwargs.get('test_cases', None)))
         return argv
 
-    def mock_keystone(self):
+    def mock_data(self):
         """
         Mock the Keystone client methods.
         """
@@ -76,24 +77,13 @@ class TestRefstackClient(unittest.TestCase):
                                          'endpoints': [{'id': 'test-id'}]}
         self.mock_identity_service_v3 = {'type': 'identity',
                                          'id': 'test-id'}
-        self.mock_ks2_client = MagicMock(
+        self.v2_config = {'auth_url': 'http://0.0.0.0:35357/v2.0/tokens',
-            name='ks_client',
+                          'auth_version': 'v2',
-            **{'auth_ref':
+                          'domain_name': 'Default',
-               {'serviceCatalog': [self.mock_identity_service_v2]}}
+                          'password': 'test',
-        )
+                          'tenant_id': 'admin_tenant_id',
-        self.mock_ks3_client = MagicMock(
+                          'tenant_name': 'tenant_name',
-            name='ks_client',
+                          'username': 'admin'}
-            **{'auth_ref':
-               {'catalog': [self.mock_identity_service_v3]}}
-        )
-        self.ks2_client_builder = self.patch(
-            'refstack_client.refstack_client.ksclient2.Client',
-            return_value=self.mock_ks2_client
-        )
-        self.ks3_client_builder = self.patch(
-            'refstack_client.refstack_client.ksclient3.Client',
-            return_value=self.mock_ks3_client
-        )
 
     def setUp(self):
         """
@@ -163,99 +153,6 @@ class TestRefstackClient(unittest.TestCase):
         expected_file = "/tempest/path/.testrepository/0"
         self.assertEqual(expected_file, output_file)
 
-    def test_get_cpid_from_keystone_with_tenant_id(self):
-        """
-        Test getting the CPID from Keystone using an admin tenant ID.
-        """
-        args = rc.parse_cli_args(self.mock_argv())
-        client = rc.RefstackClient(args)
-        client.tempest_dir = self.test_path
-        client._prep_test()
-        self.mock_keystone()
-        cpid = client._get_cpid_from_keystone(client.conf)
-        self.ks2_client_builder.assert_called_with(
-            username='admin', tenant_id='admin_tenant_id',
-            password='test', auth_url='http://0.0.0.0:35357/v2.0',
-            insecure=False
-        )
-        self.assertEqual('test-id', cpid)
-
-    def test_get_cpid_from_keystone_with_tenant_name(self):
-        """
-        Test getting the CPID from Keystone using an admin tenant name.
-        """
-        args = rc.parse_cli_args(self.mock_argv())
-        client = rc.RefstackClient(args)
-        client.tempest_dir = self.test_path
-        client._prep_test()
-        client.conf.remove_option('identity', 'tenant_id')
-        client.conf.set('identity', 'tenant_name', 'tenant_name')
-        self.mock_keystone()
-        cpid = client._get_cpid_from_keystone(client.conf)
-        self.ks2_client_builder.assert_called_with(
-            username='admin', tenant_name='tenant_name',
-            password='test', auth_url='http://0.0.0.0:35357/v2.0',
-            insecure=False
-        )
-        self.assertEqual('test-id', cpid)
-
-    def test_get_cpid_from_keystone_by_tenant_name_from_account_file(self):
-        """
-        Test getting a CPID from Keystone using an admin tenant name
-        from an accounts file.
-        """
-
-        args = rc.parse_cli_args(self.mock_argv())
-        client = rc.RefstackClient(args)
-        client.tempest_dir = self.test_path
-        client._prep_test()
-        client.conf.add_section('auth')
-        client.conf.set('auth',
-                        'test_accounts_file',
-                        '%s/test-accounts.yaml' % self.test_path)
-        self.mock_keystone()
-        cpid = client._get_cpid_from_keystone(client.conf)
-        self.ks2_client_builder.assert_called_with(
-            username='admin', tenant_name='tenant_name',
-            password='test', auth_url='http://0.0.0.0:35357/v2.0',
-            insecure=False
-        )
-        self.assertEqual('test-id', cpid)
-
-    def test_get_cpid_from_keystone_by_tenant_id_from_account_file(self):
-        """
-        Test getting a CPID from Keystone using an admin tenant ID
-        from an accounts file.
-        """
-
-        accounts = [
-            {
-                'username': 'admin',
-                'tenant_id': 'tenant_id',
-                'password': 'test'
-            }
-        ]
-        self.patch(
-            'refstack_client.refstack_client.read_accounts_yaml',
-            return_value=accounts)
-
-        args = rc.parse_cli_args(self.mock_argv())
-        client = rc.RefstackClient(args)
-        client.tempest_dir = self.test_path
-        client._prep_test()
-        client.conf.add_section('auth')
-        client.conf.set('auth',
-                        'test_accounts_file',
-                        '%s/test-accounts.yaml' % self.test_path)
-        self.mock_keystone()
-        cpid = client._get_cpid_from_keystone(client.conf)
-        self.ks2_client_builder.assert_called_with(
-            username='admin', tenant_id='tenant_id',
-            password='test', auth_url='http://0.0.0.0:35357/v2.0',
-            insecure=False
-        )
-        self.assertEqual('test-id', cpid)
-
     def test_get_cpid_account_file_not_found(self):
         """
         Test that the client will exit if an accounts file is specified,
@@ -271,11 +168,11 @@ class TestRefstackClient(unittest.TestCase):
                         'test_accounts_file',
                         '%s/some-file.yaml' % self.test_path)
 
-        self.mock_keystone()
+        self.mock_data()
         with self.assertRaises(SystemExit):
-            client._get_cpid_from_keystone(client.conf)
+            client._get_keystone_config(client.conf)
 
-    def test_get_cpid_account_file_empty(self):
+    def test_get_keystone_config_account_file_empty(self):
         """
         Test that the client will exit if an accounts file exists,
         but is empty.
@@ -294,119 +191,72 @@ class TestRefstackClient(unittest.TestCase):
                         'test_accounts_file',
                         '%s/some-file.yaml' % self.test_path)
 
-        self.mock_keystone()
+        self.mock_data()
         with self.assertRaises(SystemExit):
-            client._get_cpid_from_keystone(client.conf)
+            client._get_keystone_config(client.conf)
 
-    def test_get_cpid_from_keystone_insecure(self):
+    def test_get_keystone_config(self):
         """
-        Test getting the CPID from Keystone with the insecure arg passed in.
+        Test that keystone configs properly parsed.
-        """
-        argv = self.mock_argv()
-        argv.append('--insecure')
-        args = rc.parse_cli_args(argv)
-        client = rc.RefstackClient(args)
-        client.tempest_dir = self.test_path
-        client._prep_test()
-        self.mock_keystone()
-        client._get_cpid_from_keystone(client.conf)
-        self.ks2_client_builder.assert_called_with(
-            username='admin', tenant_id='admin_tenant_id',
-            password='test', auth_url='http://0.0.0.0:35357/v2.0',
-            insecure=True
-        )
-
-    def test_get_cpid_from_keystone_v3(self):
-        """
-        Test getting the CPID from Keystone API v3.
         """
         args = rc.parse_cli_args(self.mock_argv())
         client = rc.RefstackClient(args)
         client.tempest_dir = self.test_path
         client._prep_test()
-        client.conf.remove_option('identity', 'tenant_id')
         client.conf.set('identity', 'tenant_name', 'tenant_name')
-        client.conf.set('identity-feature-enabled', 'api_v3', 'true')
+        self.mock_data()
-        self.mock_keystone()
+        actual_result = client._get_keystone_config(client.conf)
-        cpid = client._get_cpid_from_keystone(client.conf)
+        expected_result = self.v2_config
-        self.ks3_client_builder.assert_called_with(
+        self.assertEqual(expected_result, actual_result)
-            username='admin', tenant_name='tenant_name',
-            password='test', auth_url='http://0.0.0.0:35357/v3',
-            insecure=False
-        )
-        self.assertEqual('test-id', cpid)
 
-    def test_get_cpid_from_keystone_v2_varying_catalogs(self):
+    def test_get_cpid_from_keystone_by_tenant_name_from_account_file(self):
         """
-        Test getting the CPID from keystone API v2 varying catalogs.
+        Test getting a CPID from Keystone using an admin tenant name
+        from an accounts file.
         """
-        argv = self.mock_argv()
+        args = rc.parse_cli_args(self.mock_argv())
-        args = rc.parse_cli_args(argv)
         client = rc.RefstackClient(args)
         client.tempest_dir = self.test_path
         client._prep_test()
+        client.conf.add_section('auth')
+        client.conf.set('auth',
+                        'test_accounts_file',
+                        '%s/test-accounts.yaml' % self.test_path)
+        self.mock_data()
+        actual_result = client._get_keystone_config(client.conf)
+        expected_result = None
+        self.assertEqual(expected_result, actual_result['tenant_id'])
+        accounts = [
+            {
+                'username': 'admin',
+                'tenant_id': 'tenant_id',
+                'password': 'test'
+            }
+        ]
+        self.patch(
+            'refstack_client.refstack_client.read_accounts_yaml',
+            return_value=accounts)
+        actual_result = client._get_keystone_config(client.conf)
+        self.assertEqual('tenant_id', actual_result['tenant_id'])
 
-        client._generate_cpid_from_endpoint = MagicMock()
+    def test_generate_keystone_data(self):
-
+        """Test that correct data is generated."""
-        # Test when the identity endpoints is empty
+        args = rc.parse_cli_args(self.mock_argv())
-        self.mock_ks2_client = MagicMock(
+        client = rc.RefstackClient(args)
-            name='ks_client',
+        client.tempest_dir = self.test_path
-            **{'auth_ref':
+        client._prep_test()
-               {'serviceCatalog': [{'type': 'identity', 'endpoints': []}]}}
+        client.conf.set('identity', 'tenant_name', 'tenant_name')
-        )
+        self.mock_data()
-        self.ks2_client_builder = self.patch(
+        configs = client._get_keystone_config(client.conf)
-            'refstack_client.refstack_client.ksclient2.Client',
+        actual_results = client._generate_keystone_data(configs)
-            return_value=self.mock_ks2_client
+        expected_results = ('v2', 'http://0.0.0.0:35357/v2.0/tokens',
-        )
+                            {'auth':
-        client._get_cpid_from_keystone(client.conf)
+                                {'passwordCredentials':
-        # Failover CPID should be generated.
+                                    {
-        client._generate_cpid_from_endpoint.assert_called_with(
+                                        'username': 'admin', 'password': 'test'
-            'http://0.0.0.0:35357/v2.0'
+                                    },
-        )
+                                 'tenantId': 'admin_tenant_id'}})
-
+        self.assertEqual(expected_results, actual_results)
-        # Test when the catalog is empty
-        self.mock_ks2_client = MagicMock(
-            name='ks_client',
-            **{'auth_ref': {'serviceCatalog': []}}
-        )
-        self.ks2_client_builder = self.patch(
-            'refstack_client.refstack_client.ksclient2.Client',
-            return_value=self.mock_ks2_client
-        )
-        client._get_cpid_from_keystone(client.conf)
-        # Failover CPID should be generated.
-        client._generate_cpid_from_endpoint.assert_called_with(
-            'http://0.0.0.0:35357/v2.0'
-        )
-
-        # Test when there is no service catalog
-        self.mock_ks2_client = MagicMock(name='ks_client', **{'auth_ref': {}})
-        self.ks2_client_builder = self.patch(
-            'refstack_client.refstack_client.ksclient2.Client',
-            return_value=self.mock_ks2_client
-        )
-        client._get_cpid_from_keystone(client.conf)
-        # Failover CPID should be generated.
-        client._generate_cpid_from_endpoint.assert_called_with(
-            'http://0.0.0.0:35357/v2.0'
-        )
-
-        # Test when catalog has other non-identity services.
-        self.mock_ks2_client = MagicMock(
-            name='ks_client',
-            **{'auth_ref':
-                {'serviceCatalog': [{'type': 'compute',
-                                     'endpoints': [{'id': 'test-id1'}]},
-                                    {'type': 'identity',
-                                     'endpoints': [{'id': 'test-id2'}]}]}
-               }
-        )
-        self.ks2_client_builder = self.patch(
-            'refstack_client.refstack_client.ksclient2.Client',
-            return_value=self.mock_ks2_client
-        )
-        cpid = client._get_cpid_from_keystone(client.conf)
-        self.assertEqual('test-id2', cpid)
 
     def test_get_cpid_from_keystone_v3_varying_catalogs(self):
         """
@@ -419,65 +269,79 @@ class TestRefstackClient(unittest.TestCase):
         client.conf.remove_option('identity', 'tenant_id')
         client.conf.set('identity', 'tenant_name', 'tenant_name')
         client.conf.set('identity-feature-enabled', 'api_v3', 'true')
-
+        self.mock_data()
+        configs = client._get_keystone_config(client.conf)
+        auth_version, auth_url, content = \
+            client._generate_keystone_data(configs)
         client._generate_cpid_from_endpoint = MagicMock()
+        requests.post = MagicMock()
+         # Test when the identity ID is None.
+        ks3_ID_None = {'auth_ref': {'catalog':
+                                    [{'type': 'identity', 'id': None}]}}
 
-        # Test when the identity ID is None.
+        @httmock.urlmatch(netloc=r'(.*\.)?127.0.0.1$', path='/v3/auth/tokens')
-        self.mock_ks3_client = MagicMock(
+        def keystone_api_v3_mock(auth_version, url, request):
-            name='ks_client',
+            return ks3_ID_None
-            **{'auth_ref': {'catalog': [{'type': 'identity', 'id': None}]}}
+        with httmock.HTTMock(keystone_api_v3_mock):
-        )
+            client._get_cpid_from_keystone(auth_version, auth_url, content)
-        self.ks3_client_builder = self.patch(
+            client._generate_cpid_from_endpoint.assert_called_with(auth_url)
-            'refstack_client.refstack_client.ksclient3.Client',
-            return_value=self.mock_ks3_client
-        )
-        client._get_cpid_from_keystone(client.conf)
-        # Failover CPID should be generated.
-        client._generate_cpid_from_endpoint.assert_called_with(
-            'http://0.0.0.0:35357/v3'
-        )
 
         # Test when the catalog is empty.
-        self.mock_ks3_client = MagicMock(
+        ks3_catalog_empty = {'auth_ref': {'catalog': []}}
-            name='ks_client',
+
-            **{'auth_ref': {'catalog': []}}
+        @httmock.urlmatch(netloc=r'(.*\.)?127.0.0.1$', path='/v3/auth/tokens')
-        )
+        def keystone_api_v3_mock(auth_version, url, request):
-        self.ks3_client_builder = self.patch(
+            return ks3_catalog_empty
-            'refstack_client.refstack_client.ksclient3.Client',
+        with httmock.HTTMock(keystone_api_v3_mock):
-            return_value=self.mock_ks3_client
+            client._get_cpid_from_keystone(auth_version, auth_url, content)
-        )
+            client._generate_cpid_from_endpoint.assert_called_with(auth_url)
-        client._get_cpid_from_keystone(client.conf)
-        # Failover CPID should be generated.
-        client._generate_cpid_from_endpoint.assert_called_with(
-            'http://0.0.0.0:35357/v3'
-        )
 
         # Test when there is no service catalog.
-        self.mock_ks3_client = MagicMock(name='ks_client', **{'auth_ref': {}})
+        ks3_no_catalog = {'auth_ref': {}}
-        self.ks3_client_builder = self.patch(
+
-            'refstack_client.refstack_client.ksclient3.Client',
+        @httmock.urlmatch(netloc=r'(.*\.)?127.0.0.1$', path='/v3/auth/tokens')
-            return_value=self.mock_ks3_client
+        def keystone_api_v3_mock(auth_version, url, request):
-        )
+            return ks3_no_catalog
-        client._get_cpid_from_keystone(client.conf)
+        with httmock.HTTMock(keystone_api_v3_mock):
-        # Failover CPID should be generated.
+            client._get_cpid_from_keystone(auth_version, auth_url, content)
-        client._generate_cpid_from_endpoint.assert_called_with(
+            client._generate_cpid_from_endpoint.assert_called_with(auth_url)
-            'http://0.0.0.0:35357/v3'
-        )
 
         #Test when catalog has other non-identity services.
-        self.mock_ks3_client = MagicMock(
+        ks3_other_services = {'token': {'catalog': [{'type': 'compute',
-            name='ks_client',
+                                        'id': 'test-id1'},
-            **{'auth_ref': {'catalog': [{'type': 'compute',
-                                         'id': 'test-id1'},
                                         {'type': 'identity',
                                          'id': 'test-id2'}]}}
-        )
+        headers = {'content-type': 'application/json'}
-        self.ks3_client_builder = self.patch(
+
-            'refstack_client.refstack_client.ksclient3.Client',
+        @httmock.urlmatch(netloc=r'(.*\.)?127.0.0.1$', path='/v3/auth/tokens')
-            return_value=self.mock_ks3_client
+        def keystone_api_v3_mock(auth_version, url, request):
-        )
+            return ks3_other_services
-        cpid = client._get_cpid_from_keystone(client.conf)
+        with httmock.HTTMock(keystone_api_v3_mock):
-        self.assertEqual('test-id2', cpid)
+            client._get_cpid_from_keystone(auth_version, auth_url, content)
+            requests.post.assert_called_with(auth_url,
+                                             data=json.dumps(content),
+                                             headers=headers,
+                                             verify=True)
+
+    def test_get_cpid_from_keystone_failure_handled(self):
+        """Test that get cpid from keystone API failure handled."""
+        args = rc.parse_cli_args(self.mock_argv())
+        client = rc.RefstackClient(args)
+        client.tempest_dir = self.test_path
+        client._prep_test()
+        client.conf.set('identity', 'tenant_name', 'tenant_name')
+        client.logger.warning = MagicMock()
+        client._generate_cpid_from_endpoint = MagicMock()
+        self.mock_data()
+        configs = client._get_keystone_config(client.conf)
+        auth_version, url, content = client._generate_keystone_data(configs)
+
+        @httmock.urlmatch(netloc=r'(.*\.)?127.0.0.1$', path='/v2/tokens')
+        def keystone_api_mock(auth_version, url, request):
+            return None
+        with httmock.HTTMock(keystone_api_mock):
+            client._get_cpid_from_keystone(auth_version, url, content)
+            client._generate_cpid_from_endpoint.assert_called_with(url)
 
     def test_generate_cpid_from_endpoint(self):
         """
@@ -589,11 +453,9 @@ class TestRefstackClient(unittest.TestCase):
 
         with httmock.HTTMock(refstack_api_mock):
             client.post_results("http://127.0.0.1", content)
-
+            client.logger.info.assert_called_with(
-        client.logger.info.assert_called_with(
+                'http://127.0.0.1/v1/results/ Response: '
-            'http://127.0.0.1/v1/results/ Response: '
+                '%s' % expected_response)
-            '%s' % expected_response
-        )
 
     def test_post_results_with_sign(self):
         """
@@ -616,10 +478,9 @@ class TestRefstackClient(unittest.TestCase):
         with httmock.HTTMock(refstack_api_mock):
             client.post_results("http://127.0.0.1", content,
                                 sign_with=self.test_path + '/rsa_key')
-        client.logger.info.assert_called_with(
+            client.logger.info.assert_called_with(
-            'http://127.0.0.1/v1/results/ Response: '
+                'http://127.0.0.1/v1/results/ Response: '
-            '%s' % expected_response
+                '%s' % expected_response)
-        )
 
     def test_run_tempest(self):
         """
@@ -634,11 +495,13 @@ class TestRefstackClient(unittest.TestCase):
             'refstack_client.refstack_client.subprocess.Popen',
             return_value=MagicMock(returncode=0))
         self.patch("os.path.isfile", return_value=True)
-        self.mock_keystone()
+        self.mock_data()
         client.get_passed_tests = MagicMock(return_value=[{'name': 'test'}])
         client.logger.info = MagicMock()
         client._save_json_results = MagicMock()
         client.post_results = MagicMock()
+        client._get_keystone_config = MagicMock(
+            return_value=self.v2_config)
         client.test()
 
         mock_popen.assert_called_with(
@@ -664,10 +527,13 @@ class TestRefstackClient(unittest.TestCase):
             'refstack_client.refstack_client.subprocess.Popen',
             return_value=MagicMock(returncode=0))
         self.patch("os.path.isfile", return_value=True)
-        self.mock_keystone()
+        self.mock_data()
         client.get_passed_tests = MagicMock(return_value=['test'])
         client.post_results = MagicMock()
         client._save_json_results = MagicMock()
+        client._get_keystone_config = MagicMock(
+            return_value=self.v2_config)
+        client._get_cpid_from_keystone = MagicMock()
         client.test()
         mock_popen.assert_called_with(
             ['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
@@ -693,10 +559,14 @@ class TestRefstackClient(unittest.TestCase):
             return_value=MagicMock(returncode=0)
         )
         self.patch("os.path.isfile", return_value=True)
-        self.mock_keystone()
+        self.mock_data()
         client.get_passed_tests = MagicMock(return_value=['test'])
         client.post_results = MagicMock()
         client._save_json_results = MagicMock()
+        client._get_keystone_config = MagicMock(
+            return_value=self.v2_config)
+        client._get_cpid_from_keystone = MagicMock(
+            return_value='test-id')
         client.test()
         mock_popen.assert_called_with(
             ['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
@@ -724,12 +594,14 @@ class TestRefstackClient(unittest.TestCase):
             'refstack_client.refstack_client.subprocess.Popen',
             return_value=MagicMock(returncode=0))
         self.patch("os.path.isfile", return_value=True)
-        self.mock_keystone()
+        self.mock_data()
         client.get_passed_tests = MagicMock(return_value=[{'name': 'test'}])
         client._save_json_results = MagicMock()
         client.post_results = MagicMock()
         lp.TestListParser.get_normalized_test_list = MagicMock(
             return_value="/tmp/some-list")
+        client._get_keystone_config = MagicMock(
+            return_value=self.v2_config)
         client.test()
 
         lp.TestListParser.get_normalized_test_list.assert_called_with(
@@ -773,9 +645,13 @@ class TestRefstackClient(unittest.TestCase):
             'refstack_client.refstack_client.subprocess.Popen',
             return_value=MagicMock(returncode=0))
         self.patch("os.path.isfile", return_value=True)
-        self.mock_keystone()
+        self.mock_data()
         client.get_passed_tests = MagicMock(return_value=['test'])
         client._save_json_results = MagicMock()
+        client._get_keystone_config = MagicMock(
+            return_value=self.v2_config)
+        client._get_cpid_from_keystone = MagicMock(
+            return_value='test-id')
         client.test()
 
         mock_popen.assert_called_with(
@@ -796,11 +672,14 @@ class TestRefstackClient(unittest.TestCase):
         """
         self.patch('refstack_client.refstack_client.subprocess.Popen',
                    return_value=MagicMock(returncode=1))
-        self.mock_keystone()
         args = rc.parse_cli_args(self.mock_argv(verbose='-vv'))
         client = rc.RefstackClient(args)
         client.tempest_dir = self.test_path
+        self.mock_data()
         client.logger.error = MagicMock()
+        client._get_keystone_config = MagicMock(
+            return_value=self.v2_config)
+        client._get_cpid_from_keystone = MagicMock()
         client.test()
         self.assertTrue(client.logger.error.called)
 

requirements.txt

@@ -1,4 +1,3 @@
-python-keystoneclient>=0.10.0
 gitpython>=0.3.2.RC1
 python-subunit>=0.0.18
 pycrypto>=2.6.1