Replacing functional test authentication hookup

This change, replaces the authentication wrapper for our functional test
calls with a simple wrapper using Keystone client. As a result, this
change removes our dependence on Tempest trunk to run our functional
tests. Unfortunately, this was done primarally due to the uncompability
between Tempest's oslo.log and the oslo_log that we use in Barbican that
was causing our gates to fail and blocking merges across the project.

Change-Id: I0eee6a34d1ab5ca654e737d95c1e124465dc9c14

contrib/devstack/lib/barbican

@@ -94,6 +94,9 @@ function configure_barbican {
     cp $BARBICAN_DIR/etc/barbican/barbican-admin-paste.ini $BARBICAN_CONF_DIR
     cp -R $BARBICAN_DIR/etc/barbican/vassals $BARBICAN_CONF_DIR
 
+    # Copy functional test config
+    cp $BARBICAN_DIR/etc/barbican/barbican-functional.conf $BARBICAN_CONF_DIR
+
     # Set the logging to INFO
     iniset $BARBICAN_CONF DEFAULT verbose True
 

etc/barbican/barbican-functional.conf

@@ -0,0 +1,21 @@
+[DEFAULT]
+
+[identity]
+# Replace these with values that represent your identity configuration
+uri=http://localhost:5000/v3
+version=v3
+
+username=admin
+project_name=admin
+password=secretadmin
+domain_name=Default
+
+[keymanager]
+
+# use this to run the functional tests against a
+# different barbican server than the one that is
+# specified in the service catalog.  To use what is
+# in the service catalog, just comment this out
+# or leave it blank.
+# override_url=http://localhost:9311/v1
+# override_url_version=v1

functionaltests/__init__.py

@@ -1,25 +0,0 @@
-"""
-Copyright 2015 Rackspace
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-import os
-
-from tempest import config
-
-CONF = config.CONF
-
-# Use local tempest conf if one is available.
-# This usually means we're running tests outside of devstack
-if os.path.exists('./etc/dev_tempest.conf'):
-    CONF.set_config_path('./etc/dev_tempest.conf')

functionaltests/api/base.py

@@ -13,22 +13,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-import os
+import logging
 
 import oslotest.base as oslotest
-from tempest import auth
-from tempest import config
-from tempest import manager as tempest_manager
-from tempest.openstack.common import log as logging
 
 from functionaltests.common import client
+from functionaltests.common import config
 
-CONF = config.CONF
+CONF = config.get_config()
-
-# Use local tempest conf if one is available.
-# This usually means we're running tests outside of devstack
-if os.path.exists('./etc/dev_tempest.conf'):
-    CONF.set_config_path('./etc/dev_tempest.conf')
 
 
 class TestCase(oslotest.BaseTestCase):
@@ -48,23 +40,7 @@ class TestCase(oslotest.BaseTestCase):
         self.LOG.info('Starting: %s', self._testMethodName)
         super(TestCase, self).setUp()
 
-        # determine which type of credentials to use
+        self.client = client.BarbicanClient()
-        if 'v3' in CONF.identity.auth_version:
-            credentials = BarbicanV3Credentials()
-        else:
-            credentials = BarbicanV2Credentials()
-
-        # tempest changed how you access the auth_provider so we will
-        # try the "new way" first (a top-level function in the tempest_manager)
-        # If that fails we will fall back on the "old way" calling a method
-        # within tempest_manager.Manager.
-        try:
-            auth_provider = tempest_manager.get_auth_provider(credentials)
-        except AttributeError:
-            mgr = tempest_manager.Manager(credentials=credentials)
-            auth_provider = mgr.get_auth_provider(credentials)
-
-        self.client = client.BarbicanClient(auth_provider)
 
     def tearDown(self):
         super(TestCase, self).tearDown()
@@ -77,31 +53,3 @@ class TestCase(oslotest.BaseTestCase):
             case_name=cls.__name__
         )
         return name
-
-
-class BarbicanV2Credentials(auth.KeystoneV2Credentials):
-
-    def __init__(self):
-        credentials = dict(
-            username=CONF.identity.admin_username,
-            password=CONF.identity.admin_password
-        )
-        # Some identity v2 implementations don't need the tenant name, so
-        # only include it here if the user provided it in the config file.
-        if CONF.identity.admin_tenant_name:
-            credentials['tenant_name'] = CONF.identity.admin_tenant_name
-
-        super(BarbicanV2Credentials, self).__init__(**credentials)
-
-
-class BarbicanV3Credentials(auth.KeystoneV3Credentials):
-
-    def __init__(self):
-        credentials = dict(
-            username=CONF.identity.admin_username,
-            password=CONF.identity.admin_password,
-            project_name=CONF.identity.admin_tenant_name,
-            domain_name=CONF.identity.admin_domain_name,
-        )
-
-        super(BarbicanV3Credentials, self).__init__(**credentials)

functionaltests/api/v1/behaviors/base_behaviors.py

@@ -13,10 +13,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+import logging
 import os
 
-from tempest.openstack.common import log as logging
-
 
 class BaseBehaviors(object):
 

functionaltests/api/v1/models/base_models.py

@@ -14,9 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 import json
+import logging
 
 import six
-from tempest.openstack.common import log as logging
 
 LOG = logging.getLogger(__name__)
 

functionaltests/common/__init__.py

@@ -1,30 +0,0 @@
-"""
-Copyright 2015 Rackspace
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-import os
-
-from oslo.config import cfg
-from tempest import config
-
-CONF = config.CONF
-
-# Use local tempest conf if one is available.
-# This usually means we're running tests outside of devstack
-if os.path.exists('./etc/dev_tempest.conf'):
-    CONF.set_config_path('./etc/dev_tempest.conf')
-
-CONF.register_group(cfg.OptGroup('keymanager'))
-CONF.register_opt(cfg.StrOpt('override-url'), group='keymanager')
-CONF.register_opt(cfg.StrOpt('override-url-version'), group='keymanager')

functionaltests/common/auth.py

@@ -0,0 +1,93 @@
+"""
+Copyright 2015 Rackspace
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from keystoneclient.v2_0 import client as v2_client
+from keystoneclient.v3 import client as v3_client
+from requests import auth
+
+STORED_AUTHENTICATION = None
+
+
+class FunctionalTestAuth(auth.AuthBase):
+
+    def __init__(self, endpoint, version, username, password, project_name):
+        self.endpoint = endpoint
+        self.version = version
+        self.username = username
+        self.password = password
+        self.project_name = project_name
+
+        self._client = None
+
+    @property
+    def service_catalog(self):
+        if not self._client:
+            self.authenticate()
+        return self.stored_auth.get(self.username, {}).get('service_catalog')
+
+    @property
+    def auth_client(self):
+        if not self._client:
+            self.authenticate()
+        return self._client
+
+    @property
+    def stored_auth(self):
+        global STORED_AUTHENTICATION
+        if not STORED_AUTHENTICATION:
+            STORED_AUTHENTICATION = {}
+        return STORED_AUTHENTICATION
+
+    def _auth_with_keystone_client(self):
+        if self.version.lower() == 'v2':
+            self._client = v2_client.Client(
+                username=self.username,
+                password=self.password,
+                tenant_name=self.project_name,
+                auth_url=self.endpoint
+            )
+            return (self._client.auth_token, self._client.tenant_id)
+
+        elif self.version.lower() == 'v3':
+            self._client = v3_client.Client(
+                username=self.username,
+                password=self.password,
+                project_name=self.project_name,
+                auth_url=self.endpoint
+            )
+            return (self._client.auth_token, self._client.project_id)
+        else:
+            raise Exception('Unknown authentication version')
+
+    def authenticate(self):
+        creds = self.stored_auth.get(self.username)
+
+        if not creds:
+            token, project_id = self._auth_with_keystone_client()
+            self.stored_auth[self.username] = {
+                'token': token,
+                'project_id': project_id,
+                'service_catalog': self._client.service_catalog
+            }
+
+        return self.stored_auth[self.username]
+
+    def __call__(self, r):
+        creds = self.authenticate()
+
+        # modify and return the request
+        r.headers['X-Project-Id'] = creds.get('project_id')
+        r.headers['X-Auth-Token'] = creds.get('token')
+        return r

functionaltests/common/client.py

@@ -13,62 +13,31 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+import logging
 import os
 
 import requests
-from requests import auth
 from six.moves import urllib
-from tempest.common.utils import misc as misc_utils
+from tempest_lib.common.utils import misc as misc_utils
-from tempest import config
+
-from tempest.openstack.common import log as logging
+from functionaltests.common import auth
+from functionaltests.common import config
 
 LOG = logging.getLogger(__name__)
 
-CONF = config.CONF
+CONF = config.get_config()
-
-# Use local tempest conf if one is available.
-# This usually means we're running tests outside of devstack.
-if os.path.exists('./etc/dev_tempest.conf'):
-    CONF.set_config_path('./etc/dev_tempest.conf')
-
-
-class BarbicanClientAuth(auth.AuthBase):
-    """Implementation of Requests Auth for Barbican http calls."""
-
-    def __init__(self, auth_provider):
-        credentials = auth_provider.fill_credentials()
-
-        self.username = credentials.username
-        self.password = credentials.password
-
-        if 'v3' in CONF.identity.auth_version:
-            self.project_name = credentials.project_name
-            self.project_id = credentials.project_id
-        else:
-            self.tenant_name = credentials.tenant_name
-            self.project_id = credentials.tenant_id
-
-        try:
-            self.token = auth_provider.get_token()
-        except ValueError:
-            # hockeynut - some auth providers will allow the v3 expiration
-            # date format which includes milliseconds.  This change will retry
-            # the call to get the auth token with the milliseconds included in
-            # the date format string.
-            auth_provider.EXPIRY_DATE_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ'
-            self.token = auth_provider.get_token()
-
-    def __call__(self, r):
-        r.headers['X-Project-Id'] = self.project_id
-        r.headers['X-Auth-Token'] = self.token
-        return r
 
 
 class BarbicanClient(object):
 
-    def __init__(self, auth_provider, api_version='v1'):
+    def __init__(self, api_version='v1'):
-        self._auth = BarbicanClientAuth(auth_provider)
+        self._auth = auth.FunctionalTestAuth(
-        self._auth_provider = auth_provider
+            endpoint=CONF.identity.uri,
+            version=CONF.identity.version,
+            username=CONF.identity.username,
+            password=CONF.identity.password,
+            project_name=CONF.identity.project_name
+        )
         self.timeout = 10
         self.api_version = api_version
         self.default_headers = {
@@ -157,15 +126,23 @@ class BarbicanClient(object):
     def get_base_url(self, include_version=True):
         if CONF.keymanager.override_url:
             return self._get_base_url_from_config(include_version)
-        filters = {
-            'service': 'key-manager',
-            'region': self.region,
-        }
 
-        base_url = self._auth_provider.base_url(filters)
+        endpoint = self._auth.service_catalog.get_endpoints(
+            service_type='key-manager',
+            service_name='barbican',
+            region_name='RegionOne',
+            endpoint_type='public'
+        )
 
-        if include_version:
+        base_url = endpoint['key-manager'][0].get('url')
+
+        # Make sure we handle the edge cases around Keystone providing
+        # endpoints with or without versions
+        if include_version and self.api_version not in base_url:
             base_url = urllib.parse.urljoin(base_url, self.api_version)
+        elif not include_version and self.api_version in base_url:
+            base_url, _ = os.path.split(base_url)
+
         return self._get_url_w_trailing_slash(base_url)
 
     def get_list_of_models(self, item_list, model_type):
@@ -194,7 +171,7 @@ class BarbicanClient(object):
                 use_auth=True, response_model_type=None, request_model=None,
                 params=None):
         """Prepares and sends http request through Requests."""
-        if 'http' not in url:
+        if url and 'http' not in url:
             url = urllib.parse.urljoin(self.get_base_url(), url)
 
         # Duplicate Base headers and add extras (if needed)

functionaltests/common/config.py

@@ -0,0 +1,68 @@
+"""
+Copyright 2015 Rackspace
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import os
+
+from oslo_config import cfg
+
+TEST_CONF = None
+
+
+def setup_config(config_file=''):
+    global TEST_CONF
+    TEST_CONF = cfg.ConfigOpts()
+
+    identity_group = cfg.OptGroup(name='identity')
+    identity_options = [
+        cfg.StrOpt('uri', default='http://localhost:5000/v3'),
+        cfg.StrOpt('version', default='v3'),
+        cfg.StrOpt('username', default='admin'),
+        cfg.StrOpt('password', default='secretadmin'),
+        cfg.StrOpt('project_name', default='admin'),
+        cfg.StrOpt('domain_name', default='Default'),
+        cfg.StrOpt('region', default='RegionOne')
+    ]
+    TEST_CONF.register_group(identity_group)
+    TEST_CONF.register_opts(identity_options, group=identity_group)
+
+    keymanager_group = cfg.OptGroup(name='keymanager')
+    keymanager_options = [
+        cfg.StrOpt('override_url', default=''),
+        cfg.StrOpt('override_url_version', default='')
+    ]
+    TEST_CONF.register_group(keymanager_group)
+    TEST_CONF.register_opts(keymanager_options, group=keymanager_group)
+
+    # Figure out which config to load
+    config_to_load = []
+    local_config = './etc/barbican/barbican-functional.conf'
+    if os.path.isfile(config_file):
+        config_to_load.append(config_file)
+    elif os.path.isfile(local_config):
+        config_to_load.append(local_config)
+    else:
+        config_to_load.append('/etc/barbican/barbican-functional.conf')
+
+    # Actually parse config
+    TEST_CONF(
+        (),  # Required to load a anonymous config
+        default_config_files=config_to_load
+    )
+
+
+def get_config():
+    if not TEST_CONF:
+        setup_config()
+    return TEST_CONF

test-requirements.txt

@@ -11,6 +11,8 @@ testrepository>=0.0.18
 testtools>=0.9.36,!=1.2.0
 fixtures>=0.3.14
 requests>=2.2.0,!=2.4.0
+python-keystoneclient>=1.1.0
+tempest-lib>=0.3.0
 
 # Documentation build requirements
 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3

tox.ini

@@ -48,7 +48,6 @@ commands=
 deps =
     {[testenv]deps}
     nose
-    git+https://github.com/openstack/tempest.git
 commands = nosetests {toxinidir}/functionaltests --match='{posargs}'
 
 [flake8]