Port slow, overly assertive v1 functional tests to integration tests

This patch ports existing v1 api functional tests to a newly added group
of integration tests. The new tests take about 1/8th of the time of the
previous tests, thanks to not forking off api and registry server
processes.

One original functional test is preserved, to ensure that we are still
testing the basic v1 functional wireup.

Related to bp refactoring-better-faster-stronger-functional-tests

Change-Id: Ifcc76b4bf4e484a52558538e5398cad882e48e4c

glance/common/config.py

@@ -148,12 +148,16 @@ def setup_logging():
     root_logger.addHandler(handler)
 
 
-def _get_deployment_flavor():
+def _get_deployment_flavor(flavor=None):
     """
     Retrieve the paste_deploy.flavor config item, formatted appropriately
     for appending to the application name.
+
+    :param flavor: if specified, use this setting rather than the
+                   paste_deploy.flavor configuration setting
     """
-    flavor = CONF.paste_deploy.flavor
+    if not flavor:
+        flavor = CONF.paste_deploy.flavor
     return '' if not flavor else ('-' + flavor)
 
 
@@ -183,14 +187,16 @@ def _get_deployment_config_file():
     return os.path.abspath(path)
 
 
-def load_paste_app(app_name=None):
+def load_paste_app(app_name=None, flavor=None, conf_file=None):
     """
     Builds and returns a WSGI app from a paste config file.
 
     We assume the last config file specified in the supplied ConfigOpts
-    object is the paste config file.
+    object is the paste config file, if conf_file is None.
 
     :param app_name: name of the application to load
+    :param flavor: name of the variant of the application to load
+    :param conf_file: path to the paste config file
 
     :raises RuntimeError when config file cannot be located or application
             cannot be loaded from config file
@@ -200,9 +206,10 @@ def load_paste_app(app_name=None):
 
     # append the deployment flavor to the application name,
     # in order to identify the appropriate paste pipeline
-    app_name += _get_deployment_flavor()
+    app_name += _get_deployment_flavor(flavor)
 
-    conf_file = _get_deployment_config_file()
+    if not conf_file:
+        conf_file = _get_deployment_config_file()
 
     try:
         logger = logging.getLogger(__name__)

glance/db/simple/api.py

@@ -56,6 +56,20 @@ def reset():
 
 
 def setup_db_env(*args, **kwargs):
+    """
+    Setup global environment configuration variables.
+
+    We have no connection-oriented environment variables, so this is a NOOP.
+    """
+    pass
+
+
+def clear_db_env(*args, **kwargs):
+    """
+    Setup global environment configuration variables.
+
+    We have no connection-oriented environment variables, so this is a NOOP.
+    """
     pass
 
 

glance/db/sqlalchemy/api.py

@@ -109,7 +109,7 @@ def _ping_listener(dbapi_conn, connection_rec, connection_proxy):
 
 def setup_db_env():
     """
-    Setup configuration for database
+    Setup global configuration for database.
     """
     global sa_logger, _IDLE_TIMEOUT, _MAX_RETRIES, _RETRY_INTERVAL, _CONNECTION
 
@@ -122,6 +122,17 @@ def setup_db_env():
         sa_logger.setLevel(logging.DEBUG)
 
 
+def clear_db_env():
+    """
+    Unset global configuration variables for database.
+    """
+    global _ENGINE, _MAKER, _MAX_RETRIES, _RETRY_INTERVAL, _CONNECTION
+    _ENGINE = None
+    _MAKER = None
+    _MAX_RETRIES = None
+    _RETRY_INTERVAL = None
+
+
 def _check_mutate_authorization(context, image_ref):
     if not is_image_mutable(context, image_ref):
         LOG.info(_("Attempted to modify image user did not own."))

glance/tests/integration/legacy_functional/base.py

@@ -0,0 +1,205 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import atexit
+import os.path
+import tempfile
+
+import fixtures
+from oslo.config import cfg
+
+from glance import tests as glance_tests
+import glance.common.client
+from glance.common import config
+import glance.db.sqlalchemy.api
+import glance.db.sqlalchemy.migration
+import glance.registry.client.v1.client
+import glance.store
+from glance.tests import utils as test_utils
+
+
+TESTING_API_PASTE_CONF = """
+[pipeline:glance-api]
+pipeline = versionnegotiation gzip unauthenticated-context rootapp
+
+[pipeline:glance-api-caching]
+pipeline = versionnegotiation gzip unauthenticated-context cache rootapp
+
+[pipeline:glance-api-cachemanagement]
+pipeline =
+    versionnegotiation
+    gzip
+    unauthenticated-context
+    cache
+    cache_manage
+    rootapp
+
+[pipeline:glance-api-fakeauth]
+pipeline = versionnegotiation gzip fakeauth context rootapp
+
+[pipeline:glance-api-noauth]
+pipeline = versionnegotiation gzip context rootapp
+
+[composite:rootapp]
+paste.composite_factory = glance.api:root_app_factory
+/: apiversions
+/v1: apiv1app
+/v2: apiv2app
+
+[app:apiversions]
+paste.app_factory = glance.api.versions:create_resource
+
+[app:apiv1app]
+paste.app_factory = glance.api.v1.router:API.factory
+
+[app:apiv2app]
+paste.app_factory = glance.api.v2.router:API.factory
+
+[filter:versionnegotiation]
+paste.filter_factory =
+ glance.api.middleware.version_negotiation:VersionNegotiationFilter.factory
+
+[filter:gzip]
+paste.filter_factory = glance.api.middleware.gzip:GzipMiddleware.factory
+
+[filter:cache]
+paste.filter_factory = glance.api.middleware.cache:CacheFilter.factory
+
+[filter:cache_manage]
+paste.filter_factory =
+ glance.api.middleware.cache_manage:CacheManageFilter.factory
+
+[filter:context]
+paste.filter_factory = glance.api.middleware.context:ContextMiddleware.factory
+
+[filter:unauthenticated-context]
+paste.filter_factory =
+ glance.api.middleware.context:UnauthenticatedContextMiddleware.factory
+
+[filter:fakeauth]
+paste.filter_factory = glance.tests.utils:FakeAuthMiddleware.factory
+"""
+
+TESTING_REGISTRY_PASTE_CONF = """
+[pipeline:glance-registry]
+pipeline = unauthenticated-context registryapp
+
+[pipeline:glance-registry-fakeauth]
+pipeline = fakeauth context registryapp
+
+[app:registryapp]
+paste.app_factory = glance.registry.api.v1:API.factory
+
+[filter:context]
+paste.filter_factory = glance.api.middleware.context:ContextMiddleware.factory
+
+[filter:unauthenticated-context]
+paste.filter_factory =
+ glance.api.middleware.context:UnauthenticatedContextMiddleware.factory
+
+[filter:fakeauth]
+paste.filter_factory = glance.tests.utils:FakeAuthMiddleware.factory
+"""
+
+CONF = cfg.CONF
+CONF.import_opt('filesystem_store_datadir', 'glance.store.filesystem')
+
+
+class ApiTest(test_utils.BaseTestCase):
+    def setUp(self):
+        super(ApiTest, self).setUp()
+        self.test_dir = self.useFixture(fixtures.TempDir()).path
+        self._configure_logging()
+        self._setup_database()
+        self._setup_stores()
+        self.glance_registry_app = self._load_paste_app(
+            'glance-registry',
+            flavor=getattr(self, 'registry_flavor', ''),
+            conf=getattr(self, 'registry_paste_conf',
+                         TESTING_REGISTRY_PASTE_CONF),
+        )
+        self._connect_registry_client()
+        self.glance_api_app = self._load_paste_app(
+            'glance-api',
+            flavor=getattr(self, 'api_flavor', ''),
+            conf=getattr(self, 'api_paste_conf', TESTING_API_PASTE_CONF),
+        )
+        self.http = test_utils.Httplib2WsgiAdapter(self.glance_api_app)
+
+    def _configure_logging(self):
+        self.config(default_log_levels=[
+                'amqplib=WARN',
+                'sqlalchemy=WARN',
+                'boto=WARN',
+                'suds=INFO',
+                'keystone=INFO',
+                'eventlet.wsgi.server=DEBUG'
+        ])
+
+    def _setup_database(self):
+        sql_connection = 'sqlite:////%s/tests.sqlite' % self.test_dir
+        self.config(sql_connection=sql_connection)
+        glance.db.sqlalchemy.api.clear_db_env()
+        glance_db_env = 'GLANCE_DB_TEST_SQLITE_FILE'
+        if glance_db_env in os.environ:
+            # use the empty db created and cached as a tempfile
+            # instead of spending the time creating a new one
+            db_location = os.environ[glance_db_env]
+            test_utils.execute('cp %s %s/tests.sqlite'
+                               % (db_location, self.test_dir))
+        else:
+            glance.db.sqlalchemy.migration.db_sync()
+
+            # copy the clean db to a temp location so that it
+            # can be reused for future tests
+            (osf, db_location) = tempfile.mkstemp()
+            os.close(osf)
+            test_utils.execute('cp %s/tests.sqlite %s'
+                               % (self.test_dir, db_location))
+            os.environ[glance_db_env] = db_location
+
+            # cleanup the temp file when the test suite is
+            # complete
+            def _delete_cached_db():
+                try:
+                    os.remove(os.environ[glance_db_env])
+                except Exception:
+                    glance_tests.logger.exception(
+                        "Error cleaning up the file %s" %
+                        os.environ[glance_db_env])
+            atexit.register(_delete_cached_db)
+
+    def _setup_stores(self):
+        image_dir = os.path.join(self.test_dir, "images")
+        self.config(filesystem_store_datadir=image_dir)
+        glance.store.create_stores()
+
+    def _load_paste_app(self, name, flavor, conf):
+        conf_file_path = os.path.join(self.test_dir, '%s-paste.ini' % name)
+        with open(conf_file_path, 'wb') as conf_file:
+            conf_file.write(conf)
+            conf_file.flush()
+        return config.load_paste_app(name, flavor=flavor,
+                                     conf_file=conf_file_path)
+
+    def _connect_registry_client(self):
+        def get_connection_type(self2):
+            def wrapped(*args, **kwargs):
+                return test_utils.HttplibWsgiAdapter(self.glance_registry_app)
+            return wrapped
+
+        self.stubs.Set(glance.common.client.BaseClient,
+                       'get_connection_type', get_connection_type)
+
+    def tearDown(self):
+        glance.db.sqlalchemy.api.clear_db_env()
+        super(ApiTest, self).tearDown()

glance/tests/utils.py

@@ -29,6 +29,7 @@ import sys
 from oslo.config import cfg
 import stubout
 import testtools
+import webob
 
 from glance.common import config
 from glance.common import exception
@@ -417,3 +418,44 @@ class FakeHTTPResponse(object):
 
     def read(self, amt):
         self.data.read(amt)
+
+
+class Httplib2WsgiAdapter(object):
+    def __init__(self, app):
+        self.app = app
+
+    def request(self, uri, method="GET", body=None, headers=None):
+        req = webob.Request.blank(uri, method=method, headers=headers)
+        req.body = body
+        resp = req.get_response(self.app)
+        return Httplib2WebobResponse(resp), resp.body
+
+
+class Httplib2WebobResponse(object):
+    def __init__(self, webob_resp):
+        self.webob_resp = webob_resp
+
+    @property
+    def status(self):
+        return self.webob_resp.status_code
+
+    def __getitem__(self, key):
+        return self.webob_resp.headers[key]
+
+    def get(self, key):
+        return self.webob_resp.headers[key]
+
+
+class HttplibWsgiAdapter(object):
+    def __init__(self, app):
+        self.app = app
+        self.req = None
+
+    def request(self, method, url, body=None, headers={}):
+        self.req = webob.Request.blank(url, method=method, headers=headers)
+        self.req.body = body
+
+    def getresponse(self):
+        response = self.req.get_response(self.app)
+        return FakeHTTPResponse(response.status_code, response.headers,
+                                response.body)