diff --git a/doc/source/authentication.rst b/doc/source/authentication.rst index 13d7730c9e..66685adb24 100644 --- a/doc/source/authentication.rst +++ b/doc/source/authentication.rst @@ -185,18 +185,18 @@ The ``context_class`` variable is needed to specify the Registry-specific request context, which contains the extra access checks used by the Registry. -Again, to enable using Keystone authentication, the application -pipeline must be modified. By default, it looks like: +Again, to enable using Keystone authentication, the appropriate +application pipeline must be selected. By default, it looks like: - [pipeline:glance-registry] - pipeline = context registryapp - -This must be changed by replacing ``context`` with ``authtoken`` and -``auth-context``:: - - [pipeline:glance-registry] + [pipeline:glance-registry-keystone] pipeline = authtoken auth-context registryapp +To enable the above application pipeline, in your main ``glance-registry.conf`` +configuration file, select the appropriate deployment flavor like so:: + + [paste_deploy] + flavor = keystone + Sharing Images With Others -------------------------- diff --git a/doc/source/configuring.rst b/doc/source/configuring.rst index c1c40fd418..fc305c6fa1 100644 --- a/doc/source/configuring.rst +++ b/doc/source/configuring.rst @@ -495,13 +495,15 @@ image files is transparent and happens using a piece of middleware that can optionally be placed in the server application pipeline. This pipeline is configured in the PasteDeploy configuration file, --paste.ini. +-paste.ini. You should not generally have to edit this file +directly, as it ships with ready-made pipelines for all common deployment +flavors. Enabling the Image Cache Middleware ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To enable the image cache middleware, you would insert the cache middleware -into your application pipeline **after** the appropriate context middleware. +To enable the image cache middleware, the cache middleware must occur in +the application pipeline **after** the appropriate context middleware. The cache middleware should be in your ``glance-api-paste.ini`` in a section titled ``[filter:cache]``. It should look like this:: @@ -510,19 +512,18 @@ titled ``[filter:cache]``. It should look like this:: paste.filter_factory = glance.common.wsgi:filter_factory glance.filter_factory = glance.api.middleware.cache:CacheFilter +A ready-made application pipeline including this filter is defined in +the ``glance-api-paste.ini`` file, looking like so:: -For example, suppose your application pipeline in the ``glance-api-paste.ini`` -file looked like so:: - - [pipeline:glance-api] - pipeline = versionnegotiation context apiv1app - -In the above application pipeline, you would add the cache middleware after the -context middleware, like so:: - - [pipeline:glance-api] + [pipeline:glance-api-caching] pipeline = versionnegotiation context cache apiv1app +To enable the above application pipeline, in your main ``glance-api.conf`` +configuration file, select the appropriate deployment flavor like so:: + + [paste_deploy] + flavor = caching + And that would give you a transparent image cache on the API server. Configuration Options Affecting the Image Cache diff --git a/etc/glance-api-paste.ini b/etc/glance-api-paste.ini index a88fede3a6..a1c85f5678 100644 --- a/etc/glance-api-paste.ini +++ b/etc/glance-api-paste.ini @@ -1,17 +1,46 @@ +# Default minimal pipeline [pipeline:glance-api] pipeline = versionnegotiation context apiv1app -# NOTE: use the following pipeline for keystone -# pipeline = versionnegotiation authtoken auth-context apiv1app -# To enable transparent caching of image files replace pipeline with below: -# pipeline = versionnegotiation context cache apiv1app -# NOTE: use the following pipeline for keystone auth (with caching) -# pipeline = versionnegotiation authtoken auth-context cache apiv1app +# Use the following pipeline for keystone auth +# i.e. in glance-api.conf: +# [paste_deploy] +# flavor = keystone +# +[pipeline:glance-api-keystone] +pipeline = versionnegotiation authtoken auth-context apiv1app -# To enable Image Cache Management API replace pipeline with below: -# pipeline = versionnegotiation context cachemanage apiv1app -# NOTE: use the following pipeline for keystone auth (with caching) -# pipeline = versionnegotiation authtoken auth-context cachemanage apiv1app +# Use the following pipeline to enable transparent caching of image files +# i.e. in glance-api.conf: +# [paste_deploy] +# flavor = caching +# +[pipeline:glance-api-caching] +pipeline = versionnegotiation context cache apiv1app + +# Use the following pipeline for keystone auth with caching +# i.e. in glance-api.conf: +# [paste_deploy] +# flavor = keystone+caching +# +[pipeline:glance-api-keystone+caching] +pipeline = versionnegotiation authtoken auth-context cache apiv1app + +# Use the following pipeline to enable the Image Cache Management API +# i.e. in glance-api.conf: +# [paste_deploy] +# flavor = cachemanagement +# +[pipeline:glance-api-cachemanagement] +pipeline = versionnegotiation context cache cachemanage apiv1app + +# Use the following pipeline for keystone auth with cache management +# i.e. in glance-api.conf: +# [paste_deploy] +# flavor = keystone+cachemanagement +# +[pipeline:glance-api-keystone+cachemanagement] +pipeline = versionnegotiation authtoken auth-context cache cachemanage apiv1app [app:apiv1app] paste.app_factory = glance.common.wsgi:app_factory diff --git a/etc/glance-registry-paste.ini b/etc/glance-registry-paste.ini index f5aabc5c4c..4aabeb4b82 100644 --- a/etc/glance-registry-paste.ini +++ b/etc/glance-registry-paste.ini @@ -1,7 +1,14 @@ +# Default minimal pipeline [pipeline:glance-registry] pipeline = context registryapp -# NOTE: use the following pipeline for keystone -# pipeline = authtoken auth-context registryapp + +# Use the following pipeline for keystone auth +# i.e. in glance-registry.conf: +# [paste_deploy] +# flavor = keystone +# +[pipeline:glance-registry-keystone] +pipeline = authtoken auth-context registryapp [app:registryapp] paste.app_factory = glance.common.wsgi:app_factory diff --git a/glance/common/config.py b/glance/common/config.py index dea2302ac0..b4d90dbf66 100644 --- a/glance/common/config.py +++ b/glance/common/config.py @@ -31,6 +31,12 @@ from glance.common import cfg from glance.common import wsgi +paste_deploy_group = cfg.OptGroup('paste_deploy') +paste_deploy_opts = [ + cfg.StrOpt('flavor'), + ] + + class GlanceConfigOpts(cfg.CommonConfigOpts): def __init__(self, default_config_files=None, **kwargs): @@ -96,6 +102,28 @@ def setup_logging(conf): root_logger.addHandler(handler) +def _register_paste_deploy_opts(conf): + """ + Idempotent registration of paste_deploy option group + + :param conf: a cfg.ConfigOpts object + """ + conf.register_group(paste_deploy_group) + conf.register_opts(paste_deploy_opts, group=paste_deploy_group) + + +def _get_deployment_flavor(conf): + """ + Retrieve the paste_deploy.flavor config item, formatted appropriately + for appending to the application name. + + :param conf: a cfg.ConfigOpts object + """ + _register_paste_deploy_opts(conf) + flavor = conf.paste_deploy.flavor + return '' if not flavor else ('-' + flavor) + + def load_paste_app(conf, app_name=None): """ Builds and returns a WSGI app from a paste config file. @@ -112,6 +140,10 @@ def load_paste_app(conf, app_name=None): if app_name is None: app_name = conf.prog + # append the deployment flavor to the application name, + # in order to identify the appropriate paste pipeline + app_name += _get_deployment_flavor(conf) + # Assume paste config is in a paste.ini file corresponding # to the last config file conf_file = os.path.abspath(conf.config_file[-1].replace(".conf", diff --git a/glance/tests/functional/__init__.py b/glance/tests/functional/__init__.py index fd6a97cc60..1448d8025d 100644 --- a/glance/tests/functional/__init__.py +++ b/glance/tests/functional/__init__.py @@ -84,6 +84,7 @@ class Server(object): self.paste_conf_base = None self.server_control = './bin/glance-control' self.exec_env = None + self.deployment_flavor = '' def write_conf(self, **kwargs): """ @@ -188,7 +189,6 @@ class ApiServer(Server): self.rbd_store_chunk_size = 4 self.delayed_delete = delayed_delete self.owner_is_tenant = True - self.cache_pipeline = "" # Set to cache for cache middleware self.image_cache_dir = os.path.join(self.test_dir, 'cache') self.image_cache_driver = 'sqlite' @@ -225,9 +225,17 @@ scrub_time = 5 scrubber_datadir = %(scrubber_datadir)s image_cache_dir = %(image_cache_dir)s image_cache_driver = %(image_cache_driver)s +[paste_deploy] +flavor = %(deployment_flavor)s """ self.paste_conf_base = """[pipeline:glance-api] -pipeline = versionnegotiation context %(cache_pipeline)s apiv1app +pipeline = versionnegotiation context apiv1app + +[pipeline:glance-api-caching] +pipeline = versionnegotiation context cache apiv1app + +[pipeline:glance-api-cachemanagement] +pipeline = versionnegotiation context cache cache_manage apiv1app [app:apiv1app] paste.app_factory = glance.common.wsgi:app_factory @@ -281,6 +289,8 @@ sql_idle_timeout = 3600 api_limit_max = 1000 limit_param_default = 25 owner_is_tenant = %(owner_is_tenant)s +[paste_deploy] +flavor = %(deployment_flavor)s """ self.paste_conf_base = """[pipeline:glance-registry] pipeline = context registryapp diff --git a/glance/tests/functional/keystone_utils.py b/glance/tests/functional/keystone_utils.py index 050aec6b15..8e459876f6 100644 --- a/glance/tests/functional/keystone_utils.py +++ b/glance/tests/functional/keystone_utils.py @@ -130,6 +130,10 @@ class AdminServer(KeystoneServer): auth_port, admin_port) +def patch_copy(base, src, offset, old, new): + base.insert(src + offset, base[src].replace(old, new)) + + def conf_patch(server, **subs): # First, pull the configuration file paste_base = server.paste_conf_base.split('\n') @@ -137,13 +141,15 @@ def conf_patch(server, **subs): # Need to find the pipeline for idx, text in enumerate(paste_base): if text.startswith('[pipeline:glance-'): - # OK, the line to modify is the next one... - modidx = idx + 1 + # OK, the lines to repeat in modified form + # are this and the next one... + modidx = idx break - # Now we need to replace the default context field... - paste_base[modidx] = paste_base[modidx].replace('context', - 'tokenauth keystone_shim') + # Now we need to add a new pipeline, replacing the default context field... + server.deployment_flavor = 'tokenauth+keystoneshim' + patch_copy(paste_base, modidx, 2, ']', '-tokenauth+keystoneshim]') + patch_copy(paste_base, modidx + 1, 2, 'context', 'tokenauth keystone_shim') # Put the conf back together and append the keystone pieces server.paste_conf_base = '\n'.join(paste_base) + """ diff --git a/glance/tests/functional/test_bin_glance_cache_manage.py b/glance/tests/functional/test_bin_glance_cache_manage.py index c9e2d69a5e..eb3c3a138b 100644 --- a/glance/tests/functional/test_bin_glance_cache_manage.py +++ b/glance/tests/functional/test_bin_glance_cache_manage.py @@ -36,11 +36,12 @@ class TestBinGlanceCacheManage(functional.FunctionalTest): """Functional tests for the bin/glance CLI tool""" def setUp(self): - self.cache_pipeline = "cache cache_manage" self.image_cache_driver = "sqlite" super(TestBinGlanceCacheManage, self).setUp() + self.api_server.deployment_flavor = "cachemanagement" + # NOTE(sirp): This is needed in case we are running the tests under an # environment in which OS_AUTH_STRATEGY=keystone. The test server we # spin up won't have keystone support, so we need to switch to the @@ -85,6 +86,7 @@ class TestBinGlanceCacheManage(functional.FunctionalTest): Test that cache index command works """ self.cleanup() + self.api_server.deployment_flavor = '' self.start_servers() # Not passing in cache_manage in pipeline... api_port = self.api_port diff --git a/glance/tests/functional/test_cache_middleware.py b/glance/tests/functional/test_cache_middleware.py index 64c509dd56..c4577108fc 100644 --- a/glance/tests/functional/test_cache_middleware.py +++ b/glance/tests/functional/test_cache_middleware.py @@ -436,11 +436,12 @@ class TestImageCacheXattr(functional.FunctionalTest, self.inited = True self.disabled = False - self.cache_pipeline = "cache" self.image_cache_driver = "xattr" super(TestImageCacheXattr, self).setUp() + self.api_server.deployment_flavor = "caching" + if not xattr_writes_supported(self.test_dir): self.inited = True self.disabled = True @@ -480,11 +481,12 @@ class TestImageCacheManageXattr(functional.FunctionalTest, self.inited = True self.disabled = False - self.cache_pipeline = "cache cache_manage" self.image_cache_driver = "xattr" super(TestImageCacheManageXattr, self).setUp() + self.api_server.deployment_flavor = "cachemanagement" + if not xattr_writes_supported(self.test_dir): self.inited = True self.disabled = True @@ -524,10 +526,11 @@ class TestImageCacheSqlite(functional.FunctionalTest, self.inited = True self.disabled = False - self.cache_pipeline = "cache" super(TestImageCacheSqlite, self).setUp() + self.api_server.deployment_flavor = "caching" + def tearDown(self): if os.path.exists(self.api_server.image_cache_dir): shutil.rmtree(self.api_server.image_cache_dir) @@ -561,11 +564,12 @@ class TestImageCacheManageSqlite(functional.FunctionalTest, self.inited = True self.disabled = False - self.cache_pipeline = "cache cache_manage" self.image_cache_driver = "sqlite" super(TestImageCacheManageSqlite, self).setUp() + self.api_server.deployment_flavor = "cachemanagement" + def tearDown(self): if os.path.exists(self.api_server.image_cache_dir): shutil.rmtree(self.api_server.image_cache_dir) diff --git a/glance/tests/unit/test_config.py b/glance/tests/unit/test_config.py index 56eb98c6ee..e5d440aaab 100644 --- a/glance/tests/unit/test_config.py +++ b/glance/tests/unit/test_config.py @@ -16,6 +16,7 @@ # under the License. import os.path +import shutil import unittest import stubout @@ -23,8 +24,9 @@ import stubout from glance.api.middleware import version_negotiation from glance.api.v1 import images from glance.api.v1 import members -from glance.common import config +from glance.common import config, context, utils from glance.image_cache import pruner +from glance.tests import utils as test_utils class TestPasteApp(unittest.TestCase): @@ -35,19 +37,39 @@ class TestPasteApp(unittest.TestCase): def tearDown(self): self.stubs.UnsetAll() - def test_load_paste_app(self): - conf = config.GlanceConfigOpts() - conf(['--config-file', - os.path.join(os.getcwd(), 'etc/glance-api.conf')]) + def _do_test_load_paste_app(self, + expected_app_type, + paste_group={}, + paste_append=None): - self.stubs.Set(config, 'setup_logging', lambda *a: None) - self.stubs.Set(images, 'create_resource', lambda *a: None) - self.stubs.Set(members, 'create_resource', lambda *a: None) + conf = test_utils.TestConfigOpts(groups=paste_group) + + def _appendto(orig, copy, str): + shutil.copy(orig, copy) + with open(copy, 'ab') as f: + f.write(str or '') + f.flush() + + paste_from = os.path.join(os.getcwd(), 'etc/glance-api-paste.ini') + paste_to = os.path.join(conf.temp_file.replace('.conf', + '-paste.ini')) + _appendto(paste_from, paste_to, paste_append) app = config.load_paste_app(conf, 'glance-api') - self.assertEquals(version_negotiation.VersionNegotiationFilter, - type(app)) + self.assertEquals(expected_app_type, type(app)) + + def test_load_paste_app(self): + type = version_negotiation.VersionNegotiationFilter + self._do_test_load_paste_app(type) + + def test_load_paste_app_with_paste_flavor(self): + paste_group = {'paste_deploy': {'flavor': 'incomplete'}} + pipeline = '[pipeline:glance-api-incomplete]\n' + \ + 'pipeline = context apiv1app' + + type = context.ContextMiddleware + self._do_test_load_paste_app(type, paste_group, paste_append=pipeline) def test_load_paste_app_with_conf_name(self): def fake_join(*args): diff --git a/glance/tests/utils.py b/glance/tests/utils.py index 3f735ca13c..51cb0cbb30 100644 --- a/glance/tests/utils.py +++ b/glance/tests/utils.py @@ -30,35 +30,66 @@ from glance.common import config class TestConfigOpts(config.GlanceConfigOpts): + """ + Support easily controllable config for unit tests, avoiding the + need to manipulate config files directly. - def __init__(self, test_values): + Configuration values are provided as a dictionary of key-value pairs, + in the simplest case feeding into the DEFAULT group only. + + Non-default groups may also populated via nested dictionaries, e.g. + + {'snafu': {'foo': 'bar', 'bells': 'whistles'}} + + equates to config of form: + + [snafu] + foo = bar + bells = whistles + + The config so provided is dumped to a temporary file, with its path + exposed via the temp_file property. + + :param test_values: dictionary of key-value pairs for the + DEFAULT group + :param groups: nested dictionary of key-value pairs for + non-default groups + """ + + def __init__(self, test_values={}, groups={}): super(TestConfigOpts, self).__init__() self._test_values = test_values + self._test_groups = groups + + self.temp_file = os.path.join(tempfile.mkdtemp(), 'testcfg.conf') + self() def __call__(self): - config_file = self._write_tmp_config_file() + self._write_tmp_config_file() try: super(TestConfigOpts, self).\ - __call__(['--config-file', config_file]) + __call__(['--config-file', self.temp_file]) finally: - os.remove(config_file) + os.remove(self.temp_file) def _write_tmp_config_file(self): contents = '[DEFAULT]\n' for key, value in self._test_values.items(): contents += '%s = %s\n' % (key, value) - (fd, path) = tempfile.mkstemp(prefix='testcfg') - try: - os.write(fd, contents) - except Exception, e: - os.close(fd) - os.remove(path) - raise e + for group, settings in self._test_groups.items(): + contents += '[%s]\n' % group + for key, value in settings.items(): + contents += '%s = %s\n' % (key, value) - os.close(fd) - return path + try: + with open(self.temp_file, 'wb') as f: + f.write(contents) + f.flush() + except Exception, e: + os.remove(self.temp_file) + raise e class skip_test(object):