separate paste-deploy configuration from parameters

PasteDeploy configuration contains class names which might change
between releases. Keeping it separate from user-configurable
parameters allows deployers to move paste-deploy ini file out of
configuration directory to a place where it can be safely overwritten
on updates e.g. under /usr/share/

DocImpact

Change-Id: I9292ca6226c8430b93565dedd45cc842742a23e2

bin/keystone-all

@@ -87,15 +87,7 @@ if __name__ == '__main__':
     if CONF.debug:
         CONF.log_opt_values(logging.getLogger(CONF.prog), logging.DEBUG)
 
-    if CONF.config_file:
+    paste_config = config.find_paste_config()
-        paste_config = CONF.config_file[0]
-    else:
-        paste_config = CONF.find_file('keystone.conf')
-        if not paste_config:
-            print ("The keystone.conf file could not be found in the "
-                   "configuration directories.")
-            CONF.print_help()
-            sys.exit(1)
 
     monkeypatch_thread = not CONF.standard_threads
     pydev_debug_url = utils.setup_remote_pydev_debug()
@@ -107,8 +99,6 @@ if __name__ == '__main__':
         monkeypatch_thread = False
     wsgi_server.monkey_patch_eventlet(monkeypatch_thread=monkeypatch_thread)
 
-    options = deploy.appconfig('config:%s' % paste_config)
-
     servers = []
     servers.append(create_server(paste_config,
                                  'admin',

doc/source/apache-httpd.rst

@@ -63,14 +63,17 @@ it goes right before::
 Files
 -----
 
-Copy the file keystone.conf to the appropriate location for your apache server, most likely::
+Copy the file httpd/wsgi-keystone.conf to the appropriate location for your apache server, most likely::
 
-    /etc/httpd/conf.d/keystone.conf
+    /etc/httpd/conf.d/wsgi-keystone.conf
 
 Create the directory ``/var/www/cgi-bin/keystone/``. You can either hardlink or softlink the files ``main`` and ``admin`` to the file ``keystone.py`` in this directory.  For a distribution appropriate place, it should probably be copied to::
 
     /usr/share/openstack/keystone/httpd/keystone.py
 
+Keystone's primary configuration file (``etc/keystone.conf``) and the PasteDeploy
+configuration file (``etc/keystone-paste.ini``) must be readable to HTTPD in
+one of the default locations described in :doc:`configuration`.
 
 SELinux
 -------

doc/source/configuration.rst

@@ -25,8 +25,8 @@ Configuring Keystone
    man/keystone-all
 
 Once Keystone is installed, it is configured via a primary configuration file
-(``etc/keystone.conf``), possibly a separate logging configuration file, and
+(``etc/keystone.conf``), a PasteDeploy configuration file (``etc/keystone-paste.ini``),
-initializing data into keystone using the command line client.
+possibly a separate logging configuration file, and initializing data into Keystone using the command line client.
 
 Starting and Stopping Keystone
 ==============================
@@ -37,7 +37,7 @@ Start Keystone services using the command::
 
 Invoking this command starts up two ``wsgi.Server`` instances, ``admin`` (the
 administration API) and ``main`` (the primary/public API interface). Both
-services are configured by ``keystone.conf`` as run in a single process.
+services are configured to run in a single process.
 
 Stop the process using ``Control-C``.
 
@@ -60,10 +60,13 @@ match if key expiry is to behave as expected.
 Configuration Files
 ===================
 
-The keystone configuration file is an ``ini`` file based on Paste_, a
+The Keystone configuration files are an ``ini`` file format based on Paste_, a
-common system used to configure python WSGI based applications. In addition to
+common system used to configure Python WSGI based applications.
-the paste configuration entries, general and driver-specific configuration
+The PasteDeploy configuration entries (WSGI pipeline definitions)
-values are organized into the following sections:
+can be provided in a separate ``keystone-paste.ini`` file, while general and
+driver-specific configuration parameters are in the primary configuration file
+``keystone.conf``. The primary configuration file is organized into the
+following sections:
 
 * ``[DEFAULT]`` - general configuration
 * ``[sql]`` - optional storage backend configuration
@@ -76,11 +79,12 @@ values are organized into the following sections:
 * ``[signing]`` - cryptographic signatures for PKI based tokens
 * ``[ssl]`` - SSL configuration
 * ``[auth]`` - Authentication plugin configuration
+* ``[paste_deploy]`` - Pointer to the PasteDeploy configuration file
 
-The Keystone configuration file is expected to be named ``keystone.conf``.
+The Keystone primary configuration file is expected to be named ``keystone.conf``.
-When starting keystone, you can specify a different configuration file to
+When starting Keystone, you can specify a different configuration file to
 use with ``--config-file``. If you do **not** specify a configuration file,
-keystone will look in the following directories for a configuration file, in
+Keystone will look in the following directories for a configuration file, in
 order:
 
 * ``~/.keystone/``
@@ -88,6 +92,8 @@ order:
 * ``/etc/keystone/``
 * ``/etc/``
 
+PasteDeploy configuration file is specified by the ``config_file`` parameter in ``[paste_deploy]`` section of the primary configuration file. If the parameter
+is not an absolute path, then Keystone looks for it in the same directories as above. If not specified, WSGI pipeline definitions are loaded from the primary configuration file.
 
 Authentication Plugins
 ----------------------
@@ -440,7 +446,7 @@ pipeline. This user crud filter allows users to use a HTTP PATCH to change
 their own password. To enable this extension you should define a
 user_crud_extension filter, insert it after the ``*_body`` middleware
 and before the ``public_service`` app in the public_api WSGI pipeline in
-keystone.conf e.g.::
+``keystone-paste.ini`` e.g.::
 
     [filter:user_crud_extension]
     paste.filter_factory = keystone.contrib.user_crud:CrudExtension.factory
@@ -463,7 +469,8 @@ Sample Configuration Files
 The ``etc/`` folder distributed with Keystone contains example configuration
 files for each Server application.
 
-* ``etc/keystone.conf``
+* ``etc/keystone.conf.sample``
+* ``etc/keystone-paste.ini``
 * ``etc/logging.conf.sample``
 * ``etc/default_catalog.templates``
 

doc/source/external-auth.rst

@@ -96,7 +96,7 @@ Pipeline configuration
 Once you have your WSGI middleware component developed you have to add it to
 your pipeline. The first step is to add the middleware to your configuration file.
 Assuming that your middleware module is ``keystone.middleware.MyMiddlewareAuth``,
-you can configure it in your ``keystone.conf`` as::
+you can configure it in your ``keystone-paste.ini`` as::
 
     [filter:my_auth]
     paste.filter_factory = keystone.middleware.MyMiddlewareAuth.factory

doc/source/installing.rst

@@ -62,6 +62,7 @@ commandline path:
 You will find sample configuration files in ``etc/``
 
 * keystone.conf
+* keystone-paste.ini
 * logging.conf
 * policy.json
 * default_catalog.templates

etc/keystone-paste.ini

@@ -0,0 +1,85 @@
+# Keystone PasteDeploy configuration file.
+
+[filter:debug]
+paste.filter_factory = keystone.common.wsgi:Debug.factory
+
+[filter:token_auth]
+paste.filter_factory = keystone.middleware:TokenAuthMiddleware.factory
+
+[filter:admin_token_auth]
+paste.filter_factory = keystone.middleware:AdminTokenAuthMiddleware.factory
+
+[filter:xml_body]
+paste.filter_factory = keystone.middleware:XmlBodyMiddleware.factory
+
+[filter:json_body]
+paste.filter_factory = keystone.middleware:JsonBodyMiddleware.factory
+
+[filter:user_crud_extension]
+paste.filter_factory = keystone.contrib.user_crud:CrudExtension.factory
+
+[filter:crud_extension]
+paste.filter_factory = keystone.contrib.admin_crud:CrudExtension.factory
+
+[filter:ec2_extension]
+paste.filter_factory = keystone.contrib.ec2:Ec2Extension.factory
+
+[filter:s3_extension]
+paste.filter_factory = keystone.contrib.s3:S3Extension.factory
+
+[filter:url_normalize]
+paste.filter_factory = keystone.middleware:NormalizingFilter.factory
+
+[filter:sizelimit]
+paste.filter_factory = keystone.middleware:RequestBodySizeLimiter.factory
+
+[filter:stats_monitoring]
+paste.filter_factory = keystone.contrib.stats:StatsMiddleware.factory
+
+[filter:stats_reporting]
+paste.filter_factory = keystone.contrib.stats:StatsExtension.factory
+
+[filter:access_log]
+paste.filter_factory = keystone.contrib.access:AccessLogMiddleware.factory
+
+[app:public_service]
+paste.app_factory = keystone.service:public_app_factory
+
+[app:service_v3]
+paste.app_factory = keystone.service:v3_app_factory
+
+[app:admin_service]
+paste.app_factory = keystone.service:admin_app_factory
+
+[pipeline:public_api]
+pipeline = access_log sizelimit url_normalize token_auth admin_token_auth xml_body json_body ec2_extension user_crud_extension public_service
+
+[pipeline:admin_api]
+pipeline = access_log sizelimit url_normalize token_auth admin_token_auth xml_body json_body ec2_extension s3_extension crud_extension admin_service
+
+[pipeline:api_v3]
+pipeline = access_log sizelimit url_normalize token_auth admin_token_auth xml_body json_body ec2_extension s3_extension service_v3
+
+[app:public_version_service]
+paste.app_factory = keystone.service:public_version_app_factory
+
+[app:admin_version_service]
+paste.app_factory = keystone.service:admin_version_app_factory
+
+[pipeline:public_version_api]
+pipeline = access_log sizelimit url_normalize xml_body public_version_service
+
+[pipeline:admin_version_api]
+pipeline = access_log sizelimit url_normalize xml_body admin_version_service
+
+[composite:main]
+use = egg:Paste#urlmap
+/v2.0 = public_api
+/v3 = api_v3
+/ = public_version_api
+
+[composite:admin]
+use = egg:Paste#urlmap
+/v2.0 = admin_api
+/v3 = api_v3
+/ = admin_version_api

etc/keystone.conf.sample

@@ -246,86 +246,6 @@ methods = password,token
 password = keystone.auth.plugins.password.Password
 token = keystone.auth.plugins.token.Token
 
-[filter:debug]
+[paste_deploy]
-paste.filter_factory = keystone.common.wsgi:Debug.factory
+# Name of the paste configuration file that defines the available pipelines
-
+config_file = keystone-paste.ini
-[filter:token_auth]
-paste.filter_factory = keystone.middleware:TokenAuthMiddleware.factory
-
-[filter:admin_token_auth]
-paste.filter_factory = keystone.middleware:AdminTokenAuthMiddleware.factory
-
-[filter:xml_body]
-paste.filter_factory = keystone.middleware:XmlBodyMiddleware.factory
-
-[filter:json_body]
-paste.filter_factory = keystone.middleware:JsonBodyMiddleware.factory
-
-[filter:user_crud_extension]
-paste.filter_factory = keystone.contrib.user_crud:CrudExtension.factory
-
-[filter:crud_extension]
-paste.filter_factory = keystone.contrib.admin_crud:CrudExtension.factory
-
-[filter:ec2_extension]
-paste.filter_factory = keystone.contrib.ec2:Ec2Extension.factory
-
-[filter:s3_extension]
-paste.filter_factory = keystone.contrib.s3:S3Extension.factory
-
-[filter:url_normalize]
-paste.filter_factory = keystone.middleware:NormalizingFilter.factory
-
-[filter:sizelimit]
-paste.filter_factory = keystone.middleware:RequestBodySizeLimiter.factory
-
-[filter:stats_monitoring]
-paste.filter_factory = keystone.contrib.stats:StatsMiddleware.factory
-
-[filter:stats_reporting]
-paste.filter_factory = keystone.contrib.stats:StatsExtension.factory
-
-[filter:access_log]
-paste.filter_factory = keystone.contrib.access:AccessLogMiddleware.factory
-
-[app:public_service]
-paste.app_factory = keystone.service:public_app_factory
-
-[app:service_v3]
-paste.app_factory = keystone.service:v3_app_factory
-
-[app:admin_service]
-paste.app_factory = keystone.service:admin_app_factory
-
-[pipeline:public_api]
-pipeline = access_log sizelimit url_normalize token_auth admin_token_auth xml_body json_body ec2_extension user_crud_extension public_service
-
-[pipeline:admin_api]
-pipeline = access_log sizelimit url_normalize token_auth admin_token_auth xml_body json_body ec2_extension s3_extension crud_extension admin_service
-
-[pipeline:api_v3]
-pipeline = access_log sizelimit url_normalize token_auth admin_token_auth xml_body json_body ec2_extension s3_extension service_v3
-
-[app:public_version_service]
-paste.app_factory = keystone.service:public_version_app_factory
-
-[app:admin_version_service]
-paste.app_factory = keystone.service:admin_version_app_factory
-
-[pipeline:public_version_api]
-pipeline = access_log sizelimit url_normalize xml_body public_version_service
-
-[pipeline:admin_version_api]
-pipeline = access_log sizelimit url_normalize xml_body admin_version_service
-
-[composite:main]
-use = egg:Paste#urlmap
-/v2.0 = public_api
-/v3 = api_v3
-/ = public_version_api
-
-[composite:admin]
-use = egg:Paste#urlmap
-/v2.0 = admin_api
-/v3 = api_v3
-/ = admin_version_api

httpd/keystone.py

@@ -7,15 +7,11 @@ from keystone import config
 
 LOG = logging.getLogger(__name__)
 CONF = config.CONF
-config_files = ['/etc/keystone/keystone.conf']
+CONF(project='keystone')
-CONF(project='keystone', default_config_files=config_files)
 
-conf = CONF.config_file[0]
 name = os.path.basename(__file__)
 
 if CONF.debug:
     CONF.log_opt_values(logging.getLogger(CONF.prog), logging.DEBUG)
 
-options = deploy.appconfig('config:%s' % CONF.config_file[0])
+deploy.loadapp('config:%s' % config.find_paste_config(), name=name)
-
-application = deploy.loadapp('config:%s' % conf, name=name)

keystone/common/config.py

@@ -407,3 +407,6 @@ def configure():
     for method_name in CONF.auth.methods:
         if method_name not in _DEFAULT_AUTH_METHODS:
             register_str(method_name, group='auth')
+
+    # PasteDeploy config file
+    register_str('config_file', group='paste_deploy', default=None)

keystone/config.py

@@ -15,7 +15,10 @@
 # under the License.
 """Wrapper for keystone.common.config that configures itself on import."""
 
+import os
+
 from keystone.common import config
+from keystone import exception
 
 
 config.configure()
@@ -31,3 +34,31 @@ register_cli_bool = config.register_cli_bool
 register_int = config.register_int
 register_cli_int = config.register_cli_int
 setup_authentication = config.setup_authentication
+
+
+def find_paste_config():
+    """Selects Keystone paste.deploy configuration file.
+
+    Keystone paste.deploy configuration file is selectd in [paste_deploy]
+    section of the main Keystone configuration file.
+    For example:
+        [paste_deploy]
+        config_file = keystone-paste.ini
+
+    :returns: The selected configuration filename
+    :raises: exception.PasteConfigNotFound
+    """
+    if CONF.paste_deploy.config_file:
+        paste_config = CONF.paste_deploy.config_file
+        paste_config_value = paste_config
+        if not os.path.isabs(paste_config):
+            paste_config = CONF.find_file(paste_config)
+    elif CONF.config_file:
+        paste_config = CONF.config_file[0]
+        paste_config_value = paste_config
+    else:
+        paste_config = CONF.find_file('keystone.conf')
+        paste_config_value = 'keystone.conf'
+    if not paste_config or not os.path.exists(paste_config):
+        raise exception.PasteConfigNotFound(config_file=paste_config_value)
+    return paste_config

keystone/exception.py

@@ -242,3 +242,9 @@ class NotImplemented(Error):
     """The action you have requested has not been implemented."""
     code = 501
     title = 'Not Implemented'
+
+
+class PasteConfigNotFound(UnexpectedError):
+    """The Keystone paste configuration file %(config_file)s could not be
+    found.
+    """

keystone/test.py

@@ -302,8 +302,8 @@ class TestCase(NoModule, unittest.TestCase):
             test_path = os.path.join(TESTSDIR, config)
             etc_path = os.path.join(ROOTDIR, 'etc', config)
             for path in [test_path, etc_path]:
-                if os.path.exists('%s.conf.sample' % path):
+                if os.path.exists('%s-paste.ini' % path):
-                    return 'config:%s.conf.sample' % path
+                    return 'config:%s-paste.ini' % path
         return config
 
     def loadapp(self, config, name='main'):

tests/test_config.py

@@ -0,0 +1,18 @@
+from keystone import config
+from keystone import exception
+from keystone import test
+
+
+CONF = config.CONF
+
+
+class ConfigTestCase(test.TestCase):
+    def test_paste_config(self):
+        self.assertEqual(config.find_paste_config(),
+                         test.etcdir('keystone-paste.ini'))
+        self.opt_in_group('paste_deploy', config_file='XYZ')
+        self.assertRaises(exception.PasteConfigNotFound,
+                          config.find_paste_config)
+        self.opt_in_group('paste_deploy', config_file='')
+        self.assertEqual(config.find_paste_config(),
+                         test.etcdir('keystone.conf.sample'))