Use oslo.config for Horizon configuration

This patch adds the infrastructure needed to move the configuration
of Horizon into oslo.config-compatible configuration file, instead of
the Django's Python-based configuration. It doesn't actually define
any configuration options, just the mechanism for loading them and the
additional types necessary to handle Horizon's complex configuration,
and the integration with oslo-config-generator.

Subsequent patches will add groups of options, making it possible to use
them in the local_settings.conf file instead of the local_settings.py
file. Note, that the options specified in the local_settings.py file
will continue to work.

Partially-Implements: blueprint ini-based-configuration
Change-Id: I2ed79ef0c6ac6d3816bba13346b7d001c46a7e80

openstack_dashboard/settings.py

@@ -16,6 +16,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import glob
 import logging
 import os
 import sys
@@ -26,6 +27,7 @@ from django.utils.translation import ugettext_lazy as _
 
 from openstack_dashboard import exceptions
 from openstack_dashboard import theme_settings
+from openstack_dashboard.utils import config
 from openstack_dashboard.utils import settings as settings_utils
 
 from horizon.utils.escape import monkeypatch_escape
@@ -339,11 +341,27 @@ OPENSTACK_PROFILER = {
     'enabled': False
 }
 
+if not LOCAL_PATH:
+    LOCAL_PATH = os.path.join(ROOT_PATH, 'local')
+LOCAL_SETTINGS_DIR_PATH = os.path.join(LOCAL_PATH, "local_settings.d")
+
+_files = glob.glob(os.path.join(LOCAL_PATH, 'local_settings.conf'))
+_files.extend(
+    sorted(glob.glob(os.path.join(LOCAL_SETTINGS_DIR_PATH, '*.conf'))))
+_config = config.load_config(_files, ROOT_PATH, LOCAL_PATH)
+
+# Apply the general configuration.
+config.apply_config(_config, globals())
+
 try:
     from local.local_settings import *  # noqa: F403,H303
 except ImportError:
     _LOG.warning("No local_settings file found.")
 
+# configure templates
+if not TEMPLATES[0]['DIRS']:
+    TEMPLATES[0]['DIRS'] = [os.path.join(ROOT_PATH, 'templates')]
+
 # configure template debugging
 TEMPLATES[0]['OPTIONS']['debug'] = DEBUG
 

openstack_dashboard/test/tests/utils.py

@@ -13,13 +13,15 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import unittest
+
 from oslo_utils import uuidutils
 
-from openstack_dashboard.test import helpers as test
+from openstack_dashboard.utils import config_types
 from openstack_dashboard.utils import filters
 
 
-class UtilsFilterTests(test.TestCase):
+class UtilsFilterTests(unittest.TestCase):
     def test_accept_valid_integer(self):
         val = 100
         ret = filters.get_int_or_uuid(val)
@@ -38,3 +40,31 @@ class UtilsFilterTests(test.TestCase):
     def test_reject_random_string(self):
         val = '55WbJTpJDf'
         self.assertRaises(ValueError, filters.get_int_or_uuid, val)
+
+
+class ConfigTypesTest(unittest.TestCase):
+    def test_literal(self):
+        literal = config_types.Literal([0])
+        self.assertEqual([1, 2, 3], literal("[1, 2, 3]"))
+        self.assertRaises(ValueError, literal, "[1, '2', 3]")
+
+        literal = config_types.Literal({0: ""})
+        self.assertEqual({1: 'a', 2: u'b'}, literal("{1: 'a', 2: u'b'}"))
+        self.assertRaises(ValueError, literal, "[1, '2', 3]")
+        self.assertRaises(ValueError, literal, "{1: 1, '2': 2}")
+
+        literal = config_types.Literal((True, 1, ""))
+        self.assertEqual((True, 13, 'x'), literal("(True, 13, 'x')"))
+        self.assertRaises(ValueError, literal, "(True, True)")
+        self.assertRaises(ValueError, literal, "(True, True, False, False)")
+        self.assertRaises(ValueError, literal, "(2, True, 'a')")
+        self.assertRaises(ValueError, literal, "(")
+
+        literal = config_types.Literal([(True, 1, lambda s: s.upper())])
+        self.assertEqual([(True, 13, 'X')], literal("[(True, 13, 'x')]"))
+
+    def test_url(self):
+        url = config_types.URL()
+        self.assertEqual('/webroot/static/', url('/webroot//static'))
+        self.assertEqual('http://webroot/static/',
+                         url('http://webroot//static'))

openstack_dashboard/utils/config.py

@@ -0,0 +1,68 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+This module contains utility functions for loading Horizon's
+configuration from .ini files using the oslo.config library.
+"""
+
+import six
+
+from oslo_config import cfg
+
+# XXX import the actual config groups here
+# from openstack_dashboard.config import config_compress
+
+
+def load_config(files=None, root_path=None, local_path=None):
+    """Load the configuration from specified files."""
+
+    config = cfg.ConfigOpts()
+    config.register_opts([
+        cfg.Opt('root_path', default=root_path),
+        cfg.Opt('local_path', default=local_path),
+    ])
+    # XXX register actual config groups here
+    # theme_group = config_theme.register_config(config)
+    if files is not None:
+        config(args=[], default_config_files=files)
+    return config
+
+
+def apply_config(config, target):
+    """Apply the configuration on the specified settings module."""
+    # TODO(rdopiera) fill with actual config groups
+    # apply_config_group(config.email, target, 'email')
+
+
+def apply_config_group(config_group, target, prefix=None):
+    for key, value in six.iteritems(config_group):
+        name = key.upper()
+        if prefix:
+            name = '_'.join([prefix.upper(), name])
+        target[name] = value
+
+
+def list_options():
+    # This is a really nasty hack to make the translatable strings
+    # work without having to initialize Django and read all the settings.
+    from django.apps import registry
+    from django.conf import settings
+
+    settings.configure()
+    registry.apps.check_apps_ready = lambda: True
+
+    config = load_config()
+    return [
+        (name, [d['opt'] for d in group._opts.values()])
+        for (name, group) in config._groups.items()
+    ]

openstack_dashboard/utils/config_types.py

@@ -0,0 +1,213 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+A set of custom types for oslo.config.
+"""
+
+import ast
+import os
+import re
+
+import six
+
+from django.utils import encoding
+from django.utils import functional
+from django.utils.module_loading import import_string
+from django.utils.translation import pgettext_lazy
+from oslo_config import types
+
+
+class Maybe(types.ConfigType):
+    """A custom option type for a value that may be None."""
+
+    def __init__(self, type_):
+        self.type_ = type_
+        type_name = getattr(type_, 'type_name', 'unknown value')
+        super(Maybe, self).__init__('optional %s' % type_name)
+
+    def __call__(self, value):
+        if value is None:
+            return None
+        return self.type_(value)
+
+    def _formatter(self, value):
+        if value is None:
+            return ''
+        return self.type_._formatter(value)
+
+
+class URL(types.ConfigType):
+    """A custom option type for a URL or part of URL."""
+
+    CLEAN_SLASH_RE = re.compile(r'(?<!:)//')
+
+    def __init__(self):
+        super(URL, self).__init__('web URL')
+
+    def __call__(self, value):
+        if not isinstance(value, six.string_types):
+            raise ValueError("Expected URL.")
+        value = re.sub(self.CLEAN_SLASH_RE, '/', value)
+        if not value.endswith('/'):
+            value += '/'
+        return value
+
+    def _formatter(self, value):
+        return self.quote_trailing_and_leading_space(value)
+
+
+class Path(types.ConfigType):
+    """A custom option type for a path to file."""
+
+    def __init__(self):
+        super(Path, self).__init__('filesystem path')
+
+    def __call__(self, value):
+        if not isinstance(value, six.string_types):
+            raise ValueError("Expected file path.")
+        return os.path.normpath(value)
+
+    def _formatter(self, value):
+        return self.quote_trailing_and_leading_space(value)
+
+
+class Translate(types.ConfigType):
+    """A custom option type for translatable strings."""
+
+    def __init__(self, hint=None):
+        self.hint = hint
+        super(Translate, self).__init__('translatable string')
+
+    def __call__(self, value):
+        if not isinstance(value, six.string_types):
+            return value
+        return pgettext_lazy(value, self.hint)
+
+    def _formatter(self, value):
+        return self.quote_trailing_and_leading_space(
+            encoding.force_text(value))
+
+
+class Literal(types.ConfigType):
+    """A custom option type for a Python literal."""
+
+    def __init__(self, spec=None):
+        self.spec = spec
+        super(Literal, self).__init__('python literal')
+
+    def __call__(self, value):
+        if isinstance(value, six.string_types):
+            try:
+                value = ast.literal_eval(value)
+            except SyntaxError as e:
+                six.raise_from(ValueError(e), e)
+        self.validate(value, self.spec)
+        return self.update(value, self.spec)
+
+    def validate(self, result, spec):  # noqa Yes, it's too complex.
+        """Validate that the result has the correct structure."""
+        if spec is None:
+            # None matches anything.
+            return
+        if isinstance(spec, dict):
+            if not isinstance(result, dict):
+                raise ValueError('Dictionary expected, but %r found.' % result)
+            if spec:
+                spec_value = next(iter(spec.values()))  # Yay Python 3!
+                for value in result.values():
+                    self.validate(value, spec_value)
+                spec_key = next(iter(spec.keys()))
+                for key in result.keys():
+                    self.validate(key, spec_key)
+        if isinstance(spec, list):
+            if not isinstance(result, list):
+                raise ValueError('List expected, but %r found.' % result)
+            if spec:
+                for value in result:
+                    self.validate(value, spec[0])
+        if isinstance(spec, tuple):
+            if not isinstance(result, tuple):
+                raise ValueError('Tuple expected, but %r found.' % result)
+            if len(result) != len(spec):
+                raise ValueError('Expected %d elements in tuple %r.' %
+                                 (len(spec), result))
+            for s, value in zip(spec, result):
+                self.validate(value, s)
+        if isinstance(spec, six.string_types):
+            if not isinstance(result, six.string_types):
+                raise ValueError('String expected, but %r found.' % result)
+        if isinstance(spec, int):
+            if not isinstance(result, int):
+                raise ValueError('Integer expected, but %r found.' % result)
+        if isinstance(spec, bool):
+            if not isinstance(result, bool):
+                raise ValueError('Boolean expected, but %r found.' % result)
+
+    def update(self, result, spec):
+        """Replace elements with results of calling callables."""
+        if isinstance(spec, dict):
+            if spec:
+                spec_value = next(iter(spec.values()))
+                for key, value in result.items():
+                    result[key] = self.update(value, spec_value)
+        if isinstance(spec, list):
+            if spec:
+                for i, value in enumerate(result):
+                    result[i] = self.update(value, spec[0])
+        if isinstance(spec, tuple):
+            return tuple(self.update(value, s)
+                         for s, value in zip(spec, result))
+        if callable(spec):
+            return spec(result)
+        return result
+
+    def _format(self, result):
+        if isinstance(result, dict):
+            return '{%s}' % ', '.join(
+                '%s: %s' % (key, self._format(value))
+                for key, value in result.items()
+            )
+        if isinstance(result, list):
+            return '[%s]' % ', '.join(self._format(value) for value in result)
+        if isinstance(result, tuple):
+            return '(%s)' % ', '.join(self._format(value) for value in result)
+        if isinstance(result, functional.Promise):
+            # Lazy translatable string.
+            return repr(encoding.force_text(result))
+        return repr(result)
+
+    def _formatter(self, value):
+        # We need to walk the lists and dicts to handle the Django lazy
+        # translatable strings inside.
+        return self._format(value)
+
+
+class Importable(types.ConfigType):
+    """A custom option type for an importable python object."""
+
+    def __init__(self):
+        super(Importable, self).__init__('importable python object')
+
+    def __call__(self, value):
+        if not isinstance(value, six.string_types):
+            # Already imported.
+            return value
+        try:
+            return import_string(value)
+        except ImportError as e:
+            six.raise_from(ValueError(e), e)
+
+    def _formatter(self, value):
+        module = value.__module__
+        name = value.__name__
+        return self.quote_trailing_and_leading_space('%s.%s' % (module, name))

setup.cfg

@@ -64,3 +64,7 @@ directory = reports
 [extract_messages]
 keywords = gettext_noop gettext_lazy ngettext_lazy:1,2 ugettext_noop ugettext_lazy ungettext_lazy:1,2 npgettext:1c,2,3 pgettext_lazy:1c,2 npgettext_lazy:1c,2,3
 add_comments = Translators:
+
+[entry_points]
+oslo.config.opts =
+    openstack_dashboard = openstack_dashboard.utils.config:list_options