Merge "Generate language list automatically"

openstack_dashboard/dashboards/settings/user/forms.py

@@ -13,6 +13,8 @@
 #    under the License.
 
 from datetime import datetime
+import gettext as gettext_module
+import itertools
 import string
 
 import babel
@@ -20,6 +22,7 @@ import babel.dates
 from django.conf import settings
 from django import shortcuts
 from django.utils import encoding
+from django.utils import lru_cache
 from django.utils import translation
 from django.utils.translation import ugettext_lazy as _
 import pytz
@@ -29,6 +32,60 @@ from horizon import messages
 from horizon.utils import functions
 
 
+def _get_language_display_name(code, desc):
+    try:
+        desc = translation.get_language_info(code)['name_local']
+        desc = string.capwords(desc)
+    except KeyError:
+        # If a language is not defined in django.conf.locale.LANG_INFO
+        # get_language_info raises KeyError
+        pass
+    return "%s (%s)" % (desc, code)
+
+
+@lru_cache.lru_cache()
+def _get_languages():
+
+    languages = []
+    processed_catalogs = set([])
+    # sorted() here is important to make processed_catalogs checking
+    # work properly.
+    for lang_code, lang_label in sorted(settings.LANGUAGES):
+        if lang_code == 'en':
+            # Add English as source language
+            languages.append(('en',
+                              _get_language_display_name('en', 'English')))
+            continue
+        found_catalogs = [
+            gettext_module.find(domain, locale_path,
+                                [translation.to_locale(lang_code)])
+            for locale_path, domain
+            in itertools.product(settings.LOCALE_PATHS,
+                                 ['django', 'djangojs'])
+        ]
+        if not all(found_catalogs):
+            continue
+        # NOTE(amotoki):
+        # Check if found catalogs are already processed or not.
+        # settings.LANGUAGES can contains languages with a same prefix
+        # like es, es-ar, es-mx. gettext_module.find() searchess Message
+        # catalog for es-ar in the order of 'es_AR' and then 'es'.
+        # If 'es' locale is translated, 'es-ar' is included in the list of
+        # found_catalogs even if 'es-ar' is not translated.
+        # In this case, the list already includes 'es' and
+        # there is no need to include 'es-ar'.
+        result = [catalog in processed_catalogs
+                  for catalog in found_catalogs]
+        if any(result):
+            continue
+        processed_catalogs |= set(found_catalogs)
+        languages.append(
+            (lang_code,
+             _get_language_display_name(lang_code, lang_label))
+        )
+    return sorted(languages)
+
+
 class UserSettingsForm(forms.SelfHandlingForm):
     max_value = getattr(settings, 'API_RESULT_LIMIT', 1000)
     language = forms.ChoiceField(label=_("Language"))
@@ -51,19 +108,7 @@ class UserSettingsForm(forms.SelfHandlingForm):
     def __init__(self, *args, **kwargs):
         super(UserSettingsForm, self).__init__(*args, **kwargs)
 
-        # Languages
+        self.fields['language'].choices = _get_languages()
-        def get_language_display_name(code, desc):
-            try:
-                desc = translation.get_language_info(code)['name_local']
-                desc = string.capwords(desc)
-            except KeyError:
-                # If a language is not defined in django.conf.locale.LANG_INFO
-                # get_language_info raises KeyError
-                pass
-            return "%s (%s)" % (desc, code)
-        languages = [(k, get_language_display_name(k, v))
-                     for k, v in settings.LANGUAGES]
-        self.fields['language'].choices = languages
 
         # Timezones
         timezones = []

openstack_dashboard/dashboards/settings/user/tests.py

@@ -14,7 +14,12 @@
 
 from django.conf import settings
 from django.core.urlresolvers import reverse
+from django.test.utils import override_settings
+from django.utils import translation
 
+import mock
+
+from openstack_dashboard.dashboards.settings.user import forms
 from openstack_dashboard.test import helpers as test
 
 
@@ -30,12 +35,146 @@ class UserSettingsTest(test.TestCase):
         self.assertContains(res, "UTC -03:00: Falkland Islands Time")
         self.assertContains(res, "UTC -10:00: United States (Honolulu) Time")
 
+    @override_settings(LOCALE_PATHS=['openstack_dashboard/locale'])
     def test_display_language(self):
         # Add an unknown language to LANGUAGES list
+        # to check it works with unknown language in the list.
         settings.LANGUAGES += (('unknown', 'Unknown Language'),)
 
         res = self.client.get(INDEX_URL)
-        # Known language
+        # In this test, we just checks language list is properly
+        # generated without an error as the result depends on
+        # existence of translation message catalogs.
         self.assertContains(res, 'English')
-        # Unknown language
+
-        self.assertContains(res, 'Unknown Language')
+
+class LanguageTest(test.TestCase):
+    """Tests for _get_languages()."""
+
+    def setUp(self):
+        super(LanguageTest, self).setUp()
+        # _get_languages is decorated by lru_cache,
+        # so we need to clear cache info before each test run.
+        forms._get_languages.cache_clear()
+
+    @staticmethod
+    def _patch_gettext_find_all_translated(*args, **kwargs):
+        domain = args[0]
+        locale_path = args[1]
+        locale = args[2][0]
+        return '%s/%s/LC_MESSAGES/%s.mo' % (locale_path, locale, domain)
+
+    @override_settings(LANGUAGES=(('de', 'Germany'),
+                                  ('en', 'English'),
+                                  ('ja', 'Japanese')),
+                       LOCALE_PATHS=['horizon/locale',
+                                     'openstack_dashboard/locale'])
+    def test_language_list_all_translated(self):
+        with mock.patch.object(
+                forms.gettext_module, 'find',
+                side_effect=LanguageTest._patch_gettext_find_all_translated):
+            languages = forms._get_languages()
+        self.assertEqual(['de', 'en', 'ja'],
+                         [code for code, name in languages])
+
+    @override_settings(LANGUAGES=(('de', 'Germany'),
+                                  ('en', 'English'),
+                                  ('fr', 'French'),
+                                  ('ja', 'Japanese')),
+                       LOCALE_PATHS=['horizon/locale',
+                                     'openstack_dashboard/locale'])
+    def test_language_list_partially_translated(self):
+        def _patch_gettext_find(*args, **kwargs):
+            domain = args[0]
+            locale_path = args[1]
+            locale = args[2][0]
+            # Assume de and fr are partially translated.
+            if locale == translation.to_locale('de'):
+                if (domain == 'django' and
+                        locale_path == 'openstack_dashboard/locale'):
+                    return
+            elif locale == translation.to_locale('fr'):
+                if (domain == 'djangojs' and locale_path == 'horizon/locale'):
+                    return
+            return '%s/%s/LC_MESSAGES/%s.mo' % (locale_path, locale,
+                                                domain)
+        with mock.patch.object(
+                forms.gettext_module, 'find',
+                side_effect=_patch_gettext_find):
+            languages = forms._get_languages()
+        self.assertEqual(['en', 'ja'],
+                         [code for code, name in languages])
+
+    @override_settings(LANGUAGES=(('de', 'Germany'),
+                                  ('ja', 'Japanese')),
+                       LOCALE_PATHS=['horizon/locale',
+                                     'openstack_dashboard/locale'])
+    def test_language_list_no_english(self):
+        with mock.patch.object(
+                forms.gettext_module, 'find',
+                side_effect=LanguageTest._patch_gettext_find_all_translated):
+            languages = forms._get_languages()
+        self.assertEqual(['de', 'ja'],
+                         [code for code, name in languages])
+
+    @override_settings(LANGUAGES=(('de', 'Germany'),
+                                  ('en', 'English'),
+                                  ('ja', 'Japanese')),
+                       LOCALE_PATHS=['horizon/locale',
+                                     'openstack_dashboard/locale'])
+    def test_language_list_with_untranslated_language(self):
+        def _patch_gettext_find(*args, **kwargs):
+            domain = args[0]
+            locale_path = args[1]
+            locale = args[2][0]
+            # Assume ja is not translated
+            if locale == translation.to_locale('ja'):
+                return
+            return '%s/%s/LC_MESSAGES/%s.mo' % (locale_path, locale, domain)
+        with mock.patch.object(
+                forms.gettext_module, 'find',
+                side_effect=_patch_gettext_find):
+            languages = forms._get_languages()
+        self.assertEqual(['de', 'en'],
+                         [code for code, name in languages])
+
+    @override_settings(LANGUAGES=(('es', 'Spanish'),
+                                  ('es-ar', 'Argentinian Spanish'),
+                                  ('es-mx', 'Mexican Spanish'),
+                                  ('en', 'English')),
+                       LOCALE_PATHS=['horizon/locale',
+                                     'openstack_dashboard/locale'])
+    def test_language_list_with_untranslated_same_prefix(self):
+        def _patch_gettext_find(*args, **kwargs):
+            domain = args[0]
+            locale_path = args[1]
+            locale = args[2][0]
+            # Assume es-ar is not translated and
+            # es-mx is partially translated.
+            # es is returned as fallback.
+            if locale == translation.to_locale('es-ar'):
+                locale = translation.to_locale('es')
+            elif (locale == translation.to_locale('es-mx') and
+                  locale_path == 'openstack_dashboard/locale'):
+                locale = translation.to_locale('es')
+            return '%s/%s/LC_MESSAGES/%s.mo' % (locale_path, locale,
+                                                domain)
+        with mock.patch.object(
+                forms.gettext_module, 'find',
+                side_effect=_patch_gettext_find):
+            languages = forms._get_languages()
+        self.assertEqual(['en', 'es'],
+                         [code for code, name in languages])
+
+    @override_settings(LANGUAGES=(('en', 'English'),
+                                  ('pt', 'Portuguese'),
+                                  ('pt-br', 'Brazilian Portuguese')),
+                       LOCALE_PATHS=['horizon/locale',
+                                     'openstack_dashboard/locale'])
+    def test_language_list_with_both_translated_same_prefix(self):
+        with mock.patch.object(
+                forms.gettext_module, 'find',
+                side_effect=LanguageTest._patch_gettext_find_all_translated):
+            languages = forms._get_languages()
+        self.assertEqual(['en', 'pt', 'pt-br'],
+                         [code for code, name in languages])

openstack_dashboard/local/local_settings.py.example

@@ -486,6 +486,30 @@ TIME_ZONE = "UTC"
 #    ('material', 'Material', 'themes/material'),
 #]
 
+# By default all languages with translation catalogs are enabled.
+# If you would like to enable a specific set of languages,
+# you can do this by setting LANGUAGES list below.
+# Each entry is a tuple of language code and language name.
+# LANGUAGES = (
+#     ('cs', 'Czech'),
+#     ('de', 'German'),
+#     ('en', 'English'),
+#     ('en-au', 'Australian English'),
+#     ('en-gb', 'British English'),
+#     ('es', 'Spanish'),
+#     ('fr', 'French'),
+#     ('id', 'Indonesian'),
+#     ('it', 'Italian'),
+#     ('ja', 'Japanese'),
+#     ('ko', 'Korean (Korea)'),
+#     ('pl', 'Polish'),
+#     ('pt-br', 'Portuguese (Brazil)'),
+#     ('ru', 'Russian'),
+#     ('tr', 'Turkish'),
+#     ('zh-cn', 'Simplified Chinese'),
+#     ('zh-tw', 'Chinese (Taiwan)'),
+# )
+
 LOGGING = {
     'version': 1,
     # When set to True this will disable all logging except

openstack_dashboard/settings.py

@@ -209,25 +209,6 @@ SESSION_COOKIE_MAX_SIZE = 4093
 #                https://bugs.launchpad.net/horizon/+bug/1349463
 SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
 
-LANGUAGES = (
-    ('cs', 'Czech'),
-    ('de', 'German'),
-    ('en', 'English'),
-    ('en-au', 'Australian English'),
-    ('en-gb', 'British English'),
-    ('es', 'Spanish'),
-    ('fr', 'French'),
-    ('id', 'Indonesian'),
-    ('it', 'Italian'),
-    ('ja', 'Japanese'),
-    ('ko', 'Korean (Korea)'),
-    ('pl', 'Polish'),
-    ('pt-br', 'Portuguese (Brazil)'),
-    ('ru', 'Russian'),
-    ('tr', 'Turkish'),
-    ('zh-cn', 'Simplified Chinese'),
-    ('zh-tw', 'Chinese (Taiwan)'),
-)
 LANGUAGE_CODE = 'en'
 LANGUAGE_COOKIE_NAME = 'horizon_language'
 USE_I18N = True

releasenotes/notes/language-list-generated-automatically-4a9bf752752d09f6.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    The available language list is now automatically generated based on
+    the availability of translation message catalogs of languages
+    instead of maintaining the language list manually.
+    If message catalogs (PO files) of some language exist for both
+    django and djangojs domains of horizon and openstack_dashboard,
+    the language will appear in the language list.
+    If you need to change the initial set of languages,
+    set ``LANGUAGES`` in ``local_settings.py``.