Sync excutils from oslo

In order to fix undesired error logs in Neutron (bug 1288188)
fixed save_and_reraise_exception() should be synced from oslo.
Oslo commit: 33a2cee6a690ecfdb2cecfe8f01b0b1dacb5bd17

Also sync gettextutils module as excutils depends on it
Latest commit in oslo: fd33d1eaa039913d8c82b94c511a3eab0c3d5789

Closes-Bug: #1294537
Related-Bug: #1288188
Change-Id: I62ab3e4e22aa000f3a8d1be26ef9e1cfb1714959

neutron/openstack/common/__init__.py

@@ -0,0 +1,17 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import six
+
+
+six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox'))

neutron/openstack/common/excutils.py

@@ -24,7 +24,7 @@ import traceback
 
 import six
 
-from neutron.openstack.common.gettextutils import _
+from neutron.openstack.common.gettextutils import _LE
 
 
 class save_and_reraise_exception(object):
@@ -49,9 +49,22 @@ class save_and_reraise_exception(object):
               decide_if_need_reraise()
               if not should_be_reraised:
                   ctxt.reraise = False
+
+    If another exception occurs and reraise flag is False,
+    the saved exception will not be logged.
+
+    If the caller wants to raise new exception during exception handling
+    he/she sets reraise to False initially with an ability to set it back to
+    True if needed::
+
+      except Exception:
+          with save_and_reraise_exception(reraise=False) as ctxt:
+              [if statements to determine whether to raise a new exception]
+              # Not raising a new exception, so reraise
+              ctxt.reraise = True
     """
-    def __init__(self):
+    def __init__(self, reraise=True):
-        self.reraise = True
+        self.reraise = reraise
 
     def __enter__(self):
         self.type_, self.value, self.tb, = sys.exc_info()
@@ -59,10 +72,11 @@ class save_and_reraise_exception(object):
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         if exc_type is not None:
-            logging.error(_('Original exception being dropped: %s'),
+            if self.reraise:
-                          traceback.format_exception(self.type_,
+                logging.error(_LE('Original exception being dropped: %s'),
-                                                     self.value,
+                              traceback.format_exception(self.type_,
-                                                     self.tb))
+                                                         self.value,
+                                                         self.tb))
             return False
         if self.reraise:
             six.reraise(self.type_, self.value, self.tb)
@@ -88,8 +102,8 @@ def forever_retry_uncaught_exceptions(infunc):
                 if (cur_time - last_log_time > 60 or
                         this_exc_message != last_exc_message):
                     logging.exception(
-                        _('Unexpected exception occurred %d time(s)... '
+                        _LE('Unexpected exception occurred %d time(s)... '
-                          'retrying.') % exc_count)
+                            'retrying.') % exc_count)
                     last_log_time = cur_time
                     last_exc_message = this_exc_message
                     exc_count = 0

neutron/openstack/common/gettextutils.py

@@ -23,11 +23,11 @@ Usual usage in an openstack.common module:
 """
 
 import copy
+import functools
 import gettext
 import locale
 from logging import handlers
 import os
-import re
 
 from babel import localedata
 import six
@@ -35,6 +35,17 @@ import six
 _localedir = os.environ.get('neutron'.upper() + '_LOCALEDIR')
 _t = gettext.translation('neutron', localedir=_localedir, fallback=True)
 
+# We use separate translation catalogs for each log level, so set up a
+# mapping between the log level name and the translator. The domain
+# for the log level is project_name + "-log-" + log_level so messages
+# for each level end up in their own catalog.
+_t_log_levels = dict(
+    (level, gettext.translation('neutron' + '-log-' + level,
+                                localedir=_localedir,
+                                fallback=True))
+    for level in ['info', 'warning', 'error', 'critical']
+)
+
 _AVAILABLE_LANGUAGES = {}
 USE_LAZY = False
 
@@ -60,6 +71,28 @@ def _(msg):
         return _t.ugettext(msg)
 
 
+def _log_translation(msg, level):
+    """Build a single translation of a log message
+    """
+    if USE_LAZY:
+        return Message(msg, domain='neutron' + '-log-' + level)
+    else:
+        translator = _t_log_levels[level]
+        if six.PY3:
+            return translator.gettext(msg)
+        return translator.ugettext(msg)
+
+# Translators for log levels.
+#
+# The abbreviated names are meant to reflect the usual use of a short
+# name like '_'. The "L" is for "log" and the other letter comes from
+# the level.
+_LI = functools.partial(_log_translation, level='info')
+_LW = functools.partial(_log_translation, level='warning')
+_LE = functools.partial(_log_translation, level='error')
+_LC = functools.partial(_log_translation, level='critical')
+
+
 def install(domain, lazy=False):
     """Install a _() function using the given translation domain.
 
@@ -118,7 +151,8 @@ class Message(six.text_type):
     and can be treated as such.
     """
 
-    def __new__(cls, msgid, msgtext=None, params=None, domain='neutron', *args):
+    def __new__(cls, msgid, msgtext=None, params=None,
+                domain='neutron', *args):
         """Create a new Message object.
 
         In order for translation to work gettext requires a message ID, this
@@ -213,47 +247,22 @@ class Message(six.text_type):
         if other is None:
             params = (other,)
         elif isinstance(other, dict):
-            params = self._trim_dictionary_parameters(other)
+            # Merge the dictionaries
+            # Copy each item in case one does not support deep copy.
+            params = {}
+            if isinstance(self.params, dict):
+                for key, val in self.params.items():
+                    params[key] = self._copy_param(val)
+            for key, val in other.items():
+                params[key] = self._copy_param(val)
         else:
             params = self._copy_param(other)
         return params
 
-    def _trim_dictionary_parameters(self, dict_param):
-        """Return a dict that only has matching entries in the msgid."""
-        # NOTE(luisg): Here we trim down the dictionary passed as parameters
-        # to avoid carrying a lot of unnecessary weight around in the message
-        # object, for example if someone passes in Message() % locals() but
-        # only some params are used, and additionally we prevent errors for
-        # non-deepcopyable objects by unicoding() them.
-
-        # Look for %(param) keys in msgid;
-        # Skip %% and deal with the case where % is first character on the line
-        keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', self.msgid)
-
-        # If we don't find any %(param) keys but have a %s
-        if not keys and re.findall('(?:[^%]|^)%[a-z]', self.msgid):
-            # Apparently the full dictionary is the parameter
-            params = self._copy_param(dict_param)
-        else:
-            params = {}
-            # Save our existing parameters as defaults to protect
-            # ourselves from losing values if we are called through an
-            # (erroneous) chain that builds a valid Message with
-            # arguments, and then does something like "msg % kwds"
-            # where kwds is an empty dictionary.
-            src = {}
-            if isinstance(self.params, dict):
-                src.update(self.params)
-            src.update(dict_param)
-            for key in keys:
-                params[key] = self._copy_param(src[key])
-
-        return params
-
     def _copy_param(self, param):
         try:
             return copy.deepcopy(param)
-        except TypeError:
+        except Exception:
             # Fallback to casting to unicode this will handle the
             # python code-like objects that can't be deep-copied
             return six.text_type(param)
@@ -297,9 +306,27 @@ def get_available_languages(domain):
     list_identifiers = (getattr(localedata, 'list', None) or
                         getattr(localedata, 'locale_identifiers'))
     locale_identifiers = list_identifiers()
+
     for i in locale_identifiers:
         if find(i) is not None:
             language_list.append(i)
+
+    # NOTE(luisg): Babel>=1.0,<1.3 has a bug where some OpenStack supported
+    # locales (e.g. 'zh_CN', and 'zh_TW') aren't supported even though they
+    # are perfectly legitimate locales:
+    #     https://github.com/mitsuhiko/babel/issues/37
+    # In Babel 1.3 they fixed the bug and they support these locales, but
+    # they are still not explicitly "listed" by locale_identifiers().
+    # That is  why we add the locales here explicitly if necessary so that
+    # they are listed as supported.
+    aliases = {'zh': 'zh_CN',
+               'zh_Hant_HK': 'zh_HK',
+               'zh_Hant': 'zh_TW',
+               'fil': 'tl_PH'}
+    for (locale, alias) in six.iteritems(aliases):
+        if locale in language_list and alias not in language_list:
+            language_list.append(alias)
+
     _AVAILABLE_LANGUAGES[domain] = language_list
     return copy.copy(language_list)