re-implement thread safe fnmatch

fnmatch is not thread safe for versions <= 2.7.9. We have used
it in some places without any lock for concurrency scenario. This patch
re-implements a thread safe match() which is very similar to fnmatch by
using its translate function.

Change-Id: I4b6c2ea72841201519144eb09ecf8c82b16b6143
ref: https://hg.python.org/cpython/rev/fe12c34c39eb
Closes-Bug: #1519767

ceilometer/agent/manager.py

@@ -19,7 +19,6 @@
 # under the License.
 
 import collections
-import fnmatch
 import itertools
 import random
 
@@ -241,7 +240,7 @@ class AgentManager(service_base.BaseService):
 
         def _match(pollster):
             """Find out if pollster name matches to one of the list."""
-            return any(fnmatch.fnmatch(pollster.name, pattern) for
+            return any(utils.match(pollster.name, pattern) for
                        pattern in pollster_list)
 
         if type(namespaces) is not list:

ceilometer/dispatcher/gnocchi.py

@@ -13,7 +13,6 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 import copy
-import fnmatch
 import itertools
 import operator
 import os
@@ -30,6 +29,7 @@ from ceilometer import dispatcher
 from ceilometer.dispatcher import gnocchi_client
 from ceilometer.i18n import _, _LE, _LW
 from ceilometer import keystone_client
+from ceilometer import utils
 
 CACHE_NAMESPACE = uuid.uuid4()
 LOG = log.getLogger(__name__)
@@ -110,7 +110,7 @@ class ResourcesDefinition(object):
 
     def match(self, metric_name):
         for t in self.cfg['metrics']:
-            if fnmatch.fnmatch(metric_name, t):
+            if utils.match(metric_name, t):
                 return True
         return False
 

ceilometer/event/converter.py

@@ -13,8 +13,6 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import fnmatch
-
 from debtcollector import moves
 from oslo_config import cfg
 from oslo_log import log
@@ -24,6 +22,7 @@ import six
 from ceilometer import declarative
 from ceilometer.event.storage import models
 from ceilometer.i18n import _
+from ceilometer import utils
 
 OPTS = [
     cfg.StrOpt('definitions_cfg_file',
@@ -131,13 +130,13 @@ class EventDefinition(object):
 
     def included_type(self, event_type):
         for t in self._included_types:
-            if fnmatch.fnmatch(event_type, t):
+            if utils.match(event_type, t):
                 return True
         return False
 
     def excluded_type(self, event_type):
         for t in self._excluded_types:
-            if fnmatch.fnmatch(event_type, t):
+            if utils.match(event_type, t):
                 return True
         return False
 

ceilometer/meter/notifications.py

@@ -11,7 +11,6 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import fnmatch
 import itertools
 import pkg_resources
 import six
@@ -26,6 +25,7 @@ from ceilometer.agent import plugin_base
 from ceilometer import declarative
 from ceilometer.i18n import _LE
 from ceilometer import sample
+from ceilometer import utils
 
 OPTS = [
     cfg.StrOpt('meter_definitions_cfg_file',
@@ -97,7 +97,7 @@ class MeterDefinition(object):
 
     def match_type(self, meter_name):
         for t in self._event_type:
-            if fnmatch.fnmatch(meter_name, t):
+            if utils.match(meter_name, t):
                 return True
 
     def to_samples(self, message, all_values=False):

ceilometer/pipeline.py

@@ -18,7 +18,6 @@
 # under the License.
 
 import abc
-import fnmatch
 import hashlib
 import os
 
@@ -36,6 +35,7 @@ from ceilometer.i18n import _, _LI, _LW
 from ceilometer import publisher
 from ceilometer.publisher import utils as publisher_utils
 from ceilometer import sample as sample_util
+from ceilometer import utils
 
 
 OPTS = [
@@ -272,11 +272,11 @@ class Source(object):
     def is_supported(dataset, data_name):
         # Support wildcard like storage.* and !disk.*
         # Start with negation, we consider that the order is deny, allow
-        if any(fnmatch.fnmatch(data_name, datapoint[1:])
+        if any(utils.match(data_name, datapoint[1:])
                for datapoint in dataset if datapoint[0] == '!'):
             return False
 
-        if any(fnmatch.fnmatch(data_name, datapoint)
+        if any(utils.match(data_name, datapoint)
                for datapoint in dataset if datapoint[0] != '!'):
             return True
 

ceilometer/utils.py

@@ -23,8 +23,11 @@ import calendar
 import copy
 import datetime
 import decimal
+import fnmatch
 import hashlib
+import re
 import struct
+import sys
 
 from oslo_concurrency import processutils
 from oslo_config import cfg
@@ -254,3 +257,29 @@ def kill_listeners(listeners):
     for listener in listeners:
         listener.stop()
         listener.wait()
+
+
+if sys.version_info > (2, 7, 9):
+    match = fnmatch.fnmatch
+else:
+    _MATCH_CACHE = {}
+    _MATCH_CACHE_MAX = 100
+
+    def match(string, pattern):
+        """Thread safe fnmatch re-implementation.
+
+        Standard library fnmatch in Python versions <= 2.7.9 has thread safe
+        issue, this helper function is created for such case. see:
+        https://bugs.python.org/issue23191
+        """
+        string = string.lower()
+        pattern = pattern.lower()
+
+        cached_pattern = _MATCH_CACHE.get(pattern)
+        if cached_pattern is None:
+            translated_pattern = fnmatch.translate(pattern)
+            cached_pattern = re.compile(translated_pattern)
+            if len(_MATCH_CACHE) >= _MATCH_CACHE_MAX:
+                _MATCH_CACHE.clear()
+            _MATCH_CACHE[pattern] = cached_pattern
+        return cached_pattern.match(string) is not None