Browse Source

Add support for fair locks

This adds support for a "fair" variant of the lock.  When there are
multiple entities within a single process that are blocked waiting
for the lock the fair lock will ensure that they acquire the lock
in FIFO order.

For now at least, when fair locks are in use we don't support
the "semaphores" argument.

If external locks are enabled, the inter-process ordering will be
determined by the underlying OS lock ordering and process scheduling.

Change-Id: I37577becff4978bf643c65fa9bc2d78d342ea35a
Chris Friesen 1 year ago
parent
commit
2b55da68ae

+ 26
- 0
doc/source/user/index.rst View File

@@ -47,6 +47,32 @@ sure that the names of the locks used are carefully chosen (typically by
47 47
 namespacing them to your app so that other apps will not chose the same
48 48
 names).
49 49
 
50
+Enabling fair locking
51
+=====================
52
+
53
+By default there is no requirement that the lock is ``fair``.  That is, it's
54
+possible for a thread to block waiting for the lock, then have another thread
55
+block waiting for the lock, and when the lock is released by the current owner
56
+the second waiter could acquire the lock before the first.  In an extreme case
57
+you could have a whole string of other threads acquire the lock before the
58
+first waiter acquires it, resulting in unpredictable amounts of latency.
59
+
60
+For cases where this is a problem, it's possible to specify the use of fair
61
+locks::
62
+
63
+    @lockutils.synchronized('not_thread_process_safe', fair=True)
64
+    def not_thread_process_safe():
65
+        pass
66
+
67
+When using fair locks the lock itself is slightly more expensive (which
68
+shouldn't matter in most cases), but it will ensure that all threads that
69
+block waiting for the lock will acquire it in the order that they blocked.
70
+
71
+The exception to this is when specifying both ``external`` and ``fair``
72
+locks.  In this case, the ordering *within* a given process will be fair, but
73
+the ordering *between* processes will be determined by the behaviour of the
74
+underlying OS.
75
+
50 76
 Common ways to prefix/namespace the synchronized decorator
51 77
 ==========================================================
52 78
 

+ 60
- 6
oslo_concurrency/lockutils.py View File

@@ -87,6 +87,49 @@ ReaderWriterLock = fasteners.ReaderWriterLock
87 87
 """
88 88
 
89 89
 
90
+class FairLocks(object):
91
+    """A garbage collected container of fair locks.
92
+
93
+    With a fair lock, contending lockers will get the lock in the order in
94
+    which they tried to acquire it.
95
+
96
+    This collection internally uses a weak value dictionary so that when a
97
+    lock is no longer in use (by any threads) it will automatically be
98
+    removed from this container by the garbage collector.
99
+    """
100
+
101
+    def __init__(self):
102
+        self._locks = weakref.WeakValueDictionary()
103
+        self._lock = threading.Lock()
104
+
105
+    def get(self, name):
106
+        """Gets (or creates) a lock with a given name.
107
+
108
+        :param name: The lock name to get/create (used to associate
109
+                     previously created names with the same lock).
110
+
111
+        Returns an newly constructed lock (or an existing one if it was
112
+        already created for the given name).
113
+        """
114
+        with self._lock:
115
+            try:
116
+                return self._locks[name]
117
+            except KeyError:
118
+                # The fasteners module specifies that
119
+                # ReaderWriterLock.write_lock() will give FIFO behaviour,
120
+                # so we don't need to do anything special ourselves.
121
+                rwlock = ReaderWriterLock()
122
+                self._locks[name] = rwlock
123
+                return rwlock
124
+
125
+
126
+_fair_locks = FairLocks()
127
+
128
+
129
+def internal_fair_lock(name):
130
+    return _fair_locks.get(name)
131
+
132
+
90 133
 class Semaphores(object):
91 134
     """A garbage collected container of semaphores.
92 135
 
@@ -170,7 +213,7 @@ def internal_lock(name, semaphores=None):
170 213
 
171 214
 @contextlib.contextmanager
172 215
 def lock(name, lock_file_prefix=None, external=False, lock_path=None,
173
-         do_log=True, semaphores=None, delay=0.01):
216
+         do_log=True, semaphores=None, delay=0.01, fair=False):
174 217
     """Context based lock
175 218
 
176 219
     This function yields a `threading.Semaphore` instance (if we don't use
@@ -200,16 +243,26 @@ def lock(name, lock_file_prefix=None, external=False, lock_path=None,
200 243
 
201 244
     :param delay: Delay between acquisition attempts (in seconds).
202 245
 
246
+    :param fair: Whether or not we want a "fair" lock where contending lockers
247
+        will get the lock in the order in which they tried to acquire it.
248
+
203 249
     .. versionchanged:: 0.2
204 250
        Added *do_log* optional parameter.
205 251
 
206 252
     .. versionchanged:: 0.3
207 253
        Added *delay* and *semaphores* optional parameters.
208 254
     """
209
-    int_lock = internal_lock(name, semaphores=semaphores)
255
+    if fair:
256
+        if semaphores is not None:
257
+            raise NotImplementedError(_('Specifying semaphores is not '
258
+                                        'supported when using fair locks.'))
259
+        # The fastners module specifies that write_lock() provides fairness.
260
+        int_lock = internal_fair_lock(name).write_lock()
261
+    else:
262
+        int_lock = internal_lock(name, semaphores=semaphores)
210 263
     with int_lock:
211 264
         if do_log:
212
-            LOG.debug('Acquired semaphore "%(lock)s"', {'lock': name})
265
+            LOG.debug('Acquired lock "%(lock)s"', {'lock': name})
213 266
         try:
214 267
             if external and not CONF.oslo_concurrency.disable_process_locking:
215 268
                 ext_lock = external_lock(name, lock_file_prefix, lock_path)
@@ -225,11 +278,11 @@ def lock(name, lock_file_prefix=None, external=False, lock_path=None,
225 278
                 yield int_lock
226 279
         finally:
227 280
             if do_log:
228
-                LOG.debug('Releasing semaphore "%(lock)s"', {'lock': name})
281
+                LOG.debug('Releasing lock "%(lock)s"', {'lock': name})
229 282
 
230 283
 
231 284
 def synchronized(name, lock_file_prefix=None, external=False, lock_path=None,
232
-                 semaphores=None, delay=0.01):
285
+                 semaphores=None, delay=0.01, fair=False):
233 286
     """Synchronization decorator.
234 287
 
235 288
     Decorating a method like so::
@@ -264,7 +317,8 @@ def synchronized(name, lock_file_prefix=None, external=False, lock_path=None,
264 317
             t2 = None
265 318
             try:
266 319
                 with lock(name, lock_file_prefix, external, lock_path,
267
-                          do_log=False, semaphores=semaphores, delay=delay):
320
+                          do_log=False, semaphores=semaphores, delay=delay,
321
+                          fair=fair):
268 322
                     t2 = timeutils.now()
269 323
                     LOG.debug('Lock "%(name)s" acquired by "%(function)s" :: '
270 324
                               'waited %(wait_secs)0.3fs',

+ 39
- 0
oslo_concurrency/tests/unit/test_lockutils.py View File

@@ -147,6 +147,45 @@ class LockTestCase(test_base.BaseTestCase):
147 147
         self.assertEqual(saved_sem_num, len(lockutils._semaphores),
148 148
                          "Semaphore leak detected")
149 149
 
150
+    def test_lock_internal_fair(self):
151
+        """Check that we're actually fair."""
152
+
153
+        def f(_id):
154
+            with lockutils.lock('testlock', 'test-',
155
+                                external=False, fair=True):
156
+                lock_holder.append(_id)
157
+
158
+        lock_holder = []
159
+        threads = []
160
+        # While holding the fair lock, spawn a bunch of threads that all try
161
+        # to acquire the lock.  They will all block.  Then release the lock
162
+        # and see what happens.
163
+        with lockutils.lock('testlock', 'test-', external=False, fair=True):
164
+            for i in range(10):
165
+                thread = threading.Thread(target=f, args=(i,))
166
+                threads.append(thread)
167
+                thread.start()
168
+                # Allow some time for the new thread to get queued onto the
169
+                # list of pending writers before continuing.  This is gross
170
+                # but there's no way around it without using knowledge of
171
+                # fasteners internals.
172
+                time.sleep(0.5)
173
+        # Wait for all threads.
174
+        for thread in threads:
175
+            thread.join()
176
+
177
+        self.assertEqual(10, len(lock_holder))
178
+        # Check that the threads each got the lock in fair order.
179
+        for i in range(10):
180
+            self.assertEqual(i, lock_holder[i])
181
+
182
+    def test_fair_lock_with_semaphore(self):
183
+        def do_test():
184
+            s = lockutils.Semaphores()
185
+            with lockutils.lock('testlock', 'test-', semaphores=s, fair=True):
186
+                pass
187
+        self.assertRaises(NotImplementedError, do_test)
188
+
150 189
     def test_nested_synchronized_external_works(self):
151 190
         """We can nest external syncs."""
152 191
         tempdir = tempfile.mkdtemp()

+ 15
- 0
releasenotes/notes/add-option-for-fair-locks-b6d660e97683cec6.yaml View File

@@ -0,0 +1,15 @@
1
+---
2
+prelude: >
3
+    This release includes optional support for fair locks.  When fair locks
4
+    are specified, blocking waiters will acquire the lock in the order that
5
+    they blocked.
6
+features:
7
+  - |
8
+    We now have optional support for ``fair`` locks.  When fair locks are
9
+    specified, blocking waiters will acquire the lock in the order that they
10
+    blocked.  This can be useful to ensure that existing blocked waiters do
11
+    not wait indefinitely in the face of large numbers of new attempts to
12
+    acquire the lock.  When specifying locks as both ``external`` and ``fair``,
13
+    the ordering *within* a given process will be fair, but the ordering
14
+    *between* processes will be determined by the behaviour of the underlying
15
+    OS.

Loading…
Cancel
Save