Browse Source

Merge "Cache node request zNodes"

tags/3.4.0
Zuul 5 months ago
parent
commit
c49cc46b6e
1 changed files with 30 additions and 7 deletions
  1. 30
    7
      nodepool/zk.py

+ 30
- 7
nodepool/zk.py View File

@@ -697,6 +697,7 @@ class ZooKeeper(object):
697 697
         self._became_lost = False
698 698
         self._last_retry_log = 0
699 699
         self._node_cache = None
700
+        self._request_cache = None
700 701
 
701 702
     # =======================================================================
702 703
     # Private Methods
@@ -890,6 +891,9 @@ class ZooKeeper(object):
890 891
             self._node_cache = TreeCache(self.client, self.NODE_ROOT)
891 892
             self._node_cache.start()
892 893
 
894
+            self._request_cache = TreeCache(self.client, self.REQUEST_ROOT)
895
+            self._request_cache.start()
896
+
893 897
     def disconnect(self):
894 898
         '''
895 899
         Close the ZooKeeper cluster connection.
@@ -902,6 +906,10 @@ class ZooKeeper(object):
902 906
             self._node_cache.close()
903 907
             self._node_cache = None
904 908
 
909
+        if self._request_cache is not None:
910
+            self._request_cache.close()
911
+            self._request_cache = None
912
+
905 913
         if self.client is not None and self.client.connected:
906 914
             self.client.stop()
907 915
             self.client.close()
@@ -1545,19 +1553,34 @@ class ZooKeeper(object):
1545 1553
         except kze.NoNodeError:
1546 1554
             pass
1547 1555
 
1548
-    def getNodeRequest(self, request):
1556
+    def getNodeRequest(self, request, cached=False):
1549 1557
         '''
1550 1558
         Get the data for a specific node request.
1551 1559
 
1552 1560
         :param str request: The request ID.
1561
+        :param cached: True if cached node requests should be returned.
1553 1562
 
1554 1563
         :returns: The request data, or None if the request was not found.
1555 1564
         '''
1556 1565
         path = self._requestPath(request)
1557
-        try:
1558
-            data, stat = self.client.get(path)
1559
-        except kze.NoNodeError:
1560
-            return None
1566
+        data = None
1567
+        stat = None
1568
+        if cached:
1569
+            cached_data = self._request_cache.get_data(path)
1570
+            if cached_data:
1571
+                data = cached_data.data
1572
+                stat = cached_data.stat
1573
+
1574
+        # If data is empty we either didn't use the cache or the cache didn't
1575
+        # have the request (yet). Note that even if we use caching we need to
1576
+        # do a real query if the cached data is empty because the request data
1577
+        # might not be in the cache yet when it's listed by the get_children
1578
+        # call.
1579
+        if not data:
1580
+            try:
1581
+                data, stat = self.client.get(path)
1582
+            except kze.NoNodeError:
1583
+                return None
1561 1584
 
1562 1585
         d = NodeRequest.fromDict(self._bytesToDict(data), request)
1563 1586
         d.stat = stat
@@ -1931,12 +1954,12 @@ class ZooKeeper(object):
1931 1954
             if lock_stats:
1932 1955
                 yield lock_stats
1933 1956
 
1934
-    def nodeRequestIterator(self):
1957
+    def nodeRequestIterator(self, cached=True):
1935 1958
         '''
1936 1959
         Utility generator method for iterating through all nodes requests.
1937 1960
         '''
1938 1961
         for req_id in self.getNodeRequests():
1939
-            req = self.getNodeRequest(req_id)
1962
+            req = self.getNodeRequest(req_id, cached=cached)
1940 1963
             if req:
1941 1964
                 yield req
1942 1965
 

Loading…
Cancel
Save