zuul/zuul
Code Issues Proposed changes

Report nodepool resource stats gauges in scheduler

Browse Source 
We currently report nodepool resource usage whenever we use or return
nodes.  This now happens on the executors, and they don't have a
global view of all nodes used.  The schedulers do, and they already
have a periodic stats reporting method.

Shift the reporting of node resource gauges to the scheduler.  To make
this efficient, use a tree cache for nodes.  Because node records
alone don't have enough information to tie them back to a tenant or
project, use the new user_data field on the Node object to store that
info when we mark a node in use.  Also, store the zuul system id on
the node, so that we can ensure we're only reporting nodes that belong
to us.

Update the node list in the REST API to use the cache as well, and
also filter its results by zuul system id and tenant.

Depends-On: https://review.opendev.org/807362
Change-Id: I9d0987b250b8fb54b3b937c86db327d255e54abd
This commit is contained in:
James E. Blair
2021-09-03 13:23:44 -07:00
parent b41f467340
commit aee6ef6f7f
10 changed files with 318 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files
releasenotes/notes/nodepool-4.3.0-6969ad89ea27a75e.yaml
+6
View File
@@ -0,0 +1,6 @@
---
upgrade:
- |
Nodepool 4.3.0 is now required. Zuul stores additional
information in node records in preparation for supporting multiple
schedulers.
tests/base.py
+16
View File
@@ -2846,6 +2846,9 @@ class FakeStatsd(threading.Thread):
self.wake_read, self.wake_write = os.pipe()
self.stats = []
def clear(self):
self.stats = []
def run(self):
while True:
poll = select.poll()
@@ -5167,6 +5170,19 @@ class ZuulTestCase(BaseTestCase):
for app in self.scheds:
app.sched.merger.merger_api.history.clear()
def waitUntilNodeCacheSync(self, zk_nodepool):
"""Wait until the node cache on the zk_nodepool object is in sync"""
for _ in iterate_timeout(60, 'wait for node cache sync'):
cache_state = {}
zk_state = {}
for n in self.fake_nodepool.getNodes():
zk_state[n['_oid']] = n['state']
for nid in zk_nodepool.getNodes(cached=True):
n = zk_nodepool.getNode(nid)
cache_state[n.id] = n.state
if cache_state == zk_state:
return
def __haveAllBuildsReported(self, matcher) -> bool:
for app in self.scheds.filter(matcher):
executor_client = app.sched.executor
tests/unit/test_nodepool.py
+2 -2
View File
@@ -26,8 +26,8 @@ class NodepoolWithCallback(zuul.nodepool.Nodepool):
super().__init__(*args, **kwargs)
self.provisioned_requests = []
def _handleNodeRequestEvent(self, request, event, request_id=None):
super()._handleNodeRequestEvent(request, event, request_id)
def _handleNodeRequestEvent(self, request, event):
super()._handleNodeRequestEvent(request, event)
self.provisioned_requests.append(request)
tests/unit/test_scheduler.py
+46 -13
View File
@@ -1944,6 +1944,17 @@ class TestScheduler(ZuulTestCase):
self.assertEqual(0, request.current_count)
self.assertEqual([], request.nodes)
# Some convenience variables for checking the stats.
tenant_ram_stat = 'zuul.nodepool.resources.tenant.tenant-one.ram'
project_ram_stat = ('zuul.nodepool.resources.project.'
'review_example_com/org/project.ram')
# Test that we zeroed the gauges
self.scheds.first.sched._runStats()
self.assertUnReportedStat(tenant_ram_stat, value='1024', kind='g')
self.assertUnReportedStat(project_ram_stat, value='1024', kind='g')
self.assertReportedStat(tenant_ram_stat, value='0', kind='g')
self.assertReportedStat(project_ram_stat, value='0', kind='g')
# First check that successful jobs do not autohold
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
@@ -1975,10 +1986,6 @@ class TestScheduler(ZuulTestCase):
'ram': 1024,
'instances': 1,
}
# Some convenience variables for checking these stats.
tenant_ram_stat = 'zuul.nodepool.resources.tenant.tenant-one.ram'
project_ram_stat = ('zuul.nodepool.resources.project.'
'review_example_com/org/project.ram')
B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B')
self.executor_server.failJob('project-test2', B)
@@ -1990,6 +1997,10 @@ class TestScheduler(ZuulTestCase):
build = list(self.scheds.first.sched.executor.builds.values())[0]
# We should report using the held node's resources
self.waitUntilNodeCacheSync(
self.scheds.first.sched.nodepool.zk_nodepool)
self.statsd.clear()
self.scheds.first.sched._runStats()
self.assertReportedStat(tenant_ram_stat, value='1024', kind='g')
self.assertReportedStat(project_ram_stat, value='1024', kind='g')
self.assertUnReportedStat(tenant_ram_stat, value='0', kind='g')
@@ -2029,11 +2040,15 @@ class TestScheduler(ZuulTestCase):
self.assertEqual(1, len(request2.nodes))
self.assertEqual(1, len(request2.nodes[0]["nodes"]))
# We should now report that we no longer use the nodes resources
# We should still report we use the resources
self.waitUntilNodeCacheSync(
self.scheds.first.sched.nodepool.zk_nodepool)
self.statsd.clear()
self.scheds.first.sched._runStats()
self.assertReportedStat(tenant_ram_stat, value='1024', kind='g')
self.assertReportedStat(project_ram_stat, value='1024', kind='g')
self.assertReportedStat(tenant_ram_stat, value='0', kind='g')
self.assertReportedStat(project_ram_stat, value='0', kind='g')
self.assertUnReportedStat(tenant_ram_stat, value='0', kind='g')
self.assertUnReportedStat(project_ram_stat, value='0', kind='g')
# Another failed change should not hold any more nodes
self.fake_nodepool.resources = {}
@@ -2063,6 +2078,20 @@ class TestScheduler(ZuulTestCase):
self.assertEqual(3, len(node_states))
self.assertEqual([zuul.model.STATE_USED] * 3, node_states)
# Nodepool deletes the nodes
for n in self.fake_nodepool.getNodes():
self.fake_nodepool.removeNode(n)
# We should now report that we no longer use the nodes resources
self.waitUntilNodeCacheSync(
self.scheds.first.sched.nodepool.zk_nodepool)
self.statsd.clear()
self.scheds.first.sched._runStats()
self.assertUnReportedStat(tenant_ram_stat, value='1024', kind='g')
self.assertUnReportedStat(project_ram_stat, value='1024', kind='g')
self.assertReportedStat(tenant_ram_stat, value='0', kind='g')
self.assertReportedStat(project_ram_stat, value='0', kind='g')
@simple_layout('layouts/autohold.yaml')
def test_autohold_info(self):
client = zuul.rpcclient.RPCClient('127.0.0.1',
@@ -6130,29 +6159,33 @@ For CI problems and help debugging, contact ci@example.org"""
self.executor_server.release('project-merge')
self.waitUntilSettled()
self.waitUntilNodeCacheSync(
self.scheds.first.sched.nodepool.zk_nodepool)
self.scheds.first.sched._runStats()
# Check that resource usage gauges are reported
self.assertHistory([
dict(name='project-merge', result='SUCCESS', changes='1,1'),
])
# All 3 nodes are in use
self.assertReportedStat(
'zuul.nodepool.resources.tenant.tenant-one.cores',
value='2', kind='g')
value='6', kind='g')
self.assertReportedStat(
'zuul.nodepool.resources.tenant.tenant-one.ram',
value='1024', kind='g')
value='3072', kind='g')
self.assertReportedStat(
'zuul.nodepool.resources.tenant.tenant-one.instances',
value='1', kind='g')
value='3', kind='g')
self.assertReportedStat(
'zuul.nodepool.resources.project.review_example_com/org/project.'
'cores', value='2', kind='g')
'cores', value='6', kind='g')
self.assertReportedStat(
'zuul.nodepool.resources.project.review_example_com/org/project.'
'ram', value='1024', kind='g')
'ram', value='3072', kind='g')
self.assertReportedStat(
'zuul.nodepool.resources.project.review_example_com/org/project.'
'instances', value='1', kind='g')
'instances', value='3', kind='g')
# Check that resource usage counters are reported
self.assertReportedStat(
tests/unit/test_web.py
+6 -3
View File
@@ -357,7 +357,8 @@ class TestWeb(BaseTestWeb):
'label': 'label1',
'name': 'controller',
'aliases': [],
'state': 'unknown'}],
'state': 'unknown',
'user_data': None}],
},
'override_checkout': None,
'parent': 'base',
@@ -404,7 +405,8 @@ class TestWeb(BaseTestWeb):
'label': 'label2',
'name': 'controller',
'aliases': [],
'state': 'unknown'}],
'state': 'unknown',
'user_data': None}],
},
'override_checkout': None,
'parent': 'base',
@@ -1033,7 +1035,8 @@ class TestWeb(BaseTestWeb):
'id': None,
'label': 'label1',
'name': 'controller',
'state': 'unknown'}]},
'state': 'unknown',
'user_data': None}]},
'override_branch': None,
'override_checkout': None,
'repo_state': {},
zuul/model.py
+2
View File
@@ -600,6 +600,7 @@ class Node(ConfigObject):
self.lock = None
self.hold_job = None
self.comment = None
self.user_data = None
# Attributes from Nodepool
self._state = 'unknown'
self.state_time = time.time()
@@ -651,6 +652,7 @@ class Node(ConfigObject):
d['state'] = self.state
d['hold_job'] = self.hold_job
d['comment'] = self.comment
d['user_data'] = self.user_data
for k in self._keys:
d[k] = getattr(self, k)
if internal_attributes:
zuul/nodepool.py
+106 -81
View File
@@ -25,18 +25,11 @@ from zuul.zk.exceptions import LockException
from zuul.zk.nodepool import NodeRequestEvent, ZooKeeperNodepool
def add_resources(target, source):
for key, value in source.items():
target[key] += value
def subtract_resources(target, source):
for key, value in source.items():
target[key] -= value
class Nodepool(object):
log = logging.getLogger('zuul.nodepool')
# The kind of resources we report stats on. We need a complete
# list in order to report 0 level gauges.
resource_types = ('ram', 'cores', 'instances')
def __init__(self, zk_client, system_id, statsd, scheduler=False):
self._stopped = False
@@ -50,7 +43,8 @@ class Nodepool(object):
self.zk_nodepool = ZooKeeperNodepool(
zk_client,
enable_node_request_cache=True,
node_request_event_callback=self._handleNodeRequestEvent)
node_request_event_callback=self._handleNodeRequestEvent,
enable_node_cache=True)
self.election = NodepoolEventElection(zk_client)
self.event_thread = threading.Thread(target=self.runEventElection)
self.event_thread.daemon = True
@@ -65,9 +59,10 @@ class Nodepool(object):
zk_client
)
# TODO: remove internal caches for SOS
self.current_resources_by_tenant = {}
self.current_resources_by_project = {}
def addResources(self, target, source):
for key, value in source.items():
if key in self.resource_types:
target[key] += value
def runEventElection(self):
while not self._stopped:
@@ -113,9 +108,7 @@ class Nodepool(object):
self.stop_watcher_event.clear()
self.election_won = False
def _handleNodeRequestEvent(self, request, event, request_id=None):
# TODO (felix): This callback should be wrapped by leader election, so
# that only one scheduler puts NodesProvisionedEvents in the queue.
def _handleNodeRequestEvent(self, request, event):
log = get_annotated_logger(self.log, event=request.event_id)
if request.requestor != self.system_id:
@@ -170,22 +163,6 @@ class Nodepool(object):
pipe.timing(key + '.size.%s' % len(request.labels), dt)
pipe.send()
def emitStatsResources(self):
if not self.statsd:
return
for tenant, resources in self.current_resources_by_tenant.items():
for resource, value in resources.items():
key = 'zuul.nodepool.resources.tenant.' \
'{tenant}.{resource}'
self.statsd.gauge(key, value, tenant=tenant, resource=resource)
for project, resources in self.current_resources_by_project.items():
for resource, value in resources.items():
key = 'zuul.nodepool.resources.project.' \
'{project}.{resource}'
self.statsd.gauge(
key, value, project=project, resource=resource)
def emitStatsResourceCounters(self, tenant, project, resources, duration):
if not self.statsd:
return
@@ -297,7 +274,7 @@ class Nodepool(object):
if node.lock is None:
raise Exception("Node %s is not locked" % (node,))
if node.resources:
add_resources(resources, node.resources)
self.addResources(resources, node.resources)
node.state = model.STATE_HOLD
node.hold_job = " ".join([request.tenant,
request.project,
@@ -337,48 +314,30 @@ class Nodepool(object):
# _doBuildCompletedEvent, we always want to try to unlock it.
self.zk_nodepool.unlockHoldRequest(request)
# When holding a nodeset we need to update the gauges to avoid
# leaking resources
if tenant and project and resources:
subtract_resources(
self.current_resources_by_tenant[tenant], resources)
subtract_resources(
self.current_resources_by_project[project], resources)
self.emitStatsResources()
if duration:
self.emitStatsResourceCounters(
tenant, project, resources, duration)
if tenant and project and resources and duration:
self.emitStatsResourceCounters(
tenant, project, resources, duration)
# TODO (felix): Switch back to use a build object here rather than the
# ansible_job once it's available via ZK.
def useNodeSet(self, nodeset, ansible_job=None):
self.log.info("Setting nodeset %s in use", nodeset)
resources = defaultdict(int)
user_data = None
if ansible_job:
args = ansible_job.arguments
tenant_name = args["zuul"]["tenant"]
project_name = args["zuul"]["project"]["canonical_name"]
user_data = dict(
zuul_system=self.system_id,
tenant_name=tenant_name,
project_name=project_name,
)
for node in nodeset.getNodes():
if node.lock is None:
raise Exception("Node %s is not locked", node)
node.state = model.STATE_IN_USE
node.user_data = user_data
self.zk_nodepool.storeNode(node)
if node.resources:
add_resources(resources, node.resources)
if ansible_job and resources:
args = ansible_job.arguments
# we have a buildset and thus also tenant and project so we
# can emit project specific resource usage stats
tenant_name = args["zuul"]["tenant"]
project_name = args["zuul"]["project"]["canonical_name"]
self.current_resources_by_tenant.setdefault(
tenant_name, defaultdict(int))
self.current_resources_by_project.setdefault(
project_name, defaultdict(int))
add_resources(self.current_resources_by_tenant[tenant_name],
resources)
add_resources(self.current_resources_by_project[project_name],
resources)
self.emitStatsResources()
# TODO (felix): Switch back to use a build object here rather than the
# ansible_job once it's available via ZK.
@@ -394,7 +353,7 @@ class Nodepool(object):
try:
if node.state == model.STATE_IN_USE:
if node.resources:
add_resources(resources, node.resources)
self.addResources(resources, node.resources)
node.state = model.STATE_USED
self.zk_nodepool.storeNode(node)
except Exception:
@@ -415,20 +374,9 @@ class Nodepool(object):
"for %s seconds for build %s for project %s",
nodeset, len(nodeset.nodes), duration, ansible_job, project)
# When returning a nodeset we need to update the gauges if we have a
# build. Further we calculate resource*duration and increment their
# tenant or project specific counters. With that we have both the
# current value and also counters to be able to perform accounting.
if resources:
subtract_resources(
self.current_resources_by_tenant[tenant], resources)
subtract_resources(
self.current_resources_by_project[project], resources)
self.emitStatsResources()
if duration:
self.emitStatsResourceCounters(
tenant, project, resources, duration)
if resources and duration:
self.emitStatsResourceCounters(
tenant, project, resources, duration)
def unlockNodeSet(self, nodeset):
self._unlockNodes(nodeset.getNodes())
@@ -536,3 +484,80 @@ class Nodepool(object):
req = self.zk_nodepool.getNodeRequest(req_id, cached=True)
if req.requestor == self.system_id:
yield req
def getNodes(self):
"""Get all nodes in use or held by Zuul
Note this relies entirely on the internal cache.
:returns: An iterator of Node objects in use (or held) by this Zuul
system.
"""
for node_id in self.zk_nodepool.getNodes(cached=True):
node = self.zk_nodepool.getNode(node_id)
if (node.user_data and
isinstance(node.user_data, dict) and
node.user_data.get('zuul_system') == self.system_id):
yield node
def emitStatsTotals(self, abide):
if not self.statsd:
return
total_requests = 0
tenant_requests = defaultdict(int)
resources_by_tenant = defaultdict(int)
resources_by_project = defaultdict(int)
empty_resource_dict = dict([(k, 0) for k in self.resource_types])
# Initialize zero values for gauges
for tenant in abide.tenants.values():
tenant_requests[tenant.name] = 0
resources_by_tenant[tenant.name] = empty_resource_dict.copy()
for project in tenant.all_projects:
resources_by_project[project.canonical_name] =\
empty_resource_dict.copy()
# Count node requests
for req in self.getNodeRequests():
total_requests += 1
if not req.tenant_name:
continue
tenant_requests[req.tenant_name] += 1
self.statsd.gauge('zuul.nodepool.current_requests',
total_requests)
for tenant, request_count in tenant_requests.items():
self.statsd.gauge(
"zuul.nodepool.tenant.{tenant}.current_requests",
request_count,
tenant=tenant)
# Count nodes
for node in self.getNodes():
if not node.resources:
continue
project_name = node.user_data.get('project_name')
tenant_name = node.user_data.get('tenant_name')
if not (project_name and tenant_name):
continue
if node.state not in {model.STATE_IN_USE,
model.STATE_USED,
model.STATE_HOLD}:
continue
self.addResources(resources_by_tenant[tenant_name],
node.resources)
self.addResources(resources_by_project[project_name],
node.resources)
for tenant, resources in resources_by_tenant.items():
for resource, value in resources.items():
key = 'zuul.nodepool.resources.tenant.' \
'{tenant}.{resource}'
self.statsd.gauge(key, value, tenant=tenant, resource=resource)
for project, resources in resources_by_project.items():
for resource, value in resources.items():
key = 'zuul.nodepool.resources.project.' \
'{project}.{resource}'
self.statsd.gauge(
key, value, project=project, resource=resource)
zuul/scheduler.py
+1 -23
View File
@@ -452,29 +452,7 @@ class Scheduler(threading.Thread):
self.statsd.gauge(f"{base}.management_events",
len(management_event_queues[pipeline.name]))
# export current_requests stats per tenant
tenant_requests = defaultdict(int)
# need to initialize tenants here explicitly to report a zero for
# all tenants for which no requests are in-flight
for tenant_name in self.abide.tenants.keys():
tenant_requests[tenant_name] = 0
total_requests = 0
for req in self.nodepool.getNodeRequests():
# might be None, we ignore them for this metric
total_requests += 1
if not req.tenant_name:
continue
tenant_requests[req.tenant_name] += 1
self.statsd.gauge('zuul.nodepool.current_requests',
total_requests)
for tenant, request_count in tenant_requests.items():
self.statsd.gauge(
"zuul.nodepool.tenant.{tenant}.current_requests",
request_count,
tenant=tenant)
self.nodepool.emitStatsTotals(self.abide)
def startCleanup(self):
# Run the first cleanup immediately after the first
zuul/web/__init__.py
+16 -3
View File
@@ -38,6 +38,7 @@ import zuul.rpcclient
from zuul.zk import ZooKeeperClient
from zuul.zk.components import WebComponent
from zuul.zk.nodepool import ZooKeeperNodepool
from zuul.zk.system import ZuulSystem
from zuul.lib.auth import AuthenticatorRegistry
from zuul.lib.config import get_default
@@ -243,7 +244,9 @@ class ZuulWebAPI(object):
def __init__(self, zuulweb):
self.rpc = zuulweb.rpc
self.zk_client = zuulweb.zk_client
self.zk_nodepool = ZooKeeperNodepool(self.zk_client)
self.system = ZuulSystem(self.zk_client)
self.zk_nodepool = ZooKeeperNodepool(self.zk_client,
enable_node_cache=True)
self.zuulweb = zuulweb
self.cache = {}
self.cache_time = {}
@@ -883,11 +886,21 @@ class ZuulWebAPI(object):
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
def nodes(self, tenant):
ret = []
for node in self.zk_nodepool.nodeIterator():
for node_id in self.zk_nodepool.getNodes(cached=True):
node = self.zk_nodepool.getNode(node_id)
# This returns all nodes; some of which may not be
# intended for use by Zuul, so be extra careful checking
# user_data.
if not (node.user_data and
isinstance(node.user_data, dict) and
node.user_data.get('zuul_system') ==
self.system.system_id and
node.user_data.get('tenant_name') == tenant):
continue
node_data = {}
for key in ("id", "type", "connection_type", "external_id",
"provider", "state", "state_time", "comment"):
node_data[key] = node.get(key)
node_data[key] = getattr(node, key, None)
ret.append(node_data)
resp = cherrypy.response
resp.headers['Access-Control-Allow-Origin'] = '*'
zuul/zk/nodepool.py
+117 -15
View File
@@ -22,7 +22,7 @@ from kazoo.recipe.lock import Lock
import zuul.model
from zuul.lib.jsonutil import json_dumps
from zuul.model import HoldRequest, NodeRequest
from zuul.model import HoldRequest, NodeRequest, Node
from zuul.zk import ZooKeeperBase
from zuul.zk.exceptions import LockException
@@ -32,6 +32,11 @@ class NodeRequestEvent(Enum):
DELETED = 2
class NodeEvent(Enum):
CHANGED = 0
DELETED = 2
class ZooKeeperNodepool(ZooKeeperBase):
"""
Class implementing Nodepool related ZooKeeper interface.
@@ -44,11 +49,14 @@ class ZooKeeperNodepool(ZooKeeperBase):
log = logging.getLogger("zuul.zk.nodepool.ZooKeeperNodepool")
def __init__(self, client, enable_node_request_cache=True,
node_request_event_callback=None):
def __init__(self, client,
enable_node_request_cache=False,
node_request_event_callback=None,
enable_node_cache=False):
super().__init__(client)
self.enable_node_request_cache = enable_node_request_cache
self.node_request_event_callback = node_request_event_callback
self.enable_node_cache = enable_node_cache
# The caching model we use is designed around handing out model
# data as objects. To do this, we use two caches: one is a TreeCache
# which contains raw znode data (among other details), and one for
@@ -57,6 +65,8 @@ class ZooKeeperNodepool(ZooKeeperBase):
# the data into objects more than once.
self._node_request_tree = None
self._node_request_cache = {}
self._node_tree = None
self._node_cache = {}
if self.client.connected:
self._onConnect()
@@ -68,11 +78,20 @@ class ZooKeeperNodepool(ZooKeeperBase):
self._node_request_tree.listen_fault(self._cacheFaultListener)
self._node_request_tree.listen(self.requestCacheListener)
self._node_request_tree.start()
if self.enable_node_cache:
self._node_tree = TreeCache(self.kazoo_client,
self.NODES_ROOT)
self._node_tree.listen_fault(self._cacheFaultListener)
self._node_tree.listen(self.nodeCacheListener)
self._node_tree.start()
def _onDisconnect(self):
if self._node_request_tree is not None:
self._node_request_tree.close()
self._node_request_tree = None
if self._node_tree is not None:
self._node_tree.close()
self._node_tree = None
def _launcherPath(self, launcher):
return "%s/%s" % (self.LAUNCHER_ROOT, launcher)
@@ -110,18 +129,23 @@ class ZooKeeperNodepool(ZooKeeperBase):
objs.append(Launcher.fromDict(json.loads(data.decode('utf8'))))
return objs
def getNodes(self):
def getNodes(self, cached=False):
"""
Get the current list of all nodes.
Get the current list of all node ids.
:returns: A list of nodes.
:param bool cached: Whether to use the internal cache to get the list
of ids.
:returns: A list of node ids.
"""
try:
return self.kazoo_client.get_children(self.NODES_ROOT)
except NoNodeError:
return []
if cached and self.enable_node_cache:
return list(self._node_cache.keys())
else:
try:
return self.kazoo_client.get_children(self.NODES_ROOT)
except NoNodeError:
return []
def _getNode(self, node):
def _getNodeData(self, node):
"""
Get the data for a specific node.
@@ -141,12 +165,38 @@ class ZooKeeperNodepool(ZooKeeperBase):
d['id'] = node
return d
def getNode(self, node_id):
"""
Get a Node object for a specific node.
:param str node_id: The node ID.
:returns: The Node, or None if the node was not found.
"""
node = self._node_cache.get(node_id)
if node:
return node
path = self._nodePath(node_id)
try:
data, stat = self.kazoo_client.get(path)
except NoNodeError:
return None
if not data:
return None
node = Node(None, None)
node.updateFromDict(data)
node.id = node_id
node.stat = stat
return node
def nodeIterator(self):
"""
Utility generator method for iterating through all nodes.
"""
for node_id in self.getNodes():
node = self._getNode(node_id)
node = self.getNode(node_id)
if node:
yield node
@@ -231,7 +281,7 @@ class ZooKeeperNodepool(ZooKeeperBase):
failure = False
for node_id in getHeldNodeIDs(request):
node = self._getNode(node_id)
node = self._getNodeData(node_id)
if not node or node['state'] == zuul.model.STATE_USED:
continue
@@ -383,13 +433,13 @@ class ZooKeeperNodepool(ZooKeeperBase):
and request.state != old_state
and self.node_request_event_callback):
self.node_request_event_callback(
request, NodeRequestEvent.COMPLETED, request_id=request_id)
request, NodeRequestEvent.COMPLETED)
elif event.event_type == TreeEvent.NODE_REMOVED:
request = self._node_request_cache.pop(request_id)
if self.node_request_event_callback:
self.node_request_event_callback(
request, NodeRequestEvent.DELETED, request_id=request_id)
request, NodeRequestEvent.DELETED)
def submitNodeRequest(self, node_request, priority):
"""
@@ -507,6 +557,58 @@ class ZooKeeperNodepool(ZooKeeperBase):
data = json.loads(data.decode('utf8'))
node_request.updateFromDict(data)
def nodeCacheListener(self, event):
try:
self._nodeCacheListener(event)
except Exception:
self.log.exception(
"Exception in node cache update for event: %s",
event)
def _nodeCacheListener(self, event):
if hasattr(event.event_data, 'path'):
# Ignore root node
path = event.event_data.path
if path == self.NODES_ROOT:
return
# Ignore lock nodes
if '/lock' in path:
return
# Ignore any non-node related events such as connection events here
if event.event_type not in (TreeEvent.NODE_ADDED,
TreeEvent.NODE_UPDATED,
TreeEvent.NODE_REMOVED):
return
path = event.event_data.path
node_id = path.rsplit('/', 1)[1]
if event.event_type in (TreeEvent.NODE_ADDED, TreeEvent.NODE_UPDATED):
# Nodes with empty data are invalid so skip add or update these.
if not event.event_data.data:
return
# Perform an in-place update of the cached node if possible
d = self._bytesToDict(event.event_data.data)
node = self._node_cache.get(node_id)
if node:
if event.event_data.stat.version <= node.stat.version:
# Don't update to older data
return
node.updateFromDict(d)
node.stat = event.event_data.stat
else:
node = Node(None, None)
node.updateFromDict(d)
node.id = node_id
node.stat = event.event_data.stat
self._node_cache[node_id] = node
elif event.event_type == TreeEvent.NODE_REMOVED:
node = self._node_cache.pop(node_id)
def storeNode(self, node):
"""
Store the node.