Protect getCachedChanges from concurrent modification

We observe a few exceptions in our system while iterating over
getCachedChanges [1]. This is caused by multithreaded appending new
changes to the change cache. In order to prevent this exception we
need to make lists from the values which are safe to iterate even if
the backing dict gets updated. While we're at it fix it for all other
drivers as well.

[1] Trace:
2020-03-06 12:39:51,429 ERROR zuul.GerritEventConnector: Exception moving Gerrit event:
Traceback (most recent call last):
  File "/opt/zuul/lib/python3.6/site-packages/zuul/driver/gerrit/gerritconnection.py", line 284, in run
    self._handleEvent()
  File "/opt/zuul/lib/python3.6/site-packages/zuul/driver/gerrit/gerritconnection.py", line 243, in _handleEvent
    self._getChange(event)
  File "/opt/zuul/lib/python3.6/site-packages/zuul/driver/gerrit/gerritconnection.py", line 277, in _getChange
    refresh=True, event=event)
  File "/opt/zuul/lib/python3.6/site-packages/zuul/driver/gerrit/gerritconnection.py", line 742, in _getChange
    self._updateChange(change, event, history)
  File "/opt/zuul/lib/python3.6/site-packages/zuul/driver/gerrit/gerritconnection.py", line 820, in _updateChange
    self.sched.onChangeUpdated(change, event)
  File "/opt/zuul/lib/python3.6/site-packages/zuul/scheduler.py", line 1703, in onChangeUpdated
    for other_change in source.getCachedChanges():
RuntimeError: dictionary changed size during iteration

Change-Id: I30396997441c7e7756bfb81be708389d6331bf19
This commit is contained in:
Tobias Henkel 2020-03-26 18:10:45 +01:00
parent 4eb1212b9d
commit e9120bab27
No known key found for this signature in database
GPG Key ID: 03750DEC158E5FA2
4 changed files with 5 additions and 5 deletions

View File

@ -107,8 +107,8 @@ class GerritSource(BaseSource):
return changes
def getCachedChanges(self):
for x in self.connection._change_cache.values():
for y in x.values():
for x in list(self.connection._change_cache.values()):
for y in list(x.values()):
yield y
def getProject(self, name):

View File

@ -95,7 +95,7 @@ class GithubSource(BaseSource):
return self.connection.getChangesDependingOn(change, projects, tenant)
def getCachedChanges(self):
return self.connection._change_cache.values()
return list(self.connection._change_cache.values())
def getProject(self, name):
p = self.connection.getProject(name)

View File

@ -59,7 +59,7 @@ class GitlabSource(BaseSource):
raise NotImplementedError()
def getCachedChanges(self):
return self.connection._change_cache.values()
return list(self.connection._change_cache.values())
def getProject(self, name):
p = self.connection.getProject(name)

View File

@ -90,7 +90,7 @@ class PagureSource(BaseSource):
change, projects, tenant)
def getCachedChanges(self):
return self.connection._change_cache.values()
return list(self.connection._change_cache.values())
def getProject(self, name):
p = self.connection.getProject(name)