Retry dependency update requests

It is possible for dependency updates requests to fail due to errors
with the source. Previously when this happened the change was ignored by
the pipeline and the user gets no feedback. Chances are high that
reporting back to the source will fail so we can't really notify of this
error.

Instead we retry the requests in the hope that the error is a one off
and we can continue to proceed with the originally requested job work.

Story: 2009687
Change-Id: Id010d8c6809b9f9c012b81992590e54bf5e7e1d8
This commit is contained in:
Clark Boylan 2021-11-17 10:23:38 -08:00
parent 5b2c3811bf
commit ef9d0a6f31
2 changed files with 23 additions and 1 deletions

View File

@ -741,7 +741,7 @@ class PipelineManager(metaclass=ABCMeta):
if not source:
continue
log.debug(" Found source: %s", source)
dep = source.getChangeByURL(match, event)
dep = source.getChangeByURLWithRetry(match, event)
if dep and (not dep.is_merged) and dep not in dependencies:
log.debug(" Adding dependency: %s", dep)
dependencies.append(dep)

View File

@ -13,6 +13,7 @@
# under the License.
import abc
import time
class BaseSource(object, metaclass=abc.ABCMeta):
@ -76,6 +77,27 @@ class BaseSource(object, metaclass=abc.ABCMeta):
"""
def getChangeByURLWithRetry(self, url, event):
for x in range(3):
# We retry this as we are unlikely to be able to report back
# failures if our source is broken, but if we can get the
# info on subsequent requests we can continue to do the
# requested job work.
try:
dep = self.getChangeByURL(url, event)
except Exception:
# Note that if the change isn't found dep is None.
# We do not raise in that case and do not need to handle it
# here.
retry = x != 2 and " Retrying" or ""
self.log.exception("Failed to retrieve dependency %s.%s",
url, retry)
if retry:
time.sleep(1)
else:
raise
return dep
def getChangeByKey(self, key):
"""Get the change corresponding to the supplied cache key.