jjb/jenkins-job-builder
Code Issues Proposed changes

registry: cache component loading

Browse Source 
Each component (builder, publishers...) is loaded everytime dispatch()
is called. Its main purpose is to lookup the type of the component.
However pkg_resources.EntryPoint.load() is a costly operation and it was
done every single time dispatch() is called.

Against Wikimedia configurations files [0] it is extremely noticeable.

The entrypoint is loaded solely to figure out its type. I do not see any
good reason for it to change, thus introduce a new cache
_component_type_cache.

That then let us prevent invoking load() every single time.

Run time to generate Wikimedia configuration files went from 17 seconds
to a mere 4 seconds!
[0] https://gerrit.wikimedia.org/g/integration/config/+/master/jjb/

Change-Id: I49bc5dc91e3ba69b7f15c5f5ded8a42fb61a60c2
This commit is contained in:
Antoine Musso 2019-10-28 22:51:12 +01:00
parent 9c78c09203
commit 5325b04af1
1 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
jenkins_jobs/registry.py
View File

@ -32,6 +32,7 @@ logger = logging.getLogger(__name__)
class ModuleRegistry(object): class ModuleRegistry(object):
_entry_points_cache = {} _entry_points_cache = {}
_component_type_cache = {}
def __init__(self, jjb_config, plugins_list=None): def __init__(self, jjb_config, plugins_list=None):
self.modules = [] self.modules = []
@ -128,6 +129,17 @@ class ModuleRegistry(object):
def set_parser_data(self, parser_data): def set_parser_data(self, parser_data):
self.__parser_data = parser_data self.__parser_data = parser_data
def get_component_list_type(self, entry_point):
if entry_point in self._component_type_cache:
return self._component_type_cache[entry_point]
# pkg_resources.EntryPoint.load() is costly, cache it.
component_list_type = entry_point.load().component_list_type
logging.info("Caching type %s of %s", component_list_type, entry_point)
self._component_type_cache[entry_point] = component_list_type
return component_list_type
def dispatch(self, component_type, xml_parent, component, template_data={}): def dispatch(self, component_type, xml_parent, component, template_data={}):
"""This is a method that you can call from your implementation of """This is a method that you can call from your implementation of
Base.gen_xml or component. It allows modules to define a type Base.gen_xml or component. It allows modules to define a type
@ -154,7 +166,7 @@ class ModuleRegistry(object):
) )
entry_point = self.modules_by_component_type[component_type] entry_point = self.modules_by_component_type[component_type]
component_list_type = entry_point.load().component_list_type component_list_type = self.get_component_list_type(entry_point)
if isinstance(component, dict): if isinstance(component, dict):
# The component is a singleton dictionary of name: dict(args) # The component is a singleton dictionary of name: dict(args)