zuul
/
zuul
Code Issues Proposed changes

Add inherit-files attr to evaluate parent file matchers 

When building more complex zuul job constructs,
it is impossible to utilize the `files` attribute
of a parent job to reduce the amount of regex
rules specified in child jobs. Without inheritence,
child jobs have to repeat rules for any common file
paths. This PR adds a new `inherit-files` attribute
which will make the evaluation take `files` and
`irrelevant-files` of the parent explicitly into
account.

Task: #49809
Change-Id: I847944b77fb67f9ea1cb4e688994e08de9f59a98
Signed-off-by: Gondermann <gondermann@b1-systems.de>
This commit is contained in:
Gondermann 2024-04-09 12:14:58 +02:00
parent 01e9472306
commit 2c30b8a4ee
5 changed files with 172 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
doc/source/config/job.rst
View File

@ -1092,6 +1092,17 @@ Here is an example of two job definitions:
``git commit --allow-empty`` (which can be used in order to
run all jobs).
.. attr:: inherit-files
:default: false
If this is set to ``true`` and the job has a parent, this
will additionally lookup the ``job.files`` and ``job.irrelevant-files``
regular expressions of the parent job when evaluating whether
this job should run. This acts as if the ``job.files`` and
``job.irrelevant-files`` attributes of parent and child were combined.
The evaluation stops, if a match is positive or there are no more
parent jobs with the ``job.inherit-files`` attribute set to ``true``.
.. attr:: match-on-config-updates
:default: true

99
tests/unit/test_model.py
View File

@ -312,6 +312,105 @@ class TestJob(BaseTestCase):
redact_secrets_and_keys=False)
self.assertEqual([], item.getJobs())
@mock.patch("zuul.model.zkobject.ZKObject._save")
def test_inhert_file_matcher_explicitly(self, save_mock):
base = self.pcontext.job_parser.fromYaml({
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'base',
'parent': None,
'files': ['^file1$'],
'timeout': 30,
}, None)
self.layout.addJob(base)
python27 = self.pcontext.job_parser.fromYaml({
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'python27',
'parent': 'base',
'timeout': 40,
'files': ['^file2$'],
'inherit-files': True
}, None)
self.layout.addJob(python27)
project_config = self.pcontext.project_parser.fromYaml({
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'project',
'gate': {
'jobs': [
'python27',
]
}
})
self.layout.addProjectConfig(project_config)
change = model.Change(self.project)
change.branch = 'master'
change.cache_stat = Dummy(key=Dummy(reference=uuid.uuid4().hex))
change.files = ['/COMMIT_MSG', 'file1']
item = self.queue.enqueueChanges([change], None)
self.assertTrue(base.changeMatchesFiles(change))
self.assertTrue(python27.changeMatchesFiles(change))
self.pipeline.manager.getFallbackLayout = mock.Mock(return_value=None)
with self.zk_context as ctx:
item.freezeJobGraph(self.layout, ctx,
skip_file_matcher=False,
redact_secrets_and_keys=False)
self.assertEqual([], item.getJobs())
@mock.patch("zuul.model.zkobject.ZKObject._save")
def test_inhert_irrelevant_files_explicitly(self, save_mock):
base = self.pcontext.job_parser.fromYaml({
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'base',
'parent': None,
'irrelevant-files': ['^file1$'],
'timeout': 30,
}, None)
self.layout.addJob(base)
python27 = self.pcontext.job_parser.fromYaml({
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'python27',
'parent': 'base',
'timeout': 40,
'inherit-files': True
}, None)
self.layout.addJob(python27)
project_config = self.pcontext.project_parser.fromYaml({
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'project',
'gate': {
'jobs': [
'python27',
]
}
})
self.layout.addProjectConfig(project_config)
change = model.Change(self.project)
change.branch = 'master'
change.cache_stat = Dummy(key=Dummy(reference=uuid.uuid4().hex))
change.files = ['/COMMIT_MSG', 'file1']
item = self.queue.enqueueChanges([change], None)
self.assertFalse(base.changeMatchesFiles(change))
self.assertFalse(python27.changeMatchesFiles(change))
self.pipeline.manager.getFallbackLayout = mock.Mock(return_value=None)
with self.zk_context as ctx:
item.freezeJobGraph(self.layout, ctx,
skip_file_matcher=False,
redact_secrets_and_keys=False)
self.assertEqual([], item.getJobs())
def test_job_source_project(self):
base_project = model.Project('base_project', self.source)
base_context = model.SourceContext(

9
tests/unit/test_web.py
View File

@ -366,6 +366,7 @@ class TestWeb(BaseTestWeb):
'host_variables': {},
'intermediate': False,
'irrelevant_files': [],
'inherit_files': False,
'match_on_config_updates': True,
'name': 'test-job',
'nodeset_alternatives': [{'alternatives': [],
@ -463,6 +464,7 @@ class TestWeb(BaseTestWeb):
'files': [],
'intermediate': False,
'irrelevant_files': [],
'inherit_files': False,
'match_on_config_updates': True,
'final': False,
'failure_output': [],
@ -517,6 +519,7 @@ class TestWeb(BaseTestWeb):
'files': [],
'intermediate': False,
'irrelevant_files': [],
'inherit_files': False,
'match_on_config_updates': True,
'final': False,
'failure_output': [],
@ -577,6 +580,7 @@ class TestWeb(BaseTestWeb):
'failure_output': [],
'intermediate': False,
'irrelevant_files': [],
'inherit_files': False,
'match_on_config_updates': True,
'name': 'test-job',
'override_checkout': None,
@ -704,6 +708,7 @@ class TestWeb(BaseTestWeb):
'failure_output': [],
'intermediate': False,
'irrelevant_files': [],
'inherit_files': False,
'match_on_config_updates': True,
'name': 'project-merge',
'override_checkout': None,
@ -746,6 +751,7 @@ class TestWeb(BaseTestWeb):
'failure_output': [],
'intermediate': False,
'irrelevant_files': [],
'inherit_files': False,
'match_on_config_updates': True,
'name': 'project-test1',
'override_checkout': None,
@ -788,6 +794,7 @@ class TestWeb(BaseTestWeb):
'failure_output': [],
'intermediate': False,
'irrelevant_files': [],
'inherit_files': False,
'match_on_config_updates': True,
'name': 'project-test2',
'override_checkout': None,
@ -830,6 +837,7 @@ class TestWeb(BaseTestWeb):
'failure_output': [],
'intermediate': False,
'irrelevant_files': [],
'inherit_files': False,
'match_on_config_updates': True,
'name': 'project1-project2-integration',
'override_checkout': None,
@ -900,6 +908,7 @@ class TestWeb(BaseTestWeb):
'failure_output': [],
'intermediate': False,
'irrelevant_files': [],
'inherit_files': False,
'match_on_config_updates': True,
'name': 'project-post',
'override_checkout': None,

2
zuul/configloader.py
View File

@ -574,6 +574,7 @@ class JobParser(object):
'files': to_list(str),
'secrets': to_list(vs.Any(secret, str)),
'irrelevant-files': to_list(str),
'inherit-files': bool,
# validation happens in NodeSetParser
'nodeset': vs.Any(dict, str),
'timeout': int,
@ -630,6 +631,7 @@ class JobParser(object):
'match-on-config-updates',
'workspace-scheme',
'deduplicate',
'inherit-files',
]
def __init__(self, pcontext):

61
zuul/model.py
View File

@ -2692,6 +2692,7 @@ class Job(ConfigObject):
d['override_checkout'] = self.override_checkout
d['files'] = self._files
d['irrelevant_files'] = self._irrelevant_files
d['inherit_files'] = self.inherit_files
d['variant_description'] = self.variant_description
if self.source_context:
d['source_context'] = self.source_context.toDict()
@ -2761,8 +2762,9 @@ class Job(ConfigObject):
_branches=(),
file_matcher=None,
_files=(),
irrelevant_file_matcher=None, # skip-if
irrelevant_file_matcher_list=None, # skip-if
_irrelevant_files=(),
inherit_files=False,
match_on_config_updates=True,
deduplicate='auto',
tags=frozenset(),
@ -3178,10 +3180,10 @@ class Job(ConfigObject):
def setIrrelevantFileMatcher(self, irrelevant_files):
# Set the irrelevant file matcher to match any of the change files
self._irrelevant_files = irrelevant_files
matchers = []
self.irrelevant_file_matcher_list = []
for fn in irrelevant_files:
matchers.append(change_matcher.FileMatcher(ZuulRegex(fn)))
self.irrelevant_file_matcher = change_matcher.MatchAllFiles(matchers)
self.irrelevant_file_matcher_list.append(
change_matcher.FileMatcher(ZuulRegex(fn)))
def updateVariables(self, other_vars, other_extra_vars, other_host_vars,
other_group_vars):
@ -3429,17 +3431,56 @@ class Job(ConfigObject):
return True
def changeMatchesFiles(self, change):
def changeMatchesFilesSelf(self, change, irrelevantMatcherList):
if self.file_matcher and not self.file_matcher.matches(change):
return False
# NB: This is a negative match.
if (self.irrelevant_file_matcher and
self.irrelevant_file_matcher.matches(change)):
return False
if len(irrelevantMatcherList) > 0:
# NB: This is a negative match.
irrelevantMatcher = change_matcher.MatchAllFiles(
irrelevantMatcherList)
if irrelevantMatcher.matches(change):
return False
return True
def buildIrrelevantList(self, layout):
matcherList = []
contextJob = self
while contextJob:
if contextJob.irrelevant_file_matcher_list:
matcherList.extend(contextJob.irrelevant_file_matcher_list)
if not contextJob.inherit_files:
break
if contextJob.isBase():
break
contextJob = layout.getJob(contextJob.parent)
return matcherList
def changeMatchesFiles(self, change, layout, irrelevantMatcherList=None):
if irrelevantMatcherList is None:
irrelevantMatcherList = self.buildIrrelevantList(layout)
matchesSelf = self.changeMatchesFilesSelf(change,
irrelevantMatcherList)
if not self.inherit_files:
return matchesSelf
# We want to explicitly evaluate the files and
# irrelevant-files of the parent
if not matchesSelf and not self.isBase():
parent = layout.getJob(self.parent)
return parent.changeMatchesFiles(change, layout,
irrelevantMatcherList)
return matchesSelf
class JobProject(ConfigObject):
""" A reference to a project from a job. """
@ -8244,7 +8285,7 @@ class Layout(object):
continue
updates_job_config = False
if not skip_file_matcher and \
not final_job.changeMatchesFiles(change):
not final_job.changeMatchesFiles(change, self):
matched_files = False
if final_job.match_on_config_updates:
updates_job_config = item.updatesJobConfig(