zuul/zuul
Code Issues Proposed changes

Add configuration support for negative regex

Browse Source 
The re2 library does not support negative lookahead expressions.
Expressions such as "(?!stable/)", "(?!master)", and "(?!refs/)" are
very useful branch specifiers with likely many instances in the wild.
We need to provide a migration path for these.

This updates the configuration options which currently accepts Python
regular expressions to additionally accept a nested dictionary which
allows specifying that the regex should be negated.  In the future,
other options (global, multiline, etc) could be added.

A very few options are currently already compiled with re2.  These are
left alone for now, but once the transition to re2 is complete, they
can be upgraded to use this syntax as well.

Change-Id: I509c9821993e1886cef1708ddee6d62d1a160bb0
This commit is contained in:
James E. Blair 2023-08-19 12:01:04 -07:00
parent 456205e7be
commit 3d5f87359d
33 changed files with 499 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
doc/source/developer/model-changelog.rst
View File

@ -137,3 +137,9 @@ Version 16
:Prior Zuul version: 9.0.0
:Description: Adds default_branch to the branch cache.
Affects schedulers.
Version 17
----------
:Prior Zuul version: 9.1.0
:Description: Adds ZuulRegex and adjusts SourceContext serialialization.
Affects schedulers and web.

38
doc/source/project-config.rst
View File

@ -73,6 +73,44 @@ regular expression.
Zuul uses the `RE2 library <https://github.com/google/re2/wiki/Syntax>`_
which has a restricted regular expression syntax compared to PCRE.
Some options may be specified for regular expressions. To do so, use
a dictionary to specify the regular expression in the YAML
configuration.
For example, the following are all valid values for the
:attr:`job.branches` attribute, and will all match the branch "devel":
.. code-block:: yaml
- job:
branches: devel
- job:
branches:
- devel
- job:
branches:
regex: devel
negate: false
- job:
branches:
- regex: devel
negate: false
.. attr:: <regular expression>
.. attr:: regex
The pattern for the regular expression. This uses the RE2 syntax.
.. attr:: negate
:type: bool
:default: false
Whether to negate the match.
.. _encryption:
Encryption

7
releasenotes/notes/regex-negate-8d676d4c0c25051d.yaml Normal file
View File

@ -0,0 +1,7 @@
---
features:
- |
Configuration options which accept :ref:`regular expressions
<regex>` have been updated to accept a new syntax which allows
specifying that the regular expression should be treated as a
negative match.

12
tests/fixtures/config/branch-negative/git/org_project2/.zuul.yaml vendored Normal file
View File

@ -0,0 +1,12 @@
- job:
name: test-job2
run: playbooks/test-job.yaml
- project:
name: org/project2
check:
jobs:
- test-job2:
branches:
- regex: stable
negate: true

1
tests/fixtures/config/branch-negative/git/org_project2/README vendored Normal file
View File

@ -0,0 +1 @@
test

2
tests/fixtures/config/branch-negative/git/org_project2/playbooks/test-job.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

1
tests/fixtures/config/branch-negative/main.yaml vendored
View File

@ -6,3 +6,4 @@
- project-config
untrusted-projects:
- org/project
- org/project2

85
tests/fixtures/layouts/negate-post.yaml vendored Normal file
View File

@ -0,0 +1,85 @@
- pipeline:
name: check
manager: independent
trigger:
gerrit:
- event: patchset-created
success:
gerrit:
Verified: 1
failure:
gerrit:
Verified: -1
- pipeline:
name: gate
manager: dependent
success-message: Build succeeded (gate).
trigger:
gerrit:
- event: comment-added
approval:
- Approved: 1
success:
gerrit:
Verified: 2
submit: true
failure:
gerrit:
Verified: -2
start:
gerrit:
Verified: 0
precedence: high
- pipeline:
name: post
manager: independent
trigger:
gerrit:
- event: ref-updated
ref: {'regex': '^refs/.*$', 'negate': true}
- pipeline:
name: tag
manager: independent
trigger:
gerrit:
- event: ref-updated
ref: ^refs/tags/.*$
- job:
name: base
parent: null
run: playbooks/base.yaml
nodeset:
nodes:
- label: ubuntu-xenial
name: controller
- job:
name: check-job
run: playbooks/check.yaml
- job:
name: post-job
run: playbooks/post.yaml
- job:
name: tag-job
run: playbooks/tag.yaml
- project:
name: org/project
check:
jobs:
- check-job
gate:
jobs:
- check-job
post:
jobs:
- post-job
tag:
jobs:
- tag-job

28
tests/unit/test_change_matcher.py
View File

@ -1,4 +1,5 @@
# Copyright 2015 Red Hat, Inc.
# Copyright 2023 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -14,6 +15,7 @@
from zuul import change_matcher as cm
from zuul import model
from zuul.lib.re2util import ZuulRegex
from tests.base import BaseTestCase
@ -30,22 +32,22 @@ class BaseTestMatcher(BaseTestCase):
class TestAbstractChangeMatcher(BaseTestMatcher):
def test_str(self):
matcher = cm.ProjectMatcher(self.project)
matcher = cm.ProjectMatcher(ZuulRegex(self.project))
self.assertEqual(str(matcher), '{ProjectMatcher:project}')
def test_repr(self):
matcher = cm.ProjectMatcher(self.project)
matcher = cm.ProjectMatcher(ZuulRegex(self.project))
self.assertEqual(repr(matcher), '<ProjectMatcher project>')
class TestProjectMatcher(BaseTestMatcher):
def test_matches_returns_true(self):
matcher = cm.ProjectMatcher(self.project)
matcher = cm.ProjectMatcher(ZuulRegex(self.project))
self.assertTrue(matcher.matches(self.change))
def test_matches_returns_false(self):
matcher = cm.ProjectMatcher('not_project')
matcher = cm.ProjectMatcher(ZuulRegex('not_project'))
self.assertFalse(matcher.matches(self.change))
@ -53,7 +55,7 @@ class TestBranchMatcher(BaseTestMatcher):
def setUp(self):
super(TestBranchMatcher, self).setUp()
self.matcher = cm.BranchMatcher('foo')
self.matcher = cm.BranchMatcher(ZuulRegex('foo'))
def test_matches_returns_true_on_matching_branch(self):
self.change.branch = 'foo'
@ -73,7 +75,7 @@ class TestBranchMatcher(BaseTestMatcher):
class TestAbstractMatcherCollection(BaseTestMatcher):
def test_str(self):
matcher = cm.MatchAll([cm.FileMatcher('foo')])
matcher = cm.MatchAll([cm.FileMatcher(ZuulRegex('foo'))])
self.assertEqual(str(matcher), '{MatchAll:{FileMatcher:foo}}')
def test_repr(self):
@ -93,7 +95,8 @@ class TestMatchAllFiles(BaseTestFilesMatcher):
def setUp(self):
super(TestMatchAllFiles, self).setUp()
self.matcher = cm.MatchAllFiles([cm.FileMatcher('^docs/.*$')])
self.matcher = cm.MatchAllFiles(
[cm.FileMatcher(ZuulRegex('^docs/.*$'))])
def test_matches_returns_false_when_files_attr_missing(self):
delattr(self.change, 'files')
@ -122,7 +125,8 @@ class TestMatchAnyFiles(BaseTestFilesMatcher):
def setUp(self):
super(TestMatchAnyFiles, self).setUp()
self.matcher = cm.MatchAnyFiles([cm.FileMatcher('^docs/.*$')])
self.matcher = cm.MatchAnyFiles(
[cm.FileMatcher(ZuulRegex('^docs/.*$'))])
def test_matches_returns_true_when_files_attr_missing(self):
delattr(self.change, 'files')
@ -147,20 +151,20 @@ class TestMatchAnyFiles(BaseTestFilesMatcher):
class TestMatchAll(BaseTestMatcher):
def test_matches_returns_true(self):
matcher = cm.MatchAll([cm.ProjectMatcher(self.project)])
matcher = cm.MatchAll([cm.ProjectMatcher(ZuulRegex(self.project))])
self.assertTrue(matcher.matches(self.change))
def test_matches_returns_false_for_missing_matcher(self):
matcher = cm.MatchAll([cm.ProjectMatcher('not_project')])
matcher = cm.MatchAll([cm.ProjectMatcher(ZuulRegex('not_project'))])
self.assertFalse(matcher.matches(self.change))
class TestMatchAny(BaseTestMatcher):
def test_matches_returns_true(self):
matcher = cm.MatchAny([cm.ProjectMatcher(self.project)])
matcher = cm.MatchAny([cm.ProjectMatcher(ZuulRegex(self.project))])
self.assertTrue(matcher.matches(self.change))
def test_matches_returns_false(self):
matcher = cm.MatchAny([cm.ProjectMatcher('not_project')])
matcher = cm.MatchAny([cm.ProjectMatcher(ZuulRegex('not_project'))])
self.assertFalse(matcher.matches(self.change))

8
tests/unit/test_model.py
View File

@ -1,4 +1,5 @@
# Copyright 2015 Red Hat, Inc.
# Copyright 2023 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -32,6 +33,7 @@ import zuul.lib.connections
from tests.base import BaseTestCase, FIXTURE_DIR
from zuul.lib.ansible import AnsibleManager
from zuul.lib import tracing
from zuul.lib.re2util import ZuulRegex
from zuul.model_api import MODEL_API
from zuul.zk.zkobject import LocalZKContext
from zuul.zk.components import COMPONENT_REGISTRY
@ -769,6 +771,8 @@ class TestRef(BaseTestCase):
class TestSourceContext(BaseTestCase):
def setUp(self):
super().setUp()
COMPONENT_REGISTRY.registry = Dummy()
COMPONENT_REGISTRY.registry.model_api = MODEL_API
self.connection = Dummy(connection_name='dummy_connection')
self.source = Dummy(canonical_hostname='git.example.com',
connection=self.connection)
@ -777,8 +781,8 @@ class TestSourceContext(BaseTestCase):
self.project.canonical_name, self.project.name,
self.project.connection_name, 'master', 'test', True)
self.context.implied_branches = [
change_matcher.BranchMatcher('foo'),
change_matcher.ImpliedBranchMatcher('foo'),
change_matcher.BranchMatcher(ZuulRegex('foo')),
change_matcher.ImpliedBranchMatcher(ZuulRegex('foo')),
]
def test_serialize(self):

29
tests/unit/test_model_upgrade.py
View File

@ -14,6 +14,8 @@
import json
from zuul import change_matcher
from zuul.lib.re2util import ZuulRegex
from zuul.zk.components import ComponentRegistry
from tests.base import ZuulTestCase, simple_layout, iterate_timeout
@ -320,6 +322,33 @@ class TestModelUpgrade(ZuulTestCase):
result='SUCCESS', changes='1,1'),
], ordered=False)
@model_version(16)
def test_model_16_17(self):
matcher = change_matcher.BranchMatcher(ZuulRegex('foo'))
ser = matcher.serialize()
self.assertEqual(ser, {'regex': 'foo', 'implied': False})
matcher2 = change_matcher.BranchMatcher.deserialize(ser)
self.assertEqual(matcher, matcher2)
# Upgrade our component
self.model_test_component_info.model_api = 17
component_registry = ComponentRegistry(self.zk_client)
for _ in iterate_timeout(30, "model api to update"):
if component_registry.model_api == 17:
break
matcher = change_matcher.BranchMatcher(ZuulRegex('foo'))
ser = matcher.serialize()
self.assertEqual(ser, {
'regex': {
'negate': False,
'pattern': 'foo',
},
'implied': False
})
matcher2 = change_matcher.BranchMatcher.deserialize(ser)
self.assertEqual(matcher, matcher2)
class TestGithubModelUpgrade(ZuulTestCase):
config_file = 'zuul-github-driver.conf'

29
tests/unit/test_scheduler.py
View File

@ -1705,6 +1705,35 @@ class TestScheduler(ZuulTestCase):
self.assertEqual(len(self.history), 1)
self.assertIn('project-post', job_names)
@simple_layout('layouts/negate-post.yaml')
def test_post_negative_regex(self):
"Test that post jobs run"
p = "review.example.com/org/project"
upstream = self.getUpstreamRepos([p])
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.setMerged()
A_commit = str(upstream[p].commit('master'))
self.log.debug("A commit: %s" % A_commit)
e = {
"type": "ref-updated",
"submitter": {
"name": "User Name",
},
"refUpdate": {
"oldRev": "90f173846e3af9154517b88543ffbd1691f31366",
"newRev": A_commit,
"refName": "master",
"project": "org/project",
}
}
self.fake_gerrit.addEvent(e)
self.waitUntilSettled()
job_names = [x.name for x in self.history]
self.assertEqual(len(self.history), 1)
self.assertIn('post-job', job_names)
def test_post_ignore_deletes(self):
"Test that deleting refs does not trigger post jobs"

17
tests/unit/test_v3.py
View File

@ -590,6 +590,23 @@ class TestBranchNegative(ZuulTestCase):
self.assertHistory([
dict(name='test-job', result='SUCCESS', changes='1,1')])
def test_negative_branch_match_regex(self):
# Test that a negated branch matcher regex works with implied branches.
self.create_branch('org/project2', 'stable/pike')
self.fake_gerrit.addEvent(
self.fake_gerrit.getFakeBranchCreatedEvent(
'org/project2', 'stable/pike'))
self.waitUntilSettled()
A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
B = self.fake_gerrit.addFakeChange('org/project2', 'stable/pike', 'A')
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertHistory([
dict(name='test-job2', result='SUCCESS', changes='1,1')])
class TestBranchTemplates(ZuulTestCase):
tenant_config_file = 'config/branch-templates/main.yaml'

40
zuul/change_matcher.py
View File

@ -1,4 +1,5 @@
# Copyright 2015 Red Hat, Inc.
# Copyright 2023 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -19,12 +20,21 @@ configuration.
import re
from zuul.lib.re2util import ZuulRegex
from zuul.zk.components import COMPONENT_REGISTRY
class AbstractChangeMatcher(object):
"""An abstract class that matches change attributes against regular
expressions
:arg ZuulRegex regex: A Zuul regular expression object to match
"""
def __init__(self, regex):
self._regex = regex
self.regex = re.compile(regex)
self._regex = regex.pattern
self.regex = regex
def matches(self, change):
"""Return a boolean indication of whether change matches
@ -33,13 +43,14 @@ class AbstractChangeMatcher(object):
raise NotImplementedError()
def copy(self):
return self.__class__(self._regex)
return self.__class__(self.regex)
def __deepcopy__(self, memo):
return self.copy()
def __eq__(self, other):
return str(self) == str(other)
return (self.__class__ == other.__class__ and
self.regex == other.regex)
def __ne__(self, other):
return not self.__eq__(other)
@ -83,14 +94,25 @@ class BranchMatcher(AbstractChangeMatcher):
return False
def serialize(self):
return {
"implied": self.exactmatch,
"regex": self._regex,
}
if (COMPONENT_REGISTRY.model_api < 17):
return {
"implied": self.exactmatch,
"regex": self.regex.pattern,
}
else:
return {
"implied": self.exactmatch,
"regex": self.regex.serialize(),
}
@classmethod
def deserialize(cls, data):
o = cls.__new__(cls, data['regex'])
if isinstance(data['regex'], dict):
regex = ZuulRegex.deserialize(data['regex'])
else:
# MODEL_API >= 17
regex = ZuulRegex(data['regex'])
o = cls(regex)
return o

37
zuul/configloader.py
View File

@ -1,3 +1,5 @@
# Copyright 2023 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
@ -35,7 +37,7 @@ import zuul.manager.independent
import zuul.manager.supercedent
import zuul.manager.serial
from zuul.lib.logutil import get_annotated_logger
from zuul.lib.re2util import filter_allowed_disallowed
from zuul.lib.re2util import filter_allowed_disallowed, ZuulRegex
from zuul.lib.varnames import check_varnames
from zuul.zk.components import COMPONENT_REGISTRY
from zuul.zk.config_cache import UnparsedConfigCache
@ -43,6 +45,12 @@ from zuul.zk.semaphore import SemaphoreHandler
ZUUL_CONF_ROOT = ('zuul.yaml', 'zuul.d', '.zuul.yaml', '.zuul.d')
# A voluptuous schema for a regular expression.
ZUUL_REGEX = {
vs.Required('regex'): str,
'negate': bool,
}
# Several forms accept either a single item or a list, this makes
# specifying that in the schema easy (and explicit).
@ -76,6 +84,13 @@ def check_config_path(path):
"allowed in extra-config-paths")
def make_regex(data):
if isinstance(data, dict):
return ZuulRegex(data['regex'],
negate=data.get('negate', False))
return ZuulRegex(data)
def indent(s):
return '\n'.join([' ' + x for x in s.split('\n')])
@ -586,7 +601,7 @@ def copy_safe_config(conf):
class PragmaParser(object):
pragma = {
'implied-branch-matchers': bool,
'implied-branches': to_list(str),
'implied-branches': to_list(vs.Any(ZUUL_REGEX, str)),
'_source_context': model.SourceContext,
'_start_mark': model.ZuulMark,
}
@ -615,7 +630,7 @@ class PragmaParser(object):
# (automatically generated from source file branches) are
# ImpliedBranchMatchers.
source_context.implied_branches = [
change_matcher.BranchMatcher(x)
change_matcher.BranchMatcher(make_regex(x))
for x in as_list(branches)]
@ -806,7 +821,7 @@ class JobParser(object):
'semaphore': vs.Any(semaphore, str),
'semaphores': to_list(vs.Any(semaphore, str)),
'tags': to_list(str),
'branches': to_list(str),
'branches': to_list(vs.Any(ZUUL_REGEX, str)),
'files': to_list(str),
'secrets': to_list(vs.Any(secret, str)),
'irrelevant-files': to_list(str),
@ -891,7 +906,9 @@ class JobParser(object):
job.source_context = conf['_source_context']
job.start_mark = conf['_start_mark']
job.variant_description = conf.get(
'variant-description', " ".join(as_list(conf.get('branches'))))
'variant-description', " ".join([
str(x) for x in as_list(conf.get('branches'))
]))
if project_pipeline and conf['_source_context'].trusted:
# A config project has attached this job to a
@ -1165,7 +1182,7 @@ class JobParser(object):
branches = None
if 'branches' in conf:
branches = [change_matcher.BranchMatcher(x)
branches = [change_matcher.BranchMatcher(make_regex(x))
for x in as_list(conf['branches'])]
elif not project_pipeline:
branches = self.pcontext.getImpliedBranches(job.source_context)
@ -1382,7 +1399,7 @@ class ProjectParser(object):
else:
project_config.setImpliedBranchMatchers(
[change_matcher.ImpliedBranchMatcher(
source_context.branch)])
ZuulRegex(source_context.branch))])
# Add templates
for name in conf.get('templates', []):
@ -1787,7 +1804,8 @@ class ParseContext(object):
if source_context.implied_branch_matchers is True:
if source_context.implied_branches is not None:
return source_context.implied_branches
return [change_matcher.ImpliedBranchMatcher(source_context.branch)]
return [change_matcher.ImpliedBranchMatcher(
ZuulRegex(source_context.branch))]
elif source_context.implied_branch_matchers is False:
return None
@ -1805,7 +1823,8 @@ class ParseContext(object):
if source_context.implied_branches is not None:
return source_context.implied_branches
return [change_matcher.ImpliedBranchMatcher(source_context.branch)]
return [change_matcher.ImpliedBranchMatcher(
ZuulRegex(source_context.branch))]
class TenantParser(object):

31
zuul/driver/gerrit/gerritmodel.py
View File

@ -14,14 +14,13 @@
# under the License.
import copy
import re
import time
import urllib.parse
import dateutil.parser
from zuul.model import EventFilter, RefFilter
from zuul.model import Change, TriggerEvent, FalseWithReason
from zuul.driver.util import time_to_seconds, to_list
from zuul.driver.util import time_to_seconds, to_list, make_regex
from zuul import exceptions
EMPTY_GIT_REF = '0' * 40 # git sha of all zeros, used during creates/deletes
@ -281,18 +280,18 @@ class GerritEventFilter(EventFilter):
else:
self.reject_filter = None
self._types = types
self._branches = branches
self._refs = refs
self._comments = comments
self._emails = emails
self._usernames = usernames
self.types = [re.compile(x) for x in types]
self.branches = [re.compile(x) for x in branches]
self.refs = [re.compile(x) for x in refs]
self.comments = [re.compile(x) for x in comments]
self.emails = [re.compile(x) for x in emails]
self.usernames = [re.compile(x) for x in usernames]
self._types = [x.pattern for x in types]
self._branches = [x.pattern for x in branches]
self._refs = [x.pattern for x in refs]
self._comments = [x.pattern for x in comments]
self._emails = [x.pattern for x in emails]
self._usernames = [x.pattern for x in usernames]
self.types = types
self.branches = branches
self.refs = refs
self.comments = comments
self.emails = emails
self.usernames = usernames
self.event_approvals = event_approvals
self.uuid = uuid
self.scheme = scheme
@ -566,9 +565,9 @@ class GerritRefFilter(RefFilter):
for a in approvals:
for k, v in a.items():
if k == 'username':
a['username'] = re.compile(v)
a['username'] = make_regex(v)
elif k == 'email':
a['email'] = re.compile(v)
a['email'] = make_regex(v)
elif k == 'newer-than':
a[k] = time_to_seconds(v)
elif k == 'older-than':

26
zuul/driver/gerrit/gerrittrigger.py
View File

@ -1,4 +1,5 @@
# Copyright 2012 Hewlett-Packard Development Company, L.P.
# Copyright 2023 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -17,7 +18,7 @@ import voluptuous as v
from zuul.trigger import BaseTrigger
from zuul.driver.gerrit.gerritmodel import GerritEventFilter
from zuul.driver.gerrit import gerritsource
from zuul.driver.util import scalar_or_list, to_list
from zuul.driver.util import scalar_or_list, to_list, make_regex, ZUUL_REGEX
from zuul.configloader import DeprecationWarning
@ -63,12 +64,19 @@ class GerritTrigger(BaseTrigger):
error_accumulator.addError(
GerritRejectApprovalDeprecation())
types = [make_regex(x) for x in to_list(trigger['event'])]
branches = [make_regex(x) for x in to_list(trigger.get('branch'))]
refs = [make_regex(x) for x in to_list(trigger.get('ref'))]
comments = [make_regex(x) for x in comments]
emails = [make_regex(x) for x in emails]
usernames = [make_regex(x) for x in usernames]
f = GerritEventFilter(
connection_name=connection_name,
trigger=self,
types=to_list(trigger['event']),
branches=to_list(trigger.get('branch')),
refs=to_list(trigger.get('ref')),
types=types,
branches=branches,
refs=refs,
event_approvals=approvals,
comments=comments,
emails=emails,
@ -114,13 +122,13 @@ def getSchema():
'uuid': str,
'scheme': str,
'comment_filter': scalar_or_list(str),
'comment': scalar_or_list(str),
'comment': scalar_or_list(v.Any(ZUUL_REGEX, str)),
'email_filter': scalar_or_list(str),
'email': scalar_or_list(str),
'email': scalar_or_list(v.Any(ZUUL_REGEX, str)),
'username_filter': scalar_or_list(str),
'username': scalar_or_list(str),
'branch': scalar_or_list(str),
'ref': scalar_or_list(str),
'username': scalar_or_list(v.Any(ZUUL_REGEX, str)),
'branch': scalar_or_list(v.Any(ZUUL_REGEX, str)),
'ref': scalar_or_list(v.Any(ZUUL_REGEX, str)),
'ignore-deletes': bool,
'approval': scalar_or_list(variable_dict),
'require-approval': scalar_or_list(approval),

10
zuul/driver/git/gitmodel.py
View File

@ -1,4 +1,5 @@
# Copyright 2017 Red Hat, Inc.
# Copyright 2023 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -12,8 +13,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import re
from zuul.model import TriggerEvent
from zuul.model import EventFilter
@ -44,10 +43,11 @@ class GitEventFilter(EventFilter):
super().__init__(connection_name, trigger)
self._refs = refs
self.types = types if types is not None else []
refs = refs if refs is not None else []
self.refs = [re.compile(x) for x in refs]
self._refs = [x.pattern for x in refs]
self.refs = refs
self.types = types if types is not None else []
self.ignore_deletes = ignore_deletes
def __repr__(self):

10
zuul/driver/git/gittrigger.py
View File

@ -1,4 +1,5 @@
# Copyright 2017 Red Hat, Inc.
# Copyright 2023 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -16,7 +17,7 @@ import logging
import voluptuous as v
from zuul.trigger import BaseTrigger
from zuul.driver.git.gitmodel import GitEventFilter
from zuul.driver.util import scalar_or_list, to_list
from zuul.driver.util import scalar_or_list, to_list, make_regex, ZUUL_REGEX
class GitTrigger(BaseTrigger):
@ -27,11 +28,14 @@ class GitTrigger(BaseTrigger):
error_accumulator):
efilters = []
for trigger in to_list(trigger_conf):
refs = [make_regex(x) for x in to_list(trigger.get('ref'))]
f = GitEventFilter(
connection_name=connection_name,
trigger=self,
types=to_list(trigger['event']),
refs=to_list(trigger.get('ref')),
refs=refs,
ignore_deletes=trigger.get(
'ignore-deletes', True)
)
@ -44,7 +48,7 @@ def getSchema():
git_trigger = {
v.Required('event'):
scalar_or_list(v.Any('ref-updated')),
'ref': scalar_or_list(str),
'ref': scalar_or_list(v.Any(ZUUL_REGEX, str)),
'ignore-deletes': bool,
}

24
zuul/driver/github/githubmodel.py
View File

@ -203,14 +203,14 @@ class GithubEventFilter(EventFilter):
else:
self.reject_filter = None
self._types = types
self._branches = branches
self._refs = refs
self._comments = comments
self.types = [re.compile(x) for x in types]
self.branches = [re.compile(x) for x in branches]
self.refs = [re.compile(x) for x in refs]
self.comments = [re.compile(x) for x in comments]
self._types = [x.pattern for x in types]
self._branches = [x.pattern for x in branches]
self._refs = [x.pattern for x in refs]
self._comments = [x.pattern for x in comments]
self.types = types
self.branches = branches
self.refs = refs
self.comments = comments
self.actions = actions
self.labels = labels
self.unlabels = unlabels
@ -311,6 +311,8 @@ class GithubEventFilter(EventFilter):
if self.check_runs:
check_run_found = False
for check_run in self.check_runs:
# TODO: construct as ZuulRegex in initializer when re2
# migration is complete.
if re2.fullmatch(check_run, event.check_run):
check_run_found = True
break
@ -337,6 +339,8 @@ class GithubEventFilter(EventFilter):
if self.statuses:
status_found = False
for status in self.statuses:
# TODO: construct as ZuulRegex in initializer when re2
# migration is complete.
if re2.fullmatch(status, event.status):
status_found = True
break
@ -556,6 +560,8 @@ class GithubRefFilter(RefFilter):
if self.required_statuses:
for required_status in self.required_statuses:
for status in change.status:
# TODO: construct as ZuulRegex in initializer when
# re2 migration is complete.
if re2.fullmatch(required_status, status):
return True
return FalseWithReason("Required statuses %s do not match %s" % (
@ -567,6 +573,8 @@ class GithubRefFilter(RefFilter):
# If any of the rejected statusses are present, we return false
for rstatus in self.reject_statuses:
for status in change.status:
# TODO: construct as ZuulRegex in initializer when re2
# migration is complete.
if re2.fullmatch(rstatus, status):
return FalseWithReason("Reject statuses %s match %s" % (
self.reject_statuses, change.status))

23
zuul/driver/github/githubtrigger.py
View File

@ -1,4 +1,5 @@
# Copyright 2015 Hewlett-Packard Development Company, L.P.
# Copyright 2023 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -17,7 +18,7 @@ import voluptuous as v
from zuul.trigger import BaseTrigger
from zuul.driver.github.githubmodel import GithubEventFilter
from zuul.driver.github import githubsource
from zuul.driver.util import scalar_or_list, to_list
from zuul.driver.util import scalar_or_list, to_list, make_regex, ZUUL_REGEX
class GithubTrigger(BaseTrigger):
@ -39,14 +40,20 @@ class GithubTrigger(BaseTrigger):
error_accumulator):
efilters = []
for trigger in to_list(trigger_config):
types = [make_regex(x) for x in to_list(trigger['event'])]
branches = [make_regex(x) for x in to_list(trigger.get('branch'))]
refs = [make_regex(x) for x in to_list(trigger.get('ref'))]
comments = [make_regex(x) for x in to_list(trigger.get('comment'))]
f = GithubEventFilter(
connection_name=connection_name,
trigger=self,
types=to_list(trigger['event']),
types=types,
actions=to_list(trigger.get('action')),
branches=to_list(trigger.get('branch')),
refs=to_list(trigger.get('ref')),
comments=to_list(trigger.get('comment')),
branches=branches,
refs=refs,
comments=comments,
check_runs=to_list(trigger.get('check')),
labels=to_list(trigger.get('label')),
unlabels=to_list(trigger.get('unlabel')),
@ -72,9 +79,9 @@ def getSchema():
'push',
'check_run')),
'action': scalar_or_list(str),
'branch': scalar_or_list(str),
'ref': scalar_or_list(str),
'comment': scalar_or_list(str),
'branch': scalar_or_list(v.Any(ZUUL_REGEX, str)),
'ref': scalar_or_list(v.Any(ZUUL_REGEX, str)),
'comment': scalar_or_list(v.Any(ZUUL_REGEX, str)),
'label': scalar_or_list(str),
'unlabel': scalar_or_list(str),
'state': scalar_or_list(str),

23
zuul/driver/gitlab/gitlabmodel.py
View File

@ -1,5 +1,5 @@
# Copyright 2019 Red Hat, Inc.
# Copyright 2022 Acme Gating, LLC
# Copyright 2022-2023 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -13,7 +13,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import re
from zuul.model import Change, TriggerEvent, EventFilter, RefFilter
EMPTY_GIT_REF = '0' * 40 # git sha of all zeros, used during creates/deletes
@ -152,13 +151,21 @@ class GitlabEventFilter(EventFilter):
comments=None, refs=None, labels=None, unlabels=None,
ignore_deletes=True):
super().__init__(connection_name, trigger)
self._types = types or []
self.types = [re.compile(x) for x in self._types]
types = types if types is not None else []
refs = refs if refs is not None else []
comments = comments if comments is not None else []
self._refs = [x.pattern for x in refs]
self.refs = refs
self._types = [x.pattern for x in types]
self.types = types
self._comments = [x.pattern for x in comments]
self.comments = comments
self.actions = actions or []
self._comments = comments or []
self.comments = [re.compile(x) for x in self._comments]
self._refs = refs or []
self.refs = [re.compile(x) for x in self._refs]
self.labels = labels or []
self.unlabels = unlabels or []
self.ignore_deletes = ignore_deletes

18
zuul/driver/gitlab/gitlabtrigger.py
View File

@ -1,4 +1,5 @@
# Copyright 2019 Red Hat, Inc.
# Copyright 2023 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -16,7 +17,7 @@ import logging
import voluptuous as v
from zuul.driver.gitlab.gitlabmodel import GitlabEventFilter
from zuul.trigger import BaseTrigger
from zuul.driver.util import scalar_or_list, to_list
from zuul.driver.util import scalar_or_list, to_list, make_regex, ZUUL_REGEX
class GitlabTrigger(BaseTrigger):
@ -27,13 +28,18 @@ class GitlabTrigger(BaseTrigger):
error_accumulator):
efilters = []
for trigger in to_list(trigger_config):
types = [make_regex(x) for x in to_list(trigger['event'])]
refs = [make_regex(x) for x in to_list(trigger.get('ref'))]
comments = [make_regex(x) for x in
to_list(trigger.get('comment'))]
f = GitlabEventFilter(
connection_name=connection_name,
trigger=self,
types=to_list(trigger['event']),
types=types,
actions=to_list(trigger.get('action')),
comments=to_list(trigger.get('comment')),
refs=to_list(trigger.get('ref')),
comments=comments,
refs=refs,
labels=to_list(trigger.get('labels')),
unlabels=to_list(trigger.get('unlabels')),
)
@ -53,8 +59,8 @@ def getSchema():
'gl_push',
)),
'action': scalar_or_list(str),
'comment': scalar_or_list(str),
'ref': scalar_or_list(str),
'comment': scalar_or_list(v.Any(ZUUL_REGEX, str)),
'ref': scalar_or_list(v.Any(ZUUL_REGEX, str)),
'labels': scalar_or_list(str),
'unlabels': scalar_or_list(str),
}

15
zuul/driver/pagure/paguremodel.py
View File

@ -1,4 +1,5 @@
# Copyright 2018 Red Hat, Inc.
# Copyright 2023 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -12,8 +13,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import re
from zuul.model import Change, TriggerEvent, EventFilter, RefFilter
EMPTY_GIT_REF = '0' * 40 # git sha of all zeros, used during creates/deletes
@ -142,12 +141,12 @@ class PagureEventFilter(EventFilter):
EventFilter.__init__(self, connection_name, trigger)
self._types = types
self._refs = refs
self._comments = comments
self.types = [re.compile(x) for x in types]
self.refs = [re.compile(x) for x in refs]
self.comments = [re.compile(x) for x in comments]
self._types = [x.pattern for x in types]
self._refs = [x.pattern for x in refs]
self._comments = [x.pattern for x in comments]
self.types = types
self.refs = refs
self.comments = comments
self.actions = actions
self.statuses = statuses
self.tags = tags

17
zuul/driver/pagure/paguretrigger.py
View File

@ -1,4 +1,5 @@
# Copyright 2018 Red Hat, Inc.
# Copyright 2023 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -16,7 +17,7 @@ import logging
import voluptuous as v
from zuul.trigger import BaseTrigger
from zuul.driver.pagure.paguremodel import PagureEventFilter
from zuul.driver.util import scalar_or_list, to_list
from zuul.driver.util import scalar_or_list, to_list, make_regex, ZUUL_REGEX
class PagureTrigger(BaseTrigger):
@ -27,13 +28,17 @@ class PagureTrigger(BaseTrigger):
error_accumulator):
efilters = []
for trigger in to_list(trigger_config):
types = [make_regex(x) for x in to_list(trigger['event'])]
refs = [make_regex(x) for x in to_list(trigger.get('ref'))]
comments = [make_regex(x) for x in to_list(trigger.get('comment'))]
f = PagureEventFilter(
connection_name=connection_name,
trigger=self,
types=to_list(trigger['event']),
types=types,
actions=to_list(trigger.get('action')),
refs=to_list(trigger.get('ref')),
comments=to_list(trigger.get('comment')),
refs=refs,
comments=comments,
statuses=to_list(trigger.get('status')),
tags=to_list(trigger.get('tag')),
)
@ -56,8 +61,8 @@ def getSchema():
'pg_pull_request_review',
'pg_push')),
'action': scalar_or_list(str),
'ref': scalar_or_list(str),
'comment': scalar_or_list(str),
'ref': scalar_or_list(v.Any(ZUUL_REGEX, str)),
'comment': scalar_or_list(v.Any(ZUUL_REGEX, str)),
'status': scalar_or_list(str),
'tag': scalar_or_list(str)
}

7
zuul/driver/timer/timermodel.py
View File

@ -1,4 +1,5 @@
# Copyright 2017 Red Hat, Inc.
# Copyright 2023 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -12,8 +13,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import re
from zuul.model import EventFilter, TriggerEvent
@ -21,8 +20,8 @@ class TimerEventFilter(EventFilter):
def __init__(self, connection_name, trigger, types=[], timespecs=[]):
EventFilter.__init__(self, connection_name, trigger)
self._types = types
self.types = [re.compile(x) for x in types]
self._types = [x.pattern for x in types]
self.types = types
self.timespecs = timespecs
def __repr__(self):

6
zuul/driver/timer/timertrigger.py
View File

@ -1,5 +1,6 @@
# Copyright 2012 Hewlett-Packard Development Company, L.P.
# Copyright 2013 OpenStack Foundation
# Copyright 2023 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -17,7 +18,7 @@ import voluptuous as v
from zuul.trigger import BaseTrigger
from zuul.driver.timer.timermodel import TimerEventFilter
from zuul.driver.util import to_list
from zuul.driver.util import to_list, make_regex
class TimerTrigger(BaseTrigger):
@ -27,9 +28,10 @@ class TimerTrigger(BaseTrigger):
error_accumulator):
efilters = []
for trigger in to_list(trigger_conf):
types = [make_regex('timer')]
f = TimerEventFilter(connection_name=connection_name,
trigger=self,
types=['timer'],
types=types,
timespecs=to_list(trigger['time']))
efilters.append(f)

3
zuul/driver/util.py
View File

@ -1,4 +1,5 @@
# Copyright 2017 Red Hat, Inc.
# Copyright 2023 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -16,6 +17,8 @@
import voluptuous as vs
from zuul.configloader import ZUUL_REGEX, make_regex # noqa
def time_to_seconds(s):
if s.endswith('s'):

10
zuul/driver/zuul/zuulmodel.py
View File

@ -12,8 +12,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import re
from zuul.model import EventFilter, TriggerEvent
@ -21,10 +19,10 @@ class ZuulEventFilter(EventFilter):
def __init__(self, connection_name, trigger, types=[], pipelines=[]):
EventFilter.__init__(self, connection_name, trigger)
self._types = types
self._pipelines = pipelines
self.types = [re.compile(x) for x in types]
self.pipelines = [re.compile(x) for x in pipelines]
self._types = [x.pattern for x in types]
self._pipelines = [x.pattern for x in pipelines]
self.types = types
self.pipelines = pipelines
def __repr__(self):
ret = '<ZuulEventFilter'

11
zuul/driver/zuul/zuultrigger.py
View File

@ -17,7 +17,7 @@ import logging
import voluptuous as v
from zuul.trigger import BaseTrigger
from zuul.driver.zuul.zuulmodel import ZuulEventFilter
from zuul.driver.util import scalar_or_list, to_list
from zuul.driver.util import scalar_or_list, to_list, make_regex, ZUUL_REGEX
class ZuulTrigger(BaseTrigger):
@ -33,11 +33,14 @@ class ZuulTrigger(BaseTrigger):
error_accumulator):
efilters = []
for trigger in to_list(trigger_conf):
types = [make_regex(x) for x in to_list(trigger['event'])]
pipelines = [make_regex(x) for x in
to_list(trigger.get('pipeline'))]
f = ZuulEventFilter(
connection_name=connection_name,
trigger=self,
types=to_list(trigger['event']),
pipelines=to_list(trigger.get('pipeline')),
types=types,
pipelines=pipelines,
)
efilters.append(f)
@ -49,7 +52,7 @@ def getSchema():
v.Required('event'):
scalar_or_list(v.Any('parent-change-enqueued',
'project-change-merged')),
'pipeline': scalar_or_list(str),
'pipeline': scalar_or_list(v.Any(ZUUL_REGEX, str)),
}
return zuul_trigger

44
zuul/lib/re2util.py
View File

@ -1,4 +1,5 @@
# Copyright (C) 2020 Red Hat, Inc
# Copyright (C) 2023 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -13,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import re
import re2
@ -44,3 +46,45 @@ def filter_allowed_disallowed(
if allowed:
ret.append(subject)
return ret
class ZuulRegex:
def __init__(self, pattern, negate=False):
self.pattern = pattern
self.negate = negate
# TODO: switch this to re2
self.re = re.compile(pattern)
def __eq__(self, other):
return (isinstance(other, ZuulRegex) and
self.pattern == other.pattern and
self.negate == other.negate)
def __ne__(self, other):
return not self.__eq__(other)
def match(self, subject):
if self.negate:
return not self.re.match(subject)
return self.re.match(subject)
def fullmatch(self, subject):
if self.negate:
return not self.re.fullmatch(subject)
return self.re.fullmatch(subject)
def search(self, subject):
if self.negate:
return not self.re.search(subject)
return self.re.search(subject)
def serialize(self):
return {
"pattern": self.pattern,
"negate": self.negate,
}
@classmethod
def deserialize(cls, data):
o = cls(data['pattern'], data['negate'])
return o

8
zuul/model.py
View File

@ -1,5 +1,5 @@
# Copyright 2012 Hewlett-Packard Development Company, L.P.
# Copyright 2021-2022 Acme Gating, LLC
# Copyright 2021-2023 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -42,6 +42,7 @@ import jsonpath_rw
from zuul import change_matcher
from zuul.lib.config import get_default
from zuul.lib.re2util import ZuulRegex
from zuul.lib.result_data import get_artifacts_from_result_data
from zuul.lib.logutil import get_annotated_logger
from zuul.lib.capabilities import capabilities_registry
@ -3193,7 +3194,7 @@ class Job(ConfigObject):
self._files = files
matchers = []
for fn in files:
matchers.append(change_matcher.FileMatcher(fn))
matchers.append(change_matcher.FileMatcher(ZuulRegex(fn)))
self.file_matcher = change_matcher.MatchAnyFiles(matchers)
def setIrrelevantFileMatcher(self, irrelevant_files):
@ -3201,7 +3202,7 @@ class Job(ConfigObject):
self._irrelevant_files = irrelevant_files
matchers = []
for fn in irrelevant_files:
matchers.append(change_matcher.FileMatcher(fn))
matchers.append(change_matcher.FileMatcher(ZuulRegex(fn)))
self.irrelevant_file_matcher = change_matcher.MatchAllFiles(matchers)
def updateVariables(self, other_vars, other_extra_vars, other_host_vars,
@ -3447,6 +3448,7 @@ class Job(ConfigObject):
if self.branch_matcher and not self.branch_matcher.matches(
branch_change):
return False
return True
def changeMatchesFiles(self, change):

2
zuul/model_api.py
View File

@ -14,4 +14,4 @@
# When making ZK schema changes, increment this and add a record to
# doc/source/developer/model-changelog.rst
MODEL_API = 16
MODEL_API = 17