zuul/zuul
Code Issues Proposed changes

Merge "Zuul-Web: substring search for builds, buildsets"

Browse Source
This commit is contained in:
Zuul
2024-03-19 08:34:42 +00:00
committed by Gerrit Code Review
parent fb796cc59a f9ebf6a1c9
commit 4d06f081bd
3 changed files with 121 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
tests/unit/test_database.py
View File

@@ -396,6 +396,38 @@ class TestMysqlDatabase(DBBaseTestCase):
tenant=tenant, uuid=buildset_uuid)
self.assertEqual(db_buildset.result, 'SUCCESS')
def test_sanitize_subtring_query(self):
session = self.connection.getSession()
self.assertEqual(None, session._sanitizeSubstringQuery(None))
self.assertEqual(
"job-name", session._sanitizeSubstringQuery("job-name"))
self.assertEqual(
"$%job-$%name$%", session._sanitizeSubstringQuery("%job-%name%"))
self.assertEqual(
"job$_-$%name$%", session._sanitizeSubstringQuery("job_-%name%"))
self.assertEqual(
"job$_-$%name$%", session._sanitizeSubstringQuery("job_-%name%"))
self.assertEqual(
"$$$%job$_-$%name$%",
session._sanitizeSubstringQuery("$%job_-%name%"))
def test_fuzzy_filter_op(self):
col = sqlalchemy.Column("job_name")
session = self.connection.getSession()
filter = session._getFuzzyFilterOp(col, "job-name")
self.assertEqual(type(col == "foo"), type(filter))
self.assertEqual("job-name", filter.right.value)
filter = session._getFuzzyFilterOp(col, "*job*name*")
self.assertEqual(type(col.like("foo")), type(filter))
self.assertEqual("$", filter.modifiers["escape"])
self.assertEqual("%job%name%", filter.right.value)
filter = session._getFuzzyFilterOp(col, None)
self.assertEqual(type(col.__eq__(None)), type(filter))
self.assertEqual(sqlalchemy.sql.elements.Null, type(filter.right))
class TestPostgresqlDatabase(DBBaseTestCase):
def setUp(self):

58
tests/unit/test_web.py
View File

@@ -1882,6 +1882,48 @@ class TestBuildInfo(BaseTestWeb):
"idx_min=%i" % idx_max).json()
self.assertEqual(len(builds_query), 1, builds_query)
def test_web_substring_search_builds(self):
# Generate some build records in the db.
self.add_base_changes()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
builds = self.get_url("api/tenant/tenant-one/builds?"
"project=org/project*&"
"branch=*aster").json()
self.assertEqual(len(builds), 6)
builds = self.get_url("api/tenant/tenant-one/builds?"
"job_name=*-merge").json()
self.assertEqual(len(builds), 2)
builds = self.get_url("api/tenant/tenant-one/builds?"
"job_name=*-merge&"
"project=org/project1").json()
self.assertEqual(len(builds), 1)
builds = self.get_url("api/tenant/tenant-one/builds?"
"pipeline=gat*&"
"project=org/project1&"
"job_name=*-merge&"
"job_name=*test*").json()
self.assertEqual(len(builds), 3)
self.assertTrue(
all(b["ref"]["project"] == "org/project1" for b in builds))
self.assertEqual("project-test2", builds[0]["job_name"])
self.assertEqual("project-test1", builds[1]["job_name"])
self.assertEqual("project-merge", builds[2]["job_name"])
# sql wildcard chars (% and _), as well as the escape char ($) should
# be treated literally, i.e. they are escpaed.
# The below query would return builds for
# branch LIKE "%master%" OR "%mast_r"
# if they would not be escaped
builds = self.get_url("api/tenant/tenant-one/builds?"
"branch=%master%&"
"branch=%mast_r").json()
self.assertEqual(len(builds), 0)
def test_web_list_skipped_builds(self):
# Test the exclude_result filter
# Generate some build records in the db.
@@ -2005,6 +2047,22 @@ class TestBuildInfo(BaseTestWeb):
"idx_min=%i" % idx_max).json()
self.assertEqual(len(buildsets_query), 1, buildsets_query)
def test_web_substring_search_buildsets(self):
# Generate some build records in the db.
self.add_base_changes()
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
buildsets = self.get_url("api/tenant/tenant-one/buildsets?"
"project=org/*&"
"branch=*ster*&"
"pipeline=check&"
"pipeline=*ate").json()
self.assertEqual(2, len(buildsets))
self.assertEqual("org/project1", buildsets[0]["refs"][0]["project"])
self.assertEqual("org/project", buildsets[1]["refs"][0]["project"])
def test_web_list_build_times(self):
# Generate some build records in the db.
self.add_base_changes()

38
zuul/driver/sql/sqlconnection.py
View File

@@ -119,6 +119,30 @@ class DatabaseSession(object):
self.session().close()
self.session = None
def _sanitizeSubstringQuery(self, value):
if isinstance(value, str):
escape_char = "$"
for c in (escape_char, "%", "_"):
value = value.replace(c, escape_char + c)
return value
return value
def _getFuzzyFilterOp(self, column, value):
value = self._sanitizeSubstringQuery(value)
if isinstance(value, str) and "*" in value:
return column.like(value.replace("*", "%"), escape="$")
else:
return column == value
def listFilterFuzzy(self, query, column, value):
if value is None:
return query
elif isinstance(value, list) or isinstance(value, tuple):
return query.filter(
sa.or_(*[self._getFuzzyFilterOp(column, v) for v in value])
)
return query.filter(self._getFuzzyFilterOp(column, value))
def listFilter(self, query, column, value):
if value is None:
return query
@@ -191,10 +215,10 @@ class DatabaseSession(object):
dialect_name='postgresql')
q = self.listFilter(q, buildset_table.c.tenant, tenant)
q = self.listFilter(q, buildset_table.c.pipeline, pipeline)
q = self.listFilter(q, ref_table.c.project, project)
q = self.listFilterFuzzy(q, buildset_table.c.pipeline, pipeline)
q = self.listFilterFuzzy(q, ref_table.c.project, project)
q = self.listFilter(q, ref_table.c.change, change)
q = self.listFilter(q, ref_table.c.branch, branch)
q = self.listFilterFuzzy(q, ref_table.c.branch, branch)
q = self.listFilter(q, ref_table.c.patchset, patchset)
q = self.listFilter(q, ref_table.c.ref, ref)
q = self.listFilter(q, ref_table.c.newrev, newrev)
@@ -206,7 +230,7 @@ class DatabaseSession(object):
q = self.listFilter(
q, buildset_table.c.last_build_end_time, last_build_end_time)
q = self.listFilter(q, build_table.c.uuid, uuid)
q = self.listFilter(q, build_table.c.job_name, job_name)
q = self.listFilterFuzzy(q, build_table.c.job_name, job_name)
q = self.listFilter(q, build_table.c.voting, voting)
q = self.listFilter(q, build_table.c.nodeset, nodeset)
q = self.listFilter(q, build_table.c.result, result)
@@ -390,10 +414,10 @@ class DatabaseSession(object):
dialect_name='postgresql')
q = self.listFilter(q, buildset_table.c.tenant, tenant)
q = self.listFilter(q, buildset_table.c.pipeline, pipeline)
q = self.listFilter(q, ref_table.c.project, project)
q = self.listFilterFuzzy(q, buildset_table.c.pipeline, pipeline)
q = self.listFilterFuzzy(q, ref_table.c.project, project)
q = self.listFilter(q, ref_table.c.change, change)
q = self.listFilter(q, ref_table.c.branch, branch)
q = self.listFilterFuzzy(q, ref_table.c.branch, branch)
q = self.listFilter(q, ref_table.c.patchset, patchset)
q = self.listFilter(q, ref_table.c.ref, ref)
q = self.listFilter(q, ref_table.c.newrev, newrev)