zuul
/
zuul
Code Issues Proposed changes

REST API: add idx_min, idx_max params to getBuilds, getBuildsets 

Allow filtering searches per primary index; ie return only
builds or buildsets whose primary index key is greater than idx_min
or lower than idx_max. This is expected to increase queries speed
compared to using the offset argument when it is possible to do
so, since "offset" requires the database to sift through all results until
the offset is reached.

Change-Id: I420d71d7c62dad6d118310525e97b4a546f05f99
This commit is contained in:
Matthieu Huin 2021-12-02 15:56:03 +01:00
parent 31b2c94412
commit bc211ea798
3 changed files with 62 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
tests/unit/test_web.py
View File

@ -1438,6 +1438,22 @@ class TestBuildInfo(BaseTestWeb):
resp = self.get_url("api/tenant/non-tenant/builds")
self.assertEqual(404, resp.status_code)
extrema = [int(builds[-1]['_id']), int(builds[0]['_id'])]
idx_min = min(extrema)
idx_max = max(extrema)
builds_query = self.get_url("api/tenant/tenant-one/builds?"
"idx_max=%i" % idx_min).json()
self.assertEqual(len(builds_query), 1, builds_query)
builds_query = self.get_url("api/tenant/tenant-one/builds?"
"idx_min=%i" % idx_min).json()
self.assertEqual(len(builds_query), len(builds), builds_query)
builds_query = self.get_url("api/tenant/tenant-one/builds?"
"idx_max=%i" % idx_max).json()
self.assertEqual(len(builds_query), len(builds), builds_query)
builds_query = self.get_url("api/tenant/tenant-one/builds?"
"idx_min=%i" % idx_max).json()
self.assertEqual(len(builds_query), 1, builds_query)
def test_web_badge(self):
# Generate some build records in the db.
self.add_base_changes()
@ -1498,6 +1514,22 @@ class TestBuildInfo(BaseTestWeb):
if x["job_name"] == "project-merge"][0]
self.assertEqual('SUCCESS', project_merge_build['result'])
extrema = [int(buildsets[-1]['_id']), int(buildsets[0]['_id'])]
idx_min = min(extrema)
idx_max = max(extrema)
buildsets_query = self.get_url("api/tenant/tenant-one/buildsets?"
"idx_max=%i" % idx_min).json()
self.assertEqual(len(buildsets_query), 1, buildsets_query)
buildsets_query = self.get_url("api/tenant/tenant-one/buildsets?"
"idx_min=%i" % idx_min).json()
self.assertEqual(len(buildsets_query), len(buildsets), buildsets_query)
buildsets_query = self.get_url("api/tenant/tenant-one/buildsets?"
"idx_max=%i" % idx_max).json()
self.assertEqual(len(buildsets_query), len(buildsets), buildsets_query)
buildsets_query = self.get_url("api/tenant/tenant-one/buildsets?"
"idx_min=%i" % idx_max).json()
self.assertEqual(len(buildsets_query), 1, buildsets_query)
@simple_layout('layouts/empty-check.yaml')
def test_build_error(self):
conf = textwrap.dedent(

13
zuul/driver/sql/sqlconnection.py
View File

@ -63,7 +63,7 @@ class DatabaseSession(object):
job_name=None, voting=None, nodeset=None,
result=None, provides=None, final=None, held=None,
complete=None, sort_by_buildset=False, limit=50,
offset=0):
offset=0, idx_min=None, idx_max=None):
build_table = self.connection.zuul_build_table
buildset_table = self.connection.zuul_buildset_table
@ -112,6 +112,10 @@ class DatabaseSession(object):
q = q.filter(build_table.c.result == None) # noqa
q = self.listFilter(q, provides_table.c.name, provides)
q = self.listFilter(q, build_table.c.held, held)
if idx_min:
q = q.filter(build_table.c.id >= idx_min)
if idx_max:
q = q.filter(build_table.c.id <= idx_max)
if sort_by_buildset:
# If we don't need the builds to be strictly ordered, this
@ -160,7 +164,7 @@ class DatabaseSession(object):
def getBuildsets(self, tenant=None, project=None, pipeline=None,
change=None, branch=None, patchset=None, ref=None,
newrev=None, uuid=None, result=None, complete=None,
limit=50, offset=0):
limit=50, offset=0, idx_min=None, idx_max=None):
buildset_table = self.connection.zuul_buildset_table
@ -179,6 +183,11 @@ class DatabaseSession(object):
q = self.listFilter(q, buildset_table.c.newrev, newrev)
q = self.listFilter(q, buildset_table.c.uuid, uuid)
q = self.listFilter(q, buildset_table.c.result, result)
if idx_min:
q = q.filter(buildset_table.c.id >= idx_min)
if idx_max:
q = q.filter(buildset_table.c.id <= idx_max)
if complete is True:
q = q.filter(buildset_table.c.result != None) # noqa
elif complete is False:

23
zuul/web/__init__.py
View File

@ -1237,6 +1237,7 @@ class ZuulWebAPI(object):
duration = None
ret = {
'_id': build.id,
'uuid': build.uuid,
'job_name': build.job_name,
'result': build.result,
@ -1293,7 +1294,7 @@ class ZuulWebAPI(object):
branch=None, patchset=None, ref=None, newrev=None,
uuid=None, job_name=None, voting=None, nodeset=None,
result=None, final=None, held=None, complete=None,
limit=50, skip=0):
limit=50, skip=0, idx_min=None, idx_max=None):
connection = self._get_connection()
if tenant not in self.zuulweb.abide.tenants.keys():
@ -1306,12 +1307,18 @@ class ZuulWebAPI(object):
if complete is not None:
complete = complete.lower() == 'true'
try:
_idx_max = idx_max is not None and int(idx_max) or idx_max
_idx_min = idx_min is not None and int(idx_min) or idx_min
except ValueError:
raise cherrypy.HTTPError(400, 'idx_min, idx_max must be integers')
builds = connection.getBuilds(
tenant=tenant, project=project, pipeline=pipeline, change=change,
branch=branch, patchset=patchset, ref=ref, newrev=newrev,
uuid=uuid, job_name=job_name, voting=voting, nodeset=nodeset,
result=result, final=final, held=held, complete=complete,
limit=limit, offset=skip)
limit=limit, offset=skip, idx_min=_idx_min, idx_max=_idx_max)
resp = cherrypy.response
resp.headers['Access-Control-Allow-Origin'] = '*'
@ -1333,6 +1340,7 @@ class ZuulWebAPI(object):
def buildsetToDict(self, buildset, builds=[]):
ret = {
'_id': buildset.id,
'uuid': buildset.uuid,
'result': buildset.result,
'message': buildset.message,
@ -1380,17 +1388,24 @@ class ZuulWebAPI(object):
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
def buildsets(self, tenant, project=None, pipeline=None, change=None,
branch=None, patchset=None, ref=None, newrev=None,
uuid=None, result=None, complete=None, limit=50, skip=0):
uuid=None, result=None, complete=None, limit=50, skip=0,
idx_min=None, idx_max=None):
connection = self._get_connection()
if complete:
complete = complete.lower() == 'true'
try:
_idx_max = idx_max is not None and int(idx_max) or idx_max
_idx_min = idx_min is not None and int(idx_min) or idx_min
except ValueError:
raise cherrypy.HTTPError(400, 'idx_min, idx_max must be integers')
buildsets = connection.getBuildsets(
tenant=tenant, project=project, pipeline=pipeline, change=change,
branch=branch, patchset=patchset, ref=ref, newrev=newrev,
uuid=uuid, result=result, complete=complete,
limit=limit, offset=skip)
limit=limit, offset=skip, idx_min=_idx_min, idx_max=_idx_max)
resp = cherrypy.response
resp.headers['Access-Control-Allow-Origin'] = '*'