From bc211ea7982d630e385bfdd1e9cffbaa3c8f3338 Mon Sep 17 00:00:00 2001 From: Matthieu Huin Date: Thu, 2 Dec 2021 15:56:03 +0100 Subject: [PATCH] REST API: add idx_min, idx_max params to getBuilds, getBuildsets Allow filtering searches per primary index; ie return only builds or buildsets whose primary index key is greater than idx_min or lower than idx_max. This is expected to increase queries speed compared to using the offset argument when it is possible to do so, since "offset" requires the database to sift through all results until the offset is reached. Change-Id: I420d71d7c62dad6d118310525e97b4a546f05f99 --- tests/unit/test_web.py | 32 ++++++++++++++++++++++++++++++++ zuul/driver/sql/sqlconnection.py | 13 +++++++++++-- zuul/web/__init__.py | 23 +++++++++++++++++++---- 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py index 75cf070801..0968190d2a 100644 --- a/tests/unit/test_web.py +++ b/tests/unit/test_web.py @@ -1438,6 +1438,22 @@ class TestBuildInfo(BaseTestWeb): resp = self.get_url("api/tenant/non-tenant/builds") self.assertEqual(404, resp.status_code) + extrema = [int(builds[-1]['_id']), int(builds[0]['_id'])] + idx_min = min(extrema) + idx_max = max(extrema) + builds_query = self.get_url("api/tenant/tenant-one/builds?" + "idx_max=%i" % idx_min).json() + self.assertEqual(len(builds_query), 1, builds_query) + builds_query = self.get_url("api/tenant/tenant-one/builds?" + "idx_min=%i" % idx_min).json() + self.assertEqual(len(builds_query), len(builds), builds_query) + builds_query = self.get_url("api/tenant/tenant-one/builds?" + "idx_max=%i" % idx_max).json() + self.assertEqual(len(builds_query), len(builds), builds_query) + builds_query = self.get_url("api/tenant/tenant-one/builds?" + "idx_min=%i" % idx_max).json() + self.assertEqual(len(builds_query), 1, builds_query) + def test_web_badge(self): # Generate some build records in the db. self.add_base_changes() @@ -1498,6 +1514,22 @@ class TestBuildInfo(BaseTestWeb): if x["job_name"] == "project-merge"][0] self.assertEqual('SUCCESS', project_merge_build['result']) + extrema = [int(buildsets[-1]['_id']), int(buildsets[0]['_id'])] + idx_min = min(extrema) + idx_max = max(extrema) + buildsets_query = self.get_url("api/tenant/tenant-one/buildsets?" + "idx_max=%i" % idx_min).json() + self.assertEqual(len(buildsets_query), 1, buildsets_query) + buildsets_query = self.get_url("api/tenant/tenant-one/buildsets?" + "idx_min=%i" % idx_min).json() + self.assertEqual(len(buildsets_query), len(buildsets), buildsets_query) + buildsets_query = self.get_url("api/tenant/tenant-one/buildsets?" + "idx_max=%i" % idx_max).json() + self.assertEqual(len(buildsets_query), len(buildsets), buildsets_query) + buildsets_query = self.get_url("api/tenant/tenant-one/buildsets?" + "idx_min=%i" % idx_max).json() + self.assertEqual(len(buildsets_query), 1, buildsets_query) + @simple_layout('layouts/empty-check.yaml') def test_build_error(self): conf = textwrap.dedent( diff --git a/zuul/driver/sql/sqlconnection.py b/zuul/driver/sql/sqlconnection.py index 79293dda8b..70bef86c30 100644 --- a/zuul/driver/sql/sqlconnection.py +++ b/zuul/driver/sql/sqlconnection.py @@ -63,7 +63,7 @@ class DatabaseSession(object): job_name=None, voting=None, nodeset=None, result=None, provides=None, final=None, held=None, complete=None, sort_by_buildset=False, limit=50, - offset=0): + offset=0, idx_min=None, idx_max=None): build_table = self.connection.zuul_build_table buildset_table = self.connection.zuul_buildset_table @@ -112,6 +112,10 @@ class DatabaseSession(object): q = q.filter(build_table.c.result == None) # noqa q = self.listFilter(q, provides_table.c.name, provides) q = self.listFilter(q, build_table.c.held, held) + if idx_min: + q = q.filter(build_table.c.id >= idx_min) + if idx_max: + q = q.filter(build_table.c.id <= idx_max) if sort_by_buildset: # If we don't need the builds to be strictly ordered, this @@ -160,7 +164,7 @@ class DatabaseSession(object): def getBuildsets(self, tenant=None, project=None, pipeline=None, change=None, branch=None, patchset=None, ref=None, newrev=None, uuid=None, result=None, complete=None, - limit=50, offset=0): + limit=50, offset=0, idx_min=None, idx_max=None): buildset_table = self.connection.zuul_buildset_table @@ -179,6 +183,11 @@ class DatabaseSession(object): q = self.listFilter(q, buildset_table.c.newrev, newrev) q = self.listFilter(q, buildset_table.c.uuid, uuid) q = self.listFilter(q, buildset_table.c.result, result) + if idx_min: + q = q.filter(buildset_table.c.id >= idx_min) + if idx_max: + q = q.filter(buildset_table.c.id <= idx_max) + if complete is True: q = q.filter(buildset_table.c.result != None) # noqa elif complete is False: diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py index 46cd7561c7..42457ded11 100755 --- a/zuul/web/__init__.py +++ b/zuul/web/__init__.py @@ -1237,6 +1237,7 @@ class ZuulWebAPI(object): duration = None ret = { + '_id': build.id, 'uuid': build.uuid, 'job_name': build.job_name, 'result': build.result, @@ -1293,7 +1294,7 @@ class ZuulWebAPI(object): branch=None, patchset=None, ref=None, newrev=None, uuid=None, job_name=None, voting=None, nodeset=None, result=None, final=None, held=None, complete=None, - limit=50, skip=0): + limit=50, skip=0, idx_min=None, idx_max=None): connection = self._get_connection() if tenant not in self.zuulweb.abide.tenants.keys(): @@ -1306,12 +1307,18 @@ class ZuulWebAPI(object): if complete is not None: complete = complete.lower() == 'true' + try: + _idx_max = idx_max is not None and int(idx_max) or idx_max + _idx_min = idx_min is not None and int(idx_min) or idx_min + except ValueError: + raise cherrypy.HTTPError(400, 'idx_min, idx_max must be integers') + builds = connection.getBuilds( tenant=tenant, project=project, pipeline=pipeline, change=change, branch=branch, patchset=patchset, ref=ref, newrev=newrev, uuid=uuid, job_name=job_name, voting=voting, nodeset=nodeset, result=result, final=final, held=held, complete=complete, - limit=limit, offset=skip) + limit=limit, offset=skip, idx_min=_idx_min, idx_max=_idx_max) resp = cherrypy.response resp.headers['Access-Control-Allow-Origin'] = '*' @@ -1333,6 +1340,7 @@ class ZuulWebAPI(object): def buildsetToDict(self, buildset, builds=[]): ret = { + '_id': buildset.id, 'uuid': buildset.uuid, 'result': buildset.result, 'message': buildset.message, @@ -1380,17 +1388,24 @@ class ZuulWebAPI(object): @cherrypy.tools.json_out(content_type='application/json; charset=utf-8') def buildsets(self, tenant, project=None, pipeline=None, change=None, branch=None, patchset=None, ref=None, newrev=None, - uuid=None, result=None, complete=None, limit=50, skip=0): + uuid=None, result=None, complete=None, limit=50, skip=0, + idx_min=None, idx_max=None): connection = self._get_connection() if complete: complete = complete.lower() == 'true' + try: + _idx_max = idx_max is not None and int(idx_max) or idx_max + _idx_min = idx_min is not None and int(idx_min) or idx_min + except ValueError: + raise cherrypy.HTTPError(400, 'idx_min, idx_max must be integers') + buildsets = connection.getBuildsets( tenant=tenant, project=project, pipeline=pipeline, change=change, branch=branch, patchset=patchset, ref=ref, newrev=newrev, uuid=uuid, result=result, complete=complete, - limit=limit, offset=skip) + limit=limit, offset=skip, idx_min=_idx_min, idx_max=_idx_max) resp = cherrypy.response resp.headers['Access-Control-Allow-Origin'] = '*'