From dd78ef2dcc47ffa075053f67abe1d874f2b125bd Mon Sep 17 00:00:00 2001 From: Tobias Henkel Date: Thu, 6 Aug 2015 14:14:22 +0200 Subject: [PATCH] Support pagination in simplequery for Gerrit >=2.9 Starting with Gerrit 2.9 the pagination mechanism changed to using offsets instead of sortkeys. For details please check the release notes of 2.9: https://gerrit-documentation.storage.googleapis.com/ReleaseNotes/ReleaseNotes-2.9.html#_rest_api. In order to support both versions it is necessary to check for the presence of moreChanges in the status line and then process the next chunks either with the old or the new method. Change-Id: Ieb54052debd07d86db66ef0f1d7ce1bdb42964b4 --- test-requirements.txt | 1 + .../gerrit/simple_query_pagination_new_1 | 5 ++ .../gerrit/simple_query_pagination_new_2 | 4 + .../gerrit/simple_query_pagination_old_1 | 5 ++ .../gerrit/simple_query_pagination_old_2 | 4 + .../gerrit/simple_query_pagination_old_3 | 2 + tests/test_gerrit.py | 76 +++++++++++++++++++ zuul/lib/gerrit.py | 32 ++++++-- 8 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 tests/fixtures/gerrit/simple_query_pagination_new_1 create mode 100644 tests/fixtures/gerrit/simple_query_pagination_new_2 create mode 100644 tests/fixtures/gerrit/simple_query_pagination_old_1 create mode 100644 tests/fixtures/gerrit/simple_query_pagination_old_2 create mode 100644 tests/fixtures/gerrit/simple_query_pagination_old_3 create mode 100644 tests/test_gerrit.py diff --git a/test-requirements.txt b/test-requirements.txt index c68b2db0a7..d461078b56 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,3 +11,4 @@ python-swiftclient>=1.6 testrepository>=0.0.17 testtools>=0.9.32 sphinxcontrib-programoutput +mock diff --git a/tests/fixtures/gerrit/simple_query_pagination_new_1 b/tests/fixtures/gerrit/simple_query_pagination_new_1 new file mode 100644 index 0000000000..b3fdd83087 --- /dev/null +++ b/tests/fixtures/gerrit/simple_query_pagination_new_1 @@ -0,0 +1,5 @@ +gerrit query --format json --commit-message --current-patch-set project:openstack-infra/zuul +{"project":"openstack-infra/zuul","branch":"master","topic":"(detached","id":"I173251c8b1569755124b7cb1a48b6274bf38c94b","number":"202867","subject":"Report the per-job build wait time to graphite","owner":{"name":"Timothy R. Chavez","email":"timothy.chavez@hp.com","username":"timrchavez"},"url":"https://review.openstack.org/202867","commitMessage":"Report the per-job build wait time to graphite\n\nKnowing how long a job waits to build in aggregate can give useful\ninsights into the performance and capacity of the build system. This\nchange also uses the node labels sent back from the gearman worker to\nsubmit metrics within that context.\n\nChange-Id: I173251c8b1569755124b7cb1a48b6274bf38c94b\nDepends-On: Ibca938fcf8a65facd7e39dab4eb994dfc637722a\n","createdOn":1437104683,"lastUpdated":1440760891,"open":true,"status":"NEW"} +{"project":"openstack-infra/zuul","branch":"master","topic":"ignore-deletes","id":"Iea75d05ddcb49b0bf748b72b9d2d5472d077f0c6","number":"178833","subject":"Add option to ignore ref-updated events emitted by branch deletions","owner":{"name":"K Jonathan Harker","email":"code@gentlydownthe.net","username":"jesusaurus"},"url":"https://review.openstack.org/178833","commitMessage":"Add option to ignore ref-updated events emitted by branch deletions\n\nWhen a branch is deleted, gerrit emits a ref-updated event with a newrev\nvalue of all zeros. This adds a boolean field to optionally not trigger\non these ref-updated events.\n\nChange-Id: Iea75d05ddcb49b0bf748b72b9d2d5472d077f0c6\n","createdOn":1430339761,"lastUpdated":1440735750,"open":true,"status":"NEW"} +{"project":"openstack-infra/zuul","branch":"master","topic":"undefined-projects","id":"I7912197fb86c1a7becb7f43ca36078101f632715","number":"207094","subject":"Dependencies from undefined projects","owner":{"name":"Evgeny Antyshev","email":"eantyshev@virtuozzo.com","username":"eantyshev"},"url":"https://review.openstack.org/207094","commitMessage":"Dependencies from undefined projects\n\n3rd party CI layout usually has only a few projects defined,\nso it\u0027s possible that some changes depend on projects\nwhich are unknown to Zuul scheduler.\nThese items had None as a \"item.change.project\", which\nis not handled in many places, for ex. in reconfiguration.\n\nThese cases could be handled by defining these projects in layout\nas \"foreign\" projects: no jobs, no other non-standard attributes.\nChanges to those projects are also dropped, unless\nthey came as dependencies.\n\nChange-Id: I7912197fb86c1a7becb7f43ca36078101f632715\n","createdOn":1438183395,"lastUpdated":1440667433,"open":true,"status":"NEW"} +{"type":"stats","rowCount":3,"runTimeMilliseconds":12,"moreChanges":true} \ No newline at end of file diff --git a/tests/fixtures/gerrit/simple_query_pagination_new_2 b/tests/fixtures/gerrit/simple_query_pagination_new_2 new file mode 100644 index 0000000000..9fd8d54adb --- /dev/null +++ b/tests/fixtures/gerrit/simple_query_pagination_new_2 @@ -0,0 +1,4 @@ +gerrit query --format json --commit-message --current-patch-set project:openstack-infra/zuul -S 3 +{"project":"openstack-infra/zuul","branch":"master","topic":"github-integration","id":"I95f41088ea160d4e33a507c4a413e3fa7f08906b","number":"192457","subject":"(WIP) Fix job hierarchy bug.","owner":{"name":"Wayne Warren","email":"waynr+launchpad@sdf.org","username":"waynr"},"url":"https://review.openstack.org/192457","commitMessage":"(WIP) Fix job hierarchy bug.\n\nJobTree.addJob may return \u0027None\u0027, this prevents that from happening.\n\nChange-Id: I95f41088ea160d4e33a507c4a413e3fa7f08906b\n","createdOn":1434498278,"lastUpdated":1440608984,"sortKey":"003763050002efc9","open":true,"status":"NEW"} +{"project":"openstack-infra/zuul","branch":"master","topic":"github-integration","id":"Ic5887d00ff302f67469df5154e9df10b99f1cfcd","number":"215642","subject":"(WIP) Allow using webapp from connections","owner":{"name":"Jan Hruban","email":"jan.hruban@gooddata.com","username":"hrubi"},"url":"https://review.openstack.org/215642","commitMessage":"(WIP) Allow using webapp from connections\n\nAllow connections to register their own handlers for HTTP URIs inside\nthe zuul\u0027s webapp HTTP server. That way, connections can listen for\nevents comming through HTTP.\n\nChange-Id: Ic5887d00ff302f67469df5154e9df10b99f1cfcd\n","createdOn":1440165019,"lastUpdated":1440602591,"sortKey":"0037629b00034a5a","open":true,"status":"NEW"} +{"type":"stats","rowCount":2,"runTimeMilliseconds":12,"moreChanges":false} \ No newline at end of file diff --git a/tests/fixtures/gerrit/simple_query_pagination_old_1 b/tests/fixtures/gerrit/simple_query_pagination_old_1 new file mode 100644 index 0000000000..8ff1710115 --- /dev/null +++ b/tests/fixtures/gerrit/simple_query_pagination_old_1 @@ -0,0 +1,5 @@ +gerrit query --format json --commit-message --current-patch-set project:openstack-infra/zuul +{"project":"openstack-infra/zuul","branch":"master","topic":"(detached","id":"I173251c8b1569755124b7cb1a48b6274bf38c94b","number":"202867","subject":"Report the per-job build wait time to graphite","owner":{"name":"Timothy R. Chavez","email":"timothy.chavez@hp.com","username":"timrchavez"},"url":"https://review.openstack.org/202867","commitMessage":"Report the per-job build wait time to graphite\n\nKnowing how long a job waits to build in aggregate can give useful\ninsights into the performance and capacity of the build system. This\nchange also uses the node labels sent back from the gearman worker to\nsubmit metrics within that context.\n\nChange-Id: I173251c8b1569755124b7cb1a48b6274bf38c94b\nDepends-On: Ibca938fcf8a65facd7e39dab4eb994dfc637722a\n","createdOn":1437104683,"lastUpdated":1440760891,"sortKey":"00376ce900031873","open":true,"status":"NEW"} +{"project":"openstack-infra/zuul","branch":"master","topic":"ignore-deletes","id":"Iea75d05ddcb49b0bf748b72b9d2d5472d077f0c6","number":"178833","subject":"Add option to ignore ref-updated events emitted by branch deletions","owner":{"name":"K Jonathan Harker","email":"code@gentlydownthe.net","username":"jesusaurus"},"url":"https://review.openstack.org/178833","commitMessage":"Add option to ignore ref-updated events emitted by branch deletions\n\nWhen a branch is deleted, gerrit emits a ref-updated event with a newrev\nvalue of all zeros. This adds a boolean field to optionally not trigger\non these ref-updated events.\n\nChange-Id: Iea75d05ddcb49b0bf748b72b9d2d5472d077f0c6\n","createdOn":1430339761,"lastUpdated":1440735750,"sortKey":"00376b460002ba91","open":true,"status":"NEW"} +{"project":"openstack-infra/zuul","branch":"master","topic":"undefined-projects","id":"I7912197fb86c1a7becb7f43ca36078101f632715","number":"207094","subject":"Dependencies from undefined projects","owner":{"name":"Evgeny Antyshev","email":"eantyshev@virtuozzo.com","username":"eantyshev"},"url":"https://review.openstack.org/207094","commitMessage":"Dependencies from undefined projects\n\n3rd party CI layout usually has only a few projects defined,\nso it\u0027s possible that some changes depend on projects\nwhich are unknown to Zuul scheduler.\nThese items had None as a \"item.change.project\", which\nis not handled in many places, for ex. in reconfiguration.\n\nThese cases could be handled by defining these projects in layout\nas \"foreign\" projects: no jobs, no other non-standard attributes.\nChanges to those projects are also dropped, unless\nthey came as dependencies.\n\nChange-Id: I7912197fb86c1a7becb7f43ca36078101f632715\n","createdOn":1438183395,"lastUpdated":1440667433,"sortKey":"003766d3000328f6","open":true,"status":"NEW"} +{"type":"stats","rowCount":3,"runTimeMilliseconds":12} \ No newline at end of file diff --git a/tests/fixtures/gerrit/simple_query_pagination_old_2 b/tests/fixtures/gerrit/simple_query_pagination_old_2 new file mode 100644 index 0000000000..c55cd4009a --- /dev/null +++ b/tests/fixtures/gerrit/simple_query_pagination_old_2 @@ -0,0 +1,4 @@ +gerrit query --format json --commit-message --current-patch-set project:openstack-infra/zuul resume_sortkey:'003766d3000328f6' +{"project":"openstack-infra/zuul","branch":"master","topic":"github-integration","id":"I95f41088ea160d4e33a507c4a413e3fa7f08906b","number":"192457","subject":"(WIP) Fix job hierarchy bug.","owner":{"name":"Wayne Warren","email":"waynr+launchpad@sdf.org","username":"waynr"},"url":"https://review.openstack.org/192457","commitMessage":"(WIP) Fix job hierarchy bug.\n\nJobTree.addJob may return \u0027None\u0027, this prevents that from happening.\n\nChange-Id: I95f41088ea160d4e33a507c4a413e3fa7f08906b\n","createdOn":1434498278,"lastUpdated":1440608984,"sortKey":"003763050002efc9","open":true,"status":"NEW"} +{"project":"openstack-infra/zuul","branch":"master","topic":"github-integration","id":"Ic5887d00ff302f67469df5154e9df10b99f1cfcd","number":"215642","subject":"(WIP) Allow using webapp from connections","owner":{"name":"Jan Hruban","email":"jan.hruban@gooddata.com","username":"hrubi"},"url":"https://review.openstack.org/215642","commitMessage":"(WIP) Allow using webapp from connections\n\nAllow connections to register their own handlers for HTTP URIs inside\nthe zuul\u0027s webapp HTTP server. That way, connections can listen for\nevents comming through HTTP.\n\nChange-Id: Ic5887d00ff302f67469df5154e9df10b99f1cfcd\n","createdOn":1440165019,"lastUpdated":1440602591,"sortKey":"0037629b00034a5a","open":true,"status":"NEW"} +{"type":"stats","rowCount":2,"runTimeMilliseconds":12} \ No newline at end of file diff --git a/tests/fixtures/gerrit/simple_query_pagination_old_3 b/tests/fixtures/gerrit/simple_query_pagination_old_3 new file mode 100644 index 0000000000..b8cdc4a552 --- /dev/null +++ b/tests/fixtures/gerrit/simple_query_pagination_old_3 @@ -0,0 +1,2 @@ +gerrit query --format json --commit-message --current-patch-set project:openstack-infra/zuul resume_sortkey:'0037629b00034a5a' +{"type":"stats","rowCount":0,"runTimeMilliseconds":12} \ No newline at end of file diff --git a/tests/test_gerrit.py b/tests/test_gerrit.py new file mode 100644 index 0000000000..dad2c6d487 --- /dev/null +++ b/tests/test_gerrit.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +# Copyright 2015 BMW Car IT GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import os + +try: + from unittest import mock +except ImportError: + import mock + +from tests.base import BaseTestCase +from zuul.lib.gerrit import Gerrit + +FIXTURE_DIR = os.path.join(os.path.dirname(__file__), 'fixtures/gerrit') + + +def read_fixture(file): + with open('%s/%s' % (FIXTURE_DIR, file), 'r') as fixturefile: + lines = fixturefile.readlines() + command = lines[0].replace('\n', '') + value = ''.join(lines[1:]) + return command, value + + +def read_fixtures(files): + calls = [] + values = [] + for fixture_file in files: + command, value = read_fixture(fixture_file) + calls.append(mock.call(command)) + values.append([value, '']) + return calls, values + + +class TestGerrit(BaseTestCase): + + @mock.patch('zuul.lib.gerrit.Gerrit._ssh') + def run_query(self, files, expected_patches, _ssh_mock): + gerrit = Gerrit('localhost', 'user') + + calls, values = read_fixtures(files) + _ssh_mock.side_effect = values + + result = gerrit.simpleQuery('project:openstack-infra/zuul') + + _ssh_mock.assert_has_calls(calls) + self.assertEquals(len(calls), _ssh_mock.call_count, + '_ssh should be called %d times' % len(calls)) + self.assertIsNotNone(result, 'Result is not none') + self.assertEquals(len(result), expected_patches, + 'There must be %d patches.' % expected_patches) + + def test_simple_query_pagination_new(self): + files = ['simple_query_pagination_new_1', + 'simple_query_pagination_new_2'] + expected_patches = 5 + self.run_query(files, expected_patches) + + def test_simple_query_pagination_old(self): + files = ['simple_query_pagination_old_1', + 'simple_query_pagination_old_2', + 'simple_query_pagination_old_3'] + expected_patches = 5 + self.run_query(files, expected_patches) diff --git a/zuul/lib/gerrit.py b/zuul/lib/gerrit.py index 90faf40998..5821a2d564 100644 --- a/zuul/lib/gerrit.py +++ b/zuul/lib/gerrit.py @@ -156,22 +156,42 @@ class Gerrit(object): lines = out.split('\n') if not lines: return False + + # filter out blank lines data = [json.loads(line) for line in lines - if "sortKey" in line] + if line.startswith('{')] + + # check last entry for more changes + more_changes = None + if 'moreChanges' in data[-1]: + more_changes = data[-1]['moreChanges'] + + # we have to remove the statistics line + del data[-1] + if not data: - return False + return False, more_changes self.log.debug("Received data from Gerrit query: \n%s" % (pprint.pformat(data))) - return data + return data, more_changes # gerrit returns 500 results by default, so implement paging # for large projects like nova alldata = [] - chunk = _query_chunk(query) + chunk, more_changes = _query_chunk(query) while(chunk): alldata.extend(chunk) - sortkey = "resume_sortkey:'%s'" % chunk[-1]["sortKey"] - chunk = _query_chunk("%s %s" % (query, sortkey)) + if more_changes is None: + # continue sortKey based (before Gerrit 2.9) + resume = "resume_sortkey:'%s'" % chunk[-1]["sortKey"] + elif more_changes: + # continue moreChanges based (since Gerrit 2.9) + resume = "-S %d" % len(alldata) + else: + # no more changes + break + + chunk, more_changes = _query_chunk("%s %s" % (query, resume)) return alldata def _open(self):